From 8feb3fee986917a5843981b07537bfb157dd7222 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Mon, 6 Feb 2017 16:34:11 -0500 Subject: [PATCH] Removed Wombles and Kickass, updated torrentpotato and torznab (#625) * Remove Wombles and kickass * Clean up TorrentPotato Request Gen * Opps, we need to use SupportedMovieSearchParameters xD * Consistancy * Clean up Newznab Request Gen, add year to search if cannot use IMDB * Added SpecialCharRegex to remove \ / : & --- .../IndexerTests/IndexerServiceFixture.cs | 2 - .../IndexerIntegrationTests.cs | 54 ----- .../KickassTorrentsFixture.cs | 173 --------------- .../WomblesTests/WomblesFixture.cs | 59 ------ .../NzbDrone.Core.Test.csproj | 2 - .../Datastore/Migration/127_remove_wombles.cs | 14 ++ .../Datastore/Migration/128_remove_kickass.cs | 14 ++ .../KickassTorrents/KickassTorrents.cs | 31 --- .../KickassTorrentsRequestGenerator.cs | 157 -------------- .../KickassTorrentsRssParser.cs | 44 ---- .../KickassTorrentsSettings.cs | 37 ---- .../Newznab/NewznabRequestGenerator.cs | 197 +----------------- .../Indexers/RssIndexerRequestGenerator.cs | 4 +- .../TorrentPotatoRequestGenerator.cs | 38 +--- src/NzbDrone.Core/Indexers/Torznab/Torznab.cs | 4 +- src/NzbDrone.Core/Indexers/Wombles/Wombles.cs | 36 ---- .../Indexers/Wombles/WomblesRssParser.cs | 20 -- src/NzbDrone.Core/NzbDrone.Core.csproj | 7 +- src/NzbDrone.Core/Parser/Parser.cs | 2 + 19 files changed, 47 insertions(+), 848 deletions(-) delete mode 100644 src/NzbDrone.Core.Test/IndexerTests/KickassTorrentsTests/KickassTorrentsFixture.cs delete mode 100644 src/NzbDrone.Core.Test/IndexerTests/WomblesTests/WomblesFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/127_remove_wombles.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/128_remove_kickass.cs delete mode 100644 src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs delete mode 100644 src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs delete mode 100644 src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRssParser.cs delete mode 100644 src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsSettings.cs delete mode 100644 src/NzbDrone.Core/Indexers/Wombles/Wombles.cs delete mode 100644 src/NzbDrone.Core/Indexers/Wombles/WomblesRssParser.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs index eceb25b11..ca1f67113 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerServiceFixture.cs @@ -5,7 +5,6 @@ using NUnit.Framework; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Newznab; using NzbDrone.Core.Indexers.Omgwtfnzbs; -using NzbDrone.Core.Indexers.Wombles; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Test.Framework; @@ -22,7 +21,6 @@ namespace NzbDrone.Core.Test.IndexerTests _indexers.Add(Mocker.Resolve()); _indexers.Add(Mocker.Resolve()); - _indexers.Add(Mocker.Resolve()); Mocker.SetConstant>(_indexers); } diff --git a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs index 2074e5cb2..a72bb8f57 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IntegrationTests/IndexerIntegrationTests.cs @@ -4,9 +4,7 @@ using FluentAssertions; using NUnit.Framework; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; -using NzbDrone.Core.Indexers.KickassTorrents; using NzbDrone.Core.Indexers.Nyaa; -using NzbDrone.Core.Indexers.Wombles; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; @@ -40,58 +38,6 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests }; } - [Test] - public void wombles_fetch_recent() - { - var indexer = Mocker.Resolve(); - - indexer.Definition = new IndexerDefinition - { - Name = "MyIndexer", - Settings = NullConfig.Instance - }; - - var result = indexer.FetchRecent(); - - ValidateResult(result); - } - - [Test] - [ManualTest] - [Explicit] - public void kickass_fetch_recent() - { - var indexer = Mocker.Resolve(); - - indexer.Definition = new IndexerDefinition - { - Name = "MyIndexer", - Settings = new KickassTorrentsSettings() - }; - - var result = indexer.FetchRecent(); - - ValidateTorrentResult(result, hasSize: true); - } - - [Test] - [ManualTest] - [Explicit] - public void kickass_search_single() - { - var indexer = Mocker.Resolve(); - - indexer.Definition = new IndexerDefinition - { - Name = "MyIndexer", - Settings = new KickassTorrentsSettings() - }; - - var result = indexer.Fetch(_singleSearchCriteria); - - ValidateTorrentResult(result, hasSize: true, hasMagnet: true); - } - [Test] public void nyaa_fetch_recent() { diff --git a/src/NzbDrone.Core.Test/IndexerTests/KickassTorrentsTests/KickassTorrentsFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/KickassTorrentsTests/KickassTorrentsFixture.cs deleted file mode 100644 index 8da5e572f..000000000 --- a/src/NzbDrone.Core.Test/IndexerTests/KickassTorrentsTests/KickassTorrentsFixture.cs +++ /dev/null @@ -1,173 +0,0 @@ -using Moq; -using NUnit.Framework; -using NzbDrone.Common.Http; -using NzbDrone.Core.Indexers; -using NzbDrone.Core.Indexers.KickassTorrents; -using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.Test.Framework; -using NzbDrone.Test.Common; -using System; -using System.Linq; -using FluentAssertions; -using System.Text.RegularExpressions; - -namespace NzbDrone.Core.Test.IndexerTests.KickassTorrentsTests -{ - [TestFixture] - public class KickassTorrentsFixture : CoreTest - { - [SetUp] - public void Setup() - { - Subject.Definition = new IndexerDefinition() - { - Name = "Kickass Torrents", - Settings = new KickassTorrentsSettings() { VerifiedOnly = false } - }; - } - - [Test] - public void should_parse_recent_feed_from_KickassTorrents() - { - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents.xml"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - releases.First().Should().BeOfType(); - - var torrentInfo = (TorrentInfo) releases.First(); - - torrentInfo.Title.Should().Be("Doctor Stranger.E03.140512.HDTV.H264.720p-iPOP.avi [CTRG]"); - torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); - torrentInfo.DownloadUrl.Should().Be("http://torcache.net/torrent/208C4F7866612CC88BFEBC7C496FA72C2368D1C0.torrent?title=%5Bkickass.to%5Ddoctor.stranger.e03.140512.hdtv.h264.720p.ipop.avi.ctrg"); - torrentInfo.InfoUrl.Should().Be("http://kickass.to/doctor-stranger-e03-140512-hdtv-h264-720p-ipop-avi-ctrg-t9100648.html"); - torrentInfo.CommentUrl.Should().BeNullOrEmpty(); - torrentInfo.Indexer.Should().Be(Subject.Definition.Name); - torrentInfo.PublishDate.Should().Be(DateTime.Parse("2014/05/12 16:16:49")); - torrentInfo.Size.Should().Be(1205364736); - torrentInfo.InfoHash.Should().Be("208C4F7866612CC88BFEBC7C496FA72C2368D1C0"); - torrentInfo.MagnetUrl.Should().Be("magnet:?xt=urn:btih:208C4F7866612CC88BFEBC7C496FA72C2368D1C0&dn=doctor+stranger+e03+140512+hdtv+h264+720p+ipop+avi+ctrg&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce"); - } - - [Test] - public void should_return_empty_list_on_404() - { - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), new byte[0], System.Net.HttpStatusCode.NotFound)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(0); - - ExceptionVerification.IgnoreWarns(); - } - - [Test] - public void should_not_return_unverified_releases_if_not_configured() - { - ((KickassTorrentsSettings) Subject.Definition.Settings).VerifiedOnly = true; - - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents.xml"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(4); - } - - [Test] - public void should_set_seeders_to_null() - { - // Atm, Kickass supplies 0 as seeders and leechers on the rss feed (but not the site), so set it to null if there aren't any peers. - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents.xml"); - - recentFeed = recentFeed.Replace("Mon, 12 May 2014 16:16:49 +0000", string.Format("{0:R}", DateTime.UtcNow)); - recentFeed = Regex.Replace(recentFeed, @"(seeds|peers)\>\d*", "$1>0"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - releases.First().Should().BeOfType(); - - var torrentInfo = (TorrentInfo)releases.First(); - - torrentInfo.Peers.Should().NotHaveValue(); - torrentInfo.Seeders.Should().NotHaveValue(); - } - - [Test] - public void should_not_set_seeders_to_null_if_has_peers() - { - // Atm, Kickass supplies 0 as seeders and leechers on the rss feed (but not the site), so set it to null if there aren't any peers. - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents.xml"); - - recentFeed = recentFeed.Replace("Mon, 12 May 2014 16:16:49 +0000", string.Format("{0:R}", DateTime.UtcNow)); - recentFeed = Regex.Replace(recentFeed, @"(seeds)\>\d*", "$1>0"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - releases.First().Should().BeOfType(); - - var torrentInfo = (TorrentInfo)releases.First(); - - torrentInfo.Peers.Should().Be(311); - torrentInfo.Seeders.Should().Be(0); - } - - [Test] - public void should_not_set_seeders_to_null_if_older_than_12_hours() - { - // Atm, Kickass supplies 0 as seeders and leechers on the rss feed (but not the site), so set it to null if there aren't any peers. - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents.xml"); - - recentFeed = Regex.Replace(recentFeed, @"(seeds|peers)\>\d*", "$1>0"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - releases.First().Should().BeOfType(); - - var torrentInfo = (TorrentInfo)releases.First(); - - torrentInfo.Peers.Should().Be(0); - torrentInfo.Seeders.Should().Be(0); - } - - - [Test] - public void should_handle_xml_with_html_accents() - { - var recentFeed = ReadAllText(@"Files/Indexers/KickassTorrents/KickassTorrents_accents.xml"); - - Mocker.GetMock() - .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - } - } -} diff --git a/src/NzbDrone.Core.Test/IndexerTests/WomblesTests/WomblesFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/WomblesTests/WomblesFixture.cs deleted file mode 100644 index 0c48c1529..000000000 --- a/src/NzbDrone.Core.Test/IndexerTests/WomblesTests/WomblesFixture.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System; -using System.Linq; -using FluentAssertions; -using Moq; -using NUnit.Framework; -using NzbDrone.Common.Http; -using NzbDrone.Core.Indexers; -using NzbDrone.Core.Indexers.Wombles; -using NzbDrone.Core.Test.Framework; -using NzbDrone.Core.ThingiProvider; - -namespace NzbDrone.Core.Test.IndexerTests.WomblesTests -{ - - [TestFixture] - public class TorrentRssIndexerFixture : CoreTest - { - [SetUp] - public void Setup() - { - - Subject.Definition = new IndexerDefinition() - { - Name = "Wombles", - Settings = new NullConfig(), - }; - } - - private void GivenRecentFeedResponse(string rssXmlFile) - { - var recentFeed = ReadAllText(@"Files/Indexers/" + rssXmlFile); - - Mocker.GetMock() - .Setup(o => o.Execute(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - } - - [Test] - public void should_parse_recent_feed_from_wombles() - { - GivenRecentFeedResponse("Wombles/wombles.xml"); - - var releases = Subject.FetchRecent(); - - releases.Should().HaveCount(5); - - var releaseInfo = releases.First(); - - releaseInfo.Title.Should().Be("One.Child.S01E01.720p.HDTV.x264-TLA"); - releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet); - releaseInfo.DownloadUrl.Should().Be("http://indexer.local/nzb/bb4/One.Child.S01E01.720p.HDTV.x264-TLA.nzb"); - releaseInfo.InfoUrl.Should().BeNullOrEmpty(); - releaseInfo.CommentUrl.Should().BeNullOrEmpty(); - releaseInfo.Indexer.Should().Be(Subject.Definition.Name); - releaseInfo.PublishDate.Should().Be(DateTime.Parse("2016-02-17 23:03:52 +0000").ToUniversalTime()); - releaseInfo.Size.Should().Be(956*1024*1024); - } - } -} diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 8cf473015..7dcf7ae33 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -257,11 +257,9 @@ - - diff --git a/src/NzbDrone.Core/Datastore/Migration/127_remove_wombles.cs b/src/NzbDrone.Core/Datastore/Migration/127_remove_wombles.cs new file mode 100644 index 000000000..f72e80238 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/127_remove_wombles.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(127)] + public class remove_wombles : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Delete.FromTable("Indexers").Row(new { Implementation = "Wombles" }); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/128_remove_kickass.cs b/src/NzbDrone.Core/Datastore/Migration/128_remove_kickass.cs new file mode 100644 index 000000000..c290ac406 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/128_remove_kickass.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(128)] + public class remove_kickass : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Delete.FromTable("Indexers").Row(new { Implementation = "Kickass Torrents" }); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs deleted file mode 100644 index 328b7ddde..000000000 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrents.cs +++ /dev/null @@ -1,31 +0,0 @@ -using NLog; -using NzbDrone.Common.Http; -using NzbDrone.Core.Configuration; -using NzbDrone.Core.Parser; - -namespace NzbDrone.Core.Indexers.KickassTorrents -{ - public class KickassTorrents : HttpIndexerBase - { - public override string Name => "Kickass Torrents"; - - public override DownloadProtocol Protocol => DownloadProtocol.Torrent; - public override int PageSize => 25; - - public KickassTorrents(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, indexerStatusService, configService, parsingService, logger) - { - - } - - public override IIndexerRequestGenerator GetRequestGenerator() - { - return new KickassTorrentsRequestGenerator() { Settings = Settings, PageSize = PageSize }; - } - - public override IParseIndexerResponse GetParser() - { - return new KickassTorrentsRssParser() { Settings = Settings }; - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs deleted file mode 100644 index 9caaa1685..000000000 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRequestGenerator.cs +++ /dev/null @@ -1,157 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using NzbDrone.Common.Http; -using NzbDrone.Core.IndexerSearch.Definitions; - -namespace NzbDrone.Core.Indexers.KickassTorrents -{ - public class KickassTorrentsRequestGenerator : IIndexerRequestGenerator - { - public KickassTorrentsSettings Settings { get; set; } - - public int MaxPages { get; set; } - public int PageSize { get; set; } - - public KickassTorrentsRequestGenerator() - { - MaxPages = 30; - PageSize = 25; - } - - public virtual IndexerPageableRequestChain GetRecentRequests() - { - var pageableRequests = new IndexerPageableRequestChain(); - - pageableRequests.Add(GetPagedRequests(MaxPages, "tv")); - - return pageableRequests; - } - - public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) - { - var pageableRequests = new IndexerPageableRequestChain(); - - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - "category:tv", - string.Format("season:{0}", searchCriteria.SeasonNumber), - string.Format("episode:{0}", searchCriteria.EpisodeNumber))); - - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - string.Format("S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber), - "category:tv")); - } - - return pageableRequests; - } - - public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) - { - var pageableRequests = new IndexerPageableRequestChain(); - - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - "category:tv", - string.Format("season:{0}", searchCriteria.SeasonNumber))); - - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - "category:tv", - string.Format("S{0:00}", searchCriteria.SeasonNumber))); - - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - "category:tv", - string.Format("Season {0}", searchCriteria.SeasonNumber))); - } - - return pageableRequests; - } - - public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) - { - var pageableRequests = new IndexerPageableRequestChain(); - - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - string.Format("{0:yyyy-MM-dd}", searchCriteria.AirDate), - "category:tv")); - } - - return pageableRequests; - } - - public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) - { - return new IndexerPageableRequestChain(); - } - - public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) - { - var pageableRequests = new IndexerPageableRequestChain(); - - foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) - { - pageableRequests.Add(GetPagedRequests(MaxPages, "usearch", - PrepareQuery(queryTitle), - "category:tv")); - } - - return pageableRequests; - } - - private IEnumerable GetPagedRequests(int maxPages, string rssType, params string[] searchParameters) - { - string searchUrl = null; - - if (searchParameters.Any()) - { - // Prevent adding a '/' if no search parameters are specified - if (Settings.VerifiedOnly) - { - searchUrl = string.Format("/{0} verified:1", string.Join(" ", searchParameters)); - } - else - { - searchUrl = string.Format("/{0}", string.Join(" ", searchParameters)).Trim(); - } - } - - if (PageSize == 0) - { - var request = new IndexerRequest(string.Format("{0}/{1}{2}/?rss=1&field=time_add&sorder=desc", Settings.BaseUrl.TrimEnd('/'), rssType, searchUrl), HttpAccept.Rss); - request.HttpRequest.SuppressHttpError = true; - - yield return request; - } - else - { - for (var page = 0; page < maxPages; page++) - { - var request = new IndexerRequest(string.Format("{0}/{1}{2}/{3}/?rss=1&field=time_add&sorder=desc", Settings.BaseUrl.TrimEnd('/'), rssType, searchUrl, page + 1), HttpAccept.Rss); - request.HttpRequest.SuppressHttpError = true; - - yield return request; - } - } - } - - private string PrepareQuery(string query) - { - return query.Replace('+', ' '); - } - - public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) - { - return new IndexerPageableRequestChain(); - } - } -} diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRssParser.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRssParser.cs deleted file mode 100644 index 7e9e58235..000000000 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsRssParser.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.Linq; -using System.Xml.Linq; -using NzbDrone.Common.Extensions; -using NzbDrone.Core.Parser.Model; - -namespace NzbDrone.Core.Indexers.KickassTorrents -{ - public class KickassTorrentsRssParser : EzrssTorrentRssParser - { - public KickassTorrentsSettings Settings { get; set; } - - protected override bool PreProcess(IndexerResponse indexerResponse) - { - if (indexerResponse.HttpResponse.StatusCode == System.Net.HttpStatusCode.NotFound) - { - return false; - } - - return base.PreProcess(indexerResponse); - } - - protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) - { - var verified = item.FindDecendants("verified").SingleOrDefault(); - - if (Settings != null && Settings.VerifiedOnly && (string)verified == "0") - { - return null; - } - - // Atm, Kickass supplies 0 as seeders and leechers on the rss feed for recent releases, so set it to null if there aren't any peers. - // But only for releases younger than 12h (the real number seems to be close to 14h, but it depends on a number of factors). - var torrentInfo = releaseInfo as TorrentInfo; - if (torrentInfo.Peers.HasValue && torrentInfo.Peers.Value == 0 && torrentInfo.PublishDate > DateTime.UtcNow.AddHours(-12)) - { - torrentInfo.Seeders = null; - torrentInfo.Peers = null; - } - - return base.PostProcess(item, releaseInfo); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsSettings.cs b/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsSettings.cs deleted file mode 100644 index c795fb5bc..000000000 --- a/src/NzbDrone.Core/Indexers/KickassTorrents/KickassTorrentsSettings.cs +++ /dev/null @@ -1,37 +0,0 @@ -using FluentValidation; -using NzbDrone.Core.Annotations; -using NzbDrone.Core.ThingiProvider; -using NzbDrone.Core.Validation; - -namespace NzbDrone.Core.Indexers.KickassTorrents -{ - public class KickassTorrentsSettingsValidator : AbstractValidator - { - public KickassTorrentsSettingsValidator() - { - RuleFor(c => c.BaseUrl).ValidRootUrl(); - } - } - - public class KickassTorrentsSettings : IProviderConfig - { - private static readonly KickassTorrentsSettingsValidator Validator = new KickassTorrentsSettingsValidator(); - - public KickassTorrentsSettings() - { - BaseUrl = ""; - VerifiedOnly = true; - } - - [FieldDefinition(0, Label = "Website URL", HelpText = "Please verify that the url you enter is a trustworthy site.")] - public string BaseUrl { get; set; } - - [FieldDefinition(1, Label = "Verified Only", Type = FieldType.Checkbox, HelpText = "By setting this to No you will likely get more junk and unconfirmed releases, so use it with caution.")] - public bool VerifiedOnly { get; set; } - - public NzbDroneValidationResult Validate() - { - return new NzbDroneValidationResult(Validator.Validate(this)); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 040276ed2..1ad2dd579 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -22,69 +22,6 @@ namespace NzbDrone.Core.Indexers.Newznab PageSize = 100; } - private bool SupportsSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportedSearchParameters != null && - capabilities.SupportedSearchParameters.Contains("q"); - } - } - - private bool SupportsTvSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportedTvSearchParameters != null && - capabilities.SupportedTvSearchParameters.Contains("q") && - capabilities.SupportedTvSearchParameters.Contains("season") && - capabilities.SupportedTvSearchParameters.Contains("ep"); - } - } - - private bool SupportsTvdbSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportedTvSearchParameters != null && - capabilities.SupportedTvSearchParameters.Contains("tvdbid") && - capabilities.SupportedTvSearchParameters.Contains("season") && - capabilities.SupportedTvSearchParameters.Contains("ep"); - } - } - - private bool SupportsTvRageSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportedTvSearchParameters != null && - capabilities.SupportedTvSearchParameters.Contains("rid") && - capabilities.SupportedTvSearchParameters.Contains("season") && - capabilities.SupportedTvSearchParameters.Contains("ep"); - } - } - - private bool SupportsTvMazeSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportedTvSearchParameters != null && - capabilities.SupportedTvSearchParameters.Contains("tvmazeid") && - capabilities.SupportedTvSearchParameters.Contains("season") && - capabilities.SupportedTvSearchParameters.Contains("ep"); - } - } - private bool SupportsMovieSearch { get @@ -96,16 +33,6 @@ namespace NzbDrone.Core.Indexers.Newznab } } - private bool SupportsAggregatedIdSearch - { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.SupportsAggregateIdSearch; - } - } - public virtual IndexerPageableRequestChain GetRecentRequests() { var pageableRequests = new IndexerPageableRequestChain(); @@ -126,14 +53,11 @@ namespace NzbDrone.Core.Indexers.Newznab if (SupportsMovieSearch) { - pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "movie", - string.Format("&imdbid={0}", searchCriteria.Movie.ImdbId.Substring(2)))); //strip off the "tt" - VERY HACKY + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "movie", $"&imdbid={searchCriteria.Movie.ImdbId.Substring(2)}")); } else { - //Let's try anyways with q parameter, worst case nothing found. - pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", - string.Format("&q={0}", Parser.Parser.NormalizeTitle(searchCriteria.Movie.Title)))); + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", $"&q={Parser.Parser.NormalizeTitle(searchCriteria.Movie.Title)}%20{searchCriteria.Movie.Year}")); } return pageableRequests; @@ -141,134 +65,27 @@ namespace NzbDrone.Core.Indexers.Newznab public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, - string.Format("&season={0}&ep={1}", - searchCriteria.SeasonNumber, - searchCriteria.EpisodeNumber)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, - string.Format("&season={0}", - searchCriteria.SeasonNumber)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - AddTvIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria, - string.Format("&season={0:yyyy}&ep={0:MM}/{0:dd}", - searchCriteria.AirDate)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - if (SupportsSearch) - { - foreach (var queryTitle in searchCriteria.QueryTitles) - { - pageableRequests.Add(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", - string.Format("&q={0}+{1:00}", - NewsnabifyTitle(queryTitle), - searchCriteria.AbsoluteEpisodeNumber))); - } - } - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - if (SupportsSearch) - { - foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) - { - var query = queryTitle.Replace('+', ' '); - query = System.Web.HttpUtility.UrlEncode(query); - - pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "search", - string.Format("&q={0}", - query))); - } - } - - return pageableRequests; - } - - private void AddTvIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable categories, SearchCriteriaBase searchCriteria, string parameters) - { - var includeTvdbSearch = SupportsTvdbSearch && searchCriteria.Series.TvdbId > 0; - var includeTvRageSearch = SupportsTvRageSearch && searchCriteria.Series.TvRageId > 0; - var includeTvMazeSearch = SupportsTvMazeSearch && searchCriteria.Series.TvMazeId > 0; - - if (SupportsAggregatedIdSearch && (includeTvdbSearch || includeTvRageSearch || includeTvMazeSearch)) - { - var ids = ""; - - if (includeTvdbSearch) - { - ids += "&tvdbid=" + searchCriteria.Series.TvdbId; - } - - if (includeTvRageSearch) - { - ids += "&rid=" + searchCriteria.Series.TvRageId; - } - - if (includeTvMazeSearch) - { - ids += "&tvmazeid=" + searchCriteria.Series.TvMazeId; - } - - chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", ids + parameters)); - } - else - { - if (includeTvdbSearch) - { - chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", - string.Format("&tvdbid={0}{1}", searchCriteria.Series.TvdbId, parameters))); - } - else if (includeTvRageSearch) - { - chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", - string.Format("&rid={0}{1}", searchCriteria.Series.TvRageId, parameters))); - } - - else if (includeTvMazeSearch) - { - chain.Add(GetPagedRequests(maxPages, categories, "tvsearch", - string.Format("&tvmazeid={0}{1}", searchCriteria.Series.TvMazeId, parameters))); - } - } - - if (SupportsTvSearch) - { - chain.AddTier(); - foreach (var queryTitle in searchCriteria.QueryTitles) - { - chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - string.Format("&q={0}{1}", - NewsnabifyTitle(queryTitle), - parameters))); - } - } + return new IndexerPageableRequestChain(); } private IEnumerable GetPagedRequests(int maxPages, IEnumerable categories, string searchType, string parameters) diff --git a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs index f9de0d54c..bac3efd51 100644 --- a/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/RssIndexerRequestGenerator.cs @@ -23,9 +23,9 @@ namespace NzbDrone.Core.Indexers return pageableRequests; } - public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { - throw new NotImplementedException(); + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) diff --git a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs index 808a40b89..acf2cf015 100644 --- a/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/TorrentPotato/TorrentPotatoRequestGenerator.cs @@ -27,29 +27,17 @@ namespace NzbDrone.Core.Indexers.TorrentPotato public virtual IndexerPageableRequestChain GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}E{1:00}", searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(SeasonSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "S{0:00}", searchCriteria.SeasonNumber)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, "\"{0:yyyy MM dd}\"", searchCriteria.AirDate)); - - return pageableRequests; + return new IndexerPageableRequestChain(); } public virtual IndexerPageableRequestChain GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) @@ -59,17 +47,7 @@ namespace NzbDrone.Core.Indexers.TorrentPotato public virtual IndexerPageableRequestChain GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) { - var pageableRequests = new IndexerPageableRequestChain(); - - foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) - { - var query = queryTitle.Replace('+', ' '); - query = System.Web.HttpUtility.UrlEncode(query); - - pageableRequests.Add(GetPagedRequests("search", searchCriteria.Series.TvdbId, query)); - } - - return pageableRequests; + return new IndexerPageableRequestChain(); } private IEnumerable GetPagedRequests(string mode, int? tvdbId, string query, params object[] args) @@ -114,7 +92,7 @@ namespace NzbDrone.Core.Indexers.TorrentPotato } else { - requestBuilder.AddQueryParam("search", searchCriteria.Movie.Title); + requestBuilder.AddQueryParam("search", $"{searchCriteria.Movie.Title} {searchCriteria.Movie.Year}"); } yield return new IndexerRequest(requestBuilder.Build()); @@ -122,15 +100,9 @@ namespace NzbDrone.Core.Indexers.TorrentPotato public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { - - var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetMovieRequest(searchCriteria)); - return pageableRequests; - - } } } diff --git a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs index b17573758..6aab4c280 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs @@ -96,8 +96,8 @@ namespace NzbDrone.Core.Indexers.Torznab return null; } - if (capabilities.SupportedTvSearchParameters != null && - new[] { "q", "imdbid" }.Any(v => capabilities.SupportedTvSearchParameters.Contains(v))) + if (capabilities.SupportedMovieSearchParameters != null && + new[] { "q", "imdbid" }.Any(v => capabilities.SupportedMovieSearchParameters.Contains(v))) { return null; } diff --git a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs b/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs deleted file mode 100644 index eab281ee8..000000000 --- a/src/NzbDrone.Core/Indexers/Wombles/Wombles.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using NLog; -using NzbDrone.Common.Http; -using NzbDrone.Core.Configuration; -using NzbDrone.Core.IndexerSearch.Definitions; -using NzbDrone.Core.Parser; -using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.ThingiProvider; - -namespace NzbDrone.Core.Indexers.Wombles -{ - public class Wombles : HttpIndexerBase - { - public override string Name => "Womble's"; - - public override DownloadProtocol Protocol => DownloadProtocol.Usenet; - public override bool SupportsSearch => false; - - public override IParseIndexerResponse GetParser() - { - return new WomblesRssParser(); - } - - public override IIndexerRequestGenerator GetRequestGenerator() - { - return new RssIndexerRequestGenerator("http://newshost.co.za/rss/?sec=Movies&fr=false"); - } - - public Wombles(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) - : base(httpClient, indexerStatusService, configService, parsingService, logger) - { - - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/Wombles/WomblesRssParser.cs b/src/NzbDrone.Core/Indexers/Wombles/WomblesRssParser.cs deleted file mode 100644 index 3c357aaf5..000000000 --- a/src/NzbDrone.Core/Indexers/Wombles/WomblesRssParser.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Xml.Linq; - -namespace NzbDrone.Core.Indexers.Wombles -{ - public class WomblesRssParser : RssParser - { - public WomblesRssParser() - { - ParseSizeInDescription = true; - } - - protected override DateTime GetPublishDate(XElement item) - { - var dateString = item.TryGetValue("pubDate") + " +0000"; - - return XElementExtensions.ParseDate(dateString); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 031b6a03a..32293bb7e 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -124,6 +124,7 @@ + @@ -667,10 +668,6 @@ - - - - @@ -714,8 +711,6 @@ - - diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 6b2e6e705..ddaabf37e 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -296,6 +296,7 @@ namespace NzbDrone.Core.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex WordDelimiterRegex = new Regex(@"(\s|\.|,|_|-|=|\|)+", RegexOptions.Compiled); + private static readonly Regex SpecialCharRegex = new Regex(@"(\&|\:|\\|\/)+", RegexOptions.Compiled); private static readonly Regex PunctuationRegex = new Regex(@"[^\w\s]", RegexOptions.Compiled); private static readonly Regex CommonWordRegex = new Regex(@"\b(a|an|the|and|or|of)\b\s?", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex SpecialEpisodeWordRegex = new Regex(@"\b(part|special|edition|christmas)\b\s?", RegexOptions.IgnoreCase | RegexOptions.Compiled); @@ -621,6 +622,7 @@ namespace NzbDrone.Core.Parser title = PunctuationRegex.Replace(title, string.Empty); title = CommonWordRegex.Replace(title, string.Empty); title = DuplicateSpacesRegex.Replace(title, " "); + title = SpecialCharRegex.Replace(title, string.Empty); return title.Trim().ToLower(); }