From 85be0be455a09923dbb8de49be7985a3e21c3685 Mon Sep 17 00:00:00 2001 From: Qstick Date: Mon, 31 May 2021 21:51:43 -0400 Subject: [PATCH] Fixed: Push Downloads to client fails for download overrides --- .../Download/DownloadClientProviderFixture.cs | 227 ++++++++++++++++++ .../DownloadClientStatusServiceFixture.cs | 161 +++++++++++++ .../IndexerTests/TestIndexer.cs | 7 +- .../DownloadStation/UsenetDownloadStation.cs | 3 +- .../Download/Clients/NzbVortex/NzbVortex.cs | 3 +- .../Download/Clients/Nzbget/Nzbget.cs | 3 +- .../Download/Clients/Pneumatic/Pneumatic.cs | 17 +- .../Download/Clients/Sabnzbd/Sabnzbd.cs | 3 +- .../Download/Clients/rTorrent/RTorrent.cs | 2 +- .../Download/DownloadClientBase.cs | 3 +- .../DownloadMappingService.cs | 2 +- src/NzbDrone.Core/Download/DownloadService.cs | 8 +- src/NzbDrone.Core/Download/IDownloadClient.cs | 3 +- .../Download/NzbValidationService.cs | 10 +- .../Download/TorrentClientBase.cs | 72 +----- .../Download/UsenetClientBase.cs | 46 +--- .../DownloadClientRejectedReleaseException.cs | 15 +- .../Exceptions/ReleaseDownloadException.cs | 16 +- .../Exceptions/ReleaseUnavailableException.cs | 18 +- .../IndexerSearch/NzbSearchService.cs | 1 + .../Indexers/Definitions/AnimeBytes.cs | 2 +- .../Indexers/Definitions/AnimeTorrents.cs | 2 +- .../Definitions/Avistaz/AvistazBase.cs | 2 +- .../Indexers/Definitions/BakaBT.cs | 2 +- .../Indexers/Definitions/BeyondHD.cs | 2 +- .../BroadcastheNet/BroadcastheNet.cs | 2 +- .../Definitions/Cardigann/Cardigann.cs | 33 ++- .../Indexers/Definitions/DigitalCore.cs | 2 +- .../Indexers/Definitions/FileList/FileList.cs | 2 +- .../Indexers/Definitions/Gazelle/Gazelle.cs | 2 +- .../Indexers/Definitions/HDBits/HDBits.cs | 2 +- .../Indexers/Definitions/HDTorrents.cs | 2 +- .../Definitions/Headphones/Headphones.cs | 7 +- .../Indexers/Definitions/IPTorrents.cs | 2 +- .../Indexers/Definitions/ImmortalSeed.cs | 2 +- .../Indexers/Definitions/Milkie.cs | 2 +- .../Indexers/Definitions/MyAnonamouse.cs | 2 +- .../Indexers/Definitions/Newznab/Newznab.cs | 7 +- .../PassThePopcorn/PassThePopcorn.cs | 2 +- .../Indexers/Definitions/PreToMe.cs | 2 +- .../Indexers/Definitions/Rarbg/Rarbg.cs | 2 +- .../Indexers/Definitions/RevolutionTT.cs | 2 +- .../Indexers/Definitions/SubsPlease.cs | 2 +- .../Indexers/Definitions/SuperBits.cs | 2 +- .../Indexers/Definitions/ThePirateBay.cs | 2 +- .../Indexers/Definitions/TorrentDay.cs | 2 +- .../Indexers/Definitions/TorrentLeech.cs | 2 +- .../TorrentPotato/TorrentPotato.cs | 2 +- .../Indexers/Definitions/TorrentSeeds.cs | 2 +- .../Indexers/Definitions/Torznab/Torznab.cs | 2 +- .../Indexers/Definitions/UNIT3D/Unit3dBase.cs | 2 +- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 38 --- src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 +- .../Indexers/TorrentIndexerBase.cs | 90 +++++++ .../Indexers/UsenetIndexerBase.cs | 85 +++++++ 55 files changed, 702 insertions(+), 236 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Download/DownloadClientProviderFixture.cs create mode 100644 src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs rename src/NzbDrone.Core/{Indexers => Download}/DownloadMappingService.cs (98%) create mode 100644 src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs create mode 100644 src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientProviderFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientProviderFixture.cs new file mode 100644 index 000000000..1387ec3fd --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/DownloadClientProviderFixture.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Download; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Download +{ + [TestFixture] + public class DownloadClientProviderFixture : CoreTest + { + private List _downloadClients; + private List _blockedProviders; + private int _nextId; + + [SetUp] + public void SetUp() + { + _downloadClients = new List(); + _blockedProviders = new List(); + _nextId = 1; + + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_downloadClients); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedProviders); + } + + private Mock WithUsenetClient(int priority = 0) + { + var mock = new Mock(MockBehavior.Default); + mock.SetupGet(s => s.Definition) + .Returns(Builder + .CreateNew() + .With(v => v.Id = _nextId++) + .With(v => v.Priority = priority) + .Build()); + + _downloadClients.Add(mock.Object); + + mock.SetupGet(v => v.Protocol).Returns(DownloadProtocol.Usenet); + + return mock; + } + + private Mock WithTorrentClient(int priority = 0) + { + var mock = new Mock(MockBehavior.Default); + mock.SetupGet(s => s.Definition) + .Returns(Builder + .CreateNew() + .With(v => v.Id = _nextId++) + .With(v => v.Priority = priority) + .Build()); + + _downloadClients.Add(mock.Object); + + mock.SetupGet(v => v.Protocol).Returns(DownloadProtocol.Torrent); + + return mock; + } + + private void GivenBlockedClient(int id) + { + _blockedProviders.Add(new DownloadClientStatus + { + ProviderId = id, + DisabledTill = DateTime.UtcNow.AddHours(3) + }); + } + + [Test] + public void should_roundrobin_over_usenet_client() + { + WithUsenetClient(); + WithUsenetClient(); + WithUsenetClient(); + WithTorrentClient(); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + + client1.Definition.Id.Should().Be(1); + client2.Definition.Id.Should().Be(2); + client3.Definition.Id.Should().Be(3); + client4.Definition.Id.Should().Be(1); + client5.Definition.Id.Should().Be(2); + } + + [Test] + public void should_roundrobin_over_torrent_client() + { + WithUsenetClient(); + WithTorrentClient(); + WithTorrentClient(); + WithTorrentClient(); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(2); + client2.Definition.Id.Should().Be(3); + client3.Definition.Id.Should().Be(4); + client4.Definition.Id.Should().Be(2); + client5.Definition.Id.Should().Be(3); + } + + [Test] + public void should_roundrobin_over_protocol_separately() + { + WithUsenetClient(); + WithTorrentClient(); + WithTorrentClient(); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Usenet); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(1); + client2.Definition.Id.Should().Be(2); + client3.Definition.Id.Should().Be(3); + client4.Definition.Id.Should().Be(2); + } + + [Test] + public void should_skip_blocked_torrent_client() + { + WithUsenetClient(); + WithTorrentClient(); + WithTorrentClient(); + WithTorrentClient(); + + GivenBlockedClient(3); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(2); + client2.Definition.Id.Should().Be(4); + client3.Definition.Id.Should().Be(2); + client4.Definition.Id.Should().Be(4); + } + + [Test] + public void should_not_skip_blocked_torrent_client_if_all_blocked() + { + WithUsenetClient(); + WithTorrentClient(); + WithTorrentClient(); + WithTorrentClient(); + + GivenBlockedClient(2); + GivenBlockedClient(3); + GivenBlockedClient(4); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(2); + client2.Definition.Id.Should().Be(3); + client3.Definition.Id.Should().Be(4); + client4.Definition.Id.Should().Be(2); + } + + [Test] + public void should_skip_secondary_prio_torrent_client() + { + WithUsenetClient(); + WithTorrentClient(2); + WithTorrentClient(); + WithTorrentClient(); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(3); + client2.Definition.Id.Should().Be(4); + client3.Definition.Id.Should().Be(3); + client4.Definition.Id.Should().Be(4); + } + + [Test] + public void should_not_skip_secondary_prio_torrent_client_if_primary_blocked() + { + WithUsenetClient(); + WithTorrentClient(2); + WithTorrentClient(2); + WithTorrentClient(); + + GivenBlockedClient(4); + + var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent); + + client1.Definition.Id.Should().Be(2); + client2.Definition.Id.Should().Be(3); + client3.Definition.Id.Should().Be(2); + client4.Definition.Id.Should().Be(3); + } + } +} diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs new file mode 100644 index 000000000..a608321d9 --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs @@ -0,0 +1,161 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Core.Download; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Download +{ + public class DownloadClientStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); + } + + private DownloadClientStatus WithStatus(DownloadClientStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + + return status; + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_consider_blocked_within_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_consider_blocked_after_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + } + + [Test] + public void should_not_escalate_further_till_after_5_minutes_since_initial_failure() + { + var origStatus = WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + + origStatus.EscalationLevel.Should().Be(3); + } + + [Test] + public void should_escalate_further_after_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + + status.EscalationLevel.Should().BeGreaterThan(3); + } + + [Test] + public void should_not_escalate_beyond_3_hours() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1)); + } + } +} diff --git a/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs b/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs index 1d7522ec8..4a96c882a 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs @@ -1,12 +1,13 @@ using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Test.IndexerTests { - public class TestIndexer : HttpIndexerBase + public class TestIndexer : UsenetIndexerBase { public override string Name => "Test Indexer"; public override string BaseUrl => "http://testindexer.com"; @@ -18,8 +19,8 @@ namespace NzbDrone.Core.Test.IndexerTests public int _supportedPageSize; public override int PageSize => _supportedPageSize; - public TestIndexer(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) - : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + public TestIndexer(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger) { } diff --git a/src/NzbDrone.Core/Download/Clients/DownloadStation/UsenetDownloadStation.cs b/src/NzbDrone.Core/Download/Clients/DownloadStation/UsenetDownloadStation.cs index 1cba414c1..c5c51433d 100644 --- a/src/NzbDrone.Core/Download/Clients/DownloadStation/UsenetDownloadStation.cs +++ b/src/NzbDrone.Core/Download/Clients/DownloadStation/UsenetDownloadStation.cs @@ -31,9 +31,8 @@ namespace NzbDrone.Core.Download.Clients.DownloadStation IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, - IValidateNzbs nzbValidationService, Logger logger) - : base(httpClient, configService, diskProvider, nzbValidationService, logger) + : base(httpClient, configService, diskProvider, logger) { _dsInfoProxy = dsInfoProxy; _dsTaskProxy = dsTaskProxy; diff --git a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs index 731d1ce6e..f2d68823d 100644 --- a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs +++ b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs @@ -20,9 +20,8 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, - IValidateNzbs nzbValidationService, Logger logger) - : base(httpClient, configService, diskProvider, nzbValidationService, logger) + : base(httpClient, configService, diskProvider, logger) { _proxy = proxy; } diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs index 474f4954f..ecdbcf1cd 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs @@ -25,9 +25,8 @@ namespace NzbDrone.Core.Download.Clients.Nzbget IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, - IValidateNzbs nzbValidationService, Logger logger) - : base(httpClient, configService, diskProvider, nzbValidationService, logger) + : base(httpClient, configService, diskProvider, logger) { _proxy = proxy; } diff --git a/src/NzbDrone.Core/Download/Clients/Pneumatic/Pneumatic.cs b/src/NzbDrone.Core/Download/Clients/Pneumatic/Pneumatic.cs index 3c7367c55..ee19dd95c 100644 --- a/src/NzbDrone.Core/Download/Clients/Pneumatic/Pneumatic.cs +++ b/src/NzbDrone.Core/Download/Clients/Pneumatic/Pneumatic.cs @@ -1,5 +1,7 @@ +using System; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; @@ -14,24 +16,20 @@ namespace NzbDrone.Core.Download.Clients.Pneumatic { public class Pneumatic : DownloadClientBase { - private readonly IHttpClient _httpClient; - - public Pneumatic(IHttpClient httpClient, - IConfigService configService, + public Pneumatic(IConfigService configService, IDiskProvider diskProvider, Logger logger) : base(configService, diskProvider, logger) { - _httpClient = httpClient; } public override string Name => "Pneumatic"; public override DownloadProtocol Protocol => DownloadProtocol.Usenet; - public override string Download(ReleaseInfo release, bool redirect) + public override async Task Download(ReleaseInfo release, bool redirect, IIndexer indexer) { - var url = release.DownloadUrl; + var url = new Uri(release.DownloadUrl); var title = release.Title; title = StringUtil.CleanFileName(title); @@ -40,7 +38,10 @@ namespace NzbDrone.Core.Download.Clients.Pneumatic var nzbFile = Path.Combine(Settings.NzbFolder, title + ".nzb"); _logger.Debug("Downloading NZB from: {0} to: {1}", url, nzbFile); - _httpClient.DownloadFile(url, nzbFile); + + var nzbData = await indexer.Download(url); + + File.WriteAllBytes(nzbFile, nzbData); _logger.Debug("NZB Download succeeded, saved to: {0}", nzbFile); diff --git a/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs b/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs index 25076a1fc..ff3f23136 100644 --- a/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs +++ b/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs @@ -22,9 +22,8 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, - IValidateNzbs nzbValidationService, Logger logger) - : base(httpClient, configService, diskProvider, nzbValidationService, logger) + : base(httpClient, configService, diskProvider, logger) { _proxy = proxy; } diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs index eeee3eba4..091ac1242 100644 --- a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs @@ -66,7 +66,7 @@ namespace NzbDrone.Core.Download.Clients.RTorrent { _logger.Debug("rTorrent didn't add the torrent within {0} seconds: {1}.", tries * retryDelay / 1000, filename); - throw new ReleaseDownloadException(release, "Downloading torrent failed"); + throw new ReleaseDownloadException("Downloading torrent failed"); } return hash; diff --git a/src/NzbDrone.Core/Download/DownloadClientBase.cs b/src/NzbDrone.Core/Download/DownloadClientBase.cs index 26b37d249..395cd491b 100644 --- a/src/NzbDrone.Core/Download/DownloadClientBase.cs +++ b/src/NzbDrone.Core/Download/DownloadClientBase.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using FluentValidation.Results; using NLog; using NzbDrone.Common.Disk; @@ -54,7 +55,7 @@ namespace NzbDrone.Core.Download get; } - public abstract string Download(ReleaseInfo release, bool redirect); + public abstract Task Download(ReleaseInfo release, bool redirect, IIndexer indexer); public ValidationResult Test() { diff --git a/src/NzbDrone.Core/Indexers/DownloadMappingService.cs b/src/NzbDrone.Core/Download/DownloadMappingService.cs similarity index 98% rename from src/NzbDrone.Core/Indexers/DownloadMappingService.cs rename to src/NzbDrone.Core/Download/DownloadMappingService.cs index 2b34684ef..a3dbcc539 100644 --- a/src/NzbDrone.Core/Indexers/DownloadMappingService.cs +++ b/src/NzbDrone.Core/Download/DownloadMappingService.cs @@ -6,7 +6,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.Security; -namespace NzbDrone.Core.Indexers +namespace NzbDrone.Core.Download { public interface IDownloadMappingService { diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index 662add61b..6d5e5df7c 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -17,7 +17,7 @@ namespace NzbDrone.Core.Download { public interface IDownloadService { - void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect); + Task SendReportToClient(ReleaseInfo release, string source, string host, bool redirect); Task DownloadReport(string link, int indexerId, string source, string host, string title); void RecordRedirect(string link, int indexerId, string source, string host, string title); } @@ -49,7 +49,7 @@ namespace NzbDrone.Core.Download _logger = logger; } - public void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect) + public async Task SendReportToClient(ReleaseInfo release, string source, string host, bool redirect) { var downloadTitle = release.Title; var downloadClient = _downloadClientProvider.GetDownloadClient(release.DownloadProtocol); @@ -69,10 +69,12 @@ namespace NzbDrone.Core.Download _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2)); } + var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId)); + string downloadClientId; try { - downloadClientId = downloadClient.Download(release, redirect); + downloadClientId = await downloadClient.Download(release, redirect, indexer); _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id); _indexerStatusService.RecordSuccess(release.IndexerId); } diff --git a/src/NzbDrone.Core/Download/IDownloadClient.cs b/src/NzbDrone.Core/Download/IDownloadClient.cs index bd6757399..0b6d0664f 100644 --- a/src/NzbDrone.Core/Download/IDownloadClient.cs +++ b/src/NzbDrone.Core/Download/IDownloadClient.cs @@ -1,3 +1,4 @@ +using System.Threading.Tasks; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; @@ -7,6 +8,6 @@ namespace NzbDrone.Core.Download public interface IDownloadClient : IProvider { DownloadProtocol Protocol { get; } - string Download(ReleaseInfo release, bool redirect); + Task Download(ReleaseInfo release, bool redirect, IIndexer indexer); } } diff --git a/src/NzbDrone.Core/Download/NzbValidationService.cs b/src/NzbDrone.Core/Download/NzbValidationService.cs index e3cbff710..90de08eca 100644 --- a/src/NzbDrone.Core/Download/NzbValidationService.cs +++ b/src/NzbDrone.Core/Download/NzbValidationService.cs @@ -8,12 +8,12 @@ namespace NzbDrone.Core.Download { public interface IValidateNzbs { - void Validate(string filename, byte[] fileContent); + void Validate(byte[] fileContent); } public class NzbValidationService : IValidateNzbs { - public void Validate(string filename, byte[] fileContent) + public void Validate(byte[] fileContent) { var reader = new StreamReader(new MemoryStream(fileContent)); @@ -24,7 +24,7 @@ namespace NzbDrone.Core.Download if (nzb == null) { - throw new InvalidNzbException("Invalid NZB: No Root element [{0}]", filename); + throw new InvalidNzbException("Invalid NZB: No Root element"); } // nZEDb has an bug in their error reporting code spitting out invalid http status codes @@ -37,7 +37,7 @@ namespace NzbDrone.Core.Download if (!nzb.Name.LocalName.Equals("nzb")) { - throw new InvalidNzbException("Invalid NZB: Unexpected root element. Expected 'nzb' found '{0}' [{1}]", nzb.Name.LocalName, filename); + throw new InvalidNzbException("Invalid NZB: Unexpected root element. Expected 'nzb' found '{0}'", nzb.Name.LocalName); } var ns = nzb.Name.Namespace; @@ -45,7 +45,7 @@ namespace NzbDrone.Core.Download if (files.Empty()) { - throw new InvalidNzbException("Invalid NZB: No files [{0}]", filename); + throw new InvalidNzbException("Invalid NZB: No files"); } } } diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index cb4b508e0..b1ac80612 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -1,5 +1,6 @@ using System; using System.Net; +using System.Threading.Tasks; using MonoTorrent; using NLog; using NzbDrone.Common.Disk; @@ -39,7 +40,7 @@ namespace NzbDrone.Core.Download protected abstract string AddFromTorrentFile(ReleaseInfo release, string hash, string filename, byte[] fileContent); protected abstract string AddFromTorrentLink(ReleaseInfo release, string hash, string torrentLink); - public override string Download(ReleaseInfo release, bool redirect) + public override async Task Download(ReleaseInfo release, bool redirect, IIndexer indexer) { var torrentInfo = release as TorrentInfo; @@ -66,7 +67,7 @@ namespace NzbDrone.Core.Download { try { - return DownloadFromWebUrl(release, torrentUrl); + return await DownloadFromWebUrl(release, indexer, torrentUrl); } catch (Exception ex) { @@ -87,7 +88,7 @@ namespace NzbDrone.Core.Download } catch (NotSupportedException ex) { - throw new ReleaseDownloadException(release, "Magnet not supported by download client. ({0})", ex.Message); + throw new ReleaseDownloadException("Magnet not supported by download client. ({0})", ex.Message); } } } @@ -103,7 +104,7 @@ namespace NzbDrone.Core.Download { if (torrentUrl.IsNullOrWhiteSpace()) { - throw new ReleaseDownloadException(release, "Magnet not supported by download client. ({0})", ex.Message); + throw new ReleaseDownloadException("Magnet not supported by download client. ({0})", ex.Message); } _logger.Debug("Magnet not supported by download client, trying torrent. ({0})", ex.Message); @@ -112,75 +113,18 @@ namespace NzbDrone.Core.Download if (torrentUrl.IsNotNullOrWhiteSpace()) { - return DownloadFromWebUrl(release, torrentUrl); + return await DownloadFromWebUrl(release, indexer, torrentUrl); } } return null; } - private string DownloadFromWebUrl(ReleaseInfo release, string torrentUrl) + private async Task DownloadFromWebUrl(ReleaseInfo release, IIndexer indexer, string torrentUrl) { byte[] torrentFile = null; - try - { - var request = new HttpRequest(torrentUrl); - request.Headers.Accept = "application/x-bittorrent"; - request.AllowAutoRedirect = false; - - var response = _httpClient.Get(request); - - if (response.StatusCode == HttpStatusCode.MovedPermanently || - response.StatusCode == HttpStatusCode.Found || - response.StatusCode == HttpStatusCode.SeeOther) - { - var locationHeader = response.Headers.GetSingleValue("Location"); - - _logger.Trace("Torrent request is being redirected to: {0}", locationHeader); - - if (locationHeader != null) - { - if (locationHeader.StartsWith("magnet:")) - { - return DownloadFromMagnetUrl(release, locationHeader); - } - - return DownloadFromWebUrl(release, locationHeader); - } - - throw new WebException("Remote website tried to redirect without providing a location."); - } - - torrentFile = response.ResponseData; - - _logger.Debug("Downloading torrent for release '{0}' finished ({1} bytes from {2})", release.Title, torrentFile.Length, torrentUrl); - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release '{0}' failed since it no longer exists ({1})", release.Title, torrentUrl); - throw new ReleaseUnavailableException(release, "Downloading torrent failed", ex); - } - - if ((int)ex.Response.StatusCode == 429) - { - _logger.Error("API Grab Limit reached for {0}", torrentUrl); - } - else - { - _logger.Error(ex, "Downloading torrent file for release '{0}' failed ({1})", release.Title, torrentUrl); - } - - throw new ReleaseDownloadException(release, "Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release '{0}' failed ({1})", release.Title, torrentUrl); - - throw new ReleaseDownloadException(release, "Downloading torrent failed", ex); - } + torrentFile = await indexer.Download(new Uri(torrentUrl)); var filename = string.Format("{0}.torrent", StringUtil.CleanFileName(release.Title)); var hash = _torrentFileInfoReader.GetHashFromTorrentFile(torrentFile); diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index 98e4eff07..2541c6059 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -1,9 +1,9 @@ -using System.Net; +using System; +using System.Threading.Tasks; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; @@ -15,17 +15,14 @@ namespace NzbDrone.Core.Download where TSettings : IProviderConfig, new() { protected readonly IHttpClient _httpClient; - private readonly IValidateNzbs _nzbValidationService; protected UsenetClientBase(IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, - IValidateNzbs nzbValidationService, Logger logger) : base(configService, diskProvider, logger) { _httpClient = httpClient; - _nzbValidationService = nzbValidationService; } public override DownloadProtocol Protocol => DownloadProtocol.Usenet; @@ -33,9 +30,9 @@ namespace NzbDrone.Core.Download protected abstract string AddFromNzbFile(ReleaseInfo release, string filename, byte[] fileContents); protected abstract string AddFromLink(ReleaseInfo release); - public override string Download(ReleaseInfo release, bool redirect) + public override async Task Download(ReleaseInfo release, bool redirect, IIndexer indexer) { - var url = release.DownloadUrl; + var url = new Uri(release.DownloadUrl); if (redirect) { @@ -46,40 +43,7 @@ namespace NzbDrone.Core.Download byte[] nzbData; - try - { - var request = new HttpRequest(url); - nzbData = _httpClient.Get(request).ResponseData; - - _logger.Debug("Downloaded nzb for release '{0}' finished ({1} bytes from {2})", release.Title, nzbData.Length, url); - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading nzb file for release '{0}' failed since it no longer exists ({1})", release.Title, url); - throw new ReleaseUnavailableException(release, "Downloading nzb failed", ex); - } - - if ((int)ex.Response.StatusCode == 429) - { - _logger.Error("API Grab Limit reached for {0}", url); - } - else - { - _logger.Error(ex, "Downloading nzb for release '{0}' failed ({1})", release.Title, url); - } - - throw new ReleaseDownloadException(release, "Downloading nzb failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading nzb for release '{0}' failed ({1})", release.Title, url); - - throw new ReleaseDownloadException(release, "Downloading nzb failed", ex); - } - - _nzbValidationService.Validate(filename, nzbData); + nzbData = await indexer.Download(url); _logger.Info("Adding report [{0}] to the queue.", release.Title); return AddFromNzbFile(release, filename, nzbData); diff --git a/src/NzbDrone.Core/Exceptions/DownloadClientRejectedReleaseException.cs b/src/NzbDrone.Core/Exceptions/DownloadClientRejectedReleaseException.cs index e7cee87d8..e55fc9705 100644 --- a/src/NzbDrone.Core/Exceptions/DownloadClientRejectedReleaseException.cs +++ b/src/NzbDrone.Core/Exceptions/DownloadClientRejectedReleaseException.cs @@ -1,28 +1,33 @@ -using System; +using System; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Exceptions { public class DownloadClientRejectedReleaseException : ReleaseDownloadException { + public ReleaseInfo Release { get; set; } public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, params object[] args) - : base(release, message, args) + : base(message, args) { + Release = release; } public DownloadClientRejectedReleaseException(ReleaseInfo release, string message) - : base(release, message) + : base(message) { + Release = release; } public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, Exception innerException, params object[] args) - : base(release, message, innerException, args) + : base(message, innerException, args) { + Release = release; } public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, Exception innerException) - : base(release, message, innerException) + : base(message, innerException) { + Release = release; } } } diff --git a/src/NzbDrone.Core/Exceptions/ReleaseDownloadException.cs b/src/NzbDrone.Core/Exceptions/ReleaseDownloadException.cs index b891b6d3b..037fe09c1 100644 --- a/src/NzbDrone.Core/Exceptions/ReleaseDownloadException.cs +++ b/src/NzbDrone.Core/Exceptions/ReleaseDownloadException.cs @@ -1,4 +1,4 @@ -using System; +using System; using NzbDrone.Common.Exceptions; using NzbDrone.Core.Parser.Model; @@ -6,30 +6,24 @@ namespace NzbDrone.Core.Exceptions { public class ReleaseDownloadException : NzbDroneException { - public ReleaseInfo Release { get; set; } - - public ReleaseDownloadException(ReleaseInfo release, string message, params object[] args) + public ReleaseDownloadException(string message, params object[] args) : base(message, args) { - Release = release; } - public ReleaseDownloadException(ReleaseInfo release, string message) + public ReleaseDownloadException(string message) : base(message) { - Release = release; } - public ReleaseDownloadException(ReleaseInfo release, string message, Exception innerException, params object[] args) + public ReleaseDownloadException(string message, Exception innerException, params object[] args) : base(message, innerException, args) { - Release = release; } - public ReleaseDownloadException(ReleaseInfo release, string message, Exception innerException) + public ReleaseDownloadException(string message, Exception innerException) : base(message, innerException) { - Release = release; } } } diff --git a/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs index 41442c1ea..2af8c9a40 100644 --- a/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs +++ b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs @@ -1,27 +1,27 @@ -using System; +using System; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Exceptions { public class ReleaseUnavailableException : ReleaseDownloadException { - public ReleaseUnavailableException(ReleaseInfo release, string message, params object[] args) - : base(release, message, args) + public ReleaseUnavailableException(string message, params object[] args) + : base(message, args) { } - public ReleaseUnavailableException(ReleaseInfo release, string message) - : base(release, message) + public ReleaseUnavailableException(string message) + : base(message) { } - public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException, params object[] args) - : base(release, message, innerException, args) + public ReleaseUnavailableException(string message, Exception innerException, params object[] args) + : base(message, innerException, args) { } - public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException) - : base(release, message, innerException) + public ReleaseUnavailableException(string message, Exception innerException) + : base(message, innerException) { } } diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index ed101fae4..b59510f5a 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using NLog; using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.IndexerSearch.Definitions; diff --git a/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs b/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs index 875a5ce1f..b740edf5a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs @@ -23,7 +23,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class AnimeBytes : HttpIndexerBase + public class AnimeBytes : TorrentIndexerBase { public override string Name => "AnimeBytes"; public override string BaseUrl => "https://animebytes.tv/"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs b/src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs index 2cbf6d0de..e2a071ac0 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class AnimeTorrents : HttpIndexerBase + public class AnimeTorrents : TorrentIndexerBase { public override string Name => "AnimeTorrents"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazBase.cs index 2c7d71eb7..36aa474b7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazBase.cs @@ -9,7 +9,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.Definitions.Avistaz { - public abstract class AvistazBase : HttpIndexerBase + public abstract class AvistazBase : TorrentIndexerBase { public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public override string BaseUrl => ""; diff --git a/src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs b/src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs index e3baec54d..96d2deb89 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class BakaBT : HttpIndexerBase + public class BakaBT : TorrentIndexerBase { public override string Name => "BakaBT"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs b/src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs index 18c876a14..f9c87efd9 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class BeyondHD : HttpIndexerBase + public class BeyondHD : TorrentIndexerBase { public override string Name => "BeyondHD"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs b/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs index 53160130b..64bf2b287 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs @@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.BroadcastheNet { - public class BroadcastheNet : HttpIndexerBase + public class BroadcastheNet : TorrentIndexerBase { public override string Name => "BroadcasTheNet"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 1f54d3e16..1aa04a197 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net; using System.Text; using System.Threading.Tasks; using FluentValidation.Results; @@ -7,6 +8,7 @@ using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -14,7 +16,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Cardigann { - public class Cardigann : HttpIndexerBase + public class Cardigann : TorrentIndexerBase { private readonly IIndexerDefinitionUpdateService _definitionService; private readonly ICached _generatorCache; @@ -147,6 +149,7 @@ namespace NzbDrone.Core.Indexers.Cardigann if (request.Url.Scheme == "magnet") { + ValidateMagnet(request.Url.FullUri); return Encoding.UTF8.GetBytes(request.Url.FullUri); } @@ -159,10 +162,36 @@ namespace NzbDrone.Core.Indexers.Cardigann var response = await _httpClient.ExecuteAsync(request); downloadBytes = response.ResponseData; } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri); + throw new ReleaseUnavailableException("Downloading torrent failed", ex); + } + + if ((int)ex.Response.StatusCode == 429) + { + _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri); + } + else + { + _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); + } + + throw new ReleaseDownloadException("Downloading torrent failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); + + throw new ReleaseDownloadException("Downloading torrent failed", ex); + } catch (Exception) { _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Download failed"); + _logger.Error("Downloading torrent failed"); + throw; } return downloadBytes; diff --git a/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs b/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs index cb8a515a3..1144b724b 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class DigitalCore : HttpIndexerBase + public class DigitalCore : TorrentIndexerBase { public override string Name => "DigitalCore"; public override string BaseUrl => "https://digitalcore.club/"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs index 244b49cf8..dab71f775 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs @@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.FileList { - public class FileList : HttpIndexerBase + public class FileList : TorrentIndexerBase { public override string Name => "FileList.io"; public override string BaseUrl => "https://filelist.io"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs index b285c73b5..eb11cc192 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs @@ -7,7 +7,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.Gazelle { - public abstract class Gazelle : HttpIndexerBase + public abstract class Gazelle : TorrentIndexerBase { public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public override string BaseUrl => ""; diff --git a/src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBits.cs b/src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBits.cs index d2880e49d..af141d2e3 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBits.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBits.cs @@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.HDBits { - public class HDBits : HttpIndexerBase + public class HDBits : TorrentIndexerBase { public override string Name => "HDBits"; public override string BaseUrl => "https://hdbits.org"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs b/src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs index a969345e5..e59dd3fff 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs @@ -20,7 +20,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class HDTorrents : HttpIndexerBase + public class HDTorrents : TorrentIndexerBase { public override string Name => "HD-Torrents"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs index 84de801b2..102ea2bab 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs @@ -6,11 +6,12 @@ using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.Headphones { - public class Headphones : HttpIndexerBase + public class Headphones : UsenetIndexerBase { public override string Name => "Headphones VIP"; @@ -35,8 +36,8 @@ namespace NzbDrone.Core.Indexers.Headphones return new HeadphonesRssParser(Capabilities.Categories); } - public Headphones(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) - : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + public Headphones(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger) { } diff --git a/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs b/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs index 621eab2db..0c0217379 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs @@ -18,7 +18,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class IPTorrents : HttpIndexerBase + public class IPTorrents : TorrentIndexerBase { public override string Name => "IPTorrents"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/ImmortalSeed.cs b/src/NzbDrone.Core/Indexers/Definitions/ImmortalSeed.cs index 19d7deec2..92a639166 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/ImmortalSeed.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/ImmortalSeed.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class ImmortalSeed : HttpIndexerBase + public class ImmortalSeed : TorrentIndexerBase { public override string Name => "ImmortalSeed"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Milkie.cs b/src/NzbDrone.Core/Indexers/Definitions/Milkie.cs index ef73fde51..a0be7a212 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Milkie.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Milkie.cs @@ -16,7 +16,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class Milkie : HttpIndexerBase + public class Milkie : TorrentIndexerBase { public override string Name => "Milkie"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs b/src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs index 8b52148d7..fbc2dc93a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs @@ -18,7 +18,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class MyAnonamouse : HttpIndexerBase + public class MyAnonamouse : TorrentIndexerBase { public override string Name => "MyAnonamouse"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 3e30a5652..1c9638c8e 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -7,13 +7,14 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Newznab { - public class Newznab : HttpIndexerBase + public class Newznab : UsenetIndexerBase { private readonly INewznabCapabilitiesProvider _capabilitiesProvider; @@ -108,8 +109,8 @@ namespace NzbDrone.Core.Indexers.Newznab } } - public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) - : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger) { _capabilitiesProvider = capabilitiesProvider; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcorn.cs b/src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcorn.cs index c6929ab2f..b3471937e 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcorn.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcorn.cs @@ -7,7 +7,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.PassThePopcorn { - public class PassThePopcorn : HttpIndexerBase + public class PassThePopcorn : TorrentIndexerBase { public override string Name => "PassThePopcorn"; public override string BaseUrl => "https://passthepopcorn.me"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/PreToMe.cs b/src/NzbDrone.Core/Indexers/Definitions/PreToMe.cs index f90561510..39d28138c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/PreToMe.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/PreToMe.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class PreToMe : HttpIndexerBase + public class PreToMe : TorrentIndexerBase { public override string Name => "PreToMe"; public override string BaseUrl => "https://pretome.info/"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs index 0c99a156c..2386d8f74 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs @@ -11,7 +11,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Rarbg { - public class Rarbg : HttpIndexerBase + public class Rarbg : TorrentIndexerBase { private readonly IRarbgTokenProvider _tokenProvider; diff --git a/src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs b/src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs index 340d74722..09bccbb6b 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class RevolutionTT : HttpIndexerBase + public class RevolutionTT : TorrentIndexerBase { public override string Name => "RevolutionTT"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs b/src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs index 06579773e..86f08824d 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs @@ -18,7 +18,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class SubsPlease : HttpIndexerBase + public class SubsPlease : TorrentIndexerBase { public override string Name => "SubsPlease"; public override string BaseUrl => "https://subsplease.org/"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/SuperBits.cs b/src/NzbDrone.Core/Indexers/Definitions/SuperBits.cs index 963c579a9..a83e91083 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SuperBits.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SuperBits.cs @@ -19,7 +19,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class SuperBits : HttpIndexerBase + public class SuperBits : TorrentIndexerBase { public override string Name => "SuperBits"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/ThePirateBay.cs b/src/NzbDrone.Core/Indexers/Definitions/ThePirateBay.cs index 54fb468e0..49e45eec0 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/ThePirateBay.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/ThePirateBay.cs @@ -18,7 +18,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class ThePirateBay : HttpIndexerBase + public class ThePirateBay : TorrentIndexerBase { public override string Name => "ThePirateBay"; public override string BaseUrl => "https://thepiratebay.org/"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs index 9be07a848..c12a13c5c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs @@ -17,7 +17,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class TorrentDay : HttpIndexerBase + public class TorrentDay : TorrentIndexerBase { public override string Name => "TorrentDay"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs index cf4dc2de6..e1abf3998 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs @@ -21,7 +21,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class TorrentLeech : HttpIndexerBase + public class TorrentLeech : TorrentIndexerBase { public override string Name => "TorrentLeech"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs index da0df9110..f585e4449 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs @@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.TorrentPotato { - public class TorrentPotato : HttpIndexerBase + public class TorrentPotato : TorrentIndexerBase { public override string Name => "TorrentPotato"; public override string BaseUrl => "http://127.0.0.1"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs index a73ea5e74..9d9a8791a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs @@ -20,7 +20,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { - public class TorrentSeeds : HttpIndexerBase + public class TorrentSeeds : TorrentIndexerBase { public override string Name => "TorrentSeeds"; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs index 2de54a63b..bfb51ba8d 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs @@ -14,7 +14,7 @@ using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Torznab { - public class Torznab : HttpIndexerBase + public class Torznab : TorrentIndexerBase { private readonly INewznabCapabilitiesProvider _capabilitiesProvider; diff --git a/src/NzbDrone.Core/Indexers/Definitions/UNIT3D/Unit3dBase.cs b/src/NzbDrone.Core/Indexers/Definitions/UNIT3D/Unit3dBase.cs index 537b8d09c..0d1f071a1 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/UNIT3D/Unit3dBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/UNIT3D/Unit3dBase.cs @@ -5,7 +5,7 @@ using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers.Definitions.UNIT3D { - public abstract class Unit3dBase : HttpIndexerBase + public abstract class Unit3dBase : TorrentIndexerBase { public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public override string BaseUrl => ""; diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 1e4ed8363..e599a7a70 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Net; using System.Text; @@ -10,7 +9,6 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.Indexers.Exceptions; @@ -105,42 +103,6 @@ namespace NzbDrone.Core.Indexers return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } - public override async Task Download(Uri link) - { - Cookies = GetCookies(); - - if (link.Scheme == "magnet") - { - return Encoding.UTF8.GetBytes(link.OriginalString); - } - - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - - var downloadBytes = Array.Empty(); - - try - { - var response = await _httpClient.ExecuteAsync(request); - downloadBytes = response.ResponseData; - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Download failed"); - throw; - } - - return downloadBytes; - } - protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator) { //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index e6feea055..1ef3badea 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -77,7 +77,7 @@ namespace NzbDrone.Core.Indexers public abstract Task Fetch(TvSearchCriteria searchCriteria); public abstract Task Fetch(BookSearchCriteria searchCriteria); public abstract Task Fetch(BasicSearchCriteria searchCriteria); - public abstract Task Download(Uri searchCriteria); + public abstract Task Download(Uri link); public abstract IndexerCapabilities GetCapabilities(); diff --git a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs new file mode 100644 index 000000000..5f61c2d18 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs @@ -0,0 +1,90 @@ +using System; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using MonoTorrent; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Indexers +{ + public abstract class TorrentIndexerBase : HttpIndexerBase + where TSettings : IProviderConfig, new() + { + protected TorrentIndexerBase(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + { + } + + public override async Task Download(Uri link) + { + Cookies = GetCookies(); + + if (link.Scheme == "magnet") + { + ValidateMagnet(link.OriginalString); + return Encoding.UTF8.GetBytes(link.OriginalString); + } + + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var request = requestBuilder.Build(); + request.AllowAutoRedirect = FollowRedirect; + + byte[] torrentData; + + try + { + var response = await _httpClient.ExecuteAsync(request); + torrentData = response.ResponseData; + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Downloading torrent failed", ex); + } + + if ((int)ex.Response.StatusCode == 429) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Downloading torrent failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Downloading torrent failed", ex); + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Downloading torrent failed"); + throw; + } + + return torrentData; + } + + protected void ValidateMagnet(string link) + { + MagnetLink.Parse(link); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs new file mode 100644 index 000000000..3622d55c7 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs @@ -0,0 +1,85 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Indexers +{ + public abstract class UsenetIndexerBase : HttpIndexerBase + where TSettings : IProviderConfig, new() + { + private readonly IValidateNzbs _nzbValidationService; + + protected UsenetIndexerBase(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + { + _nzbValidationService = nzbValidationService; + } + + public override async Task Download(Uri link) + { + Cookies = GetCookies(); + + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var request = requestBuilder.Build(); + request.AllowAutoRedirect = FollowRedirect; + + byte[] nzbData; + + try + { + var response = await _httpClient.ExecuteAsync(request); + nzbData = response.ResponseData; + + _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri); + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Downloading nzb failed", ex); + } + + if ((int)ex.Response.StatusCode == 429) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Downloading nzb failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Downloading nzb failed", ex); + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Downloading nzb failed"); + throw; + } + + _nzbValidationService.Validate(nzbData); + + return nzbData; + } + } +}