From fa051257e3de9606257fc899dcd91af2568849cb Mon Sep 17 00:00:00 2001 From: Qstick Date: Thu, 16 Nov 2017 12:32:31 -0500 Subject: [PATCH] Fixed: Releases no longer available on the indexer should be removed from the pending queue --- .../DownloadApprovedFixture.cs | 22 +++++++++++++++ .../Download/DownloadServiceFixture.cs | 15 ++++++++++ src/NzbDrone.Core/Download/DownloadService.cs | 5 ++++ .../Download/ProcessDownloadDecisions.cs | 9 +++++- .../Download/TorrentClientBase.cs | 8 +++++- .../Download/UsenetClientBase.cs | 8 +++++- .../Exceptions/ArtistNotFoundException.cs | 18 ++++++------ .../Exceptions/ReleaseUnavailableException.cs | 28 +++++++++++++++++++ .../Exceptions/SeriesNotFoundException.cs | 27 ------------------ src/NzbDrone.Core/NzbDrone.Core.csproj | 2 +- 10 files changed, 102 insertions(+), 40 deletions(-) create mode 100644 src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs delete mode 100644 src/NzbDrone.Core/Exceptions/SeriesNotFoundException.cs diff --git a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs index cac1446ad..b4bf3759b 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; @@ -262,5 +263,26 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet)), Times.Once()); Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Torrent)), Times.Once()); } + + [Test] + public void should_add_to_rejected_if_release_unavailable_on_indexer() + { + var albums = new List { GetAlbum(1) }; + var remoteAlbum = GetRemoteAlbum(albums, new QualityModel(Quality.MP3_320)); + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteAlbum)); + + Mocker.GetMock() + .Setup(s => s.DownloadReport(It.IsAny())) + .Throws(new ReleaseUnavailableException(remoteAlbum.Release, "That 404 Error is not just a Quirk")); + + var result = Subject.ProcessDecisions(decisions); + + result.Grabbed.Should().BeEmpty(); + result.Rejected.Should().NotBeEmpty(); + + ExceptionVerification.ExpectedWarns(1); + } } } diff --git a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs index 1cc7d81f7..58b73dbcd 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs @@ -179,6 +179,21 @@ namespace NzbDrone.Core.Test.Download .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Never()); } + [Test] + public void Download_report_should_not_trigger_indexer_backoff_on_indexer_404_error() + { + var mock = WithUsenetClient(); + mock.Setup(s => s.Download(It.IsAny())) + .Callback(v => { + throw new ReleaseUnavailableException(v.Release, "Error", new WebException()); + }); + + Assert.Throws(() => Subject.DownloadReport(_parseResult)); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Never()); + } + [Test] public void should_not_attempt_download_if_client_isnt_configured() { diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index 450836a4a..62d5e6fd8 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -74,6 +74,11 @@ namespace NzbDrone.Core.Download _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id); _indexerStatusService.RecordSuccess(remoteAlbum.Release.IndexerId); } + catch (ReleaseUnavailableException) + { + _logger.Trace("Release {0} no longer available on indexer.", remoteAlbum); + throw; + } catch (ReleaseDownloadException ex) { var http429 = ex.InnerException as TooManyRequestsException; diff --git a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs index d2338732c..42e713b91 100644 --- a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs +++ b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs @@ -6,6 +6,7 @@ using NLog; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; namespace NzbDrone.Core.Download @@ -40,6 +41,7 @@ namespace NzbDrone.Core.Download var grabbed = new List(); var pending = new List(); var failed = new List(); + var rejected = decisions.Where(d => d.Rejected).ToList(); var usenetFailed = false; var torrentFailed = false; @@ -74,6 +76,11 @@ namespace NzbDrone.Core.Download _downloadService.DownloadReport(remoteAlbum); grabbed.Add(report); } + catch (ReleaseUnavailableException) + { + _logger.Warn("Failed to download release from indexer, no longer available. " + remoteAlbum); + rejected.Add(report); + } catch (Exception ex) { if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException) @@ -99,7 +106,7 @@ namespace NzbDrone.Core.Download pending.AddRange(ProcessFailedGrabs(grabbed, failed)); - return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList()); + return new ProcessedDecisions(grabbed, pending, rejected); } internal List GetQualifiedReports(IEnumerable decisions) diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index 8e89979f6..500a570db 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using MonoTorrent; using NzbDrone.Common.Disk; @@ -160,6 +160,12 @@ namespace NzbDrone.Core.Download } catch (HttpException ex) { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading torrent file for album '{0}' failed since it no longer exists ({1})", remoteAlbum.Release.Title, torrentUrl); + throw new ReleaseUnavailableException(remoteAlbum.Release, "Downloading torrent failed", ex); + } + if ((int)ex.Response.StatusCode == 429) { _logger.Error("API Grab Limit reached for {0}", torrentUrl); diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index 29fdefb55..d5aed8f21 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Exceptions; @@ -46,6 +46,12 @@ namespace NzbDrone.Core.Download } catch (HttpException ex) { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading nzb file for album '{0}' failed since it no longer exists ({1})", remoteAlbum.Release.Title, url); + throw new ReleaseUnavailableException(remoteAlbum.Release, "Downloading torrent failed", ex); + } + if ((int)ex.Response.StatusCode == 429) { _logger.Error("API Grab Limit reached for {0}", url); diff --git a/src/NzbDrone.Core/Exceptions/ArtistNotFoundException.cs b/src/NzbDrone.Core/Exceptions/ArtistNotFoundException.cs index 58c7ea61f..321777c79 100644 --- a/src/NzbDrone.Core/Exceptions/ArtistNotFoundException.cs +++ b/src/NzbDrone.Core/Exceptions/ArtistNotFoundException.cs @@ -1,4 +1,4 @@ -using NzbDrone.Common.Exceptions; +using NzbDrone.Common.Exceptions; using System; using System.Collections.Generic; using System.Linq; @@ -8,24 +8,24 @@ namespace NzbDrone.Core.Exceptions { public class ArtistNotFoundException : NzbDroneException { - public string SpotifyId { get; set; } + public string MusicBrainzId { get; set; } - public ArtistNotFoundException(string spotifyId) - : base(string.Format("Artist with SpotifyId {0} was not found, it may have been removed from Spotify.", spotifyId)) + public ArtistNotFoundException(string musicbrainzId) + : base(string.Format("Artist with MusicBrainz {0} was not found, it may have been removed from MusicBrainz.", musicbrainzId)) { - SpotifyId = spotifyId; + MusicBrainzId = musicbrainzId; } - public ArtistNotFoundException(string spotifyId, string message, params object[] args) + public ArtistNotFoundException(string musicbrainzId, string message, params object[] args) : base(message, args) { - SpotifyId = spotifyId; + MusicBrainzId = musicbrainzId; } - public ArtistNotFoundException(string spotifyId, string message) + public ArtistNotFoundException(string musicbrainzId, string message) : base(message) { - SpotifyId = spotifyId; + MusicBrainzId = musicbrainzId; } } } diff --git a/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs new file mode 100644 index 000000000..41442c1ea --- /dev/null +++ b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs @@ -0,0 +1,28 @@ +using System; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Exceptions +{ + public class ReleaseUnavailableException : ReleaseDownloadException + { + public ReleaseUnavailableException(ReleaseInfo release, string message, params object[] args) + : base(release, message, args) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message) + : base(release, message) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException, params object[] args) + : base(release, message, innerException, args) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException) + : base(release, message, innerException) + { + } + } +} diff --git a/src/NzbDrone.Core/Exceptions/SeriesNotFoundException.cs b/src/NzbDrone.Core/Exceptions/SeriesNotFoundException.cs deleted file mode 100644 index b329bde8d..000000000 --- a/src/NzbDrone.Core/Exceptions/SeriesNotFoundException.cs +++ /dev/null @@ -1,27 +0,0 @@ -using NzbDrone.Common.Exceptions; - -namespace NzbDrone.Core.Exceptions -{ - public class SeriesNotFoundException : NzbDroneException - { - public int TvdbSeriesId { get; set; } - - public SeriesNotFoundException(int tvdbSeriesId) - : base(string.Format("Series with tvdbid {0} was not found, it may have been removed from TheTVDB.", tvdbSeriesId)) - { - TvdbSeriesId = tvdbSeriesId; - } - - public SeriesNotFoundException(int tvdbSeriesId, string message, params object[] args) - : base(message, args) - { - TvdbSeriesId = tvdbSeriesId; - } - - public SeriesNotFoundException(int tvdbSeriesId, string message) - : base(message) - { - TvdbSeriesId = tvdbSeriesId; - } - } -} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 7ec81e99a..c3870e151 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -417,7 +417,7 @@ - +