From 4f37a36619600ebffaa5aedd8a1458e0a2aaf76e Mon Sep 17 00:00:00 2001 From: Leonardo Galli Date: Sat, 21 Jan 2017 20:57:29 +0100 Subject: [PATCH] Updated HttpNetImporterBase. Still needs work to correctly handle failures. --- .../NetImport/RSSImportFixture.cs | 34 ++-- .../NetImport/RSSImportParserFixture.cs | 36 ++++ .../NzbDrone.Core.Test.csproj | 1 + .../NetImport/HttpNetImportBase.cs | 188 +++++++++++++++++- 4 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 src/NzbDrone.Core.Test/NetImport/RSSImportParserFixture.cs diff --git a/src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs b/src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs index f724a1b8d..de13c40bf 100644 --- a/src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs +++ b/src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs @@ -1,32 +1,42 @@ -using System.Linq; -using System.Text; +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; using FluentAssertions; +using Moq; using NUnit.Framework; using NzbDrone.Common.Http; -using NzbDrone.Core.Indexers; +using NzbDrone.Core.Datastore; using NzbDrone.Core.NetImport; using NzbDrone.Core.NetImport.RSSImport; using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.NetImport { - public class RSSImportTest : CoreTest + [TestFixture] + public class RSSImportFixture : CoreTest { - private NetImportResponse CreateResponse(string url, string content) - { - var httpRequest = new HttpRequest(url); - var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content)); - return new NetImportResponse(new NetImportRequest(httpRequest), httpResponse); + [SetUp] + public void Setup() + { + Subject.Definition = Subject.DefaultDefinitions.First(); } + private void GivenRecentFeedResponse(string rssXmlFile) + { + var recentFeed = ReadAllText(@"Files/" + rssXmlFile); + Mocker.GetMock() + .Setup(o => o.Execute(It.IsAny())) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + } [Test] - public void should_handle_relative_url() + public void should_fetch_imdb_list() { - var xml = ReadAllText("Files/imdb_watchlist.xml"); + GivenRecentFeedResponse("imdb_watchlist.xml"); - var result = Subject.ParseResponse(CreateResponse("http://my.indexer.com/api?q=My+Favourite+Show", xml)); + var result = Subject.Fetch(); result.First().Title.Should().Be("Think Like a Man Too"); result.First().ImdbId.Should().Be("tt2239832"); diff --git a/src/NzbDrone.Core.Test/NetImport/RSSImportParserFixture.cs b/src/NzbDrone.Core.Test/NetImport/RSSImportParserFixture.cs new file mode 100644 index 000000000..cde97c653 --- /dev/null +++ b/src/NzbDrone.Core.Test/NetImport/RSSImportParserFixture.cs @@ -0,0 +1,36 @@ +using System.Linq; +using System.Text; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.NetImport; +using NzbDrone.Core.NetImport.RSSImport; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.NetImport +{ + public class RSSImportTest : CoreTest + { + private NetImportResponse CreateResponse(string url, string content) + { + var httpRequest = new HttpRequest(url); + var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content)); + + return new NetImportResponse(new NetImportRequest(httpRequest), httpResponse); + } + + + [Test] + public void should_parse_xml_of_imdb() + { + var xml = ReadAllText("Files/imdb_watchlist.xml"); + + var result = Subject.ParseResponse(CreateResponse("http://my.indexer.com/api?q=My+Favourite+Show", xml)); + + result.First().Title.Should().Be("Think Like a Man Too"); + result.First().ImdbId.Should().Be("tt2239832"); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index d65fed883..a563c9f31 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -285,6 +285,7 @@ + diff --git a/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs b/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs index 9d5582a9c..5b82fca5f 100644 --- a/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs +++ b/src/NzbDrone.Core/NetImport/HttpNetImportBase.cs @@ -26,6 +26,10 @@ namespace NzbDrone.Core.NetImport public override bool Enabled => true; + public bool SupportsPaging => PageSize > 0; + + public virtual int PageSize => 0; + public virtual TimeSpan RateLimit => TimeSpan.FromSeconds(2); public abstract INetImportRequestGenerator GetRequestGenerator(); @@ -39,7 +43,189 @@ namespace NzbDrone.Core.NetImport public override IList Fetch() { - return new List(); + var generator = GetRequestGenerator(); + + return FetchMovies(generator.GetMovies()); + } + + protected virtual IList FetchMovies(NetImportPageableRequestChain pageableRequestChain, bool isRecent = false) + { + var movies = new List(); + var url = string.Empty; + + var parser = GetParser(); + + try + { + var fullyUpdated = false; + Movie lastMovie = null; + if (isRecent) + { + //lastReleaseInfo = _indexerStatusService.GetLastRssSyncReleaseInfo(Definition.Id); + } + + for (int i = 0; i < pageableRequestChain.Tiers; i++) + { + var pageableRequests = pageableRequestChain.GetTier(i); + + foreach (var pageableRequest in pageableRequests) + { + var pagedReleases = new List(); + + foreach (var request in pageableRequest) + { + url = request.Url.FullUri; + + var page = FetchPage(request, parser); + + pagedReleases.AddRange(page); + + if (isRecent && page.Any()) + { + if (lastMovie == null) + { + fullyUpdated = true; + break; + }/* + var oldestReleaseDate = page.Select(v => v.PublishDate).Min(); + if (oldestReleaseDate < lastReleaseInfo.PublishDate || page.Any(v => v.DownloadUrl == lastReleaseInfo.DownloadUrl)) + { + fullyUpdated = true; + break; + } + + if (pagedReleases.Count >= MaxNumResultsPerQuery && + oldestReleaseDate < DateTime.UtcNow - TimeSpan.FromHours(24)) + { + fullyUpdated = false; + break; + }*///update later + } + else if (pagedReleases.Count >= MaxNumResultsPerQuery) + { + break; + } + + if (!IsFullPage(page)) + { + break; + } + } + + movies.AddRange(pagedReleases); + } + + if (movies.Any()) + { + break; + } + } + + if (isRecent && !movies.Empty()) + { + var ordered = movies.OrderByDescending(v => v.Title).ToList(); + + lastMovie = ordered.First(); + //_indexerStatusService.UpdateRssSyncStatus(Definition.Id, lastReleaseInfo); + } + + //_indexerStatusService.RecordSuccess(Definition.Id); + } + catch (WebException webException) + { + if (webException.Status == WebExceptionStatus.NameResolutionFailure || + webException.Status == WebExceptionStatus.ConnectFailure) + { + //_indexerStatusService.RecordConnectionFailure(Definition.Id); + } + else + { + //_indexerStatusService.RecordFailure(Definition.Id); + } + + if (webException.Message.Contains("502") || webException.Message.Contains("503") || + webException.Message.Contains("timed out")) + { + _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message); + } + else + { + _logger.Warn("{0} {1} {2}", this, url, webException.Message); + } + } + catch (HttpException httpException) + { + if ((int)httpException.Response.StatusCode == 429) + { + //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); + _logger.Warn("API Request Limit reached for {0}", this); + } + else + { + //_indexerStatusService.RecordFailure(Definition.Id); + _logger.Warn("{0} {1}", this, httpException.Message); + } + } + catch (RequestLimitReachedException) + { + //_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); + _logger.Warn("API Request Limit reached for {0}", this); + } + catch (ApiKeyException) + { + //_indexerStatusService.RecordFailure(Definition.Id); + _logger.Warn("Invalid API Key for {0} {1}", this, url); + } + catch (CloudFlareCaptchaException ex) + { + //_indexerStatusService.RecordFailure(Definition.Id); + if (ex.IsExpired) + { + _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this); + } + else + { + _logger.Error(ex, "CAPTCHA token required for {0}, check indexer settings.", this); + } + } + catch (IndexerException ex) + { + //_indexerStatusService.RecordFailure(Definition.Id); + var message = string.Format("{0} - {1}", ex.Message, url); + _logger.Warn(ex, message); + } + catch (Exception feedEx) + { + //_indexerStatusService.RecordFailure(Definition.Id); + feedEx.Data.Add("FeedUrl", url); + _logger.Error(feedEx, "An error occurred while processing feed. " + url); + } + + return movies; + } + + protected virtual bool IsFullPage(IList page) + { + return PageSize != 0 && page.Count >= PageSize; + } + + protected virtual IList FetchPage(NetImportRequest request, IParseNetImportResponse parser) + { + var response = FetchIndexerResponse(request); + + return parser.ParseResponse(response).ToList(); + } + + protected virtual NetImportResponse FetchIndexerResponse(NetImportRequest request) + { + _logger.Debug("Downloading List " + request.HttpRequest.ToString(false)); + + if (request.HttpRequest.RateLimit < RateLimit) + { + request.HttpRequest.RateLimit = RateLimit; + } + + return new NetImportResponse(request, _httpClient.Execute(request.HttpRequest)); } protected override void Test(List failures)