New: List Status Checks/Backoffs

pull/4310/head
Qstick 4 years ago
parent d10e60587b
commit b1fd924188

@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.HealthCheck.Checks;
using NzbDrone.Core.NetImport;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.HealthCheck.Checks
{
[TestFixture]
public class NetImportStatusCheckFixture : CoreTest<NetImportStatusCheck>
{
private List<INetImport> _lists = new List<INetImport>();
private List<NetImportStatus> _blockedLists = new List<NetImportStatus>();
[SetUp]
public void SetUp()
{
Mocker.GetMock<INetImportFactory>()
.Setup(v => v.GetAvailableProviders())
.Returns(_lists);
Mocker.GetMock<INetImportStatusService>()
.Setup(v => v.GetBlockedProviders())
.Returns(_blockedLists);
}
private Mock<INetImport> GivenList(int i, double backoffHours, double failureHours)
{
var id = i;
var mockList = new Mock<INetImport>();
mockList.SetupGet(s => s.Definition).Returns(new NetImportDefinition { Id = id });
mockList.SetupGet(s => s.EnableAuto).Returns(true);
_lists.Add(mockList.Object);
if (backoffHours != 0.0)
{
_blockedLists.Add(new NetImportStatus
{
ProviderId = id,
InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
EscalationLevel = 5,
DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
});
}
return mockList;
}
[Test]
public void should_not_return_error_when_no_indexers()
{
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_warning_if_indexer_unavailable()
{
GivenList(1, 10.0, 24.0);
GivenList(2, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
[Test]
public void should_return_error_if_all_indexers_unavailable()
{
GivenList(1, 10.0, 24.0);
Subject.Check().ShouldBeError();
}
[Test]
public void should_return_warning_if_few_indexers_unavailable()
{
GivenList(1, 10.0, 24.0);
GivenList(2, 10.0, 24.0);
GivenList(3, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
}
}

@ -0,0 +1,72 @@
using System;
using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Core.NetImport;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.NetImport
{
public class NetImportStatusServiceFixture : CoreTest<NetImportStatusService>
{
private DateTime _epoch;
[SetUp]
public void SetUp()
{
_epoch = DateTime.UtcNow;
Mocker.GetMock<IRuntimeInfo>()
.SetupGet(v => v.StartTime)
.Returns(_epoch - TimeSpan.FromHours(1));
}
private void WithStatus(NetImportStatus status)
{
Mocker.GetMock<INetImportStatusRepository>()
.Setup(v => v.FindByProviderId(1))
.Returns(status);
Mocker.GetMock<INetImportStatusRepository>()
.Setup(v => v.All())
.Returns(new[] { status });
}
private void VerifyUpdate()
{
Mocker.GetMock<INetImportStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<NetImportStatus>()), Times.Once());
}
private void VerifyNoUpdate()
{
Mocker.GetMock<INetImportStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<NetImportStatus>()), Times.Never());
}
[Test]
public void should_cancel_backoff_on_success()
{
WithStatus(new NetImportStatus { EscalationLevel = 2 });
Subject.RecordSuccess(1);
VerifyUpdate();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().BeNull();
}
[Test]
public void should_not_store_update_if_already_okay()
{
WithStatus(new NetImportStatus { EscalationLevel = 0 });
Subject.RecordSuccess(1);
VerifyNoUpdate();
}
}
}

@ -0,0 +1,20 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(173)]
public class net_import_status : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.TableForModel("NetImportStatus")
.WithColumn("ProviderId").AsInt32().NotNullable().Unique()
.WithColumn("InitialFailure").AsDateTime().Nullable()
.WithColumn("MostRecentFailure").AsDateTime().Nullable()
.WithColumn("EscalationLevel").AsInt32().NotNullable()
.WithColumn("DisabledTill").AsDateTime().Nullable()
.WithColumn("LastSyncListInfo").AsString().Nullable();
}
}
}

@ -137,6 +137,7 @@ namespace NzbDrone.Core.Datastore
Mapper.Entity<IndexerStatus>("IndexerStatus").RegisterModel();
Mapper.Entity<DownloadClientStatus>("DownloadClientStatus").RegisterModel();
Mapper.Entity<NetImportStatus>("NetImportStatus").RegisterModel();
Mapper.Entity<CustomFilter>("CustomFilters").RegisterModel();

@ -0,0 +1,44 @@
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.NetImport;
using NzbDrone.Core.ThingiProvider.Events;
namespace NzbDrone.Core.HealthCheck.Checks
{
[CheckOn(typeof(ProviderUpdatedEvent<INetImport>))]
[CheckOn(typeof(ProviderDeletedEvent<INetImport>))]
[CheckOn(typeof(ProviderStatusChangedEvent<INetImport>))]
public class NetImportStatusCheck : HealthCheckBase
{
private readonly INetImportFactory _providerFactory;
private readonly INetImportStatusService _providerStatusService;
public NetImportStatusCheck(INetImportFactory providerFactory, INetImportStatusService providerStatusService)
{
_providerFactory = providerFactory;
_providerStatusService = providerStatusService;
}
public override HealthCheck Check()
{
var enabledProviders = _providerFactory.GetAvailableProviders();
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new { Provider = i, Status = s })
.ToList();
if (backOffProviders.Empty())
{
return new HealthCheck(GetType());
}
if (backOffProviders.Count == enabledProviders.Count)
{
return new HealthCheck(GetType(), HealthCheckResult.Error, "All lists are unavailable due to failures", "#lists-are-unavailable-due-to-failures");
}
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Lists unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#lists-are-unavailable-due-to-failures");
}
}
}

@ -13,8 +13,8 @@ namespace NzbDrone.Core.NetImport.CouchPotato
public override bool Enabled => true;
public override bool EnableAuto => false;
public CouchPotatoImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, configService, parsingService, logger)
public CouchPotatoImport(IHttpClient httpClient, INetImportStatusService netImportStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
@ -7,6 +7,8 @@ using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Movies;
using NzbDrone.Core.NetImport.Exceptions;
using NzbDrone.Core.Parser;
@ -28,8 +30,8 @@ namespace NzbDrone.Core.NetImport
public abstract INetImportRequestGenerator GetRequestGenerator();
public abstract IParseNetImportResponse GetParser();
public HttpNetImportBase(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
: base(configService, parsingService, logger)
public HttpNetImportBase(IHttpClient httpClient, INetImportStatusService netImportStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(netImportStatusService, configService, parsingService, logger)
{
_httpClient = httpClient;
}
@ -72,10 +74,21 @@ namespace NzbDrone.Core.NetImport
break;
}
}
_netImportStatusService.RecordSuccess(Definition.Id);
}
catch (WebException webException)
{
anyFailure = true;
if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
webException.Status == WebExceptionStatus.ConnectFailure)
{
_netImportStatusService.RecordConnectionFailure(Definition.Id);
}
else
{
_netImportStatusService.RecordFailure(Definition.Id);
}
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
@ -86,23 +99,52 @@ namespace NzbDrone.Core.NetImport
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (HttpException httpException)
catch (TooManyRequestsException ex)
{
if (ex.RetryAfter != TimeSpan.Zero)
{
_netImportStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_netImportStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (HttpException ex)
{
_netImportStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
{
anyFailure = true;
if ((int)httpException.Response.StatusCode == 429)
_netImportStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (CloudFlareCaptchaException ex)
{
_netImportStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
if (ex.IsExpired)
{
_logger.Warn("API Request Limit reached for {0}", this);
_logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in import list settings.", this);
}
else
{
_logger.Warn("{0} {1}", this, httpException.Message);
_logger.Error(ex, "CAPTCHA token required for {0}, check import list settings.", this);
}
}
catch (Exception feedEx)
catch (NetImportException ex)
{
_netImportStatusService.RecordFailure(Definition.Id);
_logger.Warn(ex, "{0}", url);
}
catch (Exception ex)
{
anyFailure = true;
feedEx.Data.Add("FeedUrl", url);
_logger.Error(feedEx, "An error occurred while processing list feed {0}", url);
_netImportStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
_logger.Error(ex, "An error occurred while processing feed. {0}", url);
}
return new NetImportFetchResult { Movies = movies, AnyFailure = anyFailure };
@ -155,6 +197,16 @@ namespace NzbDrone.Core.NetImport
return new ValidationFailure(string.Empty, "No results were returned from your list, please check your settings.");
}
}
catch (RequestLimitReachedException)
{
_logger.Warn("Request limit reached");
}
catch (UnsupportedFeedException ex)
{
_logger.Warn(ex, "Net Import feed is not supported");
return new ValidationFailure(string.Empty, "Net Import feed is not supported: " + ex.Message);
}
catch (NetImportException ex)
{
_logger.Warn(ex, "Unable to connect to list");

@ -18,6 +18,7 @@ namespace NzbDrone.Core.NetImport
public abstract class NetImportBase<TSettings> : INetImport
where TSettings : IProviderConfig, new()
{
protected readonly INetImportStatusService _netImportStatusService;
protected readonly IConfigService _configService;
protected readonly IParsingService _parsingService;
protected readonly Logger _logger;
@ -30,8 +31,9 @@ namespace NzbDrone.Core.NetImport
public abstract NetImportFetchResult Fetch();
public NetImportBase(IConfigService configService, IParsingService parsingService, Logger logger)
public NetImportBase(INetImportStatusService netImportStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
{
_netImportStatusService = netImportStatusService;
_configService = configService;
_parsingService = parsingService;
_logger = logger;

@ -0,0 +1,10 @@
using NzbDrone.Core.Movies;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.NetImport
{
public class NetImportStatus : ProviderStatusBase
{
public Movie LastSyncListInfo { get; set; }
}
}

@ -0,0 +1,18 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.NetImport
{
public interface INetImportStatusRepository : IProviderStatusRepository<NetImportStatus>
{
}
public class NetImportStatusRepository : ProviderStatusRepository<NetImportStatus>, INetImportStatusRepository
{
public NetImportStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
}
}

@ -0,0 +1,40 @@
using NLog;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Movies;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.NetImport
{
public interface INetImportStatusService : IProviderStatusServiceBase<NetImportStatus>
{
Movie GetLastSyncListInfo(int importListId);
void UpdateListSyncStatus(int importListId, Movie listItemInfo);
}
public class NetImportStatusService : ProviderStatusServiceBase<INetImport, NetImportStatus>, INetImportStatusService
{
public NetImportStatusService(INetImportStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
: base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
{
}
public Movie GetLastSyncListInfo(int importListId)
{
return GetProviderStatus(importListId).LastSyncListInfo;
}
public void UpdateListSyncStatus(int importListId, Movie listItemInfo)
{
lock (_syncRoot)
{
var status = GetProviderStatus(importListId);
status.LastSyncListInfo = listItemInfo;
_providerStatusRepository.Upsert(status);
}
}
}
}

@ -15,8 +15,8 @@ namespace NzbDrone.Core.NetImport.RSSImport
public override bool Enabled => true;
public override bool EnableAuto => false;
public RSSImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, configService, parsingService, logger)
public RSSImport(IHttpClient httpClient, INetImportStatusService netImportStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -21,10 +21,11 @@ namespace NzbDrone.Core.NetImport.Radarr
public override NetImportType ListType => NetImportType.Program;
public RadarrImport(IRadarrV3Proxy radarrV3Proxy,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(configService, parsingService, logger)
: base(netImportStatusService, configService, parsingService, logger)
{
_radarrV3Proxy = radarrV3Proxy;
}

@ -16,10 +16,11 @@ namespace NzbDrone.Core.NetImport.RadarrList
public override bool EnableAuto => false;
public RadarrListImport(IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(httpClient, configService, parsingService, logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -1,4 +1,4 @@
using NLog;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
@ -13,8 +13,12 @@ namespace NzbDrone.Core.NetImport.StevenLu
public override bool Enabled => true;
public override bool EnableAuto => false;
public StevenLuImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, configService, parsingService, logger)
public StevenLuImport(IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -11,11 +11,12 @@ namespace NzbDrone.Core.NetImport.TMDb.Collection
{
public TMDbCollectionImport(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie searchForNewMovie,
Logger logger)
: base(requestBuilder, httpClient, configService, parsingService, searchForNewMovie, logger)
: base(requestBuilder, httpClient, netImportStatusService, configService, parsingService, searchForNewMovie, logger)
{
}

@ -11,11 +11,12 @@ namespace NzbDrone.Core.NetImport.TMDb.List
{
public TMDbListImport(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie searchForNewMovie,
Logger logger)
: base(requestBuilder, httpClient, configService, parsingService, searchForNewMovie, logger)
: base(requestBuilder, httpClient, netImportStatusService, configService, parsingService, searchForNewMovie, logger)
{
}

@ -11,11 +11,12 @@ namespace NzbDrone.Core.NetImport.TMDb.Person
{
public TMDbPersonImport(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie searchForNewMovie,
Logger logger)
: base(requestBuilder, httpClient, configService, parsingService, searchForNewMovie, logger)
: base(requestBuilder, httpClient, netImportStatusService, configService, parsingService, searchForNewMovie, logger)
{
}

@ -11,11 +11,12 @@ namespace NzbDrone.Core.NetImport.TMDb.Popular
{
public TMDbPopularImport(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie searchForNewMovie,
Logger logger)
: base(requestBuilder, httpClient, configService, parsingService, searchForNewMovie, logger)
: base(requestBuilder, httpClient, netImportStatusService, configService, parsingService, searchForNewMovie, logger)
{
}

@ -1,4 +1,4 @@
using NLog;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
@ -17,11 +17,12 @@ namespace NzbDrone.Core.NetImport.TMDb
protected TMDbNetImportBase(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie skyhookProxy,
Logger logger)
: base(httpClient, configService, parsingService, logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
_skyhookProxy = skyhookProxy;
_requestBuilder = requestBuilder.TMDB;

@ -13,11 +13,12 @@ namespace NzbDrone.Core.NetImport.TMDb.User
{
public TMDbUserImport(IRadarrCloudRequestBuilder requestBuilder,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
ISearchForNewMovie searchForNewMovie,
Logger logger)
: base(requestBuilder, httpClient, configService, parsingService, searchForNewMovie, logger)
: base(requestBuilder, httpClient, netImportStatusService, configService, parsingService, searchForNewMovie, logger)
{
}

@ -9,10 +9,11 @@ namespace NzbDrone.Core.NetImport.Trakt.List
{
public TraktListImport(INetImportRepository netImportRepository,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(netImportRepository, httpClient, configService, parsingService, logger)
: base(netImportRepository, httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -9,10 +9,11 @@ namespace NzbDrone.Core.NetImport.Trakt.Popular
{
public TraktPopularImport(INetImportRepository netImportRepository,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(netImportRepository, httpClient, configService, parsingService, logger)
: base(netImportRepository, httpClient, netImportStatusService, configService, parsingService, logger)
{
}

@ -18,10 +18,11 @@ namespace NzbDrone.Core.NetImport.Trakt
protected TraktImportBase(INetImportRepository netImportRepository,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(httpClient, configService, parsingService, logger)
: base(httpClient, netImportStatusService, configService, parsingService, logger)
{
_netImportRepository = netImportRepository;
}

@ -9,10 +9,11 @@ namespace NzbDrone.Core.NetImport.Trakt.User
{
public TraktUserImport(INetImportRepository netImportRepository,
IHttpClient httpClient,
INetImportStatusService netImportStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(netImportRepository, httpClient, configService, parsingService, logger)
: base(netImportRepository, httpClient, netImportStatusService, configService, parsingService, logger)
{
}

Loading…
Cancel
Save