From 8af6ea1d8f5ba7fb759d659232d8475015d1d7fd Mon Sep 17 00:00:00 2001
From: Bogdan <mynameisbogdan@users.noreply.github.com>
Date: Tue, 9 Apr 2024 21:29:05 +0300
Subject: [PATCH] New: Retry on failed indexer requests

---
 src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 38 ++++++++++++++++++-
 src/NzbDrone.Core/Prowlarr.Core.csproj        |  1 +
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
index 94752c7df..8e7f0fc1e 100644
--- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
+++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
@@ -18,6 +18,8 @@ using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
+using Polly;
+using Polly.Retry;
 
 namespace NzbDrone.Core.Indexers
 {
@@ -28,6 +30,38 @@ namespace NzbDrone.Core.Indexers
 
         protected readonly IIndexerHttpClient _httpClient;
         protected readonly IEventAggregator _eventAggregator;
+
+        protected ResiliencePipeline<HttpResponse> RetryStrategy => new ResiliencePipelineBuilder<HttpResponse>()
+            .AddRetry(new RetryStrategyOptions<HttpResponse>
+            {
+                ShouldHandle = static args => args.Outcome switch
+                {
+                    { Result.HasHttpServerError: true } => PredicateResult.True(),
+                    { Result.StatusCode: HttpStatusCode.RequestTimeout } => PredicateResult.True(),
+                    _ => PredicateResult.False()
+                },
+                Delay = RateLimit,
+                MaxRetryAttempts = 2,
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = true,
+                OnRetry = args =>
+                {
+                    var exception = args.Outcome.Exception;
+
+                    if (exception is not null)
+                    {
+                        _logger.Warn(exception, "Request for {0} failed with exception '{1}'. Retrying in {2}s.", Definition.Name, exception.Message, args.RetryDelay.TotalSeconds);
+                    }
+                    else
+                    {
+                        _logger.Warn("Request for {0} failed with status {1}. Retrying in {2}s.", Definition.Name, args.Outcome.Result?.StatusCode, args.RetryDelay.TotalSeconds);
+                    }
+
+                    return default;
+                }
+            })
+            .Build();
+
         public IDictionary<string, string> Cookies { get; set; }
 
         public override bool SupportsRss => true;
@@ -584,7 +618,9 @@ namespace NzbDrone.Core.Indexers
             request.HttpRequest.SuppressHttpError = true;
             request.HttpRequest.Encoding ??= Encoding;
 
-            var response = await _httpClient.ExecuteProxiedAsync(request.HttpRequest, Definition);
+            var response = await RetryStrategy
+                .ExecuteAsync(static async (state, _) => await state._httpClient.ExecuteProxiedAsync(state.HttpRequest, state.Definition), (_httpClient, request.HttpRequest, Definition))
+                .ConfigureAwait(false);
 
             // Check response to see if auth is needed, if needed try again
             if (CheckIfLoginNeeded(response))
diff --git a/src/NzbDrone.Core/Prowlarr.Core.csproj b/src/NzbDrone.Core/Prowlarr.Core.csproj
index b9ab1d826..29ef56865 100644
--- a/src/NzbDrone.Core/Prowlarr.Core.csproj
+++ b/src/NzbDrone.Core/Prowlarr.Core.csproj
@@ -10,6 +10,7 @@
     <PackageReference Include="Microsoft.AspNetCore.WebUtilities" Version="2.2.0" />
     <PackageReference Include="NLog.Targets.Syslog" Version="7.0.0" />
     <PackageReference Include="Npgsql" Version="7.0.6" />
+    <PackageReference Include="Polly" Version="8.3.1" />
     <PackageReference Include="Servarr.FluentMigrator.Runner" Version="3.3.2.9" />
     <PackageReference Include="Servarr.FluentMigrator.Runner.Postgres" Version="3.3.2.9" />
     <PackageReference Include="Servarr.FluentMigrator.Runner.SQLite" Version="3.3.2.9" />