New: Retry on failed downloads of torrent and nzb files

(cherry picked from commit bc20ef73bdd47b7cdad43d4c7d4b4bd534e49252)

Closes #3151

src/Directory.Packages.props

@@ -8,6 +8,7 @@
     <PackageVersion Include="DryIoc.Microsoft.DependencyInjection" Version="6.2.0" />
     <PackageVersion Include="Equ" Version="2.3.0" />
     <PackageVersion Include="FluentAssertions" Version="5.10.3" />
+    <PackageVersion Include="Polly" Version="8.2.0" />
     <PackageVersion Include="Servarr.FluentMigrator.Runner" Version="3.3.2.9" />
     <PackageVersion Include="Servarr.FluentMigrator.Runner.SQLite" Version="3.3.2.9" />
     <PackageVersion Include="Servarr.FluentMigrator.Runner.Postgres" Version="3.3.2.9" />

src/NzbDrone.Core/Download/DownloadClientBase.cs

@@ -1,15 +1,19 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Disk;
+using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
+using Polly;
+using Polly.Retry;
 
 namespace NzbDrone.Core.Download
 {
@@ -21,6 +25,37 @@ namespace NzbDrone.Core.Download
         protected readonly IRemotePathMappingService _remotePathMappingService;
         protected readonly Logger _logger;
 
+        protected ResiliencePipeline<HttpResponse> RetryStrategy => new ResiliencePipelineBuilder<HttpResponse>()
+            .AddRetry(new RetryStrategyOptions<HttpResponse>
+            {
+                ShouldHandle = static args => args.Outcome switch
+                {
+                    { Result.HasHttpServerError: true } => PredicateResult.True(),
+                    { Result.StatusCode: HttpStatusCode.RequestTimeout } => PredicateResult.True(),
+                    _ => PredicateResult.False()
+                },
+                Delay = TimeSpan.FromSeconds(3),
+                MaxRetryAttempts = 2,
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = true,
+                OnRetry = args =>
+                {
+                    var exception = args.Outcome.Exception;
+
+                    if (exception is not null)
+                    {
+                        _logger.Info(exception, "Request for {0} failed with exception '{1}'. Retrying in {2}s.", Definition.Name, exception.Message, args.RetryDelay.TotalSeconds);
+                    }
+                    else
+                    {
+                        _logger.Info("Request for {0} failed with status {1}. Retrying in {2}s.", Definition.Name, args.Outcome.Result?.StatusCode, args.RetryDelay.TotalSeconds);
+                    }
+
+                    return default;
+                }
+            })
+            .Build();
+
         public abstract string Name { get; }
 
         public Type ConfigContract => typeof(TSettings);
@@ -54,10 +89,7 @@ namespace NzbDrone.Core.Download
             return GetType().Name;
         }
 
-        public abstract DownloadProtocol Protocol
-        {
-            get;
-        }
+        public abstract DownloadProtocol Protocol { get; }
 
         public abstract Task<string> Download(RemoteBook remoteBook, IIndexer indexer);
         public abstract IEnumerable<DownloadClientItem> GetItems();

src/NzbDrone.Core/Download/TorrentClientBase.cs

@@ -133,7 +133,9 @@ namespace NzbDrone.Core.Download
                 request.Headers.Accept = "application/x-bittorrent";
                 request.AllowAutoRedirect = false;
 
-                var response = await _httpClient.GetAsync(request);
+                var response = await RetryStrategy
+                    .ExecuteAsync(static async (state, _) => await state._httpClient.GetAsync(state.request), (_httpClient, request))
+                    .ConfigureAwait(false);
 
                 if (response.StatusCode == HttpStatusCode.MovedPermanently ||
                     response.StatusCode == HttpStatusCode.Found ||

src/NzbDrone.Core/Download/UsenetClientBase.cs

@@ -47,7 +47,9 @@ namespace NzbDrone.Core.Download
                 var request = indexer?.GetDownloadRequest(url) ?? new HttpRequest(url);
                 request.RateLimitKey = remoteBook?.Release?.IndexerId.ToString();
 
-                var response = await _httpClient.GetAsync(request);
+                var response = await RetryStrategy
+                    .ExecuteAsync(static async (state, _) => await state._httpClient.GetAsync(state.request), (_httpClient, request))
+                    .ConfigureAwait(false);
 
                 nzbData = response.ResponseData;
 

src/NzbDrone.Core/Readarr.Core.csproj

@@ -5,6 +5,7 @@
   <ItemGroup>
     <PackageReference Include="Dapper" />
     <PackageReference Include="LazyCache" />
+    <PackageReference Include="Polly" />
     <PackageReference Include="System.Text.Json" />
     <PackageReference Include="System.Text.Encoding.CodePages" />
     <PackageReference Include="System.Memory" />