Fixed: Handle ratelimit response from acoustid

src/NzbDrone.Core.Test/ParserTests/FingerprintingServiceFixture.cs

@@ -216,7 +216,7 @@ FINGERPRINT=AQAHJlMURlEURcgP6cwRD43Y4Ptw9FowncWPWkf6GB9-JYdP9OgJHw8u4Apw4SsOHMdx
             Subject.Lookup(new List<LocalTrack> { localTrack }, 0.5);
             localTrack.AcoustIdResults.Should().BeNull();
 
-            ExceptionVerification.ExpectedWarns(1);
+            ExceptionVerification.ExpectedWarns(4);
         }
 
         [Test]
@@ -232,5 +232,43 @@ FINGERPRINT=AQAHJlMURlEURcgP6cwRD43Y4Ptw9FowncWPWkf6GB9-JYdP9OgJHw8u4Apw4SsOHMdx
 
             ExceptionVerification.ExpectedWarns(1);
         }
+
+        [Test]
+        public void should_retry_if_too_many_requests()
+        {
+            var error = "{\"error\": {\"code\": 14, \"message\": \"rate limit (3 requests per second) exceeded, try again later\"}, \"status\": \"error\"}";
+            var response = "{\"fingerprints\": [{\"index\": \"0\", \"results\": [{\"id\": \"a9b004fe-e161-417c-9f9e-443e4525334d\", \"recordings\": [{\"id\": \"209a4536-97ac-4e8a-aff1-1d39d029044b\"}, {\"id\": \"30f3f33e-8d0c-4e69-8539-cbd701d18f28\"}], \"score\": 0.940997}, {\"id\": \"fe0a9bec-2633-4c37-89be-b5d295b68a00\", \"score\": 0.763876}, {\"id\": \"18eab869-51dc-4948-83f7-4d8d441d5a1b\", \"score\": 0.490447}]}], \"status\": \"ok\"}";
+            Mocker.GetMock<IHttpClient>()
+                .SetupSequence(o => o.Post<LookupResponse>(It.IsAny<HttpRequest>()))
+                .Returns(new HttpResponse<LookupResponse>(new HttpResponse(new HttpRequest("dummy"), new HttpHeader(), error, HttpStatusCode.ServiceUnavailable)))
+                .Returns(new HttpResponse<LookupResponse>(new HttpResponse(new HttpRequest("dummy"), new HttpHeader(), response, HttpStatusCode.OK)));
+
+            var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Files", "Media", "nin.mp3");
+            var localTrack = new LocalTrack { Path = path };
+            Subject.Lookup(new List<LocalTrack> { localTrack }, 0.5);
+            localTrack.AcoustIdResults.Should().NotBeNull();
+            localTrack.AcoustIdResults.Should().Contain("30f3f33e-8d0c-4e69-8539-cbd701d18f28");
+
+            Mocker.GetMock<IHttpClient>()
+                .Verify(x => x.Post<LookupResponse>(It.IsAny<HttpRequest>()), Times.Exactly(2));
+        }
+
+        [Test]
+        public void should_not_retry_indefinitely_if_too_many_requests()
+        {
+            var error = "{\"error\": {\"code\": 14, \"message\": \"rate limit (3 requests per second) exceeded, try again later\"}, \"status\": \"error\"}";
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Post<LookupResponse>(It.IsAny<HttpRequest>()))
+                .Returns(new HttpResponse<LookupResponse>(new HttpResponse(new HttpRequest("dummy"), new HttpHeader(), error, HttpStatusCode.ServiceUnavailable)));
+
+            var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Files", "Media", "nin.mp3");
+            var localTrack = new LocalTrack { Path = path };
+
+            Subject.Lookup(new List<LocalTrack> { localTrack }, 0.5);
+            localTrack.AcoustIdResults.Should().BeNull();
+
+            Mocker.GetMock<IHttpClient>()
+                .Verify(x => x.Post<LookupResponse>(It.IsAny<HttpRequest>()), Times.Exactly(4));
+        }
     }
 }

src/NzbDrone.Core/Parser/FingerprintingService.cs

@@ -1,20 +1,20 @@
-using System.IO;
+using System;
-using NLog;
-using NzbDrone.Core.Parser.Model;
-using System.Diagnostics;
-using System.Linq;
-using NzbDrone.Common.Http;
-using NzbDrone.Common.Extensions;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Text;
-using NzbDrone.Common.Serializer;
-using System;
-using NzbDrone.Common.EnvironmentInfo;
-using System.Threading;
 using System.Text.RegularExpressions;
+using System.Threading;
 using Newtonsoft.Json;
+using NLog;
 using NzbDrone.Common.Cache;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Parser
 {
@@ -347,6 +347,13 @@ namespace NzbDrone.Core.Parser
                 return;
             }
 
+            var request = GenerateRequest(toLookup);
+            var response = GetResponse(request);
+            ParseResponse(response, toLookup, threshold);
+        }
+
+        public HttpRequest GenerateRequest(List<Tuple<LocalTrack, AcoustId>> toLookup)
+        {
             var httpRequest = _customerRequestBuilder.Create()
                 .WithRateLimit(0.334)
                 .Build();
@@ -364,32 +371,41 @@ namespace NzbDrone.Core.Parser
             httpRequest.SuppressHttpError = true;
             httpRequest.RequestTimeout = TimeSpan.FromSeconds(5);
 
+            return httpRequest;
+        }
+
+        public LookupResponse GetResponse(HttpRequest request, int retry = 3)
+        {
             HttpResponse<LookupResponse> httpResponse;
 
             try
             {
-                httpResponse = _httpClient.Post<LookupResponse>(httpRequest);
+                httpResponse = _httpClient.Post<LookupResponse>(request);
             }
             catch (UnexpectedHtmlContentException e)
             {
                 _logger.Warn(e, "AcoustId API gave invalid response");
-                return;
+                return retry > 0 ? GetResponse(request, retry - 1) : null;
             }
             catch (Exception e)
             {
                 _logger.Warn(e, "AcoustId API lookup failed");
-                return;
+                return null;
             }
 
             var response = httpResponse.Resource;
 
-            // The API will give errors if fingerprint isn't found or is invalid.
-            // We don't want to stop the entire import because the fingerprinting failed
-            // so just log and return.
             if (httpResponse.HasHttpError || (response != null && response.Status != "ok"))
             {
-                if (response != null && response.Error != null)
+                if (response?.Error != null)
                 {
+                    if (response.Error.Code == AcoustIdErrorCode.TooManyRequests && retry > 0)
+                    {
+                        _logger.Trace($"Too many requests, retrying in 1s");
+                        Thread.Sleep(TimeSpan.FromSeconds(1));
+                        return GetResponse(request, retry - 1);
+                    }
+
                     _logger.Debug($"Webservice error {response.Error.Code}: {response.Error.Message}");
                 }
                 else
@@ -397,9 +413,22 @@ namespace NzbDrone.Core.Parser
                     _logger.Warn("HTTP Error - {0}", httpResponse);
                 }
 
+                return null;
+            }
+
+            return response;
+        }
+
+        private void ParseResponse(LookupResponse response, List<Tuple<LocalTrack, AcoustId>> toLookup, double threshold)
+        {
+            if (response == null)
+            {
                 return;
             }
 
+            // The API will give errors if fingerprint isn't found or is invalid.
+            // We don't want to stop the entire import because the fingerprinting failed
+            // so just log and return.
             foreach (var fileResponse in response.Fingerprints)
             {
                 if (fileResponse.Results.Count == 0)
@@ -434,10 +463,33 @@ namespace NzbDrone.Core.Parser
             public List<LookupResultListItem> Fingerprints { get; set; }
         }
 
+        public enum AcoustIdErrorCode
+        {
+            // https://github.com/acoustid/acoustid-server/blob/f671339ad9ab049c4d4361d3eadb6660a8fe4dda/acoustid/api/errors.py#L10
+            UnknownFormat = 1,
+            MissingParameter = 2,
+            InvalidFingerprint = 3,
+            InvalidApikey = 4,
+            Internal = 5,
+            InvalidUserApikey = 6,
+            InvalidUuid = 7,
+            InvalidDuration = 8,
+            InvalidBitrate = 9,
+            InvalidForeignid = 10,
+            InvalidMaxDurationDiff = 11,
+            NotAllowed = 12,
+            ServiceUnavailable = 13,
+            TooManyRequests = 14,
+            InvalidMusicbrainzAccessToken = 15,
+            InsecureRequest = 16,
+            UnknownApplication = 17,
+            FingerprintNotFound = 18
+        }
+
         public class LookupError
         {
             public string Message { get; set; }
-            public int Code { get; set; }
+            public AcoustIdErrorCode Code { get; set; }
         }
 
         public class LookupResultListItem