From a579a93aab32b51ff910c5bd87980ddfcd9cc786 Mon Sep 17 00:00:00 2001
From: ta264 <ta264@users.noreply.github.com>
Date: Mon, 18 Jan 2021 20:09:28 +0000
Subject: [PATCH] Fixed: Support large calibre libraries

---
 .../Books/Calibre/CalibreProxy.cs             | 78 +++++++++++++++----
 .../Calibre/Resources/CalibreCategory.cs      | 20 +++++
 2 files changed, 84 insertions(+), 14 deletions(-)
 create mode 100644 src/NzbDrone.Core/Books/Calibre/Resources/CalibreCategory.cs

diff --git a/src/NzbDrone.Core/Books/Calibre/CalibreProxy.cs b/src/NzbDrone.Core/Books/Calibre/CalibreProxy.cs
index 77b4abb88..883536764 100644
--- a/src/NzbDrone.Core/Books/Calibre/CalibreProxy.cs
+++ b/src/NzbDrone.Core/Books/Calibre/CalibreProxy.cs
@@ -245,31 +245,81 @@ namespace NzbDrone.Core.Books.Calibre
         {
             _bookCache.Clear();
 
-            try
+            var ids = GetAllBookIds(settings);
+            var result = new List<string>();
+
+            const int count = 100;
+            var offset = 0;
+
+            while (offset < ids.Count)
             {
                 var builder = GetBuilder($"ajax/books", settings);
-                builder.LogResponseContent = false;
+                builder.AddQueryParam("ids", ids.Skip(offset).Take(count).ConcatToString(","));
 
                 var request = builder.Build();
-                var response = _httpClient.Get<Dictionary<int, CalibreBook>>(request);
-
-                var result = new List<string>();
-
-                foreach (var book in response.Resource.Values)
+                try
                 {
-                    var remotePath = book?.Formats.Values.OrderBy(f => f.LastModified).FirstOrDefault()?.Path;
-                    if (remotePath == null)
+                    var response = _httpClient.Get<Dictionary<int, CalibreBook>>(request);
+                    foreach (var book in response.Resource.Values)
                     {
-                        continue;
+                        var remotePath = book?.Formats.Values.OrderBy(f => f.LastModified).FirstOrDefault()?.Path;
+                        if (remotePath == null)
+                        {
+                            continue;
+                        }
+
+                        var localPath = _pathMapper.RemapRemoteToLocal(settings.Host, new OsPath(remotePath)).FullPath;
+                        result.Add(localPath);
+
+                        _bookCache.Set(localPath, book, TimeSpan.FromMinutes(5));
                     }
+                }
+                catch (HttpException ex)
+                {
+                    throw new CalibreException("Unable to connect to calibre library: {0}", ex, ex.Message);
+                }
 
-                    var localPath = _pathMapper.RemapRemoteToLocal(settings.Host, new OsPath(remotePath)).FullPath;
-                    result.Add(localPath);
+                offset += count;
+            }
+
+            return result;
+        }
 
-                    _bookCache.Set(localPath, book, TimeSpan.FromMinutes(5));
+        public List<int> GetAllBookIds(CalibreSettings settings)
+        {
+            // the magic string is 'allbooks' converted to hex
+            var builder = GetBuilder($"/ajax/category/616c6c626f6f6b73", settings);
+            const int count = 100;
+            var offset = 0;
+
+            var ids = new List<int>();
+
+            while (true)
+            {
+                var result = GetPaged<CalibreCategory>(builder, count, offset);
+                if (!result.Resource.BookIds.Any())
+                {
+                    break;
                 }
 
-                return result;
+                offset += count;
+                ids.AddRange(result.Resource.BookIds);
+            }
+
+            return ids;
+        }
+
+        private HttpResponse<T> GetPaged<T>(HttpRequestBuilder builder, int count, int offset)
+            where T : new()
+        {
+            builder.AddQueryParam("num", count, replace: true);
+            builder.AddQueryParam("offset", offset, replace: true);
+
+            var request = builder.Build();
+
+            try
+            {
+                return _httpClient.Get<T>(request);
             }
             catch (HttpException ex)
             {
diff --git a/src/NzbDrone.Core/Books/Calibre/Resources/CalibreCategory.cs b/src/NzbDrone.Core/Books/Calibre/Resources/CalibreCategory.cs
new file mode 100644
index 000000000..0319569d8
--- /dev/null
+++ b/src/NzbDrone.Core/Books/Calibre/Resources/CalibreCategory.cs
@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace NzbDrone.Core.Books.Calibre
+{
+    public class CalibreCategory
+    {
+        [JsonProperty("total_num")]
+        public int TotalNum { get; set; }
+        [JsonProperty("sort_order")]
+        public string SortOrder { get; set; }
+        public int Offset { get; set; }
+        public int Num { get; set; }
+        public string Sort { get; set; }
+        [JsonProperty("base_url")]
+        public string BaseUrl { get; set; }
+        [JsonProperty("book_ids")]
+        public List<int> BookIds { get; set; }
+    }
+}