Emby Improvements: Batch up the amount we get from the server.

src/Ombi.Api.Emby/EmbyApi.cs

@@ -99,9 +99,9 @@ namespace Ombi.Api.Emby
             return await Api.Request<EmbyItemContainer<MovieInformation>>(request);
         }
 
-        public async Task<EmbyItemContainer<EmbyMovie>> GetAllMovies(string apiKey, string userId, string baseUri)
+        public async Task<EmbyItemContainer<EmbyMovie>> GetAllMovies(string apiKey, int startIndex, int count, string userId, string baseUri)
         {
-            return await GetAll<EmbyMovie>("Movie", apiKey, userId, baseUri, true);
+            return await GetAll<EmbyMovie>("Movie", apiKey, userId, baseUri, true, startIndex, count);
         }
 
         public async Task<EmbyItemContainer<EmbyEpisodes>> GetAllEpisodes(string apiKey, string userId, string baseUri)
@@ -109,9 +109,9 @@ namespace Ombi.Api.Emby
             return await GetAll<EmbyEpisodes>("Episode", apiKey, userId, baseUri);
         }
 
-        public async Task<EmbyItemContainer<EmbySeries>> GetAllShows(string apiKey, string userId, string baseUri)
+        public async Task<EmbyItemContainer<EmbySeries>> GetAllShows(string apiKey, int startIndex, int count, string userId, string baseUri)
         {
-            return await GetAll<EmbySeries>("Series", apiKey, userId, baseUri);
+            return await GetAll<EmbySeries>("Series", apiKey, userId, baseUri, false, startIndex, count);
         }
 
         public async Task<SeriesInformation> GetSeriesInformation(string mediaId, string apiKey, string userId, string baseUrl)
@@ -150,6 +150,24 @@ namespace Ombi.Api.Emby
             AddHeaders(request, apiKey);
 
 
+            var obj = await Api.Request<EmbyItemContainer<T>>(request);
+            return obj;
+        }
+        private async Task<EmbyItemContainer<T>> GetAll<T>(string type, string apiKey, string userId, string baseUri, bool includeOverview, int startIndex, int count)
+        {
+            var request = new Request($"emby/users/{userId}/items", baseUri, HttpMethod.Get);
+
+            request.AddQueryString("Recursive", true.ToString());
+            request.AddQueryString("IncludeItemTypes", type);
+            request.AddQueryString("Fields", includeOverview ? "ProviderIds,Overview" : "ProviderIds");
+            request.AddQueryString("startIndex", startIndex.ToString());
+            request.AddQueryString("limit", count.ToString());
+
+            request.AddQueryString("VirtualItem", "False");
+
+            AddHeaders(request, apiKey);
+
+
             var obj = await Api.Request<EmbyItemContainer<T>>(request);
             return obj;
         }

src/Ombi.Api.Emby/IEmbyApi.cs

@@ -14,9 +14,14 @@ namespace Ombi.Api.Emby
         Task<EmbyUser> LogIn(string username, string password, string apiKey, string baseUri);
         Task<EmbyConnectUser> LoginConnectUser(string username, string password);
 
-        Task<EmbyItemContainer<EmbyMovie>> GetAllMovies(string apiKey, string userId, string baseUri);
+        Task<EmbyItemContainer<EmbyMovie>> GetAllMovies(string apiKey, int startIndex, int count, string userId,
-        Task<EmbyItemContainer<EmbyEpisodes>> GetAllEpisodes(string apiKey, string userId, string baseUri);
+            string baseUri);
-        Task<EmbyItemContainer<EmbySeries>> GetAllShows(string apiKey, string userId, string baseUri);
+
+        Task<EmbyItemContainer<EmbyEpisodes>> GetAllEpisodes(string apiKey, int startIndex, int count, string userId,
+            string baseUri);
+
+        Task<EmbyItemContainer<EmbySeries>> GetAllShows(string apiKey, int startIndex, int count, string userId,
+            string baseUri);
 
         Task<EmbyItemContainer<MovieInformation>> GetCollection(string mediaId, string apiKey, string userId,
             string baseUrl);

src/Ombi.Schedule/Jobs/Emby/EmbyContentSync.cs

@@ -71,18 +71,36 @@ namespace Ombi.Schedule.Jobs.Emby
             await _repo.ExecuteSql("DELETE FROM EmbyEpisode");
             await _repo.ExecuteSql("DELETE FROM EmbyContent");
 
-            var movies = await _api.GetAllMovies(server.ApiKey, server.AdministratorId, server.FullUri);
+            var movies = await _api.GetAllMovies(server.ApiKey,0, 200, server.AdministratorId, server.FullUri);
+            var totalCount = movies.TotalRecordCount;
+            var processed = 0;
+
             var mediaToAdd = new HashSet<EmbyContent>();
+
+            while (processed < totalCount)
+            {
                 foreach (var movie in movies.Items)
                 {
+                    processed++;
                     // Regular movie
                     await ProcessMovies(movie, mediaToAdd);
                 }
-            // TV Time
-            var tv = await _api.GetAllShows(server.ApiKey, server.AdministratorId, server.FullUri);
 
+                // Get the next batch
+                movies = await _api.GetAllMovies(server.ApiKey, processed + 1, 200, server.AdministratorId, server.FullUri);
+                await _repo.AddRange(mediaToAdd);
+                mediaToAdd.Clear();
+            }
+
+            // TV Time
+            var tv = await _api.GetAllShows(server.ApiKey, 0, 200, server.AdministratorId, server.FullUri);
+            var totalTv = tv.TotalRecordCount;
+            processed = 0;
+            while (processed < totalTv)
+            {
                 foreach (var tvShow in tv.Items)
                 {
+                    processed++;
                     if (string.IsNullOrEmpty(tvShow.ProviderIds?.Tvdb))
                     {
                         Log.Error("Provider Id on tv {0} is null", tvShow.Name);
@@ -103,6 +121,11 @@ namespace Ombi.Schedule.Jobs.Emby
                             AddedAt = DateTime.UtcNow
                         });
                 }
+                // Get the next batch
+                tv = await _api.GetAllShows(server.ApiKey, processed + 1, 200, server.AdministratorId, server.FullUri);
+                await _repo.AddRange(mediaToAdd);
+                mediaToAdd.Clear();
+            }
 
             if (mediaToAdd.Any())
                 await _repo.AddRange(mediaToAdd);

src/Ombi.Schedule/Jobs/Emby/EmbyEpisodeSync.cs

@@ -73,17 +73,22 @@ namespace Ombi.Schedule.Jobs.Emby
 
         private async Task CacheEpisodes(EmbyServers server)
         {
-            var allEpisodes = await _api.GetAllEpisodes(server.ApiKey, server.AdministratorId, server.FullUri);
+            var allEpisodes = await _api.GetAllEpisodes(server.ApiKey, 0, 200, server.AdministratorId, server.FullUri);
+            var total = allEpisodes.TotalRecordCount;
+            var processed = 0;
             var epToAdd = new List<EmbyEpisode>();
-
+            while (processed < total)
+            {
                 foreach (var ep in allEpisodes.Items)
                 {
+                    processed++;
                     // Let's make sure we have the parent request, stop those pesky forign key errors,
                     // Damn me having data integrity
                     var parent = await _repo.GetByEmbyId(ep.SeriesId);
                     if (parent == null)
                     {
-                    _logger.LogInformation("The episode {0} does not relate to a series, so we cannot save this", ep.Name);
+                        _logger.LogInformation("The episode {0} does not relate to a series, so we cannot save this",
+                            ep.Name);
                         continue;
                     }
 
@@ -106,6 +111,11 @@ namespace Ombi.Schedule.Jobs.Emby
                     }
                 }
 
+                await _repo.AddRange(epToAdd);
+                epToAdd.Clear();
+                allEpisodes = await _api.GetAllEpisodes(server.ApiKey, processed + 1, 200, server.AdministratorId, server.FullUri);
+            }
+
             if (epToAdd.Any())
             {
                 await _repo.AddRange(epToAdd);