New: Only refresh items that have updated on metadata server

src/NzbDrone.Core.Test/MusicTests/RefreshAlbumReleaseServiceFixture.cs

@@ -51,7 +51,7 @@ namespace NzbDrone.Core.Test.MusicTests
             newInfo.OldForeignReleaseIds = new List<string> { _release.ForeignReleaseId };
             newInfo.Tracks = _tracks;
 
-            Subject.RefreshEntityInfo(_release, new List<AlbumRelease> { newInfo }, false, false);
+            Subject.RefreshEntityInfo(_release, new List<AlbumRelease> { newInfo }, false, false, null);
 
             Mocker.GetMock<IReleaseService>()
                 .Verify(v => v.UpdateMany(It.Is<List<AlbumRelease>>(s => s.First().ForeignReleaseId == newInfo.ForeignReleaseId)));
@@ -120,7 +120,7 @@ namespace NzbDrone.Core.Test.MusicTests
                 .Setup(s => s.GetTracksForRefresh(_release.Id, It.IsAny<IEnumerable<string>>()))
                 .Returns(new List<Track> { oldTrack });
 
-            Subject.RefreshEntityInfo(_release, new List<AlbumRelease> { newInfo }, false, false);
+            Subject.RefreshEntityInfo(_release, new List<AlbumRelease> { newInfo }, false, false, null);
 
             Mocker.GetMock<IRefreshTrackService>()
                 .Verify(v => v.RefreshTrackInfo(It.IsAny<List<Track>>(),

src/NzbDrone.Core.Test/MusicTests/RefreshAlbumServiceFixture.cs

@@ -100,7 +100,7 @@ namespace NzbDrone.Core.Test.MusicTests
 
             GivenNewAlbumInfo(newAlbumInfo);
 
-            Subject.RefreshAlbumInfo(_albums, null, false, false);
+            Subject.RefreshAlbumInfo(_albums, null, false, false, null);
 
             Mocker.GetMock<IAlbumService>()
                 .Verify(v => v.UpdateMany(It.Is<List<Album>>(s => s.First().ForeignAlbumId == newAlbumInfo.ForeignAlbumId)));
@@ -143,7 +143,7 @@ namespace NzbDrone.Core.Test.MusicTests
 
             GivenNewAlbumInfo(newAlbumInfo);
 
-            Subject.RefreshAlbumInfo(_albums, null, false, false);
+            Subject.RefreshAlbumInfo(_albums, null, false, false, null);
 
             // check releases moved to clashing album
             Mocker.GetMock<IReleaseService>()

src/NzbDrone.Core/MetadataSource/IProvideAlbumInfo.cs

@@ -7,5 +7,6 @@ namespace NzbDrone.Core.MetadataSource
     public interface IProvideAlbumInfo
     {
         Tuple<string, Album, List<ArtistMetadata>> GetAlbumInfo(string id);
+        HashSet<string> GetChangedAlbums(DateTime startTime);
     }
 }

src/NzbDrone.Core/MetadataSource/IProvideArtistInfo.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Collections.Generic;
 using NzbDrone.Core.Music;
 
 namespace NzbDrone.Core.MetadataSource
@@ -5,5 +7,6 @@ namespace NzbDrone.Core.MetadataSource
     public interface IProvideArtistInfo
     {
         Artist GetArtistInfo(string lidarrId, int metadataProfileId);
+        HashSet<string> GetChangedArtists(DateTime startTime);
     }
 }

src/NzbDrone.Core/MetadataSource/SkyHook/Resource/RecentUpdatesResource.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+
+namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
+{
+    public class RecentUpdatesResource
+    {
+        public int Count { get; set; }
+        public bool Limited { get; set; }
+        public DateTime Since { get; set; }
+        public List<string> Items { get; set; }
+    }
+}

src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using NLog;
+using NzbDrone.Common.Cache;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Serializer;
@@ -22,6 +23,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
         private readonly IAlbumService _albumService;
         private readonly IMetadataRequestBuilder _requestBuilder;
         private readonly IMetadataProfileService _metadataProfileService;
+        private readonly ICached<HashSet<string>> _cache;
 
         private static readonly List<string> NonAudioMedia = new List<string> { "DVD", "DVD-Video", "Blu-ray", "HD-DVD", "VCD", "SVCD", "UMD", "VHS" };
         private static readonly List<string> SkippedTracks = new List<string> { "[data track]" };
@@ -31,16 +33,38 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
                             IArtistService artistService,
                             IAlbumService albumService,
                             Logger logger,
-                            IMetadataProfileService metadataProfileService)
+                            IMetadataProfileService metadataProfileService,
+                            ICacheManager cacheManager)
         {
             _httpClient = httpClient;
             _metadataProfileService = metadataProfileService;
             _requestBuilder = requestBuilder;
             _artistService = artistService;
             _albumService = albumService;
+            _cache = cacheManager.GetCache<HashSet<string>>(GetType());
             _logger = logger;
         }
 
+        public HashSet<string> GetChangedArtists(DateTime startTime)
+        {
+            var startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc);
+            var httpRequest = _requestBuilder.GetRequestBuilder().Create()
+                .SetSegment("route", "recent/artist")
+                .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds())
+                .Build();
+
+            httpRequest.SuppressHttpError = true;
+
+            var httpResponse = _httpClient.Get<RecentUpdatesResource>(httpRequest);
+
+            if (httpResponse.Resource.Limited)
+            {
+                return null;
+            }
+
+            return new HashSet<string>(httpResponse.Resource.Items);
+        }
+
         public Artist GetArtistInfo(string foreignArtistId, int metadataProfileId)
         {
             _logger.Debug("Getting Artist with LidarrAPI.MetadataID of {0}", foreignArtistId);
@@ -81,6 +105,31 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             return artist;
         }
 
+        public HashSet<string> GetChangedAlbums(DateTime startTime)
+        {
+            return _cache.Get("ChangedAlbums", () => GetChangedAlbumsUncached(startTime), TimeSpan.FromMinutes(30));
+        }
+
+        private HashSet<string> GetChangedAlbumsUncached(DateTime startTime)
+        {
+            var startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc);
+            var httpRequest = _requestBuilder.GetRequestBuilder().Create()
+                .SetSegment("route", "recent/album")
+                .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds())
+                .Build();
+
+            httpRequest.SuppressHttpError = true;
+
+            var httpResponse = _httpClient.Get<RecentUpdatesResource>(httpRequest);
+
+            if (httpResponse.Resource.Limited)
+            {
+                return null;
+            }
+
+            return new HashSet<string>(httpResponse.Resource.Items);
+        }
+
         public IEnumerable<AlbumResource> FilterAlbums(IEnumerable<AlbumResource> albums, int metadataProfileId)
         {
             var metadataProfile = _metadataProfileService.Exists(metadataProfileId) ? _metadataProfileService.Get(metadataProfileId) : _metadataProfileService.All().First();

src/NzbDrone.Core/Music/Services/RefreshAlbumReleaseService.cs

@@ -8,7 +8,7 @@ namespace NzbDrone.Core.Music
 {
     public interface IRefreshAlbumReleaseService
     {
-        bool RefreshEntityInfo(AlbumRelease entity, List<AlbumRelease> remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags);
+        bool RefreshEntityInfo(AlbumRelease entity, List<AlbumRelease> remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate);
         bool RefreshEntityInfo(List<AlbumRelease> releases, List<AlbumRelease> remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags);
     }
 
@@ -114,7 +114,7 @@ namespace NzbDrone.Core.Music
             _trackService.InsertMany(children);
         }
 
-        protected override bool RefreshChildren(SortedChildren localChildren, List<Track> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags)
+        protected override bool RefreshChildren(SortedChildren localChildren, List<Track> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             return _refreshTrackService.RefreshTrackInfo(localChildren.Added, localChildren.Updated, localChildren.Merged, localChildren.Deleted, localChildren.UpToDate, remoteChildren, forceUpdateFileTags);
         }

src/NzbDrone.Core/Music/Services/RefreshAlbumService.cs

@@ -19,7 +19,7 @@ namespace NzbDrone.Core.Music
     public interface IRefreshAlbumService
     {
         bool RefreshAlbumInfo(Album album, List<Album> remoteAlbums, bool forceUpdateFileTags);
-        bool RefreshAlbumInfo(List<Album> albums, List<Album> remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags);
+        bool RefreshAlbumInfo(List<Album> albums, List<Album> remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags, DateTime? lastUpdate);
     }
 
     public class RefreshAlbumService : RefreshEntityServiceBase<Album, AlbumRelease>, IRefreshAlbumService, IExecute<RefreshAlbumCommand>
@@ -295,7 +295,7 @@ namespace NzbDrone.Core.Music
             toMonitor.Monitored = true;
         }
 
-        protected override bool RefreshChildren(SortedChildren localChildren, List<AlbumRelease> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags)
+        protected override bool RefreshChildren(SortedChildren localChildren, List<AlbumRelease> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             var refreshList = localChildren.All;
 
@@ -314,15 +314,29 @@ namespace NzbDrone.Core.Music
             _eventAggregator.PublishEvent(new AlbumUpdatedEvent(_albumService.GetAlbum(entity.Id)));
         }
 
-        public bool RefreshAlbumInfo(List<Album> albums, List<Album> remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags)
+        public bool RefreshAlbumInfo(List<Album> albums, List<Album> remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             bool updated = false;
+
+            var updatedMusicbrainzAlbums = new HashSet<string>();
+
+            if (lastUpdate.HasValue && lastUpdate.Value.AddDays(14) > DateTime.UtcNow)
+            {
+                updatedMusicbrainzAlbums = _albumInfo.GetChangedAlbums(lastUpdate.Value);
+            }
+
             foreach (var album in albums)
             {
-                if (forceAlbumRefresh || _checkIfAlbumShouldBeRefreshed.ShouldRefresh(album))
+                if (forceAlbumRefresh ||
+                    (updatedMusicbrainzAlbums == null && _checkIfAlbumShouldBeRefreshed.ShouldRefresh(album)) ||
+                    (updatedMusicbrainzAlbums != null && updatedMusicbrainzAlbums.Contains(album.ForeignAlbumId)))
                 {
                     updated |= RefreshAlbumInfo(album, remoteAlbums, forceUpdateFileTags);
                 }
+                else
+                {
+                    _logger.Debug("Skipping refresh of album: {0}", album.Title);
+                }
             }
 
             return updated;
@@ -330,7 +344,7 @@ namespace NzbDrone.Core.Music
 
         public bool RefreshAlbumInfo(Album album, List<Album> remoteAlbums, bool forceUpdateFileTags)
         {
-            return RefreshEntityInfo(album, remoteAlbums, true, forceUpdateFileTags);
+            return RefreshEntityInfo(album, remoteAlbums, true, forceUpdateFileTags, null);
         }
 
         public void Execute(RefreshAlbumCommand message)

src/NzbDrone.Core/Music/Services/RefreshArtistService.cs

@@ -243,11 +243,11 @@ namespace NzbDrone.Core.Music
             _albumService.InsertMany(children);
         }
 
-        protected override bool RefreshChildren(SortedChildren localChildren, List<Album> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags)
+        protected override bool RefreshChildren(SortedChildren localChildren, List<Album> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             // we always want to end up refreshing the albums since we don't yet have proper data
             Ensure.That(localChildren.UpToDate.Count, () => localChildren.UpToDate.Count).IsLessThanOrEqualTo(0);
-            return _refreshAlbumService.RefreshAlbumInfo(localChildren.All, remoteChildren, forceChildRefresh, forceUpdateFileTags);
+            return _refreshAlbumService.RefreshAlbumInfo(localChildren.All, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate);
         }
 
         protected override void PublishEntityUpdatedEvent(Artist entity)
@@ -317,7 +317,7 @@ namespace NzbDrone.Core.Music
             {
                 try
                 {
-                    updated |= RefreshEntityInfo(artist, null, true, false);
+                    updated |= RefreshEntityInfo(artist, null, true, false, null);
                 }
                 catch (Exception e)
                 {
@@ -348,15 +348,24 @@ namespace NzbDrone.Core.Music
                 var artists = _artistService.GetAllArtists().OrderBy(c => c.Name).ToList();
                 var artistIds = artists.Select(x => x.Id).ToList();
 
+                var updatedMusicbrainzArtists = new HashSet<string>();
+
+                if (message.LastExecutionTime.HasValue && message.LastExecutionTime.Value.AddDays(14) > DateTime.UtcNow)
+                {
+                    updatedMusicbrainzArtists = _artistInfo.GetChangedArtists(message.LastStartTime.Value);
+                }
+
                 foreach (var artist in artists)
                 {
                     var manualTrigger = message.Trigger == CommandTrigger.Manual;
 
-                    if (manualTrigger || _checkIfArtistShouldBeRefreshed.ShouldRefresh(artist))
+                    if ((updatedMusicbrainzArtists == null && _checkIfArtistShouldBeRefreshed.ShouldRefresh(artist)) ||
+                        (updatedMusicbrainzArtists != null && updatedMusicbrainzArtists.Contains(artist.ForeignArtistId)) ||
+                        manualTrigger)
                     {
                         try
                         {
-                            updated |= RefreshEntityInfo(artist, null, manualTrigger, false);
+                            updated |= RefreshEntityInfo(artist, null, manualTrigger, false, message.LastStartTime);
                         }
                         catch (Exception e)
                         {

src/NzbDrone.Core/Music/Services/RefreshEntityServiceBase.cs

@@ -94,7 +94,7 @@ namespace NzbDrone.Core.Music
         protected abstract void PrepareNewChild(TChild child, TEntity entity);
         protected abstract void PrepareExistingChild(TChild local, TChild remote, TEntity entity);
         protected abstract void AddChildren(List<TChild> children);
-        protected abstract bool RefreshChildren(SortedChildren localChildren, List<TChild> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags);
+        protected abstract bool RefreshChildren(SortedChildren localChildren, List<TChild> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate);
 
         protected virtual void PublishEntityUpdatedEvent(TEntity entity)
         {
@@ -108,7 +108,7 @@ namespace NzbDrone.Core.Music
         {
         }
 
-        public bool RefreshEntityInfo(TEntity local, List<TEntity> remoteList, bool forceChildRefresh, bool forceUpdateFileTags)
+        public bool RefreshEntityInfo(TEntity local, List<TEntity> remoteList, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             bool updated = false;
 
@@ -177,7 +177,7 @@ namespace NzbDrone.Core.Music
             _logger.Trace($"updated: {updated} forceUpdateFileTags: {forceUpdateFileTags}");
 
             var remoteChildren = GetRemoteChildren(remote);
-            updated |= SortChildren(local, remoteChildren, forceChildRefresh, forceUpdateFileTags);
+            updated |= SortChildren(local, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate);
 
             // Do this last so entity only marked as refreshed if refresh of children completed successfully
             _logger.Trace($"Saving {typeof(TEntity).Name} {local}");
@@ -200,7 +200,7 @@ namespace NzbDrone.Core.Music
             bool updated = false;
             foreach (var entity in localList)
             {
-                updated |= RefreshEntityInfo(entity, remoteList, forceChildRefresh, forceUpdateFileTags);
+                updated |= RefreshEntityInfo(entity, remoteList, forceChildRefresh, forceUpdateFileTags, null);
             }
 
             return updated;
@@ -213,7 +213,7 @@ namespace NzbDrone.Core.Music
             return updated ? UpdateResult.UpdateTags : UpdateResult.None;
         }
 
-        protected bool SortChildren(TEntity entity, List<TChild> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags)
+        protected bool SortChildren(TEntity entity, List<TChild> remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate)
         {
             // Get existing children (and children to be) from the database
             var localChildren = GetLocalChildren(entity, remoteChildren);
@@ -278,7 +278,7 @@ namespace NzbDrone.Core.Music
             AddChildren(sortedChildren.Added);
 
             // now trigger updates
-            var updated = RefreshChildren(sortedChildren, remoteChildren, forceChildRefresh, forceUpdateFileTags);
+            var updated = RefreshChildren(sortedChildren, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate);
 
             PublishChildrenUpdatedEvent(entity, sortedChildren.Added, sortedChildren.Updated);
             return updated;