diff --git a/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumReleaseServiceFixture.cs b/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumReleaseServiceFixture.cs index b51940d93..f1df30bb2 100644 --- a/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumReleaseServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumReleaseServiceFixture.cs @@ -51,7 +51,7 @@ namespace NzbDrone.Core.Test.MusicTests newInfo.OldForeignReleaseIds = new List { _release.ForeignReleaseId }; newInfo.Tracks = _tracks; - Subject.RefreshEntityInfo(_release, new List { newInfo }, false, false); + Subject.RefreshEntityInfo(_release, new List { newInfo }, false, false, null); Mocker.GetMock() .Verify(v => v.UpdateMany(It.Is>(s => s.First().ForeignReleaseId == newInfo.ForeignReleaseId))); @@ -120,7 +120,7 @@ namespace NzbDrone.Core.Test.MusicTests .Setup(s => s.GetTracksForRefresh(_release.Id, It.IsAny>())) .Returns(new List { oldTrack }); - Subject.RefreshEntityInfo(_release, new List { newInfo }, false, false); + Subject.RefreshEntityInfo(_release, new List { newInfo }, false, false, null); Mocker.GetMock() .Verify(v => v.RefreshTrackInfo(It.IsAny>(), diff --git a/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumServiceFixture.cs b/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumServiceFixture.cs index 19f6faadb..55393fa4c 100644 --- a/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/RefreshAlbumServiceFixture.cs @@ -100,7 +100,7 @@ namespace NzbDrone.Core.Test.MusicTests GivenNewAlbumInfo(newAlbumInfo); - Subject.RefreshAlbumInfo(_albums, null, false, false); + Subject.RefreshAlbumInfo(_albums, null, false, false, null); Mocker.GetMock() .Verify(v => v.UpdateMany(It.Is>(s => s.First().ForeignAlbumId == newAlbumInfo.ForeignAlbumId))); @@ -143,7 +143,7 @@ namespace NzbDrone.Core.Test.MusicTests GivenNewAlbumInfo(newAlbumInfo); - Subject.RefreshAlbumInfo(_albums, null, false, false); + Subject.RefreshAlbumInfo(_albums, null, false, false, null); // check releases moved to clashing album Mocker.GetMock() diff --git a/src/NzbDrone.Core/MetadataSource/IProvideAlbumInfo.cs b/src/NzbDrone.Core/MetadataSource/IProvideAlbumInfo.cs index 4856d289e..26cdfef24 100644 --- a/src/NzbDrone.Core/MetadataSource/IProvideAlbumInfo.cs +++ b/src/NzbDrone.Core/MetadataSource/IProvideAlbumInfo.cs @@ -7,5 +7,6 @@ namespace NzbDrone.Core.MetadataSource public interface IProvideAlbumInfo { Tuple> GetAlbumInfo(string id); + HashSet GetChangedAlbums(DateTime startTime); } } diff --git a/src/NzbDrone.Core/MetadataSource/IProvideArtistInfo.cs b/src/NzbDrone.Core/MetadataSource/IProvideArtistInfo.cs index 0dadfa990..77f19b7a6 100644 --- a/src/NzbDrone.Core/MetadataSource/IProvideArtistInfo.cs +++ b/src/NzbDrone.Core/MetadataSource/IProvideArtistInfo.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using NzbDrone.Core.Music; namespace NzbDrone.Core.MetadataSource @@ -5,5 +7,6 @@ namespace NzbDrone.Core.MetadataSource public interface IProvideArtistInfo { Artist GetArtistInfo(string lidarrId, int metadataProfileId); + HashSet GetChangedArtists(DateTime startTime); } } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/RecentUpdatesResource.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/RecentUpdatesResource.cs new file mode 100644 index 000000000..530c9e858 --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/Resource/RecentUpdatesResource.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; + +namespace NzbDrone.Core.MetadataSource.SkyHook.Resource +{ + public class RecentUpdatesResource + { + public int Count { get; set; } + public bool Limited { get; set; } + public DateTime Since { get; set; } + public List Items { get; set; } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index dbd773d20..6074f81a2 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; using NLog; +using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; @@ -22,6 +23,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook private readonly IAlbumService _albumService; private readonly IMetadataRequestBuilder _requestBuilder; private readonly IMetadataProfileService _metadataProfileService; + private readonly ICached> _cache; private static readonly List NonAudioMedia = new List { "DVD", "DVD-Video", "Blu-ray", "HD-DVD", "VCD", "SVCD", "UMD", "VHS" }; private static readonly List SkippedTracks = new List { "[data track]" }; @@ -31,16 +33,38 @@ namespace NzbDrone.Core.MetadataSource.SkyHook IArtistService artistService, IAlbumService albumService, Logger logger, - IMetadataProfileService metadataProfileService) + IMetadataProfileService metadataProfileService, + ICacheManager cacheManager) { _httpClient = httpClient; _metadataProfileService = metadataProfileService; _requestBuilder = requestBuilder; _artistService = artistService; _albumService = albumService; + _cache = cacheManager.GetCache>(GetType()); _logger = logger; } + public HashSet GetChangedArtists(DateTime startTime) + { + var startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc); + var httpRequest = _requestBuilder.GetRequestBuilder().Create() + .SetSegment("route", "recent/artist") + .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds()) + .Build(); + + httpRequest.SuppressHttpError = true; + + var httpResponse = _httpClient.Get(httpRequest); + + if (httpResponse.Resource.Limited) + { + return null; + } + + return new HashSet(httpResponse.Resource.Items); + } + public Artist GetArtistInfo(string foreignArtistId, int metadataProfileId) { _logger.Debug("Getting Artist with LidarrAPI.MetadataID of {0}", foreignArtistId); @@ -81,6 +105,31 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return artist; } + public HashSet GetChangedAlbums(DateTime startTime) + { + return _cache.Get("ChangedAlbums", () => GetChangedAlbumsUncached(startTime), TimeSpan.FromMinutes(30)); + } + + private HashSet GetChangedAlbumsUncached(DateTime startTime) + { + var startTimeUtc = (DateTimeOffset)DateTime.SpecifyKind(startTime, DateTimeKind.Utc); + var httpRequest = _requestBuilder.GetRequestBuilder().Create() + .SetSegment("route", "recent/album") + .AddQueryParam("since", startTimeUtc.ToUnixTimeSeconds()) + .Build(); + + httpRequest.SuppressHttpError = true; + + var httpResponse = _httpClient.Get(httpRequest); + + if (httpResponse.Resource.Limited) + { + return null; + } + + return new HashSet(httpResponse.Resource.Items); + } + public IEnumerable FilterAlbums(IEnumerable albums, int metadataProfileId) { var metadataProfile = _metadataProfileService.Exists(metadataProfileId) ? _metadataProfileService.Get(metadataProfileId) : _metadataProfileService.All().First(); diff --git a/src/NzbDrone.Core/Music/Services/RefreshAlbumReleaseService.cs b/src/NzbDrone.Core/Music/Services/RefreshAlbumReleaseService.cs index bd5535dba..294610a35 100644 --- a/src/NzbDrone.Core/Music/Services/RefreshAlbumReleaseService.cs +++ b/src/NzbDrone.Core/Music/Services/RefreshAlbumReleaseService.cs @@ -8,7 +8,7 @@ namespace NzbDrone.Core.Music { public interface IRefreshAlbumReleaseService { - bool RefreshEntityInfo(AlbumRelease entity, List remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags); + bool RefreshEntityInfo(AlbumRelease entity, List remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate); bool RefreshEntityInfo(List releases, List remoteEntityList, bool forceChildRefresh, bool forceUpdateFileTags); } @@ -114,7 +114,7 @@ namespace NzbDrone.Core.Music _trackService.InsertMany(children); } - protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags) + protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { return _refreshTrackService.RefreshTrackInfo(localChildren.Added, localChildren.Updated, localChildren.Merged, localChildren.Deleted, localChildren.UpToDate, remoteChildren, forceUpdateFileTags); } diff --git a/src/NzbDrone.Core/Music/Services/RefreshAlbumService.cs b/src/NzbDrone.Core/Music/Services/RefreshAlbumService.cs index 020e093e2..aaa6169eb 100644 --- a/src/NzbDrone.Core/Music/Services/RefreshAlbumService.cs +++ b/src/NzbDrone.Core/Music/Services/RefreshAlbumService.cs @@ -19,7 +19,7 @@ namespace NzbDrone.Core.Music public interface IRefreshAlbumService { bool RefreshAlbumInfo(Album album, List remoteAlbums, bool forceUpdateFileTags); - bool RefreshAlbumInfo(List albums, List remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags); + bool RefreshAlbumInfo(List albums, List remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags, DateTime? lastUpdate); } public class RefreshAlbumService : RefreshEntityServiceBase, IRefreshAlbumService, IExecute @@ -295,7 +295,7 @@ namespace NzbDrone.Core.Music toMonitor.Monitored = true; } - protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags) + protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { var refreshList = localChildren.All; @@ -314,15 +314,29 @@ namespace NzbDrone.Core.Music _eventAggregator.PublishEvent(new AlbumUpdatedEvent(_albumService.GetAlbum(entity.Id))); } - public bool RefreshAlbumInfo(List albums, List remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags) + public bool RefreshAlbumInfo(List albums, List remoteAlbums, bool forceAlbumRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { bool updated = false; + + var updatedMusicbrainzAlbums = new HashSet(); + + if (lastUpdate.HasValue && lastUpdate.Value.AddDays(14) > DateTime.UtcNow) + { + updatedMusicbrainzAlbums = _albumInfo.GetChangedAlbums(lastUpdate.Value); + } + foreach (var album in albums) { - if (forceAlbumRefresh || _checkIfAlbumShouldBeRefreshed.ShouldRefresh(album)) + if (forceAlbumRefresh || + (updatedMusicbrainzAlbums == null && _checkIfAlbumShouldBeRefreshed.ShouldRefresh(album)) || + (updatedMusicbrainzAlbums != null && updatedMusicbrainzAlbums.Contains(album.ForeignAlbumId))) { updated |= RefreshAlbumInfo(album, remoteAlbums, forceUpdateFileTags); } + else + { + _logger.Debug("Skipping refresh of album: {0}", album.Title); + } } return updated; @@ -330,7 +344,7 @@ namespace NzbDrone.Core.Music public bool RefreshAlbumInfo(Album album, List remoteAlbums, bool forceUpdateFileTags) { - return RefreshEntityInfo(album, remoteAlbums, true, forceUpdateFileTags); + return RefreshEntityInfo(album, remoteAlbums, true, forceUpdateFileTags, null); } public void Execute(RefreshAlbumCommand message) diff --git a/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs b/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs index ba00ea9cb..bf6bbb89e 100644 --- a/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs +++ b/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs @@ -243,11 +243,11 @@ namespace NzbDrone.Core.Music _albumService.InsertMany(children); } - protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags) + protected override bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { // we always want to end up refreshing the albums since we don't yet have proper data Ensure.That(localChildren.UpToDate.Count, () => localChildren.UpToDate.Count).IsLessThanOrEqualTo(0); - return _refreshAlbumService.RefreshAlbumInfo(localChildren.All, remoteChildren, forceChildRefresh, forceUpdateFileTags); + return _refreshAlbumService.RefreshAlbumInfo(localChildren.All, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate); } protected override void PublishEntityUpdatedEvent(Artist entity) @@ -317,7 +317,7 @@ namespace NzbDrone.Core.Music { try { - updated |= RefreshEntityInfo(artist, null, true, false); + updated |= RefreshEntityInfo(artist, null, true, false, null); } catch (Exception e) { @@ -348,15 +348,24 @@ namespace NzbDrone.Core.Music var artists = _artistService.GetAllArtists().OrderBy(c => c.Name).ToList(); var artistIds = artists.Select(x => x.Id).ToList(); + var updatedMusicbrainzArtists = new HashSet(); + + if (message.LastExecutionTime.HasValue && message.LastExecutionTime.Value.AddDays(14) > DateTime.UtcNow) + { + updatedMusicbrainzArtists = _artistInfo.GetChangedArtists(message.LastStartTime.Value); + } + foreach (var artist in artists) { var manualTrigger = message.Trigger == CommandTrigger.Manual; - if (manualTrigger || _checkIfArtistShouldBeRefreshed.ShouldRefresh(artist)) + if ((updatedMusicbrainzArtists == null && _checkIfArtistShouldBeRefreshed.ShouldRefresh(artist)) || + (updatedMusicbrainzArtists != null && updatedMusicbrainzArtists.Contains(artist.ForeignArtistId)) || + manualTrigger) { try { - updated |= RefreshEntityInfo(artist, null, manualTrigger, false); + updated |= RefreshEntityInfo(artist, null, manualTrigger, false, message.LastStartTime); } catch (Exception e) { diff --git a/src/NzbDrone.Core/Music/Services/RefreshEntityServiceBase.cs b/src/NzbDrone.Core/Music/Services/RefreshEntityServiceBase.cs index b03a08361..34a9a0936 100644 --- a/src/NzbDrone.Core/Music/Services/RefreshEntityServiceBase.cs +++ b/src/NzbDrone.Core/Music/Services/RefreshEntityServiceBase.cs @@ -94,7 +94,7 @@ namespace NzbDrone.Core.Music protected abstract void PrepareNewChild(TChild child, TEntity entity); protected abstract void PrepareExistingChild(TChild local, TChild remote, TEntity entity); protected abstract void AddChildren(List children); - protected abstract bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags); + protected abstract bool RefreshChildren(SortedChildren localChildren, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate); protected virtual void PublishEntityUpdatedEvent(TEntity entity) { @@ -108,7 +108,7 @@ namespace NzbDrone.Core.Music { } - public bool RefreshEntityInfo(TEntity local, List remoteList, bool forceChildRefresh, bool forceUpdateFileTags) + public bool RefreshEntityInfo(TEntity local, List remoteList, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { bool updated = false; @@ -177,7 +177,7 @@ namespace NzbDrone.Core.Music _logger.Trace($"updated: {updated} forceUpdateFileTags: {forceUpdateFileTags}"); var remoteChildren = GetRemoteChildren(remote); - updated |= SortChildren(local, remoteChildren, forceChildRefresh, forceUpdateFileTags); + updated |= SortChildren(local, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate); // Do this last so entity only marked as refreshed if refresh of children completed successfully _logger.Trace($"Saving {typeof(TEntity).Name} {local}"); @@ -200,7 +200,7 @@ namespace NzbDrone.Core.Music bool updated = false; foreach (var entity in localList) { - updated |= RefreshEntityInfo(entity, remoteList, forceChildRefresh, forceUpdateFileTags); + updated |= RefreshEntityInfo(entity, remoteList, forceChildRefresh, forceUpdateFileTags, null); } return updated; @@ -213,7 +213,7 @@ namespace NzbDrone.Core.Music return updated ? UpdateResult.UpdateTags : UpdateResult.None; } - protected bool SortChildren(TEntity entity, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags) + protected bool SortChildren(TEntity entity, List remoteChildren, bool forceChildRefresh, bool forceUpdateFileTags, DateTime? lastUpdate) { // Get existing children (and children to be) from the database var localChildren = GetLocalChildren(entity, remoteChildren); @@ -278,7 +278,7 @@ namespace NzbDrone.Core.Music AddChildren(sortedChildren.Added); // now trigger updates - var updated = RefreshChildren(sortedChildren, remoteChildren, forceChildRefresh, forceUpdateFileTags); + var updated = RefreshChildren(sortedChildren, remoteChildren, forceChildRefresh, forceUpdateFileTags, lastUpdate); PublishChildrenUpdatedEvent(entity, sortedChildren.Added, sortedChildren.Updated); return updated;