From 149c5292f1cad922b4282180ccf7c3ee2e0726ec Mon Sep 17 00:00:00 2001 From: Leonardo Galli Date: Sat, 11 Mar 2017 00:17:09 +0100 Subject: [PATCH] PreDB Integration. Update Library is advisable --- src/NzbDrone.Api/Series/MovieLookupModule.cs | 2 +- .../135_add_haspredbentry_to_movies.cs | 15 ++ src/NzbDrone.Core/Jobs/TaskManager.cs | 2 + .../MetadataSource/IProvideMovieInfo.cs | 2 +- .../MetadataSource/PreDB/PreDBResult.cs | 14 ++ .../MetadataSource/PreDB/PreDBService.cs | 194 ++++++++++++++++++ .../MetadataSource/PreDB/PreDBSyncCommand.cs | 10 + .../MetadataSource/PreDB/PreDBSyncEvent.cs | 19 ++ .../MetadataSource/SkyHook/SkyHookProxy.cs | 23 ++- src/NzbDrone.Core/NzbDrone.Core.csproj | 6 + src/NzbDrone.Core/Tv/Movie.cs | 9 +- src/NzbDrone.Core/Tv/MovieService.cs | 2 +- src/NzbDrone.Core/Tv/RefreshMovieService.cs | 3 +- src/UI/Cells/MovieStatusCell.js | 4 - src/UI/Handlebars/Helpers/Series.js | 9 +- src/UI/Movies/Index/MoviesIndexLayout.js | 6 + 16 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/135_add_haspredbentry_to_movies.cs create mode 100644 src/NzbDrone.Core/MetadataSource/PreDB/PreDBResult.cs create mode 100644 src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs create mode 100644 src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncCommand.cs create mode 100644 src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncEvent.cs diff --git a/src/NzbDrone.Api/Series/MovieLookupModule.cs b/src/NzbDrone.Api/Series/MovieLookupModule.cs index e3ced5cb8..1b88253d9 100644 --- a/src/NzbDrone.Api/Series/MovieLookupModule.cs +++ b/src/NzbDrone.Api/Series/MovieLookupModule.cs @@ -29,7 +29,7 @@ namespace NzbDrone.Api.Movie int tmdbId = -1; if(Int32.TryParse(Request.Query.tmdbId, out tmdbId)) { - var result = _movieInfo.GetMovieInfo(tmdbId, null); + var result = _movieInfo.GetMovieInfo(tmdbId, null, true); return result.ToResource().AsResponse(); } diff --git a/src/NzbDrone.Core/Datastore/Migration/135_add_haspredbentry_to_movies.cs b/src/NzbDrone.Core/Datastore/Migration/135_add_haspredbentry_to_movies.cs new file mode 100644 index 000000000..05d8b766f --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/135_add_haspredbentry_to_movies.cs @@ -0,0 +1,15 @@ +using System.Data; +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(135)] + public class add_haspredbentry_to_movies : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("Movies").AddColumn("HasPreDBEntry").AsBoolean().WithDefaultValue(false); + } + } +} diff --git a/src/NzbDrone.Core/Jobs/TaskManager.cs b/src/NzbDrone.Core/Jobs/TaskManager.cs index 0663874eb..717fff6de 100644 --- a/src/NzbDrone.Core/Jobs/TaskManager.cs +++ b/src/NzbDrone.Core/Jobs/TaskManager.cs @@ -17,6 +17,7 @@ using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.NetImport; using NzbDrone.Core.Tv.Commands; using NzbDrone.Core.Update.Commands; +using NzbDrone.Core.MetadataSource.PreDB; namespace NzbDrone.Core.Jobs { @@ -72,6 +73,7 @@ namespace NzbDrone.Core.Jobs var defaultTasks = new[] { new ScheduledTask{ Interval = 0.25f, TypeName = typeof(CheckForFinishedDownloadCommand).FullName}, + new ScheduledTask{ Interval = 1*60, TypeName = typeof(PreDBSyncCommand).FullName}, new ScheduledTask{ Interval = 5, TypeName = typeof(MessagingCleanupCommand).FullName}, new ScheduledTask{ Interval = updateInterval, TypeName = typeof(ApplicationUpdateCommand).FullName}, // new ScheduledTask{ Interval = 3*60, TypeName = typeof(UpdateSceneMappingCommand).FullName}, diff --git a/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs b/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs index 20a2738a5..a1077f696 100644 --- a/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs +++ b/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs @@ -8,6 +8,6 @@ namespace NzbDrone.Core.MetadataSource public interface IProvideMovieInfo { Movie GetMovieInfo(string ImdbId); - Movie GetMovieInfo(int TmdbId, Profile profile); + Movie GetMovieInfo(int TmdbId, Profile profile, bool hasPreDBEntry); } } \ No newline at end of file diff --git a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBResult.cs b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBResult.cs new file mode 100644 index 000000000..075985dce --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBResult.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.MetadataSource.PreDB +{ + class PreDBResult + { + public string Title { get; set; } + public string Link { get; set; } + + } +} diff --git a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs new file mode 100644 index 000000000..8d4d2a1a2 --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBService.cs @@ -0,0 +1,194 @@ +using System.Linq; +using System.Collections.Generic; +using NLog; +using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.DecisionEngine; +using NzbDrone.Core.Download; +using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Indexers; +using System.ServiceModel.Syndication; +using System.Xml; +using NzbDrone.Common.Http; +using NzbDrone.Core.Tv; +using System; +using System.IO; +using NzbDrone.Core.Parser; +using NzbDrone.Core.IndexerSearch; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.MetadataSource.PreDB +{ + public interface IPreDBService + { + bool HasReleases(Movie movie); + } + + public class PreDBService : IPreDBService, IExecute + { + private readonly IFetchAndParseRss _rssFetcherAndParser; + private readonly IMakeDownloadDecision _downloadDecisionMaker; + private readonly IProcessDownloadDecisions _processDownloadDecisions; + private readonly IPendingReleaseService _pendingReleaseService; + private readonly IEventAggregator _eventAggregator; + private readonly IMovieService _movieService; + private readonly IHttpClient _httpClient; + private readonly IParsingService _parsingService; + private readonly Logger _logger; + + public PreDBService( + IFetchAndParseRss rssFetcherAndParser, + IMakeDownloadDecision downloadDecisionMaker, + IProcessDownloadDecisions processDownloadDecisions, + IPendingReleaseService pendingReleaseService, + IEventAggregator eventAggregator, + IMovieService movieService, + IHttpClient httpClient, + IParsingService parsingService, + Logger logger) + { + _rssFetcherAndParser = rssFetcherAndParser; + _downloadDecisionMaker = downloadDecisionMaker; + _processDownloadDecisions = processDownloadDecisions; + _pendingReleaseService = pendingReleaseService; + _eventAggregator = eventAggregator; + _movieService = movieService; + _httpClient = httpClient; + _parsingService = parsingService; + _logger = logger; + } + + private List GetResults(string category = "", string search = "") + { + var builder = new HttpRequestBuilder("http://predb.me").AddQueryParam("rss", "1"); + if (category.IsNotNullOrWhiteSpace()) + { + builder.AddQueryParam("cats", category); + } + + if (search.IsNotNullOrWhiteSpace()) + { + builder.AddQueryParam("search", search); + } + + var request = builder.Build(); + + request.AllowAutoRedirect = true; + request.SuppressHttpError = true; + + var response = _httpClient.Get(request); + + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + _logger.Warn("Non 200 StatusCode {0} encountered while searching PreDB.", response.StatusCode); + return new List(); + } + + try + { + var reader = XmlReader.Create(new StringReader(response.Content)); + + var items = SyndicationFeed.Load(reader); + + var results = new List(); + + foreach (SyndicationItem item in items.Items) + { + var result = new PreDBResult(); + result.Title = item.Title.Text; + result.Link = item.Links[0].Uri.ToString(); + results.Add(result); + } + + return results; + } + catch (Exception ex) + { + _logger.Error(ex, "Error while searching PreDB."); + } + + return new List(); + } + + private List FindMatchesToResults(List results) + { + var matches = new List(); + + foreach (PreDBResult result in results) + { + var parsedInfo = Parser.Parser.ParseMovieTitle(result.Title); + + if (parsedInfo != null) + { + var movie = _movieService.FindByTitle(parsedInfo.MovieTitle, parsedInfo.Year); + + if (movie != null) + { + matches.Add(movie); + } + } + } + + return matches; + } + + + + + private List Sync() + { + _logger.ProgressInfo("Starting PreDB Sync"); + + var results = GetResults("movies"); + + var matches = FindMatchesToResults(results); + + return matches; + } + + public void Execute(PreDBSyncCommand message) + { + var haveNewReleases = Sync(); + + foreach (Movie movie in haveNewReleases) + { + if (!movie.HasPreDBEntry) + { + movie.HasPreDBEntry = true; + _movieService.UpdateMovie(movie); + } + + if (movie.Monitored) + { + //Maybe auto search each movie once? + } + } + + _eventAggregator.PublishEvent(new PreDBSyncCompleteEvent(haveNewReleases)); + } + + public bool HasReleases(Movie movie) + { + var results = GetResults("movies", movie.Title); + + foreach (PreDBResult result in results) + { + var parsed = Parser.Parser.ParseMovieTitle(result.Title); + if (parsed == null) + { + parsed = new Parser.Model.ParsedMovieInfo { MovieTitle = result.Title, Year = 0 }; + } + var match = _parsingService.Map(parsed, "", new MovieSearchCriteria { Movie = movie }); + + if (match != null && match.Movie != null && match.Movie.Id == movie.Id) + { + return true; + } + } + + return false; + } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncCommand.cs b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncCommand.cs new file mode 100644 index 000000000..c0ed56cc3 --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncCommand.cs @@ -0,0 +1,10 @@ +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.MetadataSource.PreDB +{ + public class PreDBSyncCommand : Command + { + + public override bool SendUpdatesToClient => true; + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncEvent.cs b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncEvent.cs new file mode 100644 index 000000000..09fbe759a --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/PreDB/PreDBSyncEvent.cs @@ -0,0 +1,19 @@ +using NzbDrone.Common.Messaging; +using NzbDrone.Core.Download; +using System; +using System.Linq; +using System.Collections.Generic; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.MetadataSource.PreDB +{ + public class PreDBSyncCompleteEvent : IEvent + { + public List NewlyReleased { get; private set; } + + public PreDBSyncCompleteEvent(List newlyReleased) + { + NewlyReleased = newlyReleased; + } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index 72a6c0a54..b73370eb2 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -9,6 +9,8 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Exceptions; using NzbDrone.Core.MediaCover; using NzbDrone.Core.MetadataSource.SkyHook.Resource; +using NzbDrone.Core.MetadataSource; +using NzbDrone.Core.MetadataSource.PreDB; using NzbDrone.Core.Tv; using System.Threading; using NzbDrone.Core.Parser; @@ -25,14 +27,16 @@ namespace NzbDrone.Core.MetadataSource.SkyHook private readonly IHttpRequestBuilderFactory _movieBuilder; private readonly ITmdbConfigService _configService; private readonly IMovieService _movieService; + private readonly IPreDBService _predbService; - public SkyHookProxy(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, ITmdbConfigService configService, IMovieService movieService, Logger logger) + public SkyHookProxy(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, ITmdbConfigService configService, IMovieService movieService, IPreDBService predbService, Logger logger) { _httpClient = httpClient; _requestBuilder = requestBuilder.SkyHookTvdb; _movieBuilder = requestBuilder.TMDB; _configService = configService; _movieService = movieService; + _predbService = predbService; _logger = logger; } @@ -66,7 +70,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return new Tuple>(series, episodes.ToList()); } - public Movie GetMovieInfo(int TmdbId, Profile profile = null) + public Movie GetMovieInfo(int TmdbId, Profile profile = null, bool hasPreDBEntry = false) { var langCode = profile != null ? IsoLanguages.Get(profile.Language).TwoLetterCode : "us"; @@ -233,14 +237,27 @@ namespace NzbDrone.Core.MetadataSource.SkyHook //otherwise the title has only been announced else { - movie.Status = MovieStatusType.Announced; + movie.Status = MovieStatusType.Announced; } + //since TMDB lacks alot of information lets assume that stuff is released if its been in cinemas for longer than 3 months. if (!movie.PhysicalRelease.HasValue && (movie.Status == MovieStatusType.InCinemas) && (((DateTime.Now).Subtract(movie.InCinemas.Value)).TotalSeconds > 60*60*24*30*3)) { movie.Status = MovieStatusType.Released; } + if (!hasPreDBEntry) + { + if (_predbService.HasReleases(movie)) + { + movie.HasPreDBEntry = true; + } + else + { + movie.HasPreDBEntry = false; + } + } + //this matches with the old behavior before the creation of the MovieStatusType.InCinemas /*if (resource.status == "Released") { diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index a93d13303..2c6e5e0e2 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -99,6 +99,7 @@ + @@ -890,6 +891,10 @@ + + + + @@ -1260,6 +1265,7 @@ + diff --git a/src/NzbDrone.Core/Tv/Movie.cs b/src/NzbDrone.Core/Tv/Movie.cs index 329752a93..10d9dffc1 100644 --- a/src/NzbDrone.Core/Tv/Movie.cs +++ b/src/NzbDrone.Core/Tv/Movie.cs @@ -47,6 +47,7 @@ namespace NzbDrone.Core.Tv public HashSet Tags { get; set; } public AddMovieOptions AddOptions { get; set; } public LazyLoaded MovieFile { get; set; } + public bool HasPreDBEntry { get; set; } public int MovieFileId { get; set; } public List AlternativeTitles { get; set; } public string YouTubeTrailerId{ get; set; } @@ -73,13 +74,19 @@ namespace NzbDrone.Core.Tv else MinimumAvailabilityDate = DateTime.MaxValue; break; - //TODO: might need to handle PreDB but for now treat the same as Released + case MovieStatusType.Released: case MovieStatusType.PreDB: default: MinimumAvailabilityDate = PhysicalRelease.HasValue ? PhysicalRelease.Value : (InCinemas.HasValue ? InCinemas.Value.AddDays(90) : DateTime.MaxValue); break; } + + if (HasPreDBEntry && MinimumAvailability == MovieStatusType.PreDB) + { + return true; + } + return DateTime.Now >= MinimumAvailabilityDate.AddDays(delay); } diff --git a/src/NzbDrone.Core/Tv/MovieService.cs b/src/NzbDrone.Core/Tv/MovieService.cs index fc3eb9fb2..76b059cfe 100644 --- a/src/NzbDrone.Core/Tv/MovieService.cs +++ b/src/NzbDrone.Core/Tv/MovieService.cs @@ -101,7 +101,7 @@ namespace NzbDrone.Core.Tv ((v.MinimumAvailability == MovieStatusType.Released && v.Status >= MovieStatusType.Released) || (v.MinimumAvailability == MovieStatusType.InCinemas && v.Status >= MovieStatusType.InCinemas) || (v.MinimumAvailability == MovieStatusType.Announced && v.Status >= MovieStatusType.Announced) || - (v.MinimumAvailability == MovieStatusType.PreDB && v.Status >= MovieStatusType.Released)); + (v.MinimumAvailability == MovieStatusType.PreDB && v.Status >= MovieStatusType.Released || v.HasPreDBEntry == true)); break; } } diff --git a/src/NzbDrone.Core/Tv/RefreshMovieService.cs b/src/NzbDrone.Core/Tv/RefreshMovieService.cs index ea2827e84..8c12c9a36 100644 --- a/src/NzbDrone.Core/Tv/RefreshMovieService.cs +++ b/src/NzbDrone.Core/Tv/RefreshMovieService.cs @@ -51,7 +51,7 @@ namespace NzbDrone.Core.Tv try { - movieInfo = _movieInfo.GetMovieInfo(movie.TmdbId, movie.Profile); + movieInfo = _movieInfo.GetMovieInfo(movie.TmdbId, movie.Profile, movie.HasPreDBEntry); } catch (MovieNotFoundException) { @@ -86,6 +86,7 @@ namespace NzbDrone.Core.Tv movie.PhysicalRelease = movieInfo.PhysicalRelease; movie.YouTubeTrailerId = movieInfo.YouTubeTrailerId; movie.Studio = movieInfo.Studio; + movie.HasPreDBEntry = movieInfo.HasPreDBEntry; try { diff --git a/src/UI/Cells/MovieStatusCell.js b/src/UI/Cells/MovieStatusCell.js index f479e3f36..7731cfe79 100644 --- a/src/UI/Cells/MovieStatusCell.js +++ b/src/UI/Cells/MovieStatusCell.js @@ -12,10 +12,6 @@ module.exports = NzbDroneCell.extend({ var timeSince = new Date().getTime() - date.getTime(); var numOfMonths = timeSince / 1000 / 60 / 60 / 24 / 30; - if (status === 'released') { - this.$el.html(''); - this._setStatusWeight(3); - } if (status === 'inCinemas') { this.$el.html(''); diff --git a/src/UI/Handlebars/Helpers/Series.js b/src/UI/Handlebars/Helpers/Series.js index 2fe7e61d3..98795f366 100644 --- a/src/UI/Handlebars/Helpers/Series.js +++ b/src/UI/Handlebars/Helpers/Series.js @@ -92,7 +92,7 @@ Handlebars.registerHelper('GetStatus', function() { return new Handlebars.SafeString(' Released'); } - else if (!monitored) { + if (!monitored) { return new Handlebars.SafeString(' Not Monitored'); } }); @@ -105,16 +105,13 @@ Handlebars.registerHelper('GetBannerStatus', function() { //var timeSince = new Date().getTime() - date.getTime(); //var numOfMonths = timeSince / 1000 / 60 / 60 / 24 / 30; - if (status === "announced") { - return new Handlebars.SafeString('
 Announced
'); - } if (status === "inCinemas") { return new Handlebars.SafeString('
 In Cinemas
'); } - if (status === 'released') { - return new Handlebars.SafeString('
 Released
'); + if (status === "announced") { + return new Handlebars.SafeString('
 Announced
'); } else if (!monitored) { return new Handlebars.SafeString('
 Not Monitored
'); diff --git a/src/UI/Movies/Index/MoviesIndexLayout.js b/src/UI/Movies/Index/MoviesIndexLayout.js index a628b1501..e0d22897f 100644 --- a/src/UI/Movies/Index/MoviesIndexLayout.js +++ b/src/UI/Movies/Index/MoviesIndexLayout.js @@ -126,6 +126,12 @@ module.exports = Marionette.Layout.extend({ command : 'rsssync', errorMessage : 'RSS Sync Failed!' }, + { + title : "PreDB Sync", + icon : "icon-sonarr-refresh", + command : "predbsync", + errorMessage : "PreDB Sync Failed!" + }, { title : 'Update Library', icon : 'icon-sonarr-refresh',