From 47f66978f00dc16ecd3c8adc0e0588b5c205a61e Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 15 Mar 2018 13:24:20 +0000 Subject: [PATCH] Added the ability to refresh out backend metadata (#2078) We now can refresh the Plex Metadata in our database. For example if the Plex Agent for TV Shows is TheMovieDb, we will use that and populate the IMDB Id and TheTvDb Id's so we can accuratly match and search things. Also improved the Job settings page so we can Test CRON's and we also validate them. --- src/Ombi.Api/Api.cs | 30 ++- src/Ombi.Api/HttpRequestExtnesions.cs | 45 ++++ src/Ombi.Api/Ombi.Api.csproj | 1 + src/Ombi.Api/Request.cs | 4 + src/Ombi.DependencyInjection/IocExtensions.cs | 1 + src/Ombi.Schedule/JobSetup.cs | 65 ++--- .../Jobs/Ombi/IRefreshMetadata.cs | 9 + .../Jobs/Ombi/RefreshMetadata.cs | 249 ++++++++++++++++++ src/Ombi.Schedule/Ombi.Schedule.csproj | 2 + .../Settings/Models/JobSettings.cs | 1 + .../Settings/Models/JobSettingsHelper.cs | 4 + .../Repository/IPlexContentRepository.cs | 2 + src/Ombi.Store/Repository/IRepository.cs | 2 + .../Repository/PlexContentRepository.cs | 10 + src/Ombi.Store/Repository/Repository.cs | 2 +- src/Ombi.TheMovieDbApi/IMovieDbApi.cs | 2 + src/Ombi.TheMovieDbApi/Models/FindResult.cs | 52 ++++ src/Ombi.TheMovieDbApi/Models/TvExternals.cs | 16 ++ src/Ombi.TheMovieDbApi/TheMovieDbApi.cs | 35 ++- .../ClientApp/app/interfaces/ISettings.ts | 16 ++ .../app/services/settings.service.ts | 14 +- .../app/settings/jobs/jobs.component.html | 23 +- .../app/settings/jobs/jobs.component.ts | 23 +- .../ClientApp/app/settings/settings.module.ts | 5 +- src/Ombi/Controllers/SettingsController.cs | 72 ++++- src/Ombi/Models/CronTestModel.cs | 12 + src/Ombi/Models/CronViewModelBody.cs | 7 + src/Ombi/Models/JobSettingsViewModel.cs | 8 + src/Ombi/Ombi.csproj | 1 + 29 files changed, 667 insertions(+), 46 deletions(-) create mode 100644 src/Ombi.Api/HttpRequestExtnesions.cs create mode 100644 src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs create mode 100644 src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs create mode 100644 src/Ombi.TheMovieDbApi/Models/FindResult.cs create mode 100644 src/Ombi.TheMovieDbApi/Models/TvExternals.cs create mode 100644 src/Ombi/Models/CronTestModel.cs create mode 100644 src/Ombi/Models/CronViewModelBody.cs create mode 100644 src/Ombi/Models/JobSettingsViewModel.cs diff --git a/src/Ombi.Api/Api.cs b/src/Ombi.Api/Api.cs index c12258b8e..98fff5e0c 100644 --- a/src/Ombi.Api/Api.cs +++ b/src/Ombi.Api/Api.cs @@ -1,4 +1,7 @@ -using System.IO; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Threading.Tasks; @@ -6,6 +9,7 @@ using System.Xml.Serialization; using Newtonsoft.Json; using Microsoft.Extensions.Logging; using Ombi.Helpers; +using Polly; namespace Ombi.Api { @@ -36,6 +40,30 @@ namespace Ombi.Api if (!httpResponseMessage.IsSuccessStatusCode) { LogError(request, httpResponseMessage); + if (request.Retry) + { + + var result = Policy + .Handle() + .OrResult(r => request.StatusCodeToRetry.Contains(r.StatusCode)) + .WaitAndRetryAsync(new[] + { + TimeSpan.FromSeconds(10), + }, (exception, timeSpan, context) => + { + + Logger.LogError(LoggingEvents.Api, + $"Retrying RequestUri: {request.FullUri} Because we got Status Code: {exception?.Result?.StatusCode}"); + }); + + httpResponseMessage = await result.ExecuteAsync(async () => + { + using (var req = await httpRequestMessage.Clone()) + { + return await _client.SendAsync(req); + } + }); + } } // do something with the response diff --git a/src/Ombi.Api/HttpRequestExtnesions.cs b/src/Ombi.Api/HttpRequestExtnesions.cs new file mode 100644 index 000000000..fa2ded97d --- /dev/null +++ b/src/Ombi.Api/HttpRequestExtnesions.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Ombi.Api +{ + public static class HttpRequestExtnesions + { + public static async Task Clone(this HttpRequestMessage request) + { + var clone = new HttpRequestMessage(request.Method, request.RequestUri) + { + Content = await request.Content.Clone(), + Version = request.Version + }; + foreach (KeyValuePair prop in request.Properties) + { + clone.Properties.Add(prop); + } + foreach (KeyValuePair> header in request.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + return clone; + } + + public static async Task Clone(this HttpContent content) + { + if (content == null) return null; + + var ms = new MemoryStream(); + await content.CopyToAsync(ms); + ms.Position = 0; + + var clone = new StreamContent(ms); + foreach (KeyValuePair> header in content.Headers) + { + clone.Headers.Add(header.Key, header.Value); + } + return clone; + } + } +} \ No newline at end of file diff --git a/src/Ombi.Api/Ombi.Api.csproj b/src/Ombi.Api/Ombi.Api.csproj index e89eb54b1..325f316b8 100644 --- a/src/Ombi.Api/Ombi.Api.csproj +++ b/src/Ombi.Api/Ombi.Api.csproj @@ -11,6 +11,7 @@ + diff --git a/src/Ombi.Api/Request.cs b/src/Ombi.Api/Request.cs index 16dd38055..e4120ed9c 100644 --- a/src/Ombi.Api/Request.cs +++ b/src/Ombi.Api/Request.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net; using System.Net.Http; using System.Text; @@ -25,6 +26,9 @@ namespace Ombi.Api public string BaseUrl { get; } public HttpMethod HttpMethod { get; } + public bool Retry { get; set; } + public List StatusCodeToRetry { get; set; } = new List(); + public Action OnBeforeDeserialization { get; set; } private string FullUrl diff --git a/src/Ombi.DependencyInjection/IocExtensions.cs b/src/Ombi.DependencyInjection/IocExtensions.cs index 817dfd551..b40d49036 100644 --- a/src/Ombi.DependencyInjection/IocExtensions.cs +++ b/src/Ombi.DependencyInjection/IocExtensions.cs @@ -172,6 +172,7 @@ namespace Ombi.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); } } } diff --git a/src/Ombi.Schedule/JobSetup.cs b/src/Ombi.Schedule/JobSetup.cs index dc9d49269..e73302387 100644 --- a/src/Ombi.Schedule/JobSetup.cs +++ b/src/Ombi.Schedule/JobSetup.cs @@ -17,46 +17,49 @@ namespace Ombi.Schedule public JobSetup(IPlexContentSync plexContentSync, IRadarrSync radarrSync, IOmbiAutomaticUpdater updater, IEmbyContentSync embySync, IPlexUserImporter userImporter, IEmbyUserImporter embyUserImporter, ISonarrSync cache, ICouchPotatoSync cpCache, - ISettingsService jobsettings, ISickRageSync srSync) + ISettingsService jobsettings, ISickRageSync srSync, IRefreshMetadata refresh) { - PlexContentSync = plexContentSync; - RadarrSync = radarrSync; - Updater = updater; - EmbyContentSync = embySync; - PlexUserImporter = userImporter; - EmbyUserImporter = embyUserImporter; - SonarrSync = cache; - CpCache = cpCache; - JobSettings = jobsettings; - SrSync = srSync; + _plexContentSync = plexContentSync; + _radarrSync = radarrSync; + _updater = updater; + _embyContentSync = embySync; + _plexUserImporter = userImporter; + _embyUserImporter = embyUserImporter; + _sonarrSync = cache; + _cpCache = cpCache; + _jobSettings = jobsettings; + _srSync = srSync; + _refreshMetadata = refresh; } - private IPlexContentSync PlexContentSync { get; } - private IRadarrSync RadarrSync { get; } - private IOmbiAutomaticUpdater Updater { get; } - private IPlexUserImporter PlexUserImporter { get; } - private IEmbyContentSync EmbyContentSync { get; } - private IEmbyUserImporter EmbyUserImporter { get; } - private ISonarrSync SonarrSync { get; } - private ICouchPotatoSync CpCache { get; } - private ISickRageSync SrSync { get; } - private ISettingsService JobSettings { get; set; } + private readonly IPlexContentSync _plexContentSync; + private readonly IRadarrSync _radarrSync; + private readonly IOmbiAutomaticUpdater _updater; + private readonly IPlexUserImporter _plexUserImporter; + private readonly IEmbyContentSync _embyContentSync; + private readonly IEmbyUserImporter _embyUserImporter; + private readonly ISonarrSync _sonarrSync; + private readonly ICouchPotatoSync _cpCache; + private readonly ISickRageSync _srSync; + private readonly ISettingsService _jobSettings; + private readonly IRefreshMetadata _refreshMetadata; public void Setup() { - var s = JobSettings.GetSettings(); + var s = _jobSettings.GetSettings(); - RecurringJob.AddOrUpdate(() => EmbyContentSync.Start(), JobSettingsHelper.EmbyContent(s)); - RecurringJob.AddOrUpdate(() => SonarrSync.Start(), JobSettingsHelper.Sonarr(s)); - RecurringJob.AddOrUpdate(() => RadarrSync.CacheContent(), JobSettingsHelper.Radarr(s)); - RecurringJob.AddOrUpdate(() => PlexContentSync.CacheContent(), JobSettingsHelper.PlexContent(s)); - RecurringJob.AddOrUpdate(() => CpCache.Start(), JobSettingsHelper.CouchPotato(s)); - RecurringJob.AddOrUpdate(() => SrSync.Start(), JobSettingsHelper.SickRageSync(s)); + RecurringJob.AddOrUpdate(() => _embyContentSync.Start(), JobSettingsHelper.EmbyContent(s)); + RecurringJob.AddOrUpdate(() => _sonarrSync.Start(), JobSettingsHelper.Sonarr(s)); + RecurringJob.AddOrUpdate(() => _radarrSync.CacheContent(), JobSettingsHelper.Radarr(s)); + RecurringJob.AddOrUpdate(() => _plexContentSync.CacheContent(), JobSettingsHelper.PlexContent(s)); + RecurringJob.AddOrUpdate(() => _cpCache.Start(), JobSettingsHelper.CouchPotato(s)); + RecurringJob.AddOrUpdate(() => _srSync.Start(), JobSettingsHelper.SickRageSync(s)); + RecurringJob.AddOrUpdate(() => _refreshMetadata.Start(), JobSettingsHelper.RefreshMetadata(s)); - RecurringJob.AddOrUpdate(() => Updater.Update(null), JobSettingsHelper.Updater(s)); + RecurringJob.AddOrUpdate(() => _updater.Update(null), JobSettingsHelper.Updater(s)); - RecurringJob.AddOrUpdate(() => EmbyUserImporter.Start(), JobSettingsHelper.UserImporter(s)); - RecurringJob.AddOrUpdate(() => PlexUserImporter.Start(), JobSettingsHelper.UserImporter(s)); + RecurringJob.AddOrUpdate(() => _embyUserImporter.Start(), JobSettingsHelper.UserImporter(s)); + RecurringJob.AddOrUpdate(() => _plexUserImporter.Start(), JobSettingsHelper.UserImporter(s)); } } } diff --git a/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs b/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs new file mode 100644 index 000000000..a08db74d0 --- /dev/null +++ b/src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace Ombi.Schedule.Jobs.Ombi +{ + public interface IRefreshMetadata : IBaseJob + { + Task Start(); + } +} \ No newline at end of file diff --git a/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs b/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs new file mode 100644 index 000000000..5f5dd4635 --- /dev/null +++ b/src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs @@ -0,0 +1,249 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Ombi.Api.TheMovieDb; +using Ombi.Api.TheMovieDb.Models; +using Ombi.Api.TvMaze; +using Ombi.Core.Settings; +using Ombi.Core.Settings.Models.External; +using Ombi.Helpers; +using Ombi.Store.Entities; +using Ombi.Store.Repository; +using Ombi.Store.Repository.Requests; + +namespace Ombi.Schedule.Jobs.Ombi +{ + public class RefreshMetadata : IRefreshMetadata + { + public RefreshMetadata(IPlexContentRepository plexRepo, IEmbyContentRepository embyRepo, + ILogger log, ITvMazeApi tvApi, ISettingsService plexSettings, + IMovieDbApi movieApi) + { + _plexRepo = plexRepo; + _embyRepo = embyRepo; + _log = log; + _movieApi = movieApi; + _tvApi = tvApi; + _plexSettings = plexSettings; + } + + private readonly IPlexContentRepository _plexRepo; + private readonly IEmbyContentRepository _embyRepo; + private readonly ILogger _log; + private readonly IMovieDbApi _movieApi; + private readonly ITvMazeApi _tvApi; + private readonly ISettingsService _plexSettings; + + public async Task Start() + { + _log.LogInformation("Starting the Metadata refresh"); + try + { + var settings = await _plexSettings.GetSettingsAsync(); + if (settings.Enable) + { + await StartPlex(); + } + } + catch (Exception e) + { + _log.LogError(e, "Exception when refreshing the Plex Metadata"); + throw; + } + } + + private async Task StartPlex() + { + await StartPlexMovies(); + + // Now Tv + await StartPlexTv(); + } + + private async Task StartPlexTv() + { + var allTv = _plexRepo.GetAll().Where(x => + x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue())); + var tvCount = 0; + foreach (var show in allTv) + { + var hasImdb = show.ImdbId.HasValue(); + var hasTheMovieDb = show.TheMovieDbId.HasValue(); + var hasTvDbId = show.TvDbId.HasValue(); + + if (!hasTheMovieDb) + { + var id = await GetTheMovieDbId(hasTvDbId, hasImdb, show.TvDbId, show.ImdbId, show.Title); + show.TheMovieDbId = id; + } + + if (!hasImdb) + { + var id = await GetImdbId(hasTheMovieDb, hasTvDbId, show.Title, show.TheMovieDbId, show.TvDbId); + show.ImdbId = id; + _plexRepo.UpdateWithoutSave(show); + } + + if (!hasTvDbId) + { + var id = await GetTvDbId(hasTheMovieDb, hasImdb, show.TheMovieDbId, show.ImdbId, show.Title); + show.TvDbId = id; + _plexRepo.UpdateWithoutSave(show); + } + tvCount++; + if (tvCount >= 20) + { + await _plexRepo.SaveChangesAsync(); + tvCount = 0; + } + } + await _plexRepo.SaveChangesAsync(); + } + + private async Task StartPlexMovies() + { + var allMovies = _plexRepo.GetAll().Where(x => + x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue())); + int movieCount = 0; + foreach (var movie in allMovies) + { + var hasImdb = movie.ImdbId.HasValue(); + var hasTheMovieDb = movie.TheMovieDbId.HasValue(); + // Movies don't really use TheTvDb + + if (!hasImdb) + { + var imdbId = await GetImdbId(hasTheMovieDb, false, movie.Title, movie.TheMovieDbId, string.Empty); + movie.ImdbId = imdbId; + _plexRepo.UpdateWithoutSave(movie); + } + if (!hasTheMovieDb) + { + var id = await GetTheMovieDbId(false, hasImdb, string.Empty, movie.ImdbId, movie.Title); + movie.TheMovieDbId = id; + _plexRepo.UpdateWithoutSave(movie); + } + movieCount++; + if (movieCount >= 20) + { + await _plexRepo.SaveChangesAsync(); + movieCount = 0; + } + } + + await _plexRepo.SaveChangesAsync(); + } + + private async Task GetTheMovieDbId(bool hasTvDbId, bool hasImdb, string tvdbID, string imdbId, string title) + { + _log.LogInformation("The Media item {0} does not have a TheMovieDbId, searching for TheMovieDbId", title); + FindResult result = null; + var hasResult = false; + if (hasTvDbId) + { + result = await _movieApi.Find(tvdbID, ExternalSource.tvdb_id); + hasResult = result?.tv_results?.Length > 0; + + _log.LogInformation("Setting Show {0} because we have TvDbId, result: {1}", title, hasResult); + } + if (hasImdb && !hasResult) + { + result = await _movieApi.Find(imdbId, ExternalSource.imdb_id); + hasResult = result?.tv_results?.Length > 0; + + _log.LogInformation("Setting Show {0} because we have ImdbId, result: {1}", title, hasResult); + } + if (hasResult) + { + return result.tv_results?[0]?.id.ToString() ?? string.Empty; + } + return string.Empty; + } + + private async Task GetImdbId(bool hasTheMovieDb, bool hasTvDbId, string title, string theMovieDbId, string tvDbId) + { + _log.LogInformation("The media item {0} does not have a ImdbId, searching for ImdbId", title); + // Looks like TV Maze does not provide the moviedb id, neither does the TV endpoint on TheMovieDb + if (hasTheMovieDb) + { + _log.LogInformation("The show {0} has TheMovieDbId but not ImdbId, searching for ImdbId", title); + if (int.TryParse(theMovieDbId, out var id)) + { + var result = await _movieApi.GetTvExternals(id); + + return result.imdb_id; + } + } + + if (hasTvDbId) + { + _log.LogInformation("The show {0} has tvdbid but not ImdbId, searching for ImdbId", title); + if (int.TryParse(tvDbId, out var id)) + { + var result = await _tvApi.ShowLookupByTheTvDbId(id); + return result?.externals?.imdb; + } + } + return string.Empty; + } + + + private async Task GetTvDbId(bool hasTheMovieDb, bool hasImdb, string theMovieDbId, string imdbId, string title) + { + _log.LogInformation("The media item {0} does not have a TvDbId, searching for TvDbId", title); + if (hasTheMovieDb) + { + _log.LogInformation("The show {0} has theMovieDBId but not ImdbId, searching for ImdbId", title); + if (int.TryParse(theMovieDbId, out var id)) + { + var result = await _movieApi.GetTvExternals(id); + + return result.tvdb_id.ToString(); + } + } + + if (hasImdb) + { + _log.LogInformation("The show {0} has ImdbId but not ImdbId, searching for ImdbId", title); + var result = await _movieApi.Find(imdbId, ExternalSource.imdb_id); + if (result?.tv_results?.Length > 0) + { + var movieId = result.tv_results?[0]?.id ?? 0; + + var externalResult = await _movieApi.GetTvExternals(movieId); + + return externalResult.imdb_id; + } + } + return string.Empty; + } + + + private async Task StartEmby() + { + + } + + + private bool _disposed; + protected virtual void Dispose(bool disposing) + { + if (_disposed) + return; + + if (disposing) + { + _plexRepo?.Dispose(); + _embyRepo?.Dispose(); + } + _disposed = true; + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} \ No newline at end of file diff --git a/src/Ombi.Schedule/Ombi.Schedule.csproj b/src/Ombi.Schedule/Ombi.Schedule.csproj index cb8cef8ab..5088bc9f8 100644 --- a/src/Ombi.Schedule/Ombi.Schedule.csproj +++ b/src/Ombi.Schedule/Ombi.Schedule.csproj @@ -32,8 +32,10 @@ + + \ No newline at end of file diff --git a/src/Ombi.Settings/Settings/Models/JobSettings.cs b/src/Ombi.Settings/Settings/Models/JobSettings.cs index 7cf6e7104..ef4335fa5 100644 --- a/src/Ombi.Settings/Settings/Models/JobSettings.cs +++ b/src/Ombi.Settings/Settings/Models/JobSettings.cs @@ -10,5 +10,6 @@ public string AutomaticUpdater { get; set; } public string UserImporter { get; set; } public string SickRageSync { get; set; } + public string RefreshMetadata { get; set; } } } \ No newline at end of file diff --git a/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs b/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs index 69eaf4b33..a200c6b49 100644 --- a/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs +++ b/src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs @@ -39,6 +39,10 @@ namespace Ombi.Settings.Settings.Models { return Get(s.SickRageSync, Cron.Hourly(35)); } + public static string RefreshMetadata(JobSettings s) + { + return Get(s.RefreshMetadata, Cron.Daily(3)); + } private static string Get(string settings, string defaultCron) diff --git a/src/Ombi.Store/Repository/IPlexContentRepository.cs b/src/Ombi.Store/Repository/IPlexContentRepository.cs index 2fef89be2..381a89fa3 100644 --- a/src/Ombi.Store/Repository/IPlexContentRepository.cs +++ b/src/Ombi.Store/Repository/IPlexContentRepository.cs @@ -22,5 +22,7 @@ namespace Ombi.Store.Repository Task DeleteEpisode(PlexEpisode content); void DeleteWithoutSave(PlexServerContent content); void DeleteWithoutSave(PlexEpisode content); + Task UpdateRange(IEnumerable existingContent); + void UpdateWithoutSave(PlexServerContent existingContent); } } \ No newline at end of file diff --git a/src/Ombi.Store/Repository/IRepository.cs b/src/Ombi.Store/Repository/IRepository.cs index ed5ed28c5..c85b45d8f 100644 --- a/src/Ombi.Store/Repository/IRepository.cs +++ b/src/Ombi.Store/Repository/IRepository.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Query; using Ombi.Store.Entities; @@ -24,5 +25,6 @@ namespace Ombi.Store.Repository where TEntity : class; Task ExecuteSql(string sql); + DbSet _db { get; } } } \ No newline at end of file diff --git a/src/Ombi.Store/Repository/PlexContentRepository.cs b/src/Ombi.Store/Repository/PlexContentRepository.cs index 56fec441a..098466310 100644 --- a/src/Ombi.Store/Repository/PlexContentRepository.cs +++ b/src/Ombi.Store/Repository/PlexContentRepository.cs @@ -97,6 +97,16 @@ namespace Ombi.Store.Repository Db.PlexServerContent.Update(existingContent); await Db.SaveChangesAsync(); } + public void UpdateWithoutSave(PlexServerContent existingContent) + { + Db.PlexServerContent.Update(existingContent); + } + + public async Task UpdateRange(IEnumerable existingContent) + { + Db.PlexServerContent.UpdateRange(existingContent); + await Db.SaveChangesAsync(); + } public IQueryable GetAllEpisodes() { diff --git a/src/Ombi.Store/Repository/Repository.cs b/src/Ombi.Store/Repository/Repository.cs index b4b9f8e93..049da0356 100644 --- a/src/Ombi.Store/Repository/Repository.cs +++ b/src/Ombi.Store/Repository/Repository.cs @@ -17,7 +17,7 @@ namespace Ombi.Store.Repository _ctx = ctx; _db = _ctx.Set(); } - private readonly DbSet _db; + public DbSet _db { get; } private readonly IOmbiContext _ctx; public async Task Find(object key) diff --git a/src/Ombi.TheMovieDbApi/IMovieDbApi.cs b/src/Ombi.TheMovieDbApi/IMovieDbApi.cs index dd0d0e92c..787902a4b 100644 --- a/src/Ombi.TheMovieDbApi/IMovieDbApi.cs +++ b/src/Ombi.TheMovieDbApi/IMovieDbApi.cs @@ -15,5 +15,7 @@ namespace Ombi.Api.TheMovieDb Task> TopRated(); Task> Upcoming(); Task> SimilarMovies(int movieId); + Task Find(string externalId, ExternalSource source); + Task GetTvExternals(int theMovieDbId); } } \ No newline at end of file diff --git a/src/Ombi.TheMovieDbApi/Models/FindResult.cs b/src/Ombi.TheMovieDbApi/Models/FindResult.cs new file mode 100644 index 000000000..f76fca564 --- /dev/null +++ b/src/Ombi.TheMovieDbApi/Models/FindResult.cs @@ -0,0 +1,52 @@ +namespace Ombi.Api.TheMovieDb.Models +{ + public class FindResult + { + public Movie_Results[] movie_results { get; set; } + public object[] person_results { get; set; } + public TvResults[] tv_results { get; set; } + public object[] tv_episode_results { get; set; } + public object[] tv_season_results { get; set; } + } + + public class Movie_Results + { + public bool adult { get; set; } + public string backdrop_path { get; set; } + public int[] genre_ids { get; set; } + public int id { get; set; } + public string original_language { get; set; } + public string original_title { get; set; } + public string overview { get; set; } + public string poster_path { get; set; } + public string release_date { get; set; } + public string title { get; set; } + public bool video { get; set; } + public float vote_average { get; set; } + public int vote_count { get; set; } + } + + + public class TvResults + { + public string original_name { get; set; } + public int id { get; set; } + public string name { get; set; } + public int vote_count { get; set; } + public float vote_average { get; set; } + public string first_air_date { get; set; } + public string poster_path { get; set; } + public int[] genre_ids { get; set; } + public string original_language { get; set; } + public string backdrop_path { get; set; } + public string overview { get; set; } + public string[] origin_country { get; set; } + } + + + public enum ExternalSource + { + imdb_id, + tvdb_id + } +} \ No newline at end of file diff --git a/src/Ombi.TheMovieDbApi/Models/TvExternals.cs b/src/Ombi.TheMovieDbApi/Models/TvExternals.cs new file mode 100644 index 000000000..237ae36a7 --- /dev/null +++ b/src/Ombi.TheMovieDbApi/Models/TvExternals.cs @@ -0,0 +1,16 @@ +namespace Ombi.Api.TheMovieDb.Models +{ + public class TvExternals + { + public string imdb_id { get; set; } + public string freebase_mid { get; set; } + public string freebase_id { get; set; } + public int tvdb_id { get; set; } + public int tvrage_id { get; set; } + public string facebook_id { get; set; } + public object instagram_id { get; set; } + public object twitter_id { get; set; } + public int id { get; set; } + } + +} \ No newline at end of file diff --git a/src/Ombi.TheMovieDbApi/TheMovieDbApi.cs b/src/Ombi.TheMovieDbApi/TheMovieDbApi.cs index 1fbfe9aaf..08925e490 100644 --- a/src/Ombi.TheMovieDbApi/TheMovieDbApi.cs +++ b/src/Ombi.TheMovieDbApi/TheMovieDbApi.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Net; using System.Net.Http; using System.Threading.Tasks; using AutoMapper; @@ -25,15 +26,37 @@ namespace Ombi.Api.TheMovieDb { var request = new Request($"movie/{movieId}", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request(request); return Mapper.Map(result); } + public async Task Find(string externalId, ExternalSource source) + { + var request = new Request($"find/{externalId}", BaseUri, HttpMethod.Get); + request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); + + request.AddQueryString("external_source", source.ToString()); + + return await Api.Request(request); + } + + public async Task GetTvExternals(int theMovieDbId) + { + var request = new Request($"/tv/{theMovieDbId}/external_ids", BaseUri, HttpMethod.Get); + request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); + + return await Api.Request(request); + } + public async Task> SimilarMovies(int movieId) { var request = new Request($"movie/{movieId}/similar", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); @@ -44,6 +67,7 @@ namespace Ombi.Api.TheMovieDb var request = new Request($"movie/{movieId}", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); request.FullUri = request.FullUri.AddQueryParameter("append_to_response", "videos,release_dates"); + AddRetry(request); var result = await Api.Request(request); return Mapper.Map(result); } @@ -53,6 +77,7 @@ namespace Ombi.Api.TheMovieDb var request = new Request($"search/movie", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); request.FullUri = request.FullUri.AddQueryParameter("query", searchTerm); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); @@ -62,6 +87,7 @@ namespace Ombi.Api.TheMovieDb { var request = new Request($"movie/popular", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); } @@ -70,6 +96,7 @@ namespace Ombi.Api.TheMovieDb { var request = new Request($"movie/top_rated", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); } @@ -78,6 +105,7 @@ namespace Ombi.Api.TheMovieDb { var request = new Request($"movie/upcoming", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); } @@ -86,9 +114,14 @@ namespace Ombi.Api.TheMovieDb { var request = new Request($"movie/now_playing", BaseUri, HttpMethod.Get); request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken); + AddRetry(request); var result = await Api.Request>(request); return Mapper.Map>(result.results); } - + private static void AddRetry(Request request) + { + request.Retry = true; + request.StatusCodeToRetry.Add((HttpStatusCode)429); + } } } diff --git a/src/Ombi/ClientApp/app/interfaces/ISettings.ts b/src/Ombi/ClientApp/app/interfaces/ISettings.ts index 1ba2c4843..c79860801 100644 --- a/src/Ombi/ClientApp/app/interfaces/ISettings.ts +++ b/src/Ombi/ClientApp/app/interfaces/ISettings.ts @@ -125,6 +125,7 @@ export interface IJobSettings { automaticUpdater: string; userImporter: string; sickRageSync: string; + refreshMetadata: string; } export interface IIssueSettings extends ISettings { @@ -193,3 +194,18 @@ export interface IDogNzbSettings extends ISettings { export interface IIssueCategory extends ISettings { value: string; } + +export interface ICronTestModel { + success: boolean; + message: string; + schedule: Date[]; +} + +export interface ICronViewModelBody { + expression: string; +} + +export interface IJobSettingsViewModel { + result: boolean; + message: string; +} diff --git a/src/Ombi/ClientApp/app/services/settings.service.ts b/src/Ombi/ClientApp/app/services/settings.service.ts index 059df61e8..54f3458d5 100644 --- a/src/Ombi/ClientApp/app/services/settings.service.ts +++ b/src/Ombi/ClientApp/app/services/settings.service.ts @@ -7,6 +7,8 @@ import { IAbout, IAuthenticationSettings, ICouchPotatoSettings, + ICronTestModel, + ICronViewModelBody, ICustomizationSettings, IDiscordNotifcationSettings, IDogNzbSettings, @@ -14,6 +16,7 @@ import { IEmbySettings, IIssueSettings, IJobSettings, + IJobSettingsViewModel, ILandingPageSettings, IMattermostNotifcationSettings, IMobileNotifcationSettings, @@ -231,10 +234,15 @@ export class SettingsService extends ServiceHelpers { return this.http.get(`${this.url}/jobs`, {headers: this.headers}); } - public saveJobSettings(settings: IJobSettings): Observable { + public saveJobSettings(settings: IJobSettings): Observable { return this.http - .post(`${this.url}/jobs`, JSON.stringify(settings), {headers: this.headers}); - } + .post(`${this.url}/jobs`, JSON.stringify(settings), {headers: this.headers}); + } + + public testCron(body: ICronViewModelBody): Observable { + return this.http + .post(`${this.url}/testcron`, JSON.stringify(body), {headers: this.headers}); + } public getSickRageSettings(): Observable { return this.http.get(`${this.url}/sickrage`, {headers: this.headers}); diff --git a/src/Ombi/ClientApp/app/settings/jobs/jobs.component.html b/src/Ombi/ClientApp/app/settings/jobs/jobs.component.html index ff5e56ed8..02eb51d77 100644 --- a/src/Ombi/ClientApp/app/settings/jobs/jobs.component.html +++ b/src/Ombi/ClientApp/app/settings/jobs/jobs.component.html @@ -12,29 +12,34 @@ The Sonarr Sync is required +
The SickRage Sync is required +
The Radarr Sync is required +
The CouchPotato Sync is required +
The Automatic Update is required +
@@ -50,21 +55,37 @@ The Plex Sync is required +
The Emby Sync is required +
The User Importer is required + +
+ +
+ + + The Refresh Metadata is required +
- \ No newline at end of file + + + +
    +
  • {{item | date:'short'}}
    • +
+ \ No newline at end of file diff --git a/src/Ombi/ClientApp/app/settings/jobs/jobs.component.ts b/src/Ombi/ClientApp/app/settings/jobs/jobs.component.ts index f547c5056..380cef8de 100644 --- a/src/Ombi/ClientApp/app/settings/jobs/jobs.component.ts +++ b/src/Ombi/ClientApp/app/settings/jobs/jobs.component.ts @@ -1,7 +1,10 @@ import { Component, OnInit } from "@angular/core"; + import { FormBuilder, FormGroup, Validators } from "@angular/forms"; import { NotificationService, SettingsService } from "../../services"; +import { ICronTestModel } from "./../../interfaces"; + @Component({ templateUrl: "./jobs.component.html", }) @@ -10,6 +13,8 @@ export class JobsComponent implements OnInit { public form: FormGroup; public profilesRunning: boolean; + public testModel: ICronTestModel; + public displayTest: boolean; constructor(private readonly settingsService: SettingsService, private readonly fb: FormBuilder, @@ -26,7 +31,19 @@ export class JobsComponent implements OnInit { sonarrSync: [x.radarrSync, Validators.required], radarrSync: [x.sonarrSync, Validators.required], sickRageSync: [x.sickRageSync, Validators.required], - }); + refreshMetadata: [x.refreshMetadata, Validators.required], + }); + }); + } + + public testCron(expression: string) { + this.settingsService.testCron({ expression }).subscribe(x => { + if(x.success) { + this.testModel = x; + this.displayTest = true; + } else { + this.notificationService.error(x.message); + } }); } @@ -37,10 +54,10 @@ export class JobsComponent implements OnInit { } const settings = form.value; this.settingsService.saveJobSettings(settings).subscribe(x => { - if (x) { + if (x.result) { this.notificationService.success("Successfully saved the job settings"); } else { - this.notificationService.success("There was an error when saving the job settings"); + this.notificationService.error("There was an error when saving the job settings. " + x.message); } }); } diff --git a/src/Ombi/ClientApp/app/settings/settings.module.ts b/src/Ombi/ClientApp/app/settings/settings.module.ts index f10df8448..c95aa4362 100644 --- a/src/Ombi/ClientApp/app/settings/settings.module.ts +++ b/src/Ombi/ClientApp/app/settings/settings.module.ts @@ -41,7 +41,7 @@ import { WikiComponent } from "./wiki.component"; import { SettingsMenuComponent } from "./settingsmenu.component"; -import { AutoCompleteModule, CalendarModule, InputSwitchModule, InputTextModule, MenuModule, RadioButtonModule, TooltipModule } from "primeng/primeng"; +import { AutoCompleteModule, CalendarModule, DialogModule, InputSwitchModule, InputTextModule, MenuModule, RadioButtonModule, TooltipModule } from "primeng/primeng"; const routes: Routes = [ { path: "Ombi", component: OmbiComponent, canActivate: [AuthGuard] }, @@ -88,6 +88,7 @@ const routes: Routes = [ ClipboardModule, PipeModule, RadioButtonModule, + DialogModule, ], declarations: [ SettingsMenuComponent, @@ -139,4 +140,4 @@ const routes: Routes = [ ], }) -export class SettingsModule { } +export class SettingsModule { } \ No newline at end of file diff --git a/src/Ombi/Controllers/SettingsController.cs b/src/Ombi/Controllers/SettingsController.cs index c68a648c8..c0148c1bb 100644 --- a/src/Ombi/Controllers/SettingsController.cs +++ b/src/Ombi/Controllers/SettingsController.cs @@ -5,15 +5,18 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; using AutoMapper; using Hangfire; +using Hangfire.RecurringJobExtensions; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.PlatformAbstractions; +using NCrontab; using Ombi.Api.Emby; using Ombi.Attributes; using Ombi.Core.Models.UI; @@ -465,7 +468,8 @@ namespace Ombi.Controllers j.PlexContentSync = j.PlexContentSync.HasValue() ? j.PlexContentSync : JobSettingsHelper.PlexContent(j); j.UserImporter = j.UserImporter.HasValue() ? j.UserImporter : JobSettingsHelper.UserImporter(j); j.SickRageSync = j.SickRageSync.HasValue() ? j.SickRageSync : JobSettingsHelper.SickRageSync(j); - + j.RefreshMetadata = j.RefreshMetadata.HasValue() ? j.RefreshMetadata : JobSettingsHelper.RefreshMetadata(j); + return j; } @@ -475,9 +479,71 @@ namespace Ombi.Controllers /// The settings. /// [HttpPost("jobs")] - public async Task JobSettings([FromBody]JobSettings settings) + public async Task JobSettings([FromBody]JobSettings settings) { - return await Save(settings); + // Verify that we have correct CRON's + foreach (var propertyInfo in settings.GetType() + .GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (propertyInfo.Name.Equals("Id", StringComparison.CurrentCultureIgnoreCase)) + { + continue; + } + var expression = (string)propertyInfo.GetValue(settings, null); + + try + { + var r = CrontabSchedule.TryParse(expression); + if (r == null) + { + return new JobSettingsViewModel + { + Message = $"{propertyInfo.Name} does not have a valid CRON Expression" + }; + } + } + catch (Exception) + { + return new JobSettingsViewModel + { + Message = $"{propertyInfo.Name} does not have a valid CRON Expression" + }; + } + } + var result = await Save(settings); + + return new JobSettingsViewModel + { + Result = result + }; + } + + [HttpPost("testcron")] + public CronTestModel TestCron([FromBody] CronViewModelBody body) + { + var model = new CronTestModel(); + try + { + var time = DateTime.UtcNow; + var result = CrontabSchedule.TryParse(body.Expression); + for (int i = 0; i < 10; i++) + { + var next = result.GetNextOccurrence(time); + model.Schedule.Add(next); + time = next; + } + model.Success = true; + return model; + } + catch (Exception) + { + return new CronTestModel + { + Message = $"CRON Expression {body.Expression} is not valid" + }; + } + + } diff --git a/src/Ombi/Models/CronTestModel.cs b/src/Ombi/Models/CronTestModel.cs new file mode 100644 index 000000000..9698afbff --- /dev/null +++ b/src/Ombi/Models/CronTestModel.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; + +namespace Ombi.Models +{ + public class CronTestModel + { + public bool Success { get; set; } + public string Message { get; set; } + public List Schedule { get; set; } = new List(); + } +} \ No newline at end of file diff --git a/src/Ombi/Models/CronViewModelBody.cs b/src/Ombi/Models/CronViewModelBody.cs new file mode 100644 index 000000000..cd961eda1 --- /dev/null +++ b/src/Ombi/Models/CronViewModelBody.cs @@ -0,0 +1,7 @@ +namespace Ombi.Models +{ + public class CronViewModelBody + { + public string Expression { get; set; } + } +} \ No newline at end of file diff --git a/src/Ombi/Models/JobSettingsViewModel.cs b/src/Ombi/Models/JobSettingsViewModel.cs new file mode 100644 index 000000000..75006b86a --- /dev/null +++ b/src/Ombi/Models/JobSettingsViewModel.cs @@ -0,0 +1,8 @@ +namespace Ombi.Models +{ + public class JobSettingsViewModel + { + public bool Result { get; set; } + public string Message { get; set; } + } +} \ No newline at end of file diff --git a/src/Ombi/Ombi.csproj b/src/Ombi/Ombi.csproj index 0c2c73360..5a465fb76 100644 --- a/src/Ombi/Ombi.csproj +++ b/src/Ombi/Ombi.csproj @@ -66,6 +66,7 @@ +