Added the ability to refresh out backend metadata (#2078)

We now can refresh the Plex Metadata in our database. For example if the Plex Agent for TV Shows is TheMovieDb, we will use that and populate the IMDB Id and TheTvDb Id's so we can accuratly match and search things.

Also improved the Job settings page so we can Test CRON's and we also validate them.

src/Ombi.Api/Api.cs

@@ -1,4 +1,7 @@
-using System.IO;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Threading.Tasks;
@@ -6,6 +9,7 @@ using System.Xml.Serialization;
 using Newtonsoft.Json;
 using Microsoft.Extensions.Logging;
 using Ombi.Helpers;
+using Polly;
 
 namespace Ombi.Api
 {
@@ -36,6 +40,30 @@ namespace Ombi.Api
                 if (!httpResponseMessage.IsSuccessStatusCode)
                 {
                     LogError(request, httpResponseMessage);
+                    if (request.Retry)
+                    {
+
+                        var result = Policy
+                            .Handle<HttpRequestException>()
+                            .OrResult<HttpResponseMessage>(r => request.StatusCodeToRetry.Contains(r.StatusCode))
+                            .WaitAndRetryAsync(new[]
+                            {
+                                TimeSpan.FromSeconds(10),
+                            }, (exception, timeSpan, context) =>
+                            {
+
+                                Logger.LogError(LoggingEvents.Api,
+                                    $"Retrying RequestUri: {request.FullUri} Because we got Status Code: {exception?.Result?.StatusCode}");
+                            });
+
+                        httpResponseMessage = await result.ExecuteAsync(async () =>
+                        {
+                            using (var req = await httpRequestMessage.Clone())
+                            {
+                                return await _client.SendAsync(req);
+                            }
+                        });
+                    }
                 }
 
                 // do something with the response

src/Ombi.Api/HttpRequestExtnesions.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace Ombi.Api
+{
+    public static class HttpRequestExtnesions
+    {
+        public static async Task<HttpRequestMessage> Clone(this HttpRequestMessage request)
+        {
+            var clone = new HttpRequestMessage(request.Method, request.RequestUri)
+            {
+                Content = await request.Content.Clone(),
+                Version = request.Version
+            };
+            foreach (KeyValuePair<string, object> prop in request.Properties)
+            {
+                clone.Properties.Add(prop);
+            }
+            foreach (KeyValuePair<string, IEnumerable<string>> header in request.Headers)
+            {
+                clone.Headers.TryAddWithoutValidation(header.Key, header.Value);
+            }
+
+            return clone;
+        }
+
+        public static async Task<HttpContent> Clone(this HttpContent content)
+        {
+            if (content == null) return null;
+
+            var ms = new MemoryStream();
+            await content.CopyToAsync(ms);
+            ms.Position = 0;
+
+            var clone = new StreamContent(ms);
+            foreach (KeyValuePair<string, IEnumerable<string>> header in content.Headers)
+            {
+                clone.Headers.Add(header.Key, header.Value);
+            }
+            return clone;
+        }
+    }
+}

src/Ombi.Api/Ombi.Api.csproj

@@ -11,6 +11,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.0.1" />
     <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Polly" Version="5.8.0" />
     <PackageReference Include="System.Xml.XmlSerializer" Version="4.3.0" />
   </ItemGroup>
 

src/Ombi.Api/Request.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Net.Http;
 using System.Text;
 
@@ -25,6 +26,9 @@ namespace Ombi.Api
         public string BaseUrl { get; }
         public HttpMethod HttpMethod { get; }
 
+        public bool Retry { get; set; }
+        public List<HttpStatusCode> StatusCodeToRetry { get; set; } = new List<HttpStatusCode>();
+
         public Action<string> OnBeforeDeserialization { get; set; }
 
         private string FullUrl

src/Ombi.DependencyInjection/IocExtensions.cs

@@ -172,6 +172,7 @@ namespace Ombi.DependencyInjection
             services.AddTransient<ICouchPotatoSync, CouchPotatoSync>();
             services.AddTransient<IProcessProvider, ProcessProvider>();
             services.AddTransient<ISickRageSync, SickRageSync>();
+            services.AddTransient<IRefreshMetadata, RefreshMetadata>();
         }
     }
 }

src/Ombi.Schedule/JobSetup.cs

@@ -17,46 +17,49 @@ namespace Ombi.Schedule
         public JobSetup(IPlexContentSync plexContentSync, IRadarrSync radarrSync,
             IOmbiAutomaticUpdater updater, IEmbyContentSync embySync, IPlexUserImporter userImporter,
             IEmbyUserImporter embyUserImporter, ISonarrSync cache, ICouchPotatoSync cpCache,
-            ISettingsService<JobSettings> jobsettings, ISickRageSync srSync)
+            ISettingsService<JobSettings> jobsettings, ISickRageSync srSync, IRefreshMetadata refresh)
         {
-            PlexContentSync = plexContentSync;
-            RadarrSync = radarrSync;
-            Updater = updater;
-            EmbyContentSync = embySync;
-            PlexUserImporter = userImporter;
-            EmbyUserImporter = embyUserImporter;
-            SonarrSync = cache;
-            CpCache = cpCache;
-            JobSettings = jobsettings;
-            SrSync = srSync;
+            _plexContentSync = plexContentSync;
+            _radarrSync = radarrSync;
+            _updater = updater;
+            _embyContentSync = embySync;
+            _plexUserImporter = userImporter;
+            _embyUserImporter = embyUserImporter;
+            _sonarrSync = cache;
+            _cpCache = cpCache;
+            _jobSettings = jobsettings;
+            _srSync = srSync;
+            _refreshMetadata = refresh;
         }
 
-        private IPlexContentSync PlexContentSync { get; }
-        private IRadarrSync RadarrSync { get; }
-        private IOmbiAutomaticUpdater Updater { get; }
-        private IPlexUserImporter PlexUserImporter { get; }
-        private IEmbyContentSync EmbyContentSync { get; }
-        private IEmbyUserImporter EmbyUserImporter { get; }
-        private ISonarrSync SonarrSync { get; }
-        private ICouchPotatoSync CpCache { get; }
-        private ISickRageSync SrSync { get; }
-        private ISettingsService<JobSettings> JobSettings { get; set; }
+        private readonly IPlexContentSync _plexContentSync;
+        private readonly IRadarrSync _radarrSync;
+        private readonly IOmbiAutomaticUpdater _updater;
+        private readonly IPlexUserImporter _plexUserImporter;
+        private readonly IEmbyContentSync _embyContentSync;
+        private readonly IEmbyUserImporter _embyUserImporter;
+        private readonly ISonarrSync _sonarrSync;
+        private readonly ICouchPotatoSync _cpCache;
+        private readonly ISickRageSync _srSync;
+        private readonly ISettingsService<JobSettings> _jobSettings;
+        private readonly IRefreshMetadata _refreshMetadata;
 
         public void Setup()
         {
-            var s = JobSettings.GetSettings();
+            var s = _jobSettings.GetSettings();
 
-            RecurringJob.AddOrUpdate(() => EmbyContentSync.Start(), JobSettingsHelper.EmbyContent(s));
-            RecurringJob.AddOrUpdate(() => SonarrSync.Start(), JobSettingsHelper.Sonarr(s));
-            RecurringJob.AddOrUpdate(() => RadarrSync.CacheContent(), JobSettingsHelper.Radarr(s));
-            RecurringJob.AddOrUpdate(() => PlexContentSync.CacheContent(), JobSettingsHelper.PlexContent(s));
-            RecurringJob.AddOrUpdate(() => CpCache.Start(), JobSettingsHelper.CouchPotato(s));
-            RecurringJob.AddOrUpdate(() => SrSync.Start(), JobSettingsHelper.SickRageSync(s));
+            RecurringJob.AddOrUpdate(() => _embyContentSync.Start(), JobSettingsHelper.EmbyContent(s));
+            RecurringJob.AddOrUpdate(() => _sonarrSync.Start(), JobSettingsHelper.Sonarr(s));
+            RecurringJob.AddOrUpdate(() => _radarrSync.CacheContent(), JobSettingsHelper.Radarr(s));
+            RecurringJob.AddOrUpdate(() => _plexContentSync.CacheContent(), JobSettingsHelper.PlexContent(s));
+            RecurringJob.AddOrUpdate(() => _cpCache.Start(), JobSettingsHelper.CouchPotato(s));
+            RecurringJob.AddOrUpdate(() => _srSync.Start(), JobSettingsHelper.SickRageSync(s));
+            RecurringJob.AddOrUpdate(() => _refreshMetadata.Start(), JobSettingsHelper.RefreshMetadata(s));
 
-            RecurringJob.AddOrUpdate(() => Updater.Update(null), JobSettingsHelper.Updater(s));
+            RecurringJob.AddOrUpdate(() => _updater.Update(null), JobSettingsHelper.Updater(s));
 
-            RecurringJob.AddOrUpdate(() => EmbyUserImporter.Start(), JobSettingsHelper.UserImporter(s));
-            RecurringJob.AddOrUpdate(() => PlexUserImporter.Start(), JobSettingsHelper.UserImporter(s));
+            RecurringJob.AddOrUpdate(() => _embyUserImporter.Start(), JobSettingsHelper.UserImporter(s));
+            RecurringJob.AddOrUpdate(() => _plexUserImporter.Start(), JobSettingsHelper.UserImporter(s));
         }
     }
 }

src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs

@@ -0,0 +1,9 @@
+using System.Threading.Tasks;
+
+namespace Ombi.Schedule.Jobs.Ombi
+{
+    public interface IRefreshMetadata : IBaseJob
+    {
+        Task Start();
+    }
+}

src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs

@@ -0,0 +1,249 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Ombi.Api.TheMovieDb;
+using Ombi.Api.TheMovieDb.Models;
+using Ombi.Api.TvMaze;
+using Ombi.Core.Settings;
+using Ombi.Core.Settings.Models.External;
+using Ombi.Helpers;
+using Ombi.Store.Entities;
+using Ombi.Store.Repository;
+using Ombi.Store.Repository.Requests;
+
+namespace Ombi.Schedule.Jobs.Ombi
+{
+    public class RefreshMetadata : IRefreshMetadata
+    {
+        public RefreshMetadata(IPlexContentRepository plexRepo, IEmbyContentRepository embyRepo,
+            ILogger<RefreshMetadata> log, ITvMazeApi tvApi, ISettingsService<PlexSettings> plexSettings,
+            IMovieDbApi movieApi)
+        {
+            _plexRepo = plexRepo;
+            _embyRepo = embyRepo;
+            _log = log;
+            _movieApi = movieApi;
+            _tvApi = tvApi;
+            _plexSettings = plexSettings;
+        }
+
+        private readonly IPlexContentRepository _plexRepo;
+        private readonly IEmbyContentRepository _embyRepo;
+        private readonly ILogger _log;
+        private readonly IMovieDbApi _movieApi;
+        private readonly ITvMazeApi _tvApi;
+        private readonly ISettingsService<PlexSettings> _plexSettings;
+
+        public async Task Start()
+        {
+            _log.LogInformation("Starting the Metadata refresh");
+            try
+            {
+                var settings = await _plexSettings.GetSettingsAsync();
+                if (settings.Enable)
+                {
+                    await StartPlex();
+                }
+            }
+            catch (Exception e)
+            {
+                _log.LogError(e, "Exception when refreshing the Plex Metadata");
+                throw;
+            }
+        }
+
+        private async Task StartPlex()
+        {
+            await StartPlexMovies();
+
+            // Now Tv
+            await StartPlexTv();
+        }
+
+        private async Task StartPlexTv()
+        {
+            var allTv = _plexRepo.GetAll().Where(x =>
+                x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue()));
+            var tvCount = 0;
+            foreach (var show in allTv)
+            {
+                var hasImdb = show.ImdbId.HasValue();
+                var hasTheMovieDb = show.TheMovieDbId.HasValue();
+                var hasTvDbId = show.TvDbId.HasValue();
+
+                if (!hasTheMovieDb)
+                {
+                    var id = await GetTheMovieDbId(hasTvDbId, hasImdb, show.TvDbId, show.ImdbId, show.Title);
+                    show.TheMovieDbId = id;
+                }
+
+                if (!hasImdb)
+                {
+                    var id = await GetImdbId(hasTheMovieDb, hasTvDbId, show.Title, show.TheMovieDbId, show.TvDbId);
+                    show.ImdbId = id;
+                    _plexRepo.UpdateWithoutSave(show);
+                }
+
+                if (!hasTvDbId)
+                {
+                    var id = await GetTvDbId(hasTheMovieDb, hasImdb, show.TheMovieDbId, show.ImdbId, show.Title);
+                    show.TvDbId = id;
+                    _plexRepo.UpdateWithoutSave(show);
+                }
+                tvCount++;
+                if (tvCount >= 20)
+                {
+                    await _plexRepo.SaveChangesAsync();
+                    tvCount = 0;
+                }
+            }
+            await _plexRepo.SaveChangesAsync();
+        }
+
+        private async Task StartPlexMovies()
+        {
+            var allMovies = _plexRepo.GetAll().Where(x =>
+                x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue()));
+            int movieCount = 0;
+            foreach (var movie in allMovies)
+            {
+                var hasImdb = movie.ImdbId.HasValue();
+                var hasTheMovieDb = movie.TheMovieDbId.HasValue();
+                // Movies don't really use TheTvDb
+
+                if (!hasImdb)
+                {
+                    var imdbId = await GetImdbId(hasTheMovieDb, false, movie.Title, movie.TheMovieDbId, string.Empty);
+                    movie.ImdbId = imdbId;
+                    _plexRepo.UpdateWithoutSave(movie);
+                }
+                if (!hasTheMovieDb)
+                {
+                    var id = await GetTheMovieDbId(false, hasImdb, string.Empty, movie.ImdbId, movie.Title);
+                    movie.TheMovieDbId = id;
+                    _plexRepo.UpdateWithoutSave(movie);
+                }
+                movieCount++;
+                if (movieCount >= 20)
+                {
+                    await _plexRepo.SaveChangesAsync();
+                    movieCount = 0;
+                }
+            }
+
+            await _plexRepo.SaveChangesAsync();
+        }
+
+       private async Task<string> GetTheMovieDbId(bool hasTvDbId, bool hasImdb, string tvdbID, string imdbId, string title)
+        {
+            _log.LogInformation("The Media item {0} does not have a TheMovieDbId, searching for TheMovieDbId", title);
+            FindResult result = null;
+            var hasResult = false;
+            if (hasTvDbId)
+            {
+                result = await _movieApi.Find(tvdbID, ExternalSource.tvdb_id);
+                hasResult = result?.tv_results?.Length > 0;
+
+                _log.LogInformation("Setting Show {0} because we have TvDbId, result: {1}", title, hasResult);
+            }
+            if (hasImdb && !hasResult)
+            {
+                result = await _movieApi.Find(imdbId, ExternalSource.imdb_id);
+                hasResult = result?.tv_results?.Length > 0;
+
+                _log.LogInformation("Setting Show {0} because we have ImdbId, result: {1}", title, hasResult);
+            }
+            if (hasResult)
+            {
+                return result.tv_results?[0]?.id.ToString() ?? string.Empty;
+            }
+            return string.Empty;
+        }
+
+        private async Task<string> GetImdbId(bool hasTheMovieDb, bool hasTvDbId, string title, string theMovieDbId, string tvDbId)
+        {
+            _log.LogInformation("The media item {0} does not have a ImdbId, searching for ImdbId", title);
+            // Looks like TV Maze does not provide the moviedb id, neither does the TV endpoint on TheMovieDb
+            if (hasTheMovieDb)
+            {
+                _log.LogInformation("The show {0} has TheMovieDbId but not ImdbId, searching for ImdbId", title);
+                if (int.TryParse(theMovieDbId, out var id))
+                {
+                    var result = await _movieApi.GetTvExternals(id);
+
+                    return result.imdb_id;
+                }
+            }
+
+            if (hasTvDbId)
+            {
+                _log.LogInformation("The show {0} has tvdbid but not ImdbId, searching for ImdbId", title);
+                if (int.TryParse(tvDbId, out var id))
+                {
+                    var result = await _tvApi.ShowLookupByTheTvDbId(id);
+                    return result?.externals?.imdb;
+                }
+            }
+            return string.Empty;
+        }
+
+
+        private async Task<string> GetTvDbId(bool hasTheMovieDb, bool hasImdb, string theMovieDbId, string imdbId, string title)
+        {
+            _log.LogInformation("The media item {0} does not have a TvDbId, searching for TvDbId", title);
+            if (hasTheMovieDb)
+            {
+                _log.LogInformation("The show {0} has theMovieDBId but not ImdbId, searching for ImdbId", title);
+                if (int.TryParse(theMovieDbId, out var id))
+                {
+                    var result = await _movieApi.GetTvExternals(id);
+
+                   return result.tvdb_id.ToString();
+                }
+            }
+
+            if (hasImdb)
+            {
+                _log.LogInformation("The show {0} has ImdbId but not ImdbId, searching for ImdbId", title);
+                var result = await _movieApi.Find(imdbId, ExternalSource.imdb_id);
+                if (result?.tv_results?.Length > 0)
+                {
+                    var movieId = result.tv_results?[0]?.id ?? 0;
+
+                    var externalResult = await _movieApi.GetTvExternals(movieId);
+
+                    return externalResult.imdb_id;
+                }
+            }
+            return string.Empty;
+        }
+        
+
+        private async Task StartEmby()
+        {
+
+        }
+
+
+        private bool _disposed;
+        protected virtual void Dispose(bool disposing)
+        {
+            if (_disposed)
+                return;
+
+            if (disposing)
+            {
+                _plexRepo?.Dispose();
+                _embyRepo?.Dispose();
+            }
+            _disposed = true;
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

src/Ombi.Schedule/Ombi.Schedule.csproj

@@ -32,8 +32,10 @@
     <ProjectReference Include="..\Ombi.Api.Service\Ombi.Api.Service.csproj" />
     <ProjectReference Include="..\Ombi.Api.SickRage\Ombi.Api.SickRage.csproj" />
     <ProjectReference Include="..\Ombi.Api.Sonarr\Ombi.Api.Sonarr.csproj" />
+    <ProjectReference Include="..\Ombi.Api.TvMaze\Ombi.Api.TvMaze.csproj" />
     <ProjectReference Include="..\Ombi.Notifications\Ombi.Notifications.csproj" />
     <ProjectReference Include="..\Ombi.Settings\Ombi.Settings.csproj" />
+    <ProjectReference Include="..\Ombi.TheMovieDbApi\Ombi.Api.TheMovieDb.csproj" />
   </ItemGroup>
 
 </Project>

src/Ombi.Settings/Settings/Models/JobSettings.cs

@@ -10,5 +10,6 @@
         public string AutomaticUpdater { get; set; }
         public string UserImporter { get; set; }
         public string SickRageSync { get; set; }
+        public string RefreshMetadata { get; set; }
     }
 }

src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs

@@ -39,6 +39,10 @@ namespace Ombi.Settings.Settings.Models
         {
             return Get(s.SickRageSync, Cron.Hourly(35));
         }
+        public static string RefreshMetadata(JobSettings s)
+        {
+            return Get(s.RefreshMetadata, Cron.Daily(3));
+        }
 
 
         private static string Get(string settings, string defaultCron)

src/Ombi.Store/Repository/IPlexContentRepository.cs

@@ -22,5 +22,7 @@ namespace Ombi.Store.Repository
         Task DeleteEpisode(PlexEpisode content);
         void DeleteWithoutSave(PlexServerContent content);
         void DeleteWithoutSave(PlexEpisode content);
+        Task UpdateRange(IEnumerable<PlexServerContent> existingContent);
+        void UpdateWithoutSave(PlexServerContent existingContent);
     }
 }

src/Ombi.Store/Repository/IRepository.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Query;
 using Ombi.Store.Entities;
 
@@ -24,5 +25,6 @@ namespace Ombi.Store.Repository
             where TEntity : class;
 
         Task ExecuteSql(string sql);
+        DbSet<T> _db { get; }
     }
 }

src/Ombi.Store/Repository/PlexContentRepository.cs

@@ -97,6 +97,16 @@ namespace Ombi.Store.Repository
             Db.PlexServerContent.Update(existingContent);
             await Db.SaveChangesAsync();
         }
+        public void UpdateWithoutSave(PlexServerContent existingContent)
+        {
+            Db.PlexServerContent.Update(existingContent);
+        }
+
+        public async Task UpdateRange(IEnumerable<PlexServerContent> existingContent)
+        {
+            Db.PlexServerContent.UpdateRange(existingContent);
+            await Db.SaveChangesAsync();
+        }
 
         public IQueryable<PlexEpisode> GetAllEpisodes()
         {

src/Ombi.Store/Repository/Repository.cs

@@ -17,7 +17,7 @@ namespace Ombi.Store.Repository
             _ctx = ctx;
             _db = _ctx.Set<T>();
         }
-        private readonly DbSet<T> _db;
+        public DbSet<T> _db { get; }
         private readonly IOmbiContext _ctx;
 
         public async Task<T> Find(object key)

src/Ombi.TheMovieDbApi/IMovieDbApi.cs

@@ -15,5 +15,7 @@ namespace Ombi.Api.TheMovieDb
         Task<List<MovieSearchResult>> TopRated();
         Task<List<MovieSearchResult>> Upcoming();
         Task<List<MovieSearchResult>> SimilarMovies(int movieId);
+        Task<FindResult> Find(string externalId, ExternalSource source);
+        Task<TvExternals> GetTvExternals(int theMovieDbId);
     }
 }

src/Ombi.TheMovieDbApi/Models/FindResult.cs

@@ -0,0 +1,52 @@
+namespace Ombi.Api.TheMovieDb.Models
+{
+    public class FindResult
+    {
+        public Movie_Results[] movie_results { get; set; }
+        public object[] person_results { get; set; }
+        public TvResults[] tv_results { get; set; }
+        public object[] tv_episode_results { get; set; }
+        public object[] tv_season_results { get; set; }
+    }
+
+    public class Movie_Results
+    {
+        public bool adult { get; set; }
+        public string backdrop_path { get; set; }
+        public int[] genre_ids { get; set; }
+        public int id { get; set; }
+        public string original_language { get; set; }
+        public string original_title { get; set; }
+        public string overview { get; set; }
+        public string poster_path { get; set; }
+        public string release_date { get; set; }
+        public string title { get; set; }
+        public bool video { get; set; }
+        public float vote_average { get; set; }
+        public int vote_count { get; set; }
+    }
+
+
+    public class TvResults
+    {
+        public string original_name { get; set; }
+        public int id { get; set; }
+        public string name { get; set; }
+        public int vote_count { get; set; }
+        public float vote_average { get; set; }
+        public string first_air_date { get; set; }
+        public string poster_path { get; set; }
+        public int[] genre_ids { get; set; }
+        public string original_language { get; set; }
+        public string backdrop_path { get; set; }
+        public string overview { get; set; }
+        public string[] origin_country { get; set; }
+    }
+
+
+    public enum ExternalSource
+    {
+        imdb_id,
+        tvdb_id
+    }
+}

src/Ombi.TheMovieDbApi/Models/TvExternals.cs

@@ -0,0 +1,16 @@
+namespace Ombi.Api.TheMovieDb.Models
+{
+    public class TvExternals
+    {
+        public string imdb_id { get; set; }
+        public string freebase_mid { get; set; }
+        public string freebase_id { get; set; }
+        public int tvdb_id { get; set; }
+        public int tvrage_id { get; set; }
+        public string facebook_id { get; set; }
+        public object instagram_id { get; set; }
+        public object twitter_id { get; set; }
+        public int id { get; set; }
+    }
+
+}

src/Ombi.TheMovieDbApi/TheMovieDbApi.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Net;
 using System.Net.Http;
 using System.Threading.Tasks;
 using AutoMapper;
@@ -25,15 +26,37 @@ namespace Ombi.Api.TheMovieDb
         {
             var request = new Request($"movie/{movieId}", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
 
             var result = await Api.Request<MovieResponse>(request);
             return Mapper.Map<MovieResponseDto>(result);
         }
 
+        public async Task<FindResult> Find(string externalId, ExternalSource source)
+        {
+            var request = new Request($"find/{externalId}", BaseUri, HttpMethod.Get);
+            request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
+
+            request.AddQueryString("external_source", source.ToString());
+
+            return await Api.Request<FindResult>(request);
+        }
+        
+        public async Task<TvExternals> GetTvExternals(int theMovieDbId)
+        {
+            var request = new Request($"/tv/{theMovieDbId}/external_ids", BaseUri, HttpMethod.Get);
+            request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
+
+            return await Api.Request<TvExternals>(request);
+        }
+        
         public async Task<List<MovieSearchResult>> SimilarMovies(int movieId)
         {
             var request = new Request($"movie/{movieId}/similar", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
 
             var result = await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
@@ -44,6 +67,7 @@ namespace Ombi.Api.TheMovieDb
             var request = new Request($"movie/{movieId}", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
             request.FullUri = request.FullUri.AddQueryParameter("append_to_response", "videos,release_dates");
+            AddRetry(request);
             var result = await Api.Request<MovieResponse>(request);
             return Mapper.Map<MovieResponseDto>(result);
         }
@@ -53,6 +77,7 @@ namespace Ombi.Api.TheMovieDb
             var request = new Request($"search/movie", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
             request.FullUri = request.FullUri.AddQueryParameter("query", searchTerm);
+            AddRetry(request);
 
             var result =  await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
@@ -62,6 +87,7 @@ namespace Ombi.Api.TheMovieDb
         {
             var request = new Request($"movie/popular", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
             var result = await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
         }
@@ -70,6 +96,7 @@ namespace Ombi.Api.TheMovieDb
         {
             var request = new Request($"movie/top_rated", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
             var result = await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
         }
@@ -78,6 +105,7 @@ namespace Ombi.Api.TheMovieDb
         {
             var request = new Request($"movie/upcoming", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
             var result = await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
         }
@@ -86,9 +114,14 @@ namespace Ombi.Api.TheMovieDb
         {
             var request = new Request($"movie/now_playing", BaseUri, HttpMethod.Get);
             request.FullUri = request.FullUri.AddQueryParameter("api_key", ApiToken);
+            AddRetry(request);
             var result = await Api.Request<TheMovieDbContainer<SearchResult>>(request);
             return Mapper.Map<List<MovieSearchResult>>(result.results);
         }
-
+        private static void AddRetry(Request request)
+        {
+            request.Retry = true;
+            request.StatusCodeToRetry.Add((HttpStatusCode)429);
+        }
     }
 }

src/Ombi/ClientApp/app/interfaces/ISettings.ts

@@ -125,6 +125,7 @@ export interface IJobSettings {
   automaticUpdater: string;
   userImporter: string;
   sickRageSync: string;
+  refreshMetadata: string;
 }
 
 export interface IIssueSettings extends ISettings {
@@ -193,3 +194,18 @@ export interface IDogNzbSettings extends ISettings {
 export interface IIssueCategory extends ISettings {
   value: string;
 }
+
+export interface ICronTestModel {
+  success: boolean;
+  message: string;
+  schedule: Date[];
+}
+
+export interface ICronViewModelBody {
+  expression: string;
+}
+
+export interface IJobSettingsViewModel {
+  result: boolean;
+  message: string;
+}

src/Ombi/ClientApp/app/services/settings.service.ts

@@ -7,6 +7,8 @@ import {
     IAbout,
     IAuthenticationSettings,
     ICouchPotatoSettings,
+    ICronTestModel,
+    ICronViewModelBody,
     ICustomizationSettings,
     IDiscordNotifcationSettings,
     IDogNzbSettings,
@@ -14,6 +16,7 @@ import {
     IEmbySettings,
     IIssueSettings,
     IJobSettings,
+    IJobSettingsViewModel,
     ILandingPageSettings,
     IMattermostNotifcationSettings,
     IMobileNotifcationSettings,
@@ -231,10 +234,15 @@ export class SettingsService extends ServiceHelpers {
         return this.http.get<IJobSettings>(`${this.url}/jobs`, {headers: this.headers});
     }
 
-    public saveJobSettings(settings: IJobSettings): Observable<boolean> {
+    public saveJobSettings(settings: IJobSettings): Observable<IJobSettingsViewModel> {
         return this.http
-            .post<boolean>(`${this.url}/jobs`, JSON.stringify(settings), {headers: this.headers});
-    }   
+            .post<IJobSettingsViewModel>(`${this.url}/jobs`, JSON.stringify(settings), {headers: this.headers});
+    } 
+    
+    public testCron(body: ICronViewModelBody): Observable<ICronTestModel> {
+        return this.http
+            .post<ICronTestModel>(`${this.url}/testcron`, JSON.stringify(body), {headers: this.headers});
+    }
     
     public getSickRageSettings(): Observable<ISickRageSettings> {
         return this.http.get<ISickRageSettings>(`${this.url}/sickrage`, {headers: this.headers});

src/Ombi/ClientApp/app/settings/jobs/jobs.component.html

@@ -12,29 +12,34 @@
                     <label for="sonarrSync" class="control-label">Sonarr Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('sonarrSync').hasError('required')}" id="sonarrSync" name="sonarrSync" formControlName="sonarrSync">
                         <small *ngIf="form.get('sonarrSync').hasError('required')" class="error-text">The Sonarr Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('sonarrSync')?.value)">Test</button>
                 </div>
                 <div class="form-group">
                     <label for="sickRageSync" class="control-label">SickRage Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('sonarrSync').hasError('required')}" id="sickRageSync" name="sickRageSync" formControlName="sickRageSync">
                         <small *ngIf="form.get('sickRageSync').hasError('required')" class="error-text">The SickRage Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('sickRageSync')?.value)">Test</button>
                 </div>
 
                 <div class="form-group">
                     <label for="radarrSync" class="control-label">Radarr Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('radarrSync').hasError('required')}" id="radarrSync" name="radarrSync" formControlName="radarrSync">
                         <small *ngIf="form.get('radarrSync').hasError('required')" class="error-text">The Radarr Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('radarrSync')?.value)">Test</button>
                 </div>
 
                 <div class="form-group">
                     <label for="couchPotatoSync" class="control-label">CouchPotato Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('radarrSync').hasError('required')}" id="couchPotatoSync" name="couchPotatoSync" formControlName="couchPotatoSync">
                         <small *ngIf="form.get('couchPotatoSync').hasError('required')" class="error-text">The CouchPotato Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('couchPotatoSync')?.value)">Test</button>
                 </div>
                 
                 <div class="form-group">
                     <label for="automaticUpdater" class="control-label">Automatic Update</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('automaticUpdater').hasError('required')}" id="automaticUpdater" name="automaticUpdater" formControlName="automaticUpdater">
                         <small *ngIf="form.get('automaticUpdater').hasError('required')" class="error-text">The Automatic Update is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('automaticUpdater')?.value)">Test</button>
                 </div>
 
 
@@ -50,21 +55,37 @@
                     <label for="plexContentSync" class="control-label">Plex Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('plexContentSync').hasError('required')}" id="plexContentSync" name="plexContentSync" formControlName="plexContentSync">
                         <small *ngIf="form.get('plexContentSync').hasError('required')" class="error-text">The Plex Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('plexContentSync')?.value)">Test</button>
                 </div>
 
                 <div class="form-group">
                     <label for="embyContentSync" class="control-label">Emby Sync</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('embyContentSync').hasError('required')}" id="embyContentSync" name="embyContentSync" formControlName="embyContentSync">
                         <small *ngIf="form.get('embyContentSync').hasError('required')" class="error-text">The Emby Sync is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('embyContentSync')?.value)">Test</button>
                 </div>
 
                 <div class="form-group">
                     <label for="userImporter" class="control-label">User Importer</label>              
                         <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('userImporter').hasError('required')}" id="userImporter" name="userImporter" formControlName="userImporter">
                         <small *ngIf="form.get('userImporter').hasError('required')" class="error-text">The User Importer is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('userImporter')?.value)">Test</button>
+                </div>
+
+                <div class="form-group">
+                    <label for="userImporter" class="control-label">Refresh Metadata</label>              
+                        <input type="text" class="form-control form-control-custom" [ngClass]="{'form-error': form.get('refreshMetadata').hasError('required')}" id="refreshMetadata" name="refreshMetadata" formControlName="refreshMetadata">
+                        <small *ngIf="form.get('refreshMetadata').hasError('required')" class="error-text">The Refresh Metadata is required</small>
+                        <button type="button" class="btn btn-sm btn-primary-outline" (click)="testCron(form.get('refreshMetadata')?.value)">Test</button>
                 </div>
 
             </div>
         </form>
     </fieldset>
-</div>
+</div>
+
+<p-dialog header="CRON Schedule" [(visible)]="displayTest">
+    <ul *ngIf="testModel">
+        <li *ngFor="let item of testModel.schedule">{{item | date:'short'}}</li>
+    </ul>
+</p-dialog>

src/Ombi/ClientApp/app/settings/jobs/jobs.component.ts

@@ -1,7 +1,10 @@
 import { Component, OnInit } from "@angular/core";
+
 import { FormBuilder, FormGroup, Validators } from "@angular/forms";
 import { NotificationService, SettingsService } from "../../services";
 
+import { ICronTestModel } from "./../../interfaces";
+
 @Component({
     templateUrl: "./jobs.component.html",
 })
@@ -10,6 +13,8 @@ export class JobsComponent implements OnInit {
     public form: FormGroup;
     
     public profilesRunning: boolean;
+    public testModel: ICronTestModel;
+    public displayTest: boolean;
     
     constructor(private readonly settingsService: SettingsService,
                 private readonly fb: FormBuilder,
@@ -26,7 +31,19 @@ export class JobsComponent implements OnInit {
                 sonarrSync:               [x.radarrSync, Validators.required],
                 radarrSync:               [x.sonarrSync, Validators.required],
                 sickRageSync:             [x.sickRageSync, Validators.required],  
-            });
+                refreshMetadata:          [x.refreshMetadata, Validators.required],
+            });  
+        });
+    }
+
+    public testCron(expression: string) {
+        this.settingsService.testCron({ expression }).subscribe(x => {
+            if(x.success) {
+                this.testModel = x;
+                this.displayTest = true;              
+            } else {
+                this.notificationService.error(x.message);
+            }
         });
     }
     
@@ -37,10 +54,10 @@ export class JobsComponent implements OnInit {
         }
         const settings = form.value;
         this.settingsService.saveJobSettings(settings).subscribe(x => {
-            if (x) {
+            if (x.result) {
                 this.notificationService.success("Successfully saved the job settings");
             } else {
-                this.notificationService.success("There was an error when saving the job settings");
+                this.notificationService.error("There was an error when saving the job settings. " + x.message);
             }
         });
     }

src/Ombi/ClientApp/app/settings/settings.module.ts

@@ -41,7 +41,7 @@ import { WikiComponent } from "./wiki.component";
 
 import { SettingsMenuComponent } from "./settingsmenu.component";
 
-import { AutoCompleteModule, CalendarModule, InputSwitchModule, InputTextModule, MenuModule, RadioButtonModule, TooltipModule } from "primeng/primeng";
+import { AutoCompleteModule, CalendarModule, DialogModule, InputSwitchModule, InputTextModule, MenuModule, RadioButtonModule, TooltipModule } from "primeng/primeng";
 
 const routes: Routes = [
     { path: "Ombi", component: OmbiComponent, canActivate: [AuthGuard] },
@@ -88,6 +88,7 @@ const routes: Routes = [
         ClipboardModule,
         PipeModule,
         RadioButtonModule,
+        DialogModule,
     ],
     declarations: [
         SettingsMenuComponent,
@@ -139,4 +140,4 @@ const routes: Routes = [
     ],
 
 })
-export class SettingsModule { }
+export class SettingsModule { }

src/Ombi/Controllers/SettingsController.cs

@@ -5,15 +5,18 @@ using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
 using AutoMapper;
 using Hangfire;
+using Hangfire.RecurringJobExtensions;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.PlatformAbstractions;
+using NCrontab;
 using Ombi.Api.Emby;
 using Ombi.Attributes;
 using Ombi.Core.Models.UI;
@@ -465,7 +468,8 @@ namespace Ombi.Controllers
             j.PlexContentSync = j.PlexContentSync.HasValue() ? j.PlexContentSync : JobSettingsHelper.PlexContent(j);
             j.UserImporter = j.UserImporter.HasValue() ? j.UserImporter : JobSettingsHelper.UserImporter(j);
             j.SickRageSync = j.SickRageSync.HasValue() ? j.SickRageSync : JobSettingsHelper.SickRageSync(j);
-
+            j.RefreshMetadata = j.RefreshMetadata.HasValue() ? j.RefreshMetadata : JobSettingsHelper.RefreshMetadata(j);
+ 
             return j;
         }
 
@@ -475,9 +479,71 @@ namespace Ombi.Controllers
         /// <param name="settings">The settings.</param>
         /// <returns></returns>
         [HttpPost("jobs")]
-        public async Task<bool> JobSettings([FromBody]JobSettings settings)
+        public async Task<JobSettingsViewModel> JobSettings([FromBody]JobSettings settings)
         {
-            return await Save(settings);
+            // Verify that we have correct CRON's
+            foreach (var propertyInfo in settings.GetType()
+                .GetProperties(BindingFlags.Public | BindingFlags.Instance))
+            {
+                if (propertyInfo.Name.Equals("Id", StringComparison.CurrentCultureIgnoreCase))
+                {
+                    continue;
+                }
+                var expression = (string)propertyInfo.GetValue(settings, null);
+
+                try
+                {
+                    var r = CrontabSchedule.TryParse(expression);
+                    if (r == null)
+                    {
+                        return new JobSettingsViewModel
+                        {
+                            Message = $"{propertyInfo.Name} does not have a valid CRON Expression"
+                        };
+                    }
+                }
+                catch (Exception)
+                {
+                    return new JobSettingsViewModel
+                    {
+                        Message = $"{propertyInfo.Name} does not have a valid CRON Expression"
+                    };
+                }
+            }
+            var result = await Save(settings);
+
+            return new JobSettingsViewModel
+            {
+                Result = result
+            };
+        }
+
+        [HttpPost("testcron")]
+        public CronTestModel TestCron([FromBody] CronViewModelBody body)
+        {
+            var model = new CronTestModel();
+            try
+            {
+                var time = DateTime.UtcNow;
+                var result = CrontabSchedule.TryParse(body.Expression);
+                for (int i = 0; i < 10; i++)
+                {
+                    var next = result.GetNextOccurrence(time);
+                    model.Schedule.Add(next);
+                    time = next;
+                }
+                model.Success = true;
+                return model;
+            }
+            catch (Exception)
+            {
+                return new CronTestModel
+                {
+                    Message = $"CRON Expression {body.Expression} is not valid"
+                };
+            }
+
+
         }
 
 

src/Ombi/Models/CronTestModel.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+
+namespace Ombi.Models
+{
+    public class CronTestModel
+    {
+        public bool Success { get; set; }
+        public string Message { get; set; }
+        public List<DateTime> Schedule { get; set; } = new List<DateTime>();
+    }
+}

src/Ombi/Models/CronViewModelBody.cs

@@ -0,0 +1,7 @@
+namespace Ombi.Models
+{
+    public class CronViewModelBody
+    {
+        public string Expression { get; set; }
+    }
+}

src/Ombi/Models/JobSettingsViewModel.cs

@@ -0,0 +1,8 @@
+namespace Ombi.Models
+{
+    public class JobSettingsViewModel
+    {
+        public bool Result { get; set; }
+        public string Message { get; set; }
+    }
+}

src/Ombi/Ombi.csproj

@@ -66,6 +66,7 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.0.1" />
     <PackageReference Include="Microsoft.VisualStudio.Web.BrowserLink" Version="2.0.2" />
     <PackageReference Include="MiniProfiler.AspNetCore.Mvc" Version="4.0.0-alpha6-79" />
+    <PackageReference Include="ncrontab" Version="3.3.0" />
     <PackageReference Include="Serilog" Version="2.6.0-dev-00892" />
     <PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" />
     <PackageReference Include="Serilog.Sinks.File" Version="3.2.0" />