open-source-mirrors
/
Ombi
mirror of https://github.com/Ombi-app/Ombi
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved the way we sync the plex content and then get the metadata. #2243

Browse Source
pull/2258/head
Jamie Rees 7 years ago
parent 3ca72c7e63
commit 83f871da86
7 changed files with 138 additions and 30 deletions
Show Stats Download Patch File Download Diff File

4
src/Ombi.Schedule/Jobs/Ombi/IRefreshMetadata.cs
Unescape View File

@ -1,9 +1,11 @@
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Ombi.Schedule.Jobs.Ombi
{
public interface IRefreshMetadata : IBaseJob
{
Task Start();
Task ProcessPlexServerContent(IEnumerable<int> contentIds);
}
}

55
src/Ombi.Schedule/Jobs/Ombi/RefreshMetadata.cs
Unescape View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@ -17,7 +18,7 @@ namespace Ombi.Schedule.Jobs.Ombi
{
public RefreshMetadata(IPlexContentRepository plexRepo, IEmbyContentRepository embyRepo,
ILogger<RefreshMetadata> log, ITvMazeApi tvApi, ISettingsService<PlexSettings> plexSettings,
IMovieDbApi movieApi)
IMovieDbApi movieApi, ISettingsService<EmbySettings> embySettings)
{
_plexRepo = plexRepo;
_embyRepo = embyRepo;
@ -25,6 +26,7 @@ namespace Ombi.Schedule.Jobs.Ombi
_movieApi = movieApi;
_tvApi = tvApi;
_plexSettings = plexSettings;
_embySettings = embySettings;
}
private readonly IPlexContentRepository _plexRepo;
@ -33,6 +35,7 @@ namespace Ombi.Schedule.Jobs.Ombi
private readonly IMovieDbApi _movieApi;
private readonly ITvMazeApi _tvApi;
private readonly ISettingsService<PlexSettings> _plexSettings;
private readonly ISettingsService<EmbySettings> _embySettings;
public async Task Start()
{
@ -43,6 +46,11 @@ namespace Ombi.Schedule.Jobs.Ombi
if (settings.Enable)
{
await StartPlex();
}
var embySettings = await _embySettings.GetSettingsAsync();
if (embySettings.Enable)
{
await StartEmby();
}
}
@ -53,12 +61,45 @@ namespace Ombi.Schedule.Jobs.Ombi
}
}
public async Task ProcessPlexServerContent(IEnumerable<int> contentIds)
{
_log.LogInformation("Starting the Metadata refresh from RecentlyAddedSync");
try
{
var settings = await _plexSettings.GetSettingsAsync();
if (settings.Enable)
{
await StartPlexWithKnownContent(contentIds);
}
}
catch (Exception e)
{
_log.LogError(e, "Exception when refreshing the Plex Metadata");
throw;
}
}
private async Task StartPlexWithKnownContent(IEnumerable<int> contentids)
{
var everything = _plexRepo.GetAll().Where(x => contentids.Contains(x.Id));
var allMovies = everything.Where(x => x.Type == PlexMediaTypeEntity.Movie);
await StartPlexMovies(allMovies);
// Now Tv
var allTv = everything.Where(x => x.Type == PlexMediaTypeEntity.Show);
await StartPlexTv(allTv);
}
private async Task StartPlex()
{
await StartPlexMovies();
var allMovies = _plexRepo.GetAll().Where(x =>
x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue()));
await StartPlexMovies(allMovies);
// Now Tv
await StartPlexTv();
var allTv = _plexRepo.GetAll().Where(x =>
x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue()));
await StartPlexTv(allTv);
}
private async Task StartEmby()
@ -67,10 +108,8 @@ namespace Ombi.Schedule.Jobs.Ombi
await StartEmbyTv();
}
private async Task StartPlexTv()
private async Task StartPlexTv(IQueryable<PlexServerContent> allTv)
{
var allTv = _plexRepo.GetAll().Where(x =>
x.Type == PlexMediaTypeEntity.Show && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue() || !x.TvDbId.HasValue()));
var tvCount = 0;
foreach (var show in allTv)
{
@ -147,10 +186,8 @@ namespace Ombi.Schedule.Jobs.Ombi
await _embyRepo.SaveChangesAsync();
}
private async Task StartPlexMovies()
private async Task StartPlexMovies(IQueryable<PlexServerContent> allMovies)
{
var allMovies = _plexRepo.GetAll().Where(x =>
x.Type == PlexMediaTypeEntity.Movie && (!x.TheMovieDbId.HasValue() || !x.ImdbId.HasValue()));
int movieCount = 0;
foreach (var movie in allMovies)
{

4
src/Ombi.Schedule/Jobs/Plex/Interfaces/IPlexEpisodeSync.cs
Unescape View File

@ -1,10 +1,14 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Ombi.Api.Plex.Models;
using Ombi.Store.Entities;
namespace Ombi.Schedule.Jobs.Plex.Interfaces
{
public interface IPlexEpisodeSync : IBaseJob
{
Task Start();
Task ProcessEpsiodes(Metadata[] episodes, IQueryable<PlexEpisode> currentEpisodes);
}
}

79
src/Ombi.Schedule/Jobs/Plex/PlexContentSync.cs
Unescape View File

@ -77,11 +77,22 @@ namespace Ombi.Schedule.Jobs.Plex
Logger.LogError("Plex Settings are not valid");
return;
}
var processedContent = new HashSet<int>();
Logger.LogInformation("Starting Plex Content Cacher");
try
{
await StartTheCache(plexSettings, recentlyAddedSearch);
if (recentlyAddedSearch)
{
var result = await StartTheCache(plexSettings, true);
foreach (var r in result)
{
processedContent.Add(r);
}
}
else
{
await StartTheCache(plexSettings, false);
}
}
catch (Exception e)
{
@ -94,33 +105,58 @@ namespace Ombi.Schedule.Jobs.Plex
BackgroundJob.Enqueue(() => EpisodeSync.Start());
}
BackgroundJob.Enqueue(() => Metadata.Start());
if (processedContent.Any() && recentlyAddedSearch)
{
// Just check what we send it
BackgroundJob.Enqueue(() => Metadata.ProcessPlexServerContent(processedContent));
}
}
private async Task StartTheCache(PlexSettings plexSettings, bool recentlyAddedSearch)
private async Task<IEnumerable<int>> StartTheCache(PlexSettings plexSettings, bool recentlyAddedSearch)
{
var processedContent = new HashSet<int>();
foreach (var servers in plexSettings.Servers ?? new List<PlexServers>())
{
try
{
Logger.LogInformation("Starting to cache the content on server {0}", servers.Name);
await ProcessServer(servers, recentlyAddedSearch);
if (recentlyAddedSearch)
{
// If it's recently added search then we want the results to pass to the metadata job
// This way the metadata job is smaller in size to process, it only need to look at newly added shit
var result = await ProcessServer(servers, true);
foreach (var plexServerContent in result)
{
processedContent.Add(plexServerContent);
}
}
else
{
await ProcessServer(servers, false);
}
}
catch (Exception e)
{
Logger.LogWarning(LoggingEvents.PlexContentCacher, e, "Exception thrown when attempting to cache the Plex Content in server {0}", servers.Name);
}
}
return processedContent;
}
private async Task ProcessServer(PlexServers servers, bool recentlyAddedSearch)
private async Task<IEnumerable<int>> ProcessServer(PlexServers servers, bool recentlyAddedSearch)
{
var processedContent = new HashSet<int>();
Logger.LogInformation("Getting all content from server {0}", servers.Name);
var allContent = await GetAllContent(servers, recentlyAddedSearch);
Logger.LogInformation("We found {0} items", allContent.Count);
// Let's now process this.
var contentToAdd = new HashSet<PlexServerContent>();
var allEps = Repo.GetAllEpisodes();
foreach (var content in allContent)
{
if (content.viewGroup.Equals(PlexMediaType.Episode.ToString(), StringComparison.CurrentCultureIgnoreCase))
@ -137,8 +173,10 @@ namespace Ombi.Schedule.Jobs.Plex
continue;
}
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch);
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch, processedContent);
}
await EpisodeSync.ProcessEpsiodes(content.Metadata, allEps);
}
if (content.viewGroup.Equals(PlexMediaType.Show.ToString(), StringComparison.CurrentCultureIgnoreCase))
{
@ -146,7 +184,7 @@ namespace Ombi.Schedule.Jobs.Plex
Logger.LogInformation("Processing TV Shows");
foreach (var show in content.Metadata ?? new Metadata[] { })
{
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch);
await ProcessTvShow(servers, show, contentToAdd, recentlyAddedSearch, processedContent);
}
}
if (content.viewGroup.Equals(PlexMediaType.Movie.ToString(), StringComparison.CurrentCultureIgnoreCase))
@ -216,6 +254,10 @@ namespace Ombi.Schedule.Jobs.Plex
if (contentToAdd.Count > 500)
{
await Repo.AddRange(contentToAdd);
foreach (var c in contentToAdd)
{
processedContent.Add(c.Id);
}
contentToAdd.Clear();
}
}
@ -223,6 +265,10 @@ namespace Ombi.Schedule.Jobs.Plex
if (contentToAdd.Count > 500)
{
await Repo.AddRange(contentToAdd);
foreach (var c in contentToAdd)
{
processedContent.Add(c.Id);
}
contentToAdd.Clear();
}
}
@ -230,10 +276,16 @@ namespace Ombi.Schedule.Jobs.Plex
if (contentToAdd.Any())
{
await Repo.AddRange(contentToAdd);
foreach (var c in contentToAdd)
{
processedContent.Add(c.Id);
}
}
return processedContent;
}
private async Task ProcessTvShow(PlexServers servers, Metadata show, HashSet<PlexServerContent> contentToAdd, bool recentlyAdded)
private async Task ProcessTvShow(PlexServers servers, Metadata show, HashSet<PlexServerContent> contentToAdd, bool recentlyAdded, HashSet<int> contentProcessed)
{
var seasonList = await PlexApi.GetSeasons(servers.PlexAuthToken, servers.FullUri,
show.ratingKey);
@ -416,8 +468,17 @@ namespace Ombi.Schedule.Jobs.Plex
if (contentToAdd.Count > 500 || recentlyAdded)
{
await Repo.AddRange(contentToAdd);
foreach (var plexServerContent in contentToAdd)
{
contentProcessed.Add(plexServerContent.Id);
}
contentToAdd.Clear();
}
if (contentToAdd.Any())
{
await Repo.AddRange(contentToAdd);
}
}
/// <summary>

8
src/Ombi.Schedule/Jobs/Plex/PlexEpisodeSync.cs
Unescape View File

@ -111,7 +111,7 @@ namespace Ombi.Schedule.Jobs.Plex
// 12.03.2017 - I think we should be able to match them now
//await _repo.ExecuteSql("DELETE FROM PlexEpisode");
await ProcessEpsiodes(episodes, currentEpisodes);
await ProcessEpsiodes(episodes?.MediaContainer?.Metadata ?? new Metadata[] { }, currentEpisodes);
currentPosition += resultCount;
while (currentPosition < episodes.MediaContainer.totalSize)
@ -119,7 +119,7 @@ namespace Ombi.Schedule.Jobs.Plex
var ep = await _api.GetAllEpisodes(settings.PlexAuthToken, settings.FullUri, section.key, currentPosition,
resultCount);
await ProcessEpsiodes(ep, currentEpisodes);
await ProcessEpsiodes(ep?.MediaContainer?.Metadata ?? new Metadata[] { }, currentEpisodes);
_log.LogInformation(LoggingEvents.PlexEpisodeCacher, $"Processed {resultCount} more episodes. Total Remaining {episodes.MediaContainer.totalSize - currentPosition}");
currentPosition += resultCount;
}
@ -129,12 +129,12 @@ namespace Ombi.Schedule.Jobs.Plex
await _repo.SaveChangesAsync();
}
private async Task ProcessEpsiodes(PlexContainer episodes, IQueryable<PlexEpisode> currentEpisodes)
public async Task ProcessEpsiodes(Metadata[] episodes, IQueryable<PlexEpisode> currentEpisodes)
{
var ep = new HashSet<PlexEpisode>();
try
{
foreach (var episode in episodes?.MediaContainer?.Metadata ?? new Metadata[] { })
foreach (var episode in episodes)
{
// I don't think we need to get the metadata, we only need to get the metadata if we need the provider id (TheTvDbid). Why do we need it for episodes?
// We have the parent and grandparent rating keys to link up to the season and series

4
src/Ombi.Settings/Settings/Models/JobSettingsHelper.cs
Unescape View File

@ -25,7 +25,7 @@ namespace Ombi.Settings.Settings.Models
}
public static string PlexRecentlyAdded(JobSettings s)
{
return Get(s.PlexRecentlyAddedSync, Cron.Hourly(0));
return Get(s.PlexRecentlyAddedSync, Cron.MinuteInterval(30));
}
public static string CouchPotato(JobSettings s)
{
@ -50,7 +50,7 @@ namespace Ombi.Settings.Settings.Models
}
public static string RefreshMetadata(JobSettings s)
{
return Get(s.RefreshMetadata, Cron.DayInterval(2));
return Get(s.RefreshMetadata, Cron.DayInterval(3));
}
private static string Get(string settings, string defaultCron)

14
src/Ombi/Startup.cs
Unescape View File

@ -191,11 +191,15 @@ namespace Ombi
}
app.UseHangfireServer(new BackgroundJobServerOptions { WorkerCount = 1, ServerTimeout = TimeSpan.FromDays(1), ShutdownTimeout = TimeSpan.FromDays(1)});
app.UseHangfireDashboard(settings.BaseUrl.HasValue() ? $"{settings.BaseUrl}/hangfire" : "/hangfire",
new DashboardOptions
{
Authorization = new[] { new HangfireAuthorizationFilter() }
});
if (env.IsDevelopment())
{
app.UseHangfireDashboard(settings.BaseUrl.HasValue() ? $"{settings.BaseUrl}/hangfire" : "/hangfire",
new DashboardOptions
{
Authorization = new[] {new HangfireAuthorizationFilter()}
});
}
GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute { Attempts = 3 });
// Setup the scheduler

Write Preview
Loading…
Cancel
Save