open-source-mirrors
/
jellyfin
mirror of https://github.com/jellyfin/jellyfin
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Precache livetv program images (#11083)

Browse Source 
* Precache livetv program images

* return if cache hit

* use EnsureSuccessStatusCode

* Read proper bytes
pull/9374/merge
Cody Robibero 2 months ago committed by GitHub
parent 3bd1a5c557
commit f7f3ad9eb7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 135 additions and 45 deletions
Show Stats Download Patch File Download Diff File

16
Emby.Server.Implementations/Library/LibraryManager.cs
Unescape View File

@ -1860,7 +1860,7 @@ namespace Emby.Server.Implementations.Library
try
{
var index = item.GetImageIndex(img);
image = await ConvertImageToLocal(item, img, index).ConfigureAwait(false);
image = await ConvertImageToLocal(item, img, index, removeOnFailure: true).ConfigureAwait(false);
}
catch (ArgumentException)
{
@ -2787,7 +2787,7 @@ namespace Emby.Server.Implementations.Library
await SavePeopleMetadataAsync(people, cancellationToken).ConfigureAwait(false);
}
public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex)
public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure)
{
foreach (var url in image.Path.Split('|'))
{
@ -2806,6 +2806,7 @@ namespace Emby.Server.Implementations.Library
if (ex.StatusCode.HasValue
&& (ex.StatusCode.Value == HttpStatusCode.NotFound || ex.StatusCode.Value == HttpStatusCode.Forbidden))
{
_logger.LogDebug(ex, "Error downloading image {Url}", url);
continue;
}
@ -2813,11 +2814,14 @@ namespace Emby.Server.Implementations.Library
}
}
// Remove this image to prevent it from retrying over and over
item.RemoveImage(image);
await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false);
if (removeOnFailure)
{
// Remove this image to prevent it from retrying over and over
item.RemoveImage(image);
await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false);
}
throw new InvalidOperationException();
throw new InvalidOperationException("Unable to convert any images to local");
}
public async Task AddVirtualFolder(string name, CollectionTypeOptions? collectionType, LibraryOptions options, bool refreshLibrary)

3
MediaBrowser.Controller/Library/ILibraryManager.cs
Unescape View File

@ -517,8 +517,9 @@ namespace MediaBrowser.Controller.Library
/// <param name="item">The item.</param>
/// <param name="image">The image.</param>
/// <param name="imageIndex">Index of the image.</param>
/// <param name="removeOnFailure">Whether to remove the image from the item on failure.</param>
/// <returns>Task.</returns>
Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex);
Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure = true);
/// <summary>
/// Gets the items.

114
MediaBrowser.Providers/Manager/ProviderManager.cs
Unescape View File

@ -9,6 +9,7 @@ using System.Net.Http;
using System.Net.Mime;
using System.Threading;
using System.Threading.Tasks;
using AsyncKeyedLock;
using Jellyfin.Data.Enums;
using Jellyfin.Data.Events;
using Jellyfin.Extensions;
@ -30,6 +31,7 @@ using MediaBrowser.Model.Extensions;
using MediaBrowser.Model.IO;
using MediaBrowser.Model.Net;
using MediaBrowser.Model.Providers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Book = MediaBrowser.Controller.Entities.Book;
using Episode = MediaBrowser.Controller.Entities.TV.Episode;
@ -59,6 +61,13 @@ namespace MediaBrowser.Providers.Manager
private readonly ConcurrentDictionary<Guid, double> _activeRefreshes = new();
private readonly CancellationTokenSource _disposeCancellationTokenSource = new();
private readonly PriorityQueue<(Guid ItemId, MetadataRefreshOptions RefreshOptions), RefreshPriority> _refreshQueue = new();
private readonly IMemoryCache _memoryCache;
private readonly AsyncKeyedLocker<string> _imageSaveLock = new(o =>
{
o.PoolSize = 20;
o.PoolInitialFill = 1;
});
private IImageProvider[] _imageProviders = Array.Empty<IImageProvider>();
private IMetadataService[] _metadataServices = Array.Empty<IMetadataService>();
@ -81,6 +90,7 @@ namespace MediaBrowser.Providers.Manager
/// <param name="libraryManager">The library manager.</param>
/// <param name="baseItemManager">The BaseItem manager.</param>
/// <param name="lyricManager">The lyric manager.</param>
/// <param name="memoryCache">The memory cache.</param>
public ProviderManager(
IHttpClientFactory httpClientFactory,
ISubtitleManager subtitleManager,
@ -91,7 +101,8 @@ namespace MediaBrowser.Providers.Manager
IServerApplicationPaths appPaths,
ILibraryManager libraryManager,
IBaseItemManager baseItemManager,
ILyricManager lyricManager)
ILyricManager lyricManager,
IMemoryCache memoryCache)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
@ -103,6 +114,7 @@ namespace MediaBrowser.Providers.Manager
_subtitleManager = subtitleManager;
_baseItemManager = baseItemManager;
_lyricManager = lyricManager;
_memoryCache = memoryCache;
}
/// <inheritdoc/>
@ -150,52 +162,79 @@ namespace MediaBrowser.Providers.Manager
/// <inheritdoc/>
public async Task SaveImage(BaseItem item, string url, ImageType type, int? imageIndex, CancellationToken cancellationToken)
{
var httpClient = _httpClientFactory.CreateClient(NamedClient.Default);
using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.OK)
using (await _imageSaveLock.LockAsync(url, cancellationToken).ConfigureAwait(false))
{
throw new HttpRequestException("Invalid image received.", null, response.StatusCode);
}
if (_memoryCache.TryGetValue(url, out (string ContentType, byte[] ImageContents)? cachedValue)
&& cachedValue is not null)
{
var imageContents = cachedValue.Value.ImageContents;
var cacheStream = new MemoryStream(imageContents, 0, imageContents.Length, false);
await using (cacheStream.ConfigureAwait(false))
{
await SaveImage(
item,
cacheStream,
cachedValue.Value.ContentType,
type,
imageIndex,
cancellationToken).ConfigureAwait(false);
return;
}
}
var contentType = response.Content.Headers.ContentType?.MediaType;
var httpClient = _httpClientFactory.CreateClient(NamedClient.Default);
using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
// Workaround for tvheadend channel icons
// TODO: Isolate this hack into the tvh plugin
if (string.IsNullOrEmpty(contentType))
{
if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase))
response.EnsureSuccessStatusCode();
var contentType = response.Content.Headers.ContentType?.MediaType;
// Workaround for tvheadend channel icons
// TODO: Isolate this hack into the tvh plugin
if (string.IsNullOrEmpty(contentType))
{
contentType = "image/png";
if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase))
{
contentType = MediaTypeNames.Image.Png;
}
else
{
throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode);
}
}
else
// TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons...
if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase))
{
throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode);
throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound);
}
}
// TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons...
if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase))
{
throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound);
}
// some iptv/epg providers don't correctly report media type, extract from url if no extension found
if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType)))
{
// Strip query parameters from url to get actual path.
contentType = MimeTypes.GetMimeType(new Uri(url).GetLeftPart(UriPartial.Path));
}
// some iptv/epg providers don't correctly report media type, extract from url if no extension found
if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType)))
{
contentType = MimeTypes.GetMimeType(url);
}
if (!contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
{
throw new HttpRequestException($"Request returned {contentType} instead of an image type", null, HttpStatusCode.NotFound);
}
var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
await using (stream.ConfigureAwait(false))
{
await SaveImage(
item,
stream,
contentType,
type,
imageIndex,
cancellationToken).ConfigureAwait(false);
var responseBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var stream = new MemoryStream(responseBytes, 0, responseBytes.Length, false);
await using (stream.ConfigureAwait(false))
{
_memoryCache.Set(url, (contentType, responseBytes), TimeSpan.FromSeconds(10));
await SaveImage(
item,
stream,
contentType,
type,
imageIndex,
cancellationToken).ConfigureAwait(false);
}
}
}
@ -1115,6 +1154,7 @@ namespace MediaBrowser.Providers.Manager
}
_disposeCancellationTokenSource.Dispose();
_imageSaveLock.Dispose();
}
_disposed = true;

1
MediaBrowser.Providers/MediaBrowser.Providers.csproj
Unescape View File

@ -15,6 +15,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" />
<PackageReference Include="LrcParser" />
<PackageReference Include="MetaBrainz.MusicBrainz" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />

42
src/Jellyfin.LiveTv/Guide/GuideManager.cs
Unescape View File

@ -27,6 +27,8 @@ public class GuideManager : IGuideManager
private const string EtagKey = "ProgramEtag";
private const string ExternalServiceTag = "ExternalServiceId";
private static readonly ParallelOptions _cacheParallelOptions = new() { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 10) };
private readonly ILogger<GuideManager> _logger;
private readonly IConfigurationManager _config;
private readonly IFileSystem _fileSystem;
@ -209,6 +211,7 @@ public class GuideManager : IGuideManager
_logger.LogInformation("Refreshing guide with {0} days of guide data", guideDays);
var maxCacheDate = DateTime.UtcNow.AddDays(2);
foreach (var currentChannel in list)
{
cancellationToken.ThrowIfCancellationRequested();
@ -263,6 +266,7 @@ public class GuideManager : IGuideManager
if (newPrograms.Count > 0)
{
_libraryManager.CreateItems(newPrograms, null, cancellationToken);
await PrecacheImages(newPrograms, maxCacheDate).ConfigureAwait(false);
}
if (updatedPrograms.Count > 0)
@ -272,6 +276,7 @@ public class GuideManager : IGuideManager
currentChannel,
ItemUpdateType.MetadataImport,
cancellationToken).ConfigureAwait(false);
await PrecacheImages(updatedPrograms, maxCacheDate).ConfigureAwait(false);
}
currentChannel.IsMovie = isMovie;
@ -708,4 +713,41 @@ public class GuideManager : IGuideManager
return (item, isNew, isUpdated);
}
private async Task PrecacheImages(IReadOnlyList<BaseItem> programs, DateTime maxCacheDate)
{
await Parallel.ForEachAsync(
programs
.Where(p => p.EndDate.HasValue && p.EndDate.Value < maxCacheDate)
.DistinctBy(p => p.Id),
_cacheParallelOptions,
async (program, cancellationToken) =>
{
for (var i = 0; i < program.ImageInfos.Length; i++)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var imageInfo = program.ImageInfos[i];
if (!imageInfo.IsLocalFile)
{
try
{
program.ImageInfos[i] = await _libraryManager.ConvertImageToLocal(
program,
imageInfo,
imageIndex: 0,
removeOnFailure: false)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Unable to precache {Url}", imageInfo.Path);
}
}
}
}).ConfigureAwait(false);
}
}

4
tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs
Unescape View File

@ -17,6 +17,7 @@ using MediaBrowser.Controller.Subtitles;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.IO;
using MediaBrowser.Providers.Manager;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
@ -572,7 +573,8 @@ namespace Jellyfin.Providers.Tests.Manager
Mock.Of<IServerApplicationPaths>(),
libraryManager.Object,
baseItemManager!,
Mock.Of<ILyricManager>());
Mock.Of<ILyricManager>(),
Mock.Of<IMemoryCache>());
return providerManager;
}

Write Preview
Loading…
Cancel
Save