using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using AsyncKeyedLock; using Jellyfin.Data.Entities; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Implementations.Trickplay; /// /// ITrickplayManager implementation. /// public class TrickplayManager : ITrickplayManager { private readonly ILogger _logger; private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly EncodingHelper _encodingHelper; private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _config; private readonly IImageEncoder _imageEncoder; private readonly IDbContextFactory _dbProvider; private readonly IApplicationPaths _appPaths; private static readonly AsyncNonKeyedLocker _resourcePool = new(1); private static readonly string[] _trickplayImgExtensions = { ".jpg" }; /// /// Initializes a new instance of the class. /// /// The logger. /// The media encoder. /// The file systen. /// The encoding helper. /// The library manager. /// The server configuration manager. /// The image encoder. /// The database provider. /// The application paths. public TrickplayManager( ILogger logger, IMediaEncoder mediaEncoder, IFileSystem fileSystem, EncodingHelper encodingHelper, ILibraryManager libraryManager, IServerConfigurationManager config, IImageEncoder imageEncoder, IDbContextFactory dbProvider, IApplicationPaths appPaths) { _logger = logger; _mediaEncoder = mediaEncoder; _fileSystem = fileSystem; _encodingHelper = encodingHelper; _libraryManager = libraryManager; _config = config; _imageEncoder = imageEncoder; _dbProvider = dbProvider; _appPaths = appPaths; } /// public async Task MoveGeneratedTrickplayDataAsync(Video video, LibraryOptions? libraryOptions, CancellationToken cancellationToken) { var options = _config.Configuration.TrickplayOptions; if (!CanGenerateTrickplay(video, options.Interval)) { return; } var existingTrickplayResolutions = await GetTrickplayResolutions(video.Id).ConfigureAwait(false); foreach (var resolution in existingTrickplayResolutions) { cancellationToken.ThrowIfCancellationRequested(); var existingResolution = resolution.Key; var tileWidth = resolution.Value.TileWidth; var tileHeight = resolution.Value.TileHeight; var shouldBeSavedWithMedia = libraryOptions is null ? false : libraryOptions.SaveTrickplayWithMedia; var localOutputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, existingResolution, false); var mediaOutputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, existingResolution, true); if (shouldBeSavedWithMedia && Directory.Exists(localOutputDir)) { var localDirFiles = Directory.GetFiles(localOutputDir); var mediaDirExists = Directory.Exists(mediaOutputDir); if (localDirFiles.Length > 0 && ((mediaDirExists && Directory.GetFiles(mediaOutputDir).Length == 0) || !mediaDirExists)) { // Move images from local dir to media dir MoveContent(localOutputDir, mediaOutputDir); _logger.LogInformation("Moved trickplay images for {ItemName} to {Location}", video.Name, mediaOutputDir); } } else if (!shouldBeSavedWithMedia && Directory.Exists(mediaOutputDir)) { var mediaDirFiles = Directory.GetFiles(mediaOutputDir); var localDirExists = Directory.Exists(localOutputDir); if (mediaDirFiles.Length > 0 && ((localDirExists && Directory.GetFiles(localOutputDir).Length == 0) || !localDirExists)) { // Move images from media dir to local dir MoveContent(mediaOutputDir, localOutputDir); _logger.LogInformation("Moved trickplay images for {ItemName} to {Location}", video.Name, localOutputDir); } } } } private void MoveContent(string sourceFolder, string destinationFolder) { _fileSystem.MoveDirectory(sourceFolder, destinationFolder); var parent = Directory.GetParent(sourceFolder); if (parent is not null) { var parentContent = Directory.GetDirectories(parent.FullName); if (parentContent.Length == 0) { Directory.Delete(parent.FullName); } } } /// public async Task RefreshTrickplayDataAsync(Video video, bool replace, LibraryOptions? libraryOptions, CancellationToken cancellationToken) { _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace); var options = _config.Configuration.TrickplayOptions; if (options.Interval < 1000) { _logger.LogWarning("Trickplay image interval {Interval} is too small, reset to the minimum valid value of 1000", options.Interval); options.Interval = 1000; } foreach (var width in options.WidthResolutions) { cancellationToken.ThrowIfCancellationRequested(); await RefreshTrickplayDataInternal( video, replace, width, options, libraryOptions, cancellationToken).ConfigureAwait(false); } } private async Task RefreshTrickplayDataInternal( Video video, bool replace, int width, TrickplayOptions options, LibraryOptions? libraryOptions, CancellationToken cancellationToken) { if (!CanGenerateTrickplay(video, options.Interval)) { return; } var imgTempDir = string.Empty; using (await _resourcePool.LockAsync(cancellationToken).ConfigureAwait(false)) { try { // Extract images // Note: Media sources under parent items exist as their own video/item as well. Only use this video stream for trickplay. var mediaSource = video.GetMediaSources(false).Find(source => Guid.Parse(source.Id).Equals(video.Id)); if (mediaSource is null) { _logger.LogDebug("Found no matching media source for item {ItemId}", video.Id); return; } var mediaPath = mediaSource.Path; if (!File.Exists(mediaPath)) { _logger.LogWarning("Media not found at {Path} for item {ItemID}", mediaPath, video.Id); return; } // The width has to be even, otherwise a lot of filters will not be able to sample it var actualWidth = 2 * (width / 2); // Force using the video width when the trickplay setting has a too large width if (mediaSource.VideoStream.Width is not null && mediaSource.VideoStream.Width < width) { _logger.LogWarning("Video width {VideoWidth} is smaller than trickplay setting {TrickPlayWidth}, using video width for thumbnails", mediaSource.VideoStream.Width, width); actualWidth = 2 * ((int)mediaSource.VideoStream.Width / 2); } var tileWidth = options.TileWidth; var tileHeight = options.TileHeight; var saveWithMedia = libraryOptions is null ? false : libraryOptions.SaveTrickplayWithMedia; var outputDir = GetTrickplayDirectory(video, tileWidth, tileHeight, actualWidth, saveWithMedia); // Import existing trickplay tiles if (!replace && Directory.Exists(outputDir)) { var existingFiles = Directory.GetFiles(outputDir); if (existingFiles.Length > 0) { var hasTrickplayResolution = await HasTrickplayResolutionAsync(video.Id, actualWidth).ConfigureAwait(false); if (hasTrickplayResolution) { _logger.LogDebug("Found existing trickplay files for {ItemId}.", video.Id); return; } // Import tiles var localTrickplayInfo = new TrickplayInfo { ItemId = video.Id, Width = width, Interval = options.Interval, TileWidth = options.TileWidth, TileHeight = options.TileHeight, ThumbnailCount = existingFiles.Length, Height = 0, Bandwidth = 0 }; foreach (var tile in existingFiles) { var image = _imageEncoder.GetImageSize(tile); localTrickplayInfo.Height = Math.Max(localTrickplayInfo.Height, image.Height); var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tile).Length * 8 / localTrickplayInfo.TileWidth / localTrickplayInfo.TileHeight / (localTrickplayInfo.Interval / 1000)); localTrickplayInfo.Bandwidth = Math.Max(localTrickplayInfo.Bandwidth, bitrate); } await SaveTrickplayInfo(localTrickplayInfo).ConfigureAwait(false); _logger.LogDebug("Imported existing trickplay files for {ItemId}.", video.Id); return; } } // Generate trickplay tiles var mediaStream = mediaSource.VideoStream; var container = mediaSource.Container; _logger.LogInformation("Creating trickplay files at {Width} width, for {Path} [ID: {ItemId}]", actualWidth, mediaPath, video.Id); imgTempDir = await _mediaEncoder.ExtractVideoImagesOnIntervalAccelerated( mediaPath, container, mediaSource, mediaStream, actualWidth, TimeSpan.FromMilliseconds(options.Interval), options.EnableHwAcceleration, options.EnableHwEncoding, options.ProcessThreads, options.Qscale, options.ProcessPriority, options.EnableKeyFrameOnlyExtraction, _encodingHelper, cancellationToken).ConfigureAwait(false); if (string.IsNullOrEmpty(imgTempDir) || !Directory.Exists(imgTempDir)) { throw new InvalidOperationException("Null or invalid directory from media encoder."); } var images = _fileSystem.GetFiles(imgTempDir, _trickplayImgExtensions, false, false) .Select(i => i.FullName) .OrderBy(i => i) .ToList(); // Create tiles var trickplayInfo = CreateTiles(images, actualWidth, options, outputDir); // Save tiles info try { if (trickplayInfo is not null) { trickplayInfo.ItemId = video.Id; await SaveTrickplayInfo(trickplayInfo).ConfigureAwait(false); _logger.LogInformation("Finished creation of trickplay files for {0}", mediaPath); } else { throw new InvalidOperationException("Null trickplay tiles info from CreateTiles."); } } catch (Exception ex) { _logger.LogError(ex, "Error while saving trickplay tiles info."); // Make sure no files stay in metadata folders on failure // if tiles info wasn't saved. Directory.Delete(outputDir, true); } } catch (Exception ex) { _logger.LogError(ex, "Error creating trickplay images."); } finally { if (!string.IsNullOrEmpty(imgTempDir)) { Directory.Delete(imgTempDir, true); } } } } /// public TrickplayInfo CreateTiles(IReadOnlyList images, int width, TrickplayOptions options, string outputDir) { if (images.Count == 0) { throw new ArgumentException("Can't create trickplay from 0 images."); } var workDir = Path.Combine(_appPaths.TempDirectory, "trickplay_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(workDir); var trickplayInfo = new TrickplayInfo { Width = width, Interval = options.Interval, TileWidth = options.TileWidth, TileHeight = options.TileHeight, ThumbnailCount = images.Count, // Set during image generation Height = 0, Bandwidth = 0 }; /* * Generate trickplay tiles from sets of thumbnails */ var imageOptions = new ImageCollageOptions { Width = trickplayInfo.TileWidth, Height = trickplayInfo.TileHeight }; var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight; var requiredTiles = (int)Math.Ceiling((double)images.Count / thumbnailsPerTile); for (int i = 0; i < requiredTiles; i++) { // Set output/input paths var tilePath = Path.Combine(workDir, $"{i}.jpg"); imageOptions.OutputPath = tilePath; imageOptions.InputPaths = images.Skip(i * thumbnailsPerTile).Take(Math.Min(thumbnailsPerTile, images.Count - (i * thumbnailsPerTile))).ToList(); // Generate image and use returned height for tiles info var height = _imageEncoder.CreateTrickplayTile(imageOptions, options.JpegQuality, trickplayInfo.Width, trickplayInfo.Height != 0 ? trickplayInfo.Height : null); if (trickplayInfo.Height == 0) { trickplayInfo.Height = height; } // Update bitrate var bitrate = (int)Math.Ceiling(new FileInfo(tilePath).Length * 8m / trickplayInfo.TileWidth / trickplayInfo.TileHeight / (trickplayInfo.Interval / 1000m)); trickplayInfo.Bandwidth = Math.Max(trickplayInfo.Bandwidth, bitrate); } /* * Move trickplay tiles to output directory */ Directory.CreateDirectory(Directory.GetParent(outputDir)!.FullName); // Replace existing tiles if they already exist if (Directory.Exists(outputDir)) { Directory.Delete(outputDir, true); } _fileSystem.MoveDirectory(workDir, outputDir); return trickplayInfo; } private bool CanGenerateTrickplay(Video video, int interval) { var videoType = video.VideoType; if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay) { return false; } if (video.IsPlaceHolder) { return false; } if (video.IsShortcut) { return false; } if (!video.IsCompleteMedia) { return false; } if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks) { return false; } var libraryOptions = _libraryManager.GetLibraryOptions(video); if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction) { return false; } // Can't extract images if there are no video streams return video.GetMediaStreams().Count > 0; } /// public async Task> GetTrickplayResolutions(Guid itemId) { var trickplayResolutions = new Dictionary(); var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var trickplayInfos = await dbContext.TrickplayInfos .AsNoTracking() .Where(i => i.ItemId.Equals(itemId)) .ToListAsync() .ConfigureAwait(false); foreach (var info in trickplayInfos) { trickplayResolutions[info.Width] = info; } } return trickplayResolutions; } /// public async Task> GetTrickplayItemsAsync(int limit, int offset) { IReadOnlyList trickplayItems; var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { trickplayItems = await dbContext.TrickplayInfos .AsNoTracking() .OrderBy(i => i.ItemId) .Skip(offset) .Take(limit) .ToListAsync() .ConfigureAwait(false); } return trickplayItems; } /// public async Task SaveTrickplayInfo(TrickplayInfo info) { var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var oldInfo = await dbContext.TrickplayInfos.FindAsync(info.ItemId, info.Width).ConfigureAwait(false); if (oldInfo is not null) { dbContext.TrickplayInfos.Remove(oldInfo); } dbContext.Add(info); await dbContext.SaveChangesAsync().ConfigureAwait(false); } } /// public async Task>> GetTrickplayManifest(BaseItem item) { var trickplayManifest = new Dictionary>(); foreach (var mediaSource in item.GetMediaSources(false)) { if (mediaSource.IsRemote || !Guid.TryParse(mediaSource.Id, out var mediaSourceId)) { continue; } var trickplayResolutions = await GetTrickplayResolutions(mediaSourceId).ConfigureAwait(false); if (trickplayResolutions.Count > 0) { trickplayManifest[mediaSource.Id] = trickplayResolutions; } } return trickplayManifest; } /// public async Task GetTrickplayTilePathAsync(BaseItem item, int width, int index, bool saveWithMedia) { var trickplayResolutions = await GetTrickplayResolutions(item.Id).ConfigureAwait(false); if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo)) { return Path.Combine(GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, width, saveWithMedia), index + ".jpg"); } return string.Empty; } /// public async Task GetHlsPlaylist(Guid itemId, int width, string? apiKey) { var trickplayResolutions = await GetTrickplayResolutions(itemId).ConfigureAwait(false); if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo)) { var builder = new StringBuilder(128); if (trickplayInfo.ThumbnailCount > 0) { const string urlFormat = "{0}.jpg?MediaSourceId={1}&api_key={2}"; const string decimalFormat = "{0:0.###}"; var resolution = $"{trickplayInfo.Width}x{trickplayInfo.Height}"; var layout = $"{trickplayInfo.TileWidth}x{trickplayInfo.TileHeight}"; var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight; var thumbnailDuration = trickplayInfo.Interval / 1000d; var infDuration = thumbnailDuration * thumbnailsPerTile; var tileCount = (int)Math.Ceiling((decimal)trickplayInfo.ThumbnailCount / thumbnailsPerTile); builder .AppendLine("#EXTM3U") .Append("#EXT-X-TARGETDURATION:") .AppendLine(tileCount.ToString(CultureInfo.InvariantCulture)) .AppendLine("#EXT-X-VERSION:7") .AppendLine("#EXT-X-MEDIA-SEQUENCE:1") .AppendLine("#EXT-X-PLAYLIST-TYPE:VOD") .AppendLine("#EXT-X-IMAGES-ONLY"); for (int i = 0; i < tileCount; i++) { // All tiles prior to the last must contain full amount of thumbnails (no black). if (i == tileCount - 1) { thumbnailsPerTile = trickplayInfo.ThumbnailCount - (i * thumbnailsPerTile); infDuration = thumbnailDuration * thumbnailsPerTile; } // EXTINF builder .Append("#EXTINF:") .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, infDuration) .AppendLine(","); // EXT-X-TILES builder .Append("#EXT-X-TILES:RESOLUTION=") .Append(resolution) .Append(",LAYOUT=") .Append(layout) .Append(",DURATION=") .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, thumbnailDuration) .AppendLine(); // URL builder .AppendFormat( CultureInfo.InvariantCulture, urlFormat, i.ToString(CultureInfo.InvariantCulture), itemId.ToString("N"), apiKey) .AppendLine(); } builder.AppendLine("#EXT-X-ENDLIST"); return builder.ToString(); } } return null; } /// public string GetTrickplayDirectory(BaseItem item, int tileWidth, int tileHeight, int width, bool saveWithMedia = false) { var path = saveWithMedia ? Path.Combine(item.ContainingFolderPath, Path.ChangeExtension(item.Path, ".trickplay")) : Path.Combine(item.GetInternalMetadataPath(), "trickplay"); var subdirectory = string.Format( CultureInfo.InvariantCulture, "{0} - {1}x{2}", width.ToString(CultureInfo.InvariantCulture), tileWidth.ToString(CultureInfo.InvariantCulture), tileHeight.ToString(CultureInfo.InvariantCulture)); return Path.Combine(path, subdirectory); } private async Task HasTrickplayResolutionAsync(Guid itemId, int width) { var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { return await dbContext.TrickplayInfos .AsNoTracking() .Where(i => i.ItemId.Equals(itemId)) .AnyAsync(i => i.Width == width) .ConfigureAwait(false); } } }