From 515ee90fb96b32d89134852b95ebcd8dbb656b94 Mon Sep 17 00:00:00 2001 From: nicknsy <20588554+nicknsy@users.noreply.github.com> Date: Wed, 22 Feb 2023 18:17:54 -0800 Subject: [PATCH] Hls playlist --- .../Controllers/DynamicHlsController.cs | 26 +++- Jellyfin.Api/Helpers/DynamicHlsHelper.cs | 125 +++++++++++++++++- .../Models/StreamingDtos/VideoRequestDto.cs | 7 +- .../Trickplay/TrickplayImagesTask.cs | 1 - .../Trickplay/TrickplayManager.cs | 31 ++--- .../Trickplay/TrickplayProvider.cs | 1 - 6 files changed, 170 insertions(+), 21 deletions(-) diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index 9f2088e36e..2dbc343e96 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -407,6 +407,7 @@ public class DynamicHlsController : BaseJellyfinApiController /// Optional. The . /// Optional. The streaming options. /// Enable adaptive bitrate streaming. + /// Enable trickplay image playlists being added to master playlist. /// Video stream returned. /// A containing the playlist file. [HttpGet("Videos/{itemId}/master.m3u8")] @@ -464,7 +465,8 @@ public class DynamicHlsController : BaseJellyfinApiController [FromQuery] int? videoStreamIndex, [FromQuery] EncodingContext? context, [FromQuery] Dictionary streamOptions, - [FromQuery] bool enableAdaptiveBitrateStreaming = true) + [FromQuery] bool enableAdaptiveBitrateStreaming = true, + [FromQuery] bool enableTrickplay = true) { var streamingRequest = new HlsVideoRequestDto { @@ -518,7 +520,8 @@ public class DynamicHlsController : BaseJellyfinApiController VideoStreamIndex = videoStreamIndex, Context = context ?? EncodingContext.Streaming, StreamOptions = streamOptions, - EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming + EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming, + EnableTrickplay = enableTrickplay }; return await _dynamicHlsHelper.GetMasterHlsPlaylist(TranscodingJobType, streamingRequest, enableAdaptiveBitrateStreaming).ConfigureAwait(false); @@ -1025,6 +1028,25 @@ public class DynamicHlsController : BaseJellyfinApiController .ConfigureAwait(false); } + /// + /// Gets an image tiles playlist for trickplay. + /// + /// The item id. + /// The width of a single tile. + /// The media version id. + /// Tiles stream returned. + /// A containing the trickplay tiles file. + [HttpGet("Videos/{itemId}/tiles.m3u8")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesPlaylistFile] + public ActionResult GetTrickplayTilesHlsPlaylist( + [FromRoute, Required] Guid itemId, + [FromQuery, Required] int width, + [FromQuery, Required] string mediaSourceId) + { + return _dynamicHlsHelper.GetTilesHlsPlaylist(width, mediaSourceId); + } + /// /// Gets a video stream using HTTP live streaming. /// diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index 4486954c62..d8a16876df 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Jellyfin.Api.Extensions; using Jellyfin.Api.Models.StreamingDtos; using Jellyfin.Extensions; +using Jellyfin.Data.Entities; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Net; @@ -18,6 +19,7 @@ using MediaBrowser.Controller.Devices; using MediaBrowser.Controller.Dlna; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Net; @@ -45,6 +47,7 @@ public class DynamicHlsHelper private readonly ILogger _logger; private readonly IHttpContextAccessor _httpContextAccessor; private readonly EncodingHelper _encodingHelper; + private readonly ITrickplayManager _trickplayManager; /// /// Initializes a new instance of the class. @@ -61,6 +64,7 @@ public class DynamicHlsHelper /// Instance of the interface. /// Instance of the interface. /// Instance of . + /// Instance of . public DynamicHlsHelper( ILibraryManager libraryManager, IUserManager userManager, @@ -73,7 +77,8 @@ public class DynamicHlsHelper INetworkManager networkManager, ILogger logger, IHttpContextAccessor httpContextAccessor, - EncodingHelper encodingHelper) + EncodingHelper encodingHelper, + ITrickplayManager trickplayManager) { _libraryManager = libraryManager; _userManager = userManager; @@ -87,6 +92,7 @@ public class DynamicHlsHelper _logger = logger; _httpContextAccessor = httpContextAccessor; _encodingHelper = encodingHelper; + _trickplayManager = trickplayManager; } /// @@ -112,6 +118,81 @@ public class DynamicHlsHelper cancellationTokenSource).ConfigureAwait(false); } + /// + /// Get trickplay tiles hls playlist. + /// + /// The width of a single tile. + /// The media version id. + /// The resulting . + public ActionResult GetTilesHlsPlaylist(int width, string mediaSourceId) + { + if (_httpContextAccessor.HttpContext is null) + { + throw new ResourceNotFoundException(nameof(_httpContextAccessor.HttpContext)); + } + + var tilesResolutions = _trickplayManager.GetTilesResolutions(Guid.Parse(mediaSourceId)); + if (tilesResolutions is not null && tilesResolutions.ContainsKey(width)) + { + var builder = new StringBuilder(128); + var tilesInfo = tilesResolutions[width]; + + if (tilesInfo.TileCount > 0) + { + const string urlFormat = "{0}/Trickplay/{1}/{2}.jpg?&api_key={3}"; + const string decimalFormat = "{0:0.###}"; + + var resolution = tilesInfo.Width.ToString(CultureInfo.InvariantCulture) + "x" + tilesInfo.Height.ToString(CultureInfo.InvariantCulture); + var layout = tilesInfo.TileWidth.ToString(CultureInfo.InvariantCulture) + "x" + tilesInfo.TileHeight.ToString(CultureInfo.InvariantCulture); + var tilesPerGrid = tilesInfo.TileWidth * tilesInfo.TileHeight; + var tileDuration = (decimal)tilesInfo.Interval / 1000; + var tileGridCount = (int)Math.Ceiling((decimal)tilesInfo.TileCount / tilesPerGrid); + + builder.AppendLine("#EXTM3U"); + builder.Append("#EXT-X-TARGETDURATION:").AppendLine(tileGridCount.ToString(CultureInfo.InvariantCulture)); + builder.AppendLine("#EXT-X-VERSION:7"); + builder.AppendLine("#EXT-X-MEDIA-SEQUENCE:1"); + builder.AppendLine("#EXT-X-PLAYLIST-TYPE:VOD"); + builder.AppendLine("#EXT-X-IMAGES-ONLY"); + + for (int i = 0; i < tileGridCount; i++) + { + // All tile grids before the last one must contain full amount of tiles. + // The final grid will be 0 < count <= maxTiles + if (i == tileGridCount - 1) + { + tilesPerGrid = tilesInfo.TileCount - (i * tilesPerGrid); + } + + var infDuration = tileDuration * tilesPerGrid; + var url = string.Format( + CultureInfo.InvariantCulture, + urlFormat, + mediaSourceId, + width.ToString(CultureInfo.InvariantCulture), + i.ToString(CultureInfo.InvariantCulture), + _httpContextAccessor.HttpContext.User.GetToken()); + + // EXTINF + builder.Append("#EXTINF:").Append(string.Format(CultureInfo.InvariantCulture, decimalFormat, infDuration)) + .AppendLine(","); + + // EXT-X-TILES + builder.Append("#EXT-X-TILES:RESOLUTION=").Append(resolution).Append(",LAYOUT=").Append(layout).Append(",DURATION=") + .AppendLine(string.Format(CultureInfo.InvariantCulture, decimalFormat, tileDuration)); + + // URL + builder.AppendLine(url); + } + + builder.AppendLine("#EXT-X-ENDLIST"); + return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8")); + } + } + + return new FileContentResult(Array.Empty(), MimeTypes.GetMimeType("playlist.m3u8")); + } + private async Task GetMasterPlaylistInternal( StreamingRequestDto streamingRequest, bool isHeadRequest, @@ -299,6 +380,13 @@ public class DynamicHlsHelper AppendPlaylist(builder, state, variantUrl, newBitrate, subtitleGroup); } + if (!isLiveStream && (state.VideoRequest?.EnableTrickplay).GetValueOrDefault(false)) + { + var sourceId = Guid.Parse(state.Request.MediaSourceId); + var tilesResolutions = _trickplayManager.GetTilesResolutions(sourceId); + AddTrickplay(state, tilesResolutions, builder, _httpContextAccessor.HttpContext.User); + } + return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8")); } @@ -527,6 +615,41 @@ public class DynamicHlsHelper } } + /// + /// Appends EXT-X-IMAGE-STREAM-INF playlists for each available trickplay resolution. + /// + /// StreamState of the current stream. + /// Dictionary of widths to corresponding tiles info. + /// StringBuilder to append the field to. + /// Http user context. + private void AddTrickplay(StreamState state, Dictionary tilesResolutions, StringBuilder builder, ClaimsPrincipal user) + { + const string playlistFormat = "#EXT-X-IMAGE-STREAM-INF:BANDWIDTH={0},RESOLUTION={1}x{2},CODECS=\"jpeg\",URI=\"{3}\""; + + foreach (var resolution in tilesResolutions) + { + var width = resolution.Key; + var tilesInfo = resolution.Value; + + var url = string.Format( + CultureInfo.InvariantCulture, + "tiles.m3u8?Width={0}&MediaSourceId={1}&api_key={2}", + width.ToString(CultureInfo.InvariantCulture), + state.Request.MediaSourceId, + user.GetToken()); + + var line = string.Format( + CultureInfo.InvariantCulture, + playlistFormat, + tilesInfo.Bandwidth.ToString(CultureInfo.InvariantCulture), + tilesInfo.Width.ToString(CultureInfo.InvariantCulture), + tilesInfo.Height.ToString(CultureInfo.InvariantCulture), + url); + + builder.AppendLine(line); + } + } + /// /// Get the H.26X level of the output video stream. /// diff --git a/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs b/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs index 60c529d4ab..8548fec1a1 100644 --- a/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs +++ b/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs @@ -1,4 +1,4 @@ -namespace Jellyfin.Api.Models.StreamingDtos; +namespace Jellyfin.Api.Models.StreamingDtos; /// /// The video request dto. @@ -15,4 +15,9 @@ public class VideoRequestDto : StreamingRequestDto /// Gets or sets a value indicating whether to enable subtitles in the manifest. /// public bool EnableSubtitlesInManifest { get; set; } + + /// + /// Gets or sets a value indicating whether to enable trickplay images. + /// + public bool EnableTrickplay { get; set; } } diff --git a/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs index 3d1450a906..87ac145d75 100644 --- a/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs +++ b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs @@ -11,7 +11,6 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Globalization; using MediaBrowser.Model.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; namespace MediaBrowser.Providers.Trickplay { diff --git a/MediaBrowser.Providers/Trickplay/TrickplayManager.cs b/MediaBrowser.Providers/Trickplay/TrickplayManager.cs index f1eb389ab7..62180804f7 100644 --- a/MediaBrowser.Providers/Trickplay/TrickplayManager.cs +++ b/MediaBrowser.Providers/Trickplay/TrickplayManager.cs @@ -9,7 +9,6 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Trickplay; -using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; @@ -66,7 +65,7 @@ namespace MediaBrowser.Providers.Trickplay private async Task RefreshTrickplayData(Video video, bool replace, int width, int interval, int tileWidth, int tileHeight, bool doHwAccel, bool doHwEncode, CancellationToken cancellationToken) { - if (!CanGenerateTrickplay(video)) + if (!CanGenerateTrickplay(video, interval)) { return; } @@ -78,7 +77,7 @@ namespace MediaBrowser.Providers.Trickplay { await _resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); - if (!replace && Directory.Exists(outputDir)) + if (!replace && Directory.Exists(outputDir) && GetTilesResolutions(video.Id).ContainsKey(width)) { _logger.LogDebug("Found existing trickplay files for {ItemId}. Exiting.", video.Id); return; @@ -177,7 +176,7 @@ namespace MediaBrowser.Providers.Trickplay Interval = interval, TileWidth = tileWidth, TileHeight = tileHeight, - TileCount = (int)Math.Ceiling((decimal)images.Count / tileWidth / tileHeight), + TileCount = 0, Bandwidth = 0 }; @@ -201,7 +200,6 @@ namespace MediaBrowser.Providers.Trickplay while (i < images.Count) { var tileGrid = new SKBitmap(tilesInfo.Width * tilesInfo.TileWidth, tilesInfo.Height * tilesInfo.TileHeight); - var tileCount = 0; using (var canvas = new SKCanvas(tileGrid)) { @@ -231,7 +229,7 @@ namespace MediaBrowser.Providers.Trickplay } canvas.DrawBitmap(img, x * tilesInfo.Width, y * tilesInfo.Height); - tileCount++; + tilesInfo.TileCount++; i++; } } @@ -266,7 +264,7 @@ namespace MediaBrowser.Providers.Trickplay return tilesInfo; } - private bool CanGenerateTrickplay(Video video) + private bool CanGenerateTrickplay(Video video, int interval) { var videoType = video.VideoType; if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay) @@ -279,6 +277,16 @@ namespace MediaBrowser.Providers.Trickplay return false; } + if (video.IsShortcut) + { + return false; + } + + if (!video.IsCompleteMedia) + { + return false; + } + /* TODO config options var libraryOptions = _libraryManager.GetLibraryOptions(video); if (libraryOptions is not null) @@ -294,14 +302,7 @@ namespace MediaBrowser.Providers.Trickplay } */ - // TODO: media length is shorter than configured interval - - if (video.IsShortcut) - { - return false; - } - - if (!video.IsCompleteMedia) + if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks) { return false; } diff --git a/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs b/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs index 8606e148b1..be66dea8ad 100644 --- a/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs +++ b/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs @@ -97,7 +97,6 @@ namespace MediaBrowser.Providers.Trickplay private async Task FetchInternal(Video item, MetadataRefreshOptions options, CancellationToken cancellationToken) { - // TODO: will "search for missing metadata" always trigger this? // TODO: implement all config options --> // TODO: this is always blocking for metadata collection, make non-blocking option await _trickplayManager.RefreshTrickplayData(item, options.ReplaceAllImages, cancellationToken).ConfigureAwait(false);