Hls playlist

Jellyfin.Api/Controllers/DynamicHlsController.cs

@@ -407,6 +407,7 @@ public class DynamicHlsController : BaseJellyfinApiController
     /// <param name="context">Optional. The <see cref="EncodingContext"/>.</param>
     /// <param name="streamOptions">Optional. The streaming options.</param>
     /// <param name="enableAdaptiveBitrateStreaming">Enable adaptive bitrate streaming.</param>
+    /// <param name="enableTrickplay">Enable trickplay image playlists being added to master playlist.</param>
     /// <response code="200">Video stream returned.</response>
     /// <returns>A <see cref="FileResult"/> containing the playlist file.</returns>
     [HttpGet("Videos/{itemId}/master.m3u8")]
@@ -464,7 +465,8 @@ public class DynamicHlsController : BaseJellyfinApiController
         [FromQuery] int? videoStreamIndex,
         [FromQuery] EncodingContext? context,
         [FromQuery] Dictionary<string, string> streamOptions,
-        [FromQuery] bool enableAdaptiveBitrateStreaming = true)
+        [FromQuery] bool enableAdaptiveBitrateStreaming = true,
+        [FromQuery] bool enableTrickplay = true)
     {
         var streamingRequest = new HlsVideoRequestDto
         {
@@ -518,7 +520,8 @@ public class DynamicHlsController : BaseJellyfinApiController
             VideoStreamIndex = videoStreamIndex,
             Context = context ?? EncodingContext.Streaming,
             StreamOptions = streamOptions,
-            EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming
+            EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming,
+            EnableTrickplay = enableTrickplay
         };
 
         return await _dynamicHlsHelper.GetMasterHlsPlaylist(TranscodingJobType, streamingRequest, enableAdaptiveBitrateStreaming).ConfigureAwait(false);
@@ -1025,6 +1028,25 @@ public class DynamicHlsController : BaseJellyfinApiController
             .ConfigureAwait(false);
     }
 
+    /// <summary>
+    /// Gets an image tiles playlist for trickplay.
+    /// </summary>
+    /// <param name="itemId">The item id.</param>
+    /// <param name="width">The width of a single tile.</param>
+    /// <param name="mediaSourceId">The media version id.</param>
+    /// <response code="200">Tiles stream returned.</response>
+    /// <returns>A <see cref="FileResult"/> containing the trickplay tiles file.</returns>
+    [HttpGet("Videos/{itemId}/tiles.m3u8")]
+    [ProducesResponseType(StatusCodes.Status200OK)]
+    [ProducesPlaylistFile]
+    public ActionResult GetTrickplayTilesHlsPlaylist(
+        [FromRoute, Required] Guid itemId,
+        [FromQuery, Required] int width,
+        [FromQuery, Required] string mediaSourceId)
+    {
+        return _dynamicHlsHelper.GetTilesHlsPlaylist(width, mediaSourceId);
+    }
+
     /// <summary>
     /// Gets a video stream using HTTP live streaming.
     /// </summary>

Jellyfin.Api/Helpers/DynamicHlsHelper.cs

@@ -10,6 +10,7 @@ using System.Threading.Tasks;
 using Jellyfin.Api.Extensions;
 using Jellyfin.Api.Models.StreamingDtos;
 using Jellyfin.Extensions;
+using Jellyfin.Data.Entities;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
 using MediaBrowser.Common.Net;
@@ -18,6 +19,7 @@ using MediaBrowser.Controller.Devices;
 using MediaBrowser.Controller.Dlna;
 using MediaBrowser.Controller.Library;
 using MediaBrowser.Controller.MediaEncoding;
+using MediaBrowser.Controller.Trickplay;
 using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Net;
@@ -45,6 +47,7 @@ public class DynamicHlsHelper
     private readonly ILogger<DynamicHlsHelper> _logger;
     private readonly IHttpContextAccessor _httpContextAccessor;
     private readonly EncodingHelper _encodingHelper;
+    private readonly ITrickplayManager _trickplayManager;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="DynamicHlsHelper"/> class.
@@ -61,6 +64,7 @@ public class DynamicHlsHelper
     /// <param name="logger">Instance of the <see cref="ILogger{DynamicHlsHelper}"/> interface.</param>
     /// <param name="httpContextAccessor">Instance of the <see cref="IHttpContextAccessor"/> interface.</param>
     /// <param name="encodingHelper">Instance of <see cref="EncodingHelper"/>.</param>
+    /// <param name="trickplayManager">Instance of <see cref="ITrickplayManager"/>.</param>
     public DynamicHlsHelper(
         ILibraryManager libraryManager,
         IUserManager userManager,
@@ -73,7 +77,8 @@ public class DynamicHlsHelper
         INetworkManager networkManager,
         ILogger<DynamicHlsHelper> logger,
         IHttpContextAccessor httpContextAccessor,
-        EncodingHelper encodingHelper)
+        EncodingHelper encodingHelper,
+        ITrickplayManager trickplayManager)
     {
         _libraryManager = libraryManager;
         _userManager = userManager;
@@ -87,6 +92,7 @@ public class DynamicHlsHelper
         _logger = logger;
         _httpContextAccessor = httpContextAccessor;
         _encodingHelper = encodingHelper;
+        _trickplayManager = trickplayManager;
     }
 
     /// <summary>
@@ -112,6 +118,81 @@ public class DynamicHlsHelper
             cancellationTokenSource).ConfigureAwait(false);
     }
 
+    /// <summary>
+    /// Get trickplay tiles hls playlist.
+    /// </summary>
+    /// <param name="width">The width of a single tile.</param>
+    /// <param name="mediaSourceId">The media version id.</param>
+    /// <returns>The resulting <see cref="ActionResult"/>.</returns>
+    public ActionResult GetTilesHlsPlaylist(int width, string mediaSourceId)
+    {
+        if (_httpContextAccessor.HttpContext is null)
+        {
+            throw new ResourceNotFoundException(nameof(_httpContextAccessor.HttpContext));
+        }
+
+        var tilesResolutions = _trickplayManager.GetTilesResolutions(Guid.Parse(mediaSourceId));
+        if (tilesResolutions is not null && tilesResolutions.ContainsKey(width))
+        {
+            var builder = new StringBuilder(128);
+            var tilesInfo = tilesResolutions[width];
+
+            if (tilesInfo.TileCount > 0)
+            {
+                const string urlFormat = "{0}/Trickplay/{1}/{2}.jpg?&api_key={3}";
+                const string decimalFormat = "{0:0.###}";
+
+                var resolution = tilesInfo.Width.ToString(CultureInfo.InvariantCulture) + "x" + tilesInfo.Height.ToString(CultureInfo.InvariantCulture);
+                var layout = tilesInfo.TileWidth.ToString(CultureInfo.InvariantCulture) + "x" + tilesInfo.TileHeight.ToString(CultureInfo.InvariantCulture);
+                var tilesPerGrid = tilesInfo.TileWidth * tilesInfo.TileHeight;
+                var tileDuration = (decimal)tilesInfo.Interval / 1000;
+                var tileGridCount = (int)Math.Ceiling((decimal)tilesInfo.TileCount / tilesPerGrid);
+
+                builder.AppendLine("#EXTM3U");
+                builder.Append("#EXT-X-TARGETDURATION:").AppendLine(tileGridCount.ToString(CultureInfo.InvariantCulture));
+                builder.AppendLine("#EXT-X-VERSION:7");
+                builder.AppendLine("#EXT-X-MEDIA-SEQUENCE:1");
+                builder.AppendLine("#EXT-X-PLAYLIST-TYPE:VOD");
+                builder.AppendLine("#EXT-X-IMAGES-ONLY");
+
+                for (int i = 0; i < tileGridCount; i++)
+                {
+                    // All tile grids before the last one must contain full amount of tiles.
+                    // The final grid will be 0 < count <= maxTiles
+                    if (i == tileGridCount - 1)
+                    {
+                        tilesPerGrid = tilesInfo.TileCount - (i * tilesPerGrid);
+                    }
+
+                    var infDuration = tileDuration * tilesPerGrid;
+                    var url = string.Format(
+                        CultureInfo.InvariantCulture,
+                        urlFormat,
+                        mediaSourceId,
+                        width.ToString(CultureInfo.InvariantCulture),
+                        i.ToString(CultureInfo.InvariantCulture),
+                        _httpContextAccessor.HttpContext.User.GetToken());
+
+                    // EXTINF
+                    builder.Append("#EXTINF:").Append(string.Format(CultureInfo.InvariantCulture, decimalFormat, infDuration))
+                        .AppendLine(",");
+
+                    // EXT-X-TILES
+                    builder.Append("#EXT-X-TILES:RESOLUTION=").Append(resolution).Append(",LAYOUT=").Append(layout).Append(",DURATION=")
+                        .AppendLine(string.Format(CultureInfo.InvariantCulture, decimalFormat, tileDuration));
+
+                    // URL
+                    builder.AppendLine(url);
+                }
+
+                builder.AppendLine("#EXT-X-ENDLIST");
+                return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8"));
+            }
+        }
+
+        return new FileContentResult(Array.Empty<byte>(), MimeTypes.GetMimeType("playlist.m3u8"));
+    }
+
     private async Task<ActionResult> GetMasterPlaylistInternal(
         StreamingRequestDto streamingRequest,
         bool isHeadRequest,
@@ -299,6 +380,13 @@ public class DynamicHlsHelper
             AppendPlaylist(builder, state, variantUrl, newBitrate, subtitleGroup);
         }
 
+        if (!isLiveStream && (state.VideoRequest?.EnableTrickplay).GetValueOrDefault(false))
+        {
+            var sourceId = Guid.Parse(state.Request.MediaSourceId);
+            var tilesResolutions = _trickplayManager.GetTilesResolutions(sourceId);
+            AddTrickplay(state, tilesResolutions, builder, _httpContextAccessor.HttpContext.User);
+        }
+
         return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8"));
     }
 
@@ -527,6 +615,41 @@ public class DynamicHlsHelper
         }
     }
 
+    /// <summary>
+    /// Appends EXT-X-IMAGE-STREAM-INF playlists for each available trickplay resolution.
+    /// </summary>
+    /// <param name="state">StreamState of the current stream.</param>
+    /// <param name="tilesResolutions">Dictionary of widths to corresponding tiles info.</param>
+    /// <param name="builder">StringBuilder to append the field to.</param>
+    /// <param name="user">Http user context.</param>
+    private void AddTrickplay(StreamState state, Dictionary<int, TrickplayTilesInfo> tilesResolutions, StringBuilder builder, ClaimsPrincipal user)
+    {
+        const string playlistFormat = "#EXT-X-IMAGE-STREAM-INF:BANDWIDTH={0},RESOLUTION={1}x{2},CODECS=\"jpeg\",URI=\"{3}\"";
+
+        foreach (var resolution in tilesResolutions)
+        {
+            var width = resolution.Key;
+            var tilesInfo = resolution.Value;
+
+            var url = string.Format(
+                CultureInfo.InvariantCulture,
+                "tiles.m3u8?Width={0}&MediaSourceId={1}&api_key={2}",
+                width.ToString(CultureInfo.InvariantCulture),
+                state.Request.MediaSourceId,
+                user.GetToken());
+
+            var line = string.Format(
+                CultureInfo.InvariantCulture,
+                playlistFormat,
+                tilesInfo.Bandwidth.ToString(CultureInfo.InvariantCulture),
+                tilesInfo.Width.ToString(CultureInfo.InvariantCulture),
+                tilesInfo.Height.ToString(CultureInfo.InvariantCulture),
+                url);
+
+            builder.AppendLine(line);
+        }
+    }
+
     /// <summary>
     /// Get the H.26X level of the output video stream.
     /// </summary>

Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs

@@ -1,4 +1,4 @@
-namespace Jellyfin.Api.Models.StreamingDtos;
+namespace Jellyfin.Api.Models.StreamingDtos;
 
 /// <summary>
 /// The video request dto.
@@ -15,4 +15,9 @@ public class VideoRequestDto : StreamingRequestDto
     /// Gets or sets a value indicating whether to enable subtitles in the manifest.
     /// </summary>
     public bool EnableSubtitlesInManifest { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether to enable trickplay images.
+    /// </summary>
+    public bool EnableTrickplay { get; set; }
 }

MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs

@@ -11,7 +11,6 @@ using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Globalization;
 using MediaBrowser.Model.Tasks;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
 
 namespace MediaBrowser.Providers.Trickplay
 {

MediaBrowser.Providers/Trickplay/TrickplayManager.cs

@@ -9,7 +9,6 @@ using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.MediaEncoding;
 using MediaBrowser.Controller.Persistence;
 using MediaBrowser.Controller.Trickplay;
-using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.IO;
 using Microsoft.Extensions.Logging;
@@ -66,7 +65,7 @@ namespace MediaBrowser.Providers.Trickplay
 
         private async Task RefreshTrickplayData(Video video, bool replace, int width, int interval, int tileWidth, int tileHeight, bool doHwAccel, bool doHwEncode, CancellationToken cancellationToken)
         {
-            if (!CanGenerateTrickplay(video))
+            if (!CanGenerateTrickplay(video, interval))
             {
                 return;
             }
@@ -78,7 +77,7 @@ namespace MediaBrowser.Providers.Trickplay
             {
                 await _resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
 
-                if (!replace && Directory.Exists(outputDir))
+                if (!replace && Directory.Exists(outputDir) && GetTilesResolutions(video.Id).ContainsKey(width))
                 {
                     _logger.LogDebug("Found existing trickplay files for {ItemId}. Exiting.", video.Id);
                     return;
@@ -177,7 +176,7 @@ namespace MediaBrowser.Providers.Trickplay
                 Interval = interval,
                 TileWidth = tileWidth,
                 TileHeight = tileHeight,
-                TileCount = (int)Math.Ceiling((decimal)images.Count / tileWidth / tileHeight),
+                TileCount = 0,
                 Bandwidth = 0
             };
 
@@ -201,7 +200,6 @@ namespace MediaBrowser.Providers.Trickplay
             while (i < images.Count)
             {
                 var tileGrid = new SKBitmap(tilesInfo.Width * tilesInfo.TileWidth, tilesInfo.Height * tilesInfo.TileHeight);
-                var tileCount = 0;
 
                 using (var canvas = new SKCanvas(tileGrid))
                 {
@@ -231,7 +229,7 @@ namespace MediaBrowser.Providers.Trickplay
                             }
 
                             canvas.DrawBitmap(img, x * tilesInfo.Width, y * tilesInfo.Height);
-                            tileCount++;
+                            tilesInfo.TileCount++;
                             i++;
                         }
                     }
@@ -266,7 +264,7 @@ namespace MediaBrowser.Providers.Trickplay
             return tilesInfo;
         }
 
-        private bool CanGenerateTrickplay(Video video)
+        private bool CanGenerateTrickplay(Video video, int interval)
         {
             var videoType = video.VideoType;
             if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay)
@@ -279,6 +277,16 @@ namespace MediaBrowser.Providers.Trickplay
                 return false;
             }
 
+            if (video.IsShortcut)
+            {
+                return false;
+            }
+
+            if (!video.IsCompleteMedia)
+            {
+                return false;
+            }
+
             /* TODO config options
             var libraryOptions = _libraryManager.GetLibraryOptions(video);
             if (libraryOptions is not null)
@@ -294,14 +302,7 @@ namespace MediaBrowser.Providers.Trickplay
             }
             */
 
-            // TODO: media length is shorter than configured interval
+            if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks)
-
-            if (video.IsShortcut)
-            {
-                return false;
-            }
-
-            if (!video.IsCompleteMedia)
             {
                 return false;
             }

MediaBrowser.Providers/Trickplay/TrickplayProvider.cs

@@ -97,7 +97,6 @@ namespace MediaBrowser.Providers.Trickplay
 
         private async Task<ItemUpdateType> FetchInternal(Video item, MetadataRefreshOptions options, CancellationToken cancellationToken)
         {
-            // TODO: will "search for missing metadata" always trigger this?
             // TODO: implement all config options -->
             // TODO: this is always blocking for metadata collection, make non-blocking option
             await _trickplayManager.RefreshTrickplayData(item, options.ReplaceAllImages, cancellationToken).ConfigureAwait(false);