Feature/media segments plugin api (#12359)

CONTRIBUTORS.md

@@ -65,6 +65,7 @@
  - [joshuaboniface](https://github.com/joshuaboniface)
  - [JustAMan](https://github.com/JustAMan)
  - [justinfenn](https://github.com/justinfenn)
+ - [JPVenson](https://github.com/JPVenson)
  - [KerryRJ](https://github.com/KerryRJ)
  - [Larvitar](https://github.com/Larvitar)
  - [LeoVerto](https://github.com/LeoVerto)

Emby.Server.Implementations/Localization/Core/en-US.json

@@ -132,6 +132,8 @@
     "TaskKeyframeExtractorDescription": "Extracts keyframes from video files to create more precise HLS playlists. This task may run for a long time.",
     "TaskCleanCollectionsAndPlaylists": "Clean up collections and playlists",
     "TaskCleanCollectionsAndPlaylistsDescription": "Removes items from collections and playlists that no longer exist.",
+    "TaskExtractMediaSegments": "Media Segment Scan",
+    "TaskExtractMediaSegmentsDescription": "Extracts or obtains media segments from MediaSegment enabled plugins.",
     "TaskMoveTrickplayImages": "Migrate Trickplay Image Location",
     "TaskMoveTrickplayImagesDescription": "Moves existing trickplay files according to the library settings."
 }

Emby.Server.Implementations/ScheduledTasks/Tasks/MediaSegmentExtractionTask.cs

@@ -0,0 +1,118 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Jellyfin.Data.Enums;
+using MediaBrowser.Controller;
+using MediaBrowser.Controller.Dto;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.Globalization;
+using MediaBrowser.Model.Tasks;
+
+namespace Emby.Server.Implementations.ScheduledTasks.Tasks;
+
+/// <summary>
+/// Task to obtain media segments.
+/// </summary>
+public class MediaSegmentExtractionTask : IScheduledTask
+{
+    /// <summary>
+    /// The library manager.
+    /// </summary>
+    private readonly ILibraryManager _libraryManager;
+    private readonly ILocalizationManager _localization;
+    private readonly IMediaSegmentManager _mediaSegmentManager;
+    private static readonly BaseItemKind[] _itemTypes = [BaseItemKind.Episode, BaseItemKind.Movie, BaseItemKind.Audio, BaseItemKind.AudioBook];
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="MediaSegmentExtractionTask" /> class.
+    /// </summary>
+    /// <param name="libraryManager">The library manager.</param>
+    /// <param name="localization">The localization manager.</param>
+    /// <param name="mediaSegmentManager">The segment manager.</param>
+    public MediaSegmentExtractionTask(ILibraryManager libraryManager, ILocalizationManager localization, IMediaSegmentManager mediaSegmentManager)
+    {
+        _libraryManager = libraryManager;
+        _localization = localization;
+        _mediaSegmentManager = mediaSegmentManager;
+    }
+
+    /// <inheritdoc/>
+    public string Name => _localization.GetLocalizedString("TaskExtractMediaSegments");
+
+    /// <inheritdoc/>
+    public string Description => _localization.GetLocalizedString("TaskExtractMediaSegmentsDescription");
+
+    /// <inheritdoc/>
+    public string Category => _localization.GetLocalizedString("TasksLibraryCategory");
+
+    /// <inheritdoc/>
+    public string Key => "TaskExtractMediaSegments";
+
+    /// <inheritdoc/>
+    public async Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        progress.Report(0);
+
+        var pagesize = 100;
+
+        var query = new InternalItemsQuery
+        {
+            MediaTypes = new[] { MediaType.Video, MediaType.Audio },
+            IsVirtualItem = false,
+            IncludeItemTypes = _itemTypes,
+            DtoOptions = new DtoOptions(true),
+            SourceTypes = new[] { SourceType.Library },
+            Recursive = true,
+            Limit = pagesize
+        };
+
+        var numberOfVideos = _libraryManager.GetCount(query);
+
+        var startIndex = 0;
+        var numComplete = 0;
+
+        while (startIndex < numberOfVideos)
+        {
+            query.StartIndex = startIndex;
+
+            var baseItems = _libraryManager.GetItemList(query);
+            var currentPageCount = baseItems.Count;
+            // TODO parallelize with Parallel.ForEach?
+            for (var i = 0; i < currentPageCount; i++)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                var item = baseItems[i];
+                // Only local files supported
+                if (item.IsFileProtocol && File.Exists(item.Path))
+                {
+                    await _mediaSegmentManager.RunSegmentPluginProviders(item, false, cancellationToken).ConfigureAwait(false);
+                }
+
+                // Update progress
+                numComplete++;
+                double percent = (double)numComplete / numberOfVideos;
+                progress.Report(100 * percent);
+            }
+
+            startIndex += pagesize;
+        }
+
+        progress.Report(100);
+    }
+
+    /// <inheritdoc/>
+    public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
+    {
+        yield return new TaskTriggerInfo
+        {
+            Type = TaskTriggerInfo.TriggerInterval,
+            IntervalTicks = TimeSpan.FromHours(12).Ticks
+        };
+    }
+}

Jellyfin.Server.Implementations/MediaSegments/MediaSegmentManager.cs

@@ -1,14 +1,23 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Globalization;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
+using Jellyfin.Extensions;
+using MediaBrowser.Common.Extensions;
 using MediaBrowser.Controller;
 using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Entities.Audio;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model;
 using MediaBrowser.Model.MediaSegments;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
 
 namespace Jellyfin.Server.Implementations.MediaSegments;
 
@@ -17,15 +26,89 @@ namespace Jellyfin.Server.Implementations.MediaSegments;
 /// </summary>
 public class MediaSegmentManager : IMediaSegmentManager
 {
+    private readonly ILogger<MediaSegmentManager> _logger;
     private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
+    private readonly IMediaSegmentProvider[] _segmentProviders;
+    private readonly ILibraryManager _libraryManager;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="MediaSegmentManager"/> class.
     /// </summary>
+    /// <param name="logger">Logger.</param>
     /// <param name="dbProvider">EFCore Database factory.</param>
-    public MediaSegmentManager(IDbContextFactory<JellyfinDbContext> dbProvider)
+    /// <param name="segmentProviders">List of all media segment providers.</param>
+    /// <param name="libraryManager">Library manager.</param>
+    public MediaSegmentManager(
+        ILogger<MediaSegmentManager> logger,
+        IDbContextFactory<JellyfinDbContext> dbProvider,
+        IEnumerable<IMediaSegmentProvider> segmentProviders,
+        ILibraryManager libraryManager)
     {
+        _logger = logger;
         _dbProvider = dbProvider;
+
+        _segmentProviders = segmentProviders
+            .OrderBy(i => i is IHasOrder hasOrder ? hasOrder.Order : 0)
+            .ToArray();
+        _libraryManager = libraryManager;
+    }
+
+    /// <inheritdoc/>
+    public async Task RunSegmentPluginProviders(BaseItem baseItem, bool overwrite, CancellationToken cancellationToken)
+    {
+        var libraryOptions = _libraryManager.GetLibraryOptions(baseItem);
+        var providers = _segmentProviders
+            .Where(e => !libraryOptions.DisabledMediaSegmentProviders.Contains(GetProviderId(e.Name)))
+            .OrderBy(i =>
+                {
+                    var index = libraryOptions.MediaSegmentProvideOrder.IndexOf(i.Name);
+                    return index == -1 ? int.MaxValue : index;
+                })
+            .ToList();
+
+        _logger.LogInformation("Start media segment extraction from providers with {CountProviders} enabled", providers.Count);
+        using var db = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+
+        if (!overwrite && (await db.MediaSegments.AnyAsync(e => e.ItemId.Equals(baseItem.Id), cancellationToken).ConfigureAwait(false)))
+        {
+            _logger.LogInformation("Skip {MediaPath} as it already contains media segments", baseItem.Path);
+            return;
+        }
+
+        _logger.LogInformation("Clear existing Segments for {MediaPath}", baseItem.Path);
+
+        await db.MediaSegments.Where(e => e.ItemId.Equals(baseItem.Id)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
+
+        // no need to recreate the request object every time.
+        var requestItem = new MediaSegmentGenerationRequest() { ItemId = baseItem.Id };
+
+        foreach (var provider in providers)
+        {
+            if (!await provider.Supports(baseItem).ConfigureAwait(false))
+            {
+                _logger.LogDebug("Media Segment provider {ProviderName} does not support item with path {Path}", provider.Name, baseItem.Path);
+                continue;
+            }
+
+            _logger.LogDebug("Run Media Segment provider {ProviderName}", provider.Name);
+            try
+            {
+                var segments = await provider.GetMediaSegments(requestItem, cancellationToken)
+                    .ConfigureAwait(false);
+
+                _logger.LogInformation("Media Segment provider {ProviderName} found {CountSegments} for {MediaPath}", provider.Name, segments.Count, baseItem.Path);
+                var providerId = GetProviderId(provider.Name);
+                foreach (var segment in segments)
+                {
+                    segment.ItemId = baseItem.Id;
+                    await CreateSegmentAsync(segment, providerId).ConfigureAwait(false);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Provider {ProviderName} failed to extract segments from {MediaPath}", provider.Name, baseItem.Path);
+            }
+        }
     }
 
     /// <inheritdoc />
@@ -103,4 +186,21 @@ public class MediaSegmentManager : IMediaSegmentManager
     {
         return baseItem.MediaType is Data.Enums.MediaType.Video or Data.Enums.MediaType.Audio;
     }
+
+    /// <inheritdoc/>
+    public IEnumerable<(string Name, string Id)> GetSupportedProviders(BaseItem item)
+    {
+        if (item is not (Video or Audio))
+        {
+            return [];
+        }
+
+        return _segmentProviders
+            .Select(p => (p.Name, GetProviderId(p.Name)));
+    }
+
+    private string GetProviderId(string name)
+        => name.ToLowerInvariant()
+            .GetMD5()
+            .ToString("N", CultureInfo.InvariantCulture);
 }

MediaBrowser.Controller/MediaSegements/IMediaSegmentManager.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
@@ -13,6 +14,15 @@ namespace MediaBrowser.Controller;
 /// </summary>
 public interface IMediaSegmentManager
 {
+    /// <summary>
+    /// Uses all segment providers enabled for the <see cref="BaseItem"/>'s library to get the Media Segments.
+    /// </summary>
+    /// <param name="baseItem">The Item to evaluate.</param>
+    /// <param name="overwrite">If set, will remove existing segments and replace it with new ones otherwise will check for existing segments and if found any, stops.</param>
+    /// <param name="cancellationToken">stop request token.</param>
+    /// <returns>A task that indicates the Operation is finished.</returns>
+    Task RunSegmentPluginProviders(BaseItem baseItem, bool overwrite, CancellationToken cancellationToken);
+
     /// <summary>
     /// Returns if this item supports media segments.
     /// </summary>
@@ -50,4 +60,11 @@ public interface IMediaSegmentManager
     /// <returns>True if there are any segments stored for the item, otherwise false.</returns>
     /// TODO: this should be async but as the only caller BaseItem.GetVersionInfo isn't async, this is also not. Venson.
     bool HasSegments(Guid itemId);
+
+    /// <summary>
+    /// Gets a list of all registered Segment Providers and their IDs.
+    /// </summary>
+    /// <param name="item">The media item that should be tested for providers.</param>
+    /// <returns>A list of all providers for the tested item.</returns>
+    IEnumerable<(string Name, string Id)> GetSupportedProviders(BaseItem item);
 }

MediaBrowser.Controller/MediaSegements/IMediaSegmentProvider.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Model;
+using MediaBrowser.Model.MediaSegments;
+
+namespace MediaBrowser.Controller;
+
+/// <summary>
+/// Provides methods for Obtaining the Media Segments from an Item.
+/// </summary>
+public interface IMediaSegmentProvider
+{
+    /// <summary>
+    /// Gets the provider name.
+    /// </summary>
+    string Name { get; }
+
+    /// <summary>
+    /// Enumerates all Media Segments from an Media Item.
+    /// </summary>
+    /// <param name="request">Arguments to enumerate MediaSegments.</param>
+    /// <param name="cancellationToken">Abort token.</param>
+    /// <returns>A list of all MediaSegments found from this provider.</returns>
+    Task<IReadOnlyList<MediaSegmentDto>> GetMediaSegments(MediaSegmentGenerationRequest request, CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Should return support state for the given item.
+    /// </summary>
+    /// <param name="item">The base item to extract segments from.</param>
+    /// <returns>True if item is supported, otherwise false.</returns>
+    ValueTask<bool> Supports(BaseItem item);
+}

MediaBrowser.Model/Configuration/LibraryOptions.cs

@@ -11,6 +11,8 @@ namespace MediaBrowser.Model.Configuration
         {
             TypeOptions = Array.Empty<TypeOptions>();
             DisabledSubtitleFetchers = Array.Empty<string>();
+            DisabledMediaSegmentProviders = Array.Empty<string>();
+            MediaSegmentProvideOrder = Array.Empty<string>();
             SubtitleFetcherOrder = Array.Empty<string>();
             DisabledLocalMetadataReaders = Array.Empty<string>();
             DisabledLyricFetchers = Array.Empty<string>();
@@ -87,6 +89,10 @@ namespace MediaBrowser.Model.Configuration
 
         public string[] SubtitleFetcherOrder { get; set; }
 
+        public string[] DisabledMediaSegmentProviders { get; set; }
+
+        public string[] MediaSegmentProvideOrder { get; set; }
+
         public bool SkipSubtitlesIfEmbeddedSubtitlesPresent { get; set; }
 
         public bool SkipSubtitlesIfAudioTrackMatches { get; set; }

MediaBrowser.Model/Configuration/MetadataPluginType.cs

@@ -14,6 +14,7 @@ namespace MediaBrowser.Model.Configuration
         MetadataFetcher,
         MetadataSaver,
         SubtitleFetcher,
-        LyricFetcher
+        LyricFetcher,
+        MediaSegmentProvider
     }
 }

MediaBrowser.Model/MediaSegments/MediaSegmentGenerationRequest.cs

@@ -0,0 +1,14 @@
+using System;
+
+namespace MediaBrowser.Model;
+
+/// <summary>
+/// Model containing the arguments for enumerating the requested media item.
+/// </summary>
+public record MediaSegmentGenerationRequest
+{
+    /// <summary>
+    /// Gets the Id to the BaseItem the segments should be extracted from.
+    /// </summary>
+    public Guid ItemId { get; init; }
+}

MediaBrowser.Providers/Manager/ProviderManager.cs

@@ -62,7 +62,7 @@ namespace MediaBrowser.Providers.Manager
         private readonly CancellationTokenSource _disposeCancellationTokenSource = new();
         private readonly PriorityQueue<(Guid ItemId, MetadataRefreshOptions RefreshOptions), RefreshPriority> _refreshQueue = new();
         private readonly IMemoryCache _memoryCache;
-
+        private readonly IMediaSegmentManager _mediaSegmentManager;
         private readonly AsyncKeyedLocker<string> _imageSaveLock = new(o =>
         {
             o.PoolSize = 20;
@@ -92,6 +92,7 @@ namespace MediaBrowser.Providers.Manager
         /// <param name="baseItemManager">The BaseItem manager.</param>
         /// <param name="lyricManager">The lyric manager.</param>
         /// <param name="memoryCache">The memory cache.</param>
+        /// <param name="mediaSegmentManager">The media segment manager.</param>
         public ProviderManager(
             IHttpClientFactory httpClientFactory,
             ISubtitleManager subtitleManager,
@@ -103,7 +104,8 @@ namespace MediaBrowser.Providers.Manager
             ILibraryManager libraryManager,
             IBaseItemManager baseItemManager,
             ILyricManager lyricManager,
-            IMemoryCache memoryCache)
+            IMemoryCache memoryCache,
+            IMediaSegmentManager mediaSegmentManager)
         {
             _logger = logger;
             _httpClientFactory = httpClientFactory;
@@ -116,6 +118,7 @@ namespace MediaBrowser.Providers.Manager
             _baseItemManager = baseItemManager;
             _lyricManager = lyricManager;
             _memoryCache = memoryCache;
+            _mediaSegmentManager = mediaSegmentManager;
         }
 
         /// <inheritdoc/>
@@ -572,6 +575,14 @@ namespace MediaBrowser.Providers.Manager
                 Type = MetadataPluginType.LyricFetcher
             }));
 
+            // Media segment providers
+            var mediaSegmentProviders = _mediaSegmentManager.GetSupportedProviders(dummy);
+            pluginList.AddRange(mediaSegmentProviders.Select(i => new MetadataPlugin
+            {
+                Name = i.Name,
+                Type = MetadataPluginType.MediaSegmentProvider
+            }));
+
             summary.Plugins = pluginList.ToArray();
 
             var supportedImageTypes = imageProviders.OfType<IRemoteImageProvider>()

tests/Jellyfin.Providers.Tests/Manager/ProviderManagerTests.cs

@@ -574,7 +574,8 @@ namespace Jellyfin.Providers.Tests.Manager
                 libraryManager.Object,
                 baseItemManager!,
                 Mock.Of<ILyricManager>(),
-                Mock.Of<IMemoryCache>());
+                Mock.Of<IMemoryCache>(),
+                Mock.Of<IMediaSegmentManager>());
 
             return providerManager;
         }