From 2a681f205aba211d9eaec7866ea2f39f469fba90 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Fri, 10 Apr 2015 15:08:09 -0400 Subject: [PATCH] capture key frame info --- .../Playback/BaseStreamingService.cs | 96 ++++++++ .../Playback/Progressive/VideoService.cs | 1 - .../MediaEncoding/MediaInfoRequest.cs | 1 + .../Encoder/MediaEncoder.cs | 224 ++++++++++++++---- MediaBrowser.Model/Entities/MediaStream.cs | 9 +- .../MediaInfo/FFProbeVideoInfo.cs | 7 +- .../SqliteMediaStreamsRepository.cs | 55 ++++- 7 files changed, 340 insertions(+), 53 deletions(-) diff --git a/MediaBrowser.Api/Playback/BaseStreamingService.cs b/MediaBrowser.Api/Playback/BaseStreamingService.cs index 62bee1f9b7..728fea0e0b 100644 --- a/MediaBrowser.Api/Playback/BaseStreamingService.cs +++ b/MediaBrowser.Api/Playback/BaseStreamingService.cs @@ -1705,6 +1705,102 @@ namespace MediaBrowser.Api.Playback { state.OutputAudioCodec = "copy"; } + + if (string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase)) + { + var segmentLength = GetSegmentLength(state); + if (segmentLength.HasValue) + { + state.SegmentLength = segmentLength.Value; + } + } + } + + private int? GetSegmentLength(StreamState state) + { + var stream = state.VideoStream; + + if (stream == null) + { + return null; + } + + var frames = stream.KeyFrames; + + if (frames == null || frames.Count < 2) + { + return null; + } + + Logger.Debug("Found keyframes at {0}", string.Join(",", frames.ToArray())); + + var intervals = new List(); + for (var i = 1; i < frames.Count; i++) + { + var start = frames[i - 1]; + var end = frames[i]; + intervals.Add(end - start); + } + + Logger.Debug("Found keyframes intervals {0}", string.Join(",", intervals.ToArray())); + + var results = new List>(); + + for (var i = 1; i <= 10; i++) + { + var idealMs = i*1000; + + if (intervals.Max() < idealMs - 1000) + { + break; + } + + var segments = PredictStreamCopySegments(intervals, idealMs); + var variance = segments.Select(s => Math.Abs(idealMs - s)).Sum(); + + results.Add(new Tuple(i, variance)); + } + + if (results.Count == 0) + { + return null; + } + + return results.OrderBy(i => i.Item2).ThenBy(i => i.Item1).Select(i => i.Item1).First(); + } + + private List PredictStreamCopySegments(List intervals, int idealMs) + { + var segments = new List(); + var currentLength = 0; + + foreach (var interval in intervals) + { + if (currentLength == 0 || (currentLength + interval) <= idealMs) + { + currentLength += interval; + } + + else + { + // The segment will either be above or below the ideal. + // Need to figure out which is preferable + var offset1 = Math.Abs(idealMs - currentLength); + var offset2 = Math.Abs(idealMs - (currentLength + interval)); + + if (offset1 <= offset2) + { + segments.Add(currentLength); + currentLength = interval; + } + else + { + currentLength += interval; + } + } + } + Logger.Debug("Predicted actual segment lengths for length {0}: {1}", idealMs, string.Join(",", segments.ToArray())); + return segments; } private void AttachMediaSourceInfo(StreamState state, diff --git a/MediaBrowser.Api/Playback/Progressive/VideoService.cs b/MediaBrowser.Api/Playback/Progressive/VideoService.cs index 540c39a0c7..0ded108b1d 100644 --- a/MediaBrowser.Api/Playback/Progressive/VideoService.cs +++ b/MediaBrowser.Api/Playback/Progressive/VideoService.cs @@ -5,7 +5,6 @@ using MediaBrowser.Controller.Devices; using MediaBrowser.Controller.Dlna; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.LiveTv; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Model.IO; using ServiceStack; diff --git a/MediaBrowser.Controller/MediaEncoding/MediaInfoRequest.cs b/MediaBrowser.Controller/MediaEncoding/MediaInfoRequest.cs index ca0c2fdbb4..24df7b8854 100644 --- a/MediaBrowser.Controller/MediaEncoding/MediaInfoRequest.cs +++ b/MediaBrowser.Controller/MediaEncoding/MediaInfoRequest.cs @@ -15,6 +15,7 @@ namespace MediaBrowser.Controller.MediaEncoding public IIsoMount MountedIso { get; set; } public VideoType VideoType { get; set; } public List PlayableStreamFileNames { get; set; } + public bool ExtractKeyFrameInterval { get; set; } public MediaInfoRequest() { diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 7bcf60fd97..06e3016b9b 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using MediaBrowser.Common.IO; using MediaBrowser.Controller.Channels; using MediaBrowser.Controller.Configuration; @@ -14,6 +13,7 @@ using MediaBrowser.Model.Logging; using MediaBrowser.Model.MediaInfo; using MediaBrowser.Model.Serialization; using System; +using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; @@ -75,7 +75,7 @@ namespace MediaBrowser.MediaEncoding.Encoder protected readonly Func SubtitleEncoder; protected readonly Func MediaSourceManager; - private List _runningProcesses = new List(); + private readonly List _runningProcesses = new List(); public MediaEncoder(ILogger logger, IJsonSerializer jsonSerializer, string ffMpegPath, string ffProbePath, string version, IServerConfigurationManager configurationManager, IFileSystem fileSystem, ILiveTvManager liveTvManager, IIsoManager isoManager, ILibraryManager libraryManager, IChannelManager channelManager, ISessionManager sessionManager, Func subtitleEncoder, Func mediaSourceManager) { @@ -116,7 +116,9 @@ namespace MediaBrowser.MediaEncoding.Encoder var inputFiles = MediaEncoderHelpers.GetInputArgument(request.InputPath, request.Protocol, request.MountedIso, request.PlayableStreamFileNames); - return GetMediaInfoInternal(GetInputArgument(inputFiles, request.Protocol), request.InputPath, request.Protocol, extractChapters, + var extractKeyFrameInterval = request.ExtractKeyFrameInterval && request.Protocol == MediaProtocol.File && request.VideoType == VideoType.VideoFile; + + return GetMediaInfoInternal(GetInputArgument(inputFiles, request.Protocol), request.InputPath, request.Protocol, extractChapters, extractKeyFrameInterval, GetProbeSizeArgument(inputFiles, request.Protocol), request.MediaType == DlnaProfileType.Audio, cancellationToken); } @@ -150,12 +152,17 @@ namespace MediaBrowser.MediaEncoding.Encoder /// The primary path. /// The protocol. /// if set to true [extract chapters]. + /// if set to true [extract key frame interval]. /// The probe size argument. /// if set to true [is audio]. /// The cancellation token. /// Task{MediaInfoResult}. /// - private async Task GetMediaInfoInternal(string inputPath, string primaryPath, MediaProtocol protocol, bool extractChapters, + private async Task GetMediaInfoInternal(string inputPath, + string primaryPath, + MediaProtocol protocol, + bool extractChapters, + bool extractKeyFrameInterval, string probeSizeArgument, bool isAudio, CancellationToken cancellationToken) @@ -174,6 +181,7 @@ namespace MediaBrowser.MediaEncoding.Encoder // Must consume both or ffmpeg may hang due to deadlocks. See comments below. RedirectStandardOutput = true, RedirectStandardError = true, + RedirectStandardInput = true, FileName = FFProbePath, Arguments = string.Format(args, probeSizeArgument, inputPath).Trim(), @@ -187,12 +195,8 @@ namespace MediaBrowser.MediaEncoding.Encoder _logger.Debug("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments); - process.Exited += ProcessExited; - await _ffProbeResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); - InternalMediaInfoResult result; - try { StartProcess(process); @@ -210,19 +214,55 @@ namespace MediaBrowser.MediaEncoding.Encoder { process.BeginErrorReadLine(); - result = _jsonSerializer.DeserializeFromStream(process.StandardOutput.BaseStream); + var result = _jsonSerializer.DeserializeFromStream(process.StandardOutput.BaseStream); + + if (result != null) + { + if (result.streams != null) + { + // Normalize aspect ratio if invalid + foreach (var stream in result.streams) + { + if (string.Equals(stream.display_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase)) + { + stream.display_aspect_ratio = string.Empty; + } + if (string.Equals(stream.sample_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase)) + { + stream.sample_aspect_ratio = string.Empty; + } + } + } + + var mediaInfo = new ProbeResultNormalizer(_logger, FileSystem).GetMediaInfo(result, isAudio, primaryPath, protocol); + + if (extractKeyFrameInterval && mediaInfo.RunTimeTicks.HasValue) + { + foreach (var stream in mediaInfo.MediaStreams.Where(i => i.Type == MediaStreamType.Video) + .ToList()) + { + try + { + stream.KeyFrames = await GetKeyFrames(inputPath, stream.Index, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + + } + catch (Exception ex) + { + _logger.ErrorException("Error getting key frame interval", ex); + } + } + } + + return mediaInfo; + } } catch { - // Hate having to do this - try - { - process.Kill(); - } - catch (Exception ex1) - { - _logger.ErrorException("Error killing ffprobe", ex1); - } + StopProcess(process, 100, true); throw; } @@ -231,30 +271,108 @@ namespace MediaBrowser.MediaEncoding.Encoder _ffProbeResourcePool.Release(); } - if (result == null) + throw new ApplicationException(string.Format("FFProbe failed for {0}", inputPath)); + } + + private async Task> GetKeyFrames(string inputPath, int videoStreamIndex, CancellationToken cancellationToken) + { + const string args = "-i {0} -select_streams v:{1} -show_frames -print_format compact"; + + var process = new Process { - throw new ApplicationException(string.Format("FFProbe failed for {0}", inputPath)); + StartInfo = new ProcessStartInfo + { + CreateNoWindow = true, + UseShellExecute = false, + + // Must consume both or ffmpeg may hang due to deadlocks. See comments below. + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = true, + FileName = FFProbePath, + Arguments = string.Format(args, inputPath, videoStreamIndex.ToString(CultureInfo.InvariantCulture)).Trim(), + + WindowStyle = ProcessWindowStyle.Hidden, + ErrorDialog = false + }, + + EnableRaisingEvents = true + }; + + _logger.Debug("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments); + + StartProcess(process); + + var lines = new List(); + var outputCancellationSource = new CancellationTokenSource(4000); + + try + { + process.BeginErrorReadLine(); + + var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(outputCancellationSource.Token, cancellationToken); + + await StartReadingOutput(process.StandardOutput.BaseStream, lines, 120000, outputCancellationSource, linkedCancellationTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (cancellationToken.IsCancellationRequested) + { + throw; + } + } + finally + { + StopProcess(process, 100, true); } - cancellationToken.ThrowIfCancellationRequested(); + return lines; + } - if (result.streams != null) + private async Task StartReadingOutput(Stream source, List lines, int timeoutMs, CancellationTokenSource cancellationTokenSource, CancellationToken cancellationToken) + { + try { - // Normalize aspect ratio if invalid - foreach (var stream in result.streams) + using (var reader = new StreamReader(source)) { - if (string.Equals(stream.display_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase)) - { - stream.display_aspect_ratio = string.Empty; - } - if (string.Equals(stream.sample_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase)) + while (!reader.EndOfStream) { - stream.sample_aspect_ratio = string.Empty; + cancellationToken.ThrowIfCancellationRequested(); + + var line = await reader.ReadLineAsync().ConfigureAwait(false); + + var values = (line ?? string.Empty).Split('|') + .Where(i => !string.IsNullOrWhiteSpace(i)) + .Select(i => i.Split('=')) + .Where(i => i.Length == 2) + .ToDictionary(i => i[0], i => i[1]); + + string pktDts; + int frameMs; + if (values.TryGetValue("pkt_dts", out pktDts) && int.TryParse(pktDts, NumberStyles.Any, CultureInfo.InvariantCulture, out frameMs)) + { + string keyFrame; + if (values.TryGetValue("key_frame", out keyFrame) && string.Equals(keyFrame, "1", StringComparison.OrdinalIgnoreCase)) + { + lines.Add(frameMs); + } + + if (frameMs > timeoutMs) + { + cancellationTokenSource.Cancel(); + } + } } } } - - return new ProbeResultNormalizer(_logger, FileSystem).GetMediaInfo(result, isAudio, primaryPath, protocol); + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.ErrorException("Error reading ffprobe output", ex); + } } /// @@ -269,7 +387,14 @@ namespace MediaBrowser.MediaEncoding.Encoder /// The instance containing the event data. private void ProcessExited(object sender, EventArgs e) { - ((Process)sender).Dispose(); + var process = (Process) sender; + + lock (_runningProcesses) + { + _runningProcesses.Remove(process); + } + + process.Dispose(); } public Task ExtractAudioImage(string path, CancellationToken cancellationToken) @@ -574,6 +699,8 @@ namespace MediaBrowser.MediaEncoding.Encoder private void StartProcess(Process process) { + process.Exited += ProcessExited; + process.Start(); lock (_runningProcesses) @@ -587,27 +714,36 @@ namespace MediaBrowser.MediaEncoding.Encoder { _logger.Info("Killing ffmpeg process"); - process.StandardInput.WriteLine("q"); + try + { + process.StandardInput.WriteLine("q"); + } + catch (Exception) + { + _logger.Error("Error sending q command to process"); + } - if (!process.WaitForExit(1000)) + try { - if (enableForceKill) + if (process.WaitForExit(waitTimeMs)) { - process.Kill(); + return; } } + catch (Exception ex) + { + _logger.Error("Error in WaitForExit", ex); + } + + if (enableForceKill) + { + process.Kill(); + } } catch (Exception ex) { _logger.ErrorException("Error killing process", ex); } - finally - { - lock (_runningProcesses) - { - _runningProcesses.Remove(process); - } - } } private void StopProcesses() diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs index 0f3435174c..11eb31c27b 100644 --- a/MediaBrowser.Model/Entities/MediaStream.cs +++ b/MediaBrowser.Model/Entities/MediaStream.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Model.Dlna; +using System.Collections.Generic; +using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Extensions; using System.Diagnostics; @@ -58,6 +59,12 @@ namespace MediaBrowser.Model.Entities /// The length of the packet. public int? PacketLength { get; set; } + /// + /// Gets or sets the key frames. + /// + /// The key frames. + public List KeyFrames { get; set; } + /// /// Gets or sets the channels. /// diff --git a/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs b/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs index c433018c08..7950a8d5c7 100644 --- a/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs +++ b/MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs @@ -130,7 +130,7 @@ namespace MediaBrowser.Providers.MediaInfo return ItemUpdateType.MetadataImport; } - private const string SchemaVersion = "2"; + private const string SchemaVersion = "3"; private async Task GetMediaInfo(Video item, IIsoMount isoMount, @@ -145,7 +145,7 @@ namespace MediaBrowser.Providers.MediaInfo try { - return _json.DeserializeFromFile(cachePath); + //return _json.DeserializeFromFile(cachePath); } catch (FileNotFoundException) { @@ -167,7 +167,8 @@ namespace MediaBrowser.Providers.MediaInfo VideoType = item.VideoType, MediaType = DlnaProfileType.Video, InputPath = item.Path, - Protocol = protocol + Protocol = protocol, + ExtractKeyFrameInterval = true }, cancellationToken).ConfigureAwait(false); diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteMediaStreamsRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteMediaStreamsRepository.cs index e9d7f44ecc..76c1274b24 100644 --- a/MediaBrowser.Server.Implementations/Persistence/SqliteMediaStreamsRepository.cs +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteMediaStreamsRepository.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Controller.Persistence; +using System.Globalization; +using MediaBrowser.Controller.Persistence; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; @@ -40,7 +41,7 @@ namespace MediaBrowser.Server.Implementations.Persistence // Add PixelFormat column - createTableCommand += "(ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, IsCabac BIT NULL, PRIMARY KEY (ItemId, StreamIndex))"; + createTableCommand += "(ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, IsCabac BIT NULL, KeyFrames TEXT NULL, PRIMARY KEY (ItemId, StreamIndex))"; string[] queries = { @@ -61,6 +62,7 @@ namespace MediaBrowser.Server.Implementations.Persistence AddIsAnamorphicColumn(); AddIsCabacColumn(); AddRefFramesCommand(); + AddKeyFramesCommand(); PrepareStatements(); @@ -160,6 +162,37 @@ namespace MediaBrowser.Server.Implementations.Persistence _connection.RunQueries(new[] { builder.ToString() }, _logger); } + private void AddKeyFramesCommand() + { + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = "PRAGMA table_info(mediastreams)"; + + using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult)) + { + while (reader.Read()) + { + if (!reader.IsDBNull(1)) + { + var name = reader.GetString(1); + + if (string.Equals(name, "KeyFrames", StringComparison.OrdinalIgnoreCase)) + { + return; + } + } + } + } + } + + var builder = new StringBuilder(); + + builder.AppendLine("alter table mediastreams"); + builder.AppendLine("add column KeyFrames TEXT NULL"); + + _connection.RunQueries(new[] { builder.ToString() }, _logger); + } + private void AddIsCabacColumn() { using (var cmd = _connection.CreateCommand()) @@ -249,6 +282,7 @@ namespace MediaBrowser.Server.Implementations.Persistence "BitDepth", "IsAnamorphic", "RefFrames", + "KeyFrames", "IsCabac" }; @@ -430,7 +464,12 @@ namespace MediaBrowser.Server.Implementations.Persistence if (!reader.IsDBNull(25)) { - item.IsCabac = reader.GetBoolean(25); + item.KeyFrames = reader.GetString(25).Split(',').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => int.Parse(i, CultureInfo.InvariantCulture)).ToList(); + } + + if (!reader.IsDBNull(26)) + { + item.IsCabac = reader.GetBoolean(26); } return item; @@ -498,7 +537,15 @@ namespace MediaBrowser.Server.Implementations.Persistence _saveStreamCommand.GetParameter(22).Value = stream.BitDepth; _saveStreamCommand.GetParameter(23).Value = stream.IsAnamorphic; _saveStreamCommand.GetParameter(24).Value = stream.RefFrames; - _saveStreamCommand.GetParameter(25).Value = stream.IsCabac; + if (stream.KeyFrames != null) + { + _saveStreamCommand.GetParameter(25).Value = string.Join(",", stream.KeyFrames.Select(i => i.ToString(CultureInfo.InvariantCulture)).ToArray()); + } + else + { + _saveStreamCommand.GetParameter(25).Value = null; + } + _saveStreamCommand.GetParameter(26).Value = stream.IsCabac; _saveStreamCommand.Transaction = transaction; _saveStreamCommand.ExecuteNonQuery();