From fe1834e6be044f0fdbe68fb34122c680f29ae04d Mon Sep 17 00:00:00 2001 From: LukePulverenti Date: Thu, 14 Mar 2013 15:52:53 -0400 Subject: [PATCH] Add resume capability to GetTempFile --- .../HttpClientManager/HttpClientManager.cs | 192 +++++++++++------- .../Updates/PackageManager.cs | 9 +- .../MediaBrowser.Common.csproj | 1 + MediaBrowser.Common/Net/HttpRequestOptions.cs | 47 +++++ MediaBrowser.Common/Net/IHttpClient.cs | 20 +- .../ImageFromMediaLocationProvider.cs | 6 +- .../Html/scripts/site.js | 29 +-- Nuget/MediaBrowser.Common.Internal.nuspec | 4 +- Nuget/MediaBrowser.Common.nuspec | 2 +- Nuget/MediaBrowser.Server.Core.nuspec | 4 +- 10 files changed, 207 insertions(+), 107 deletions(-) create mode 100644 MediaBrowser.Common/Net/HttpRequestOptions.cs diff --git a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs index d653a5c9b8..56b4efd2f0 100644 --- a/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs +++ b/MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Common.Configuration; +using System.Net.Http.Headers; +using MediaBrowser.Common.Configuration; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Model.Logging; @@ -203,50 +204,96 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager /// /// Downloads the contents of a given url into a temporary location /// - /// The URL. - /// The resource pool. - /// The cancellation token. - /// The progress. - /// The user agent. + /// The options. /// Task{System.String}. /// progress + /// /// - public async Task GetTempFile(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken, IProgress progress, string userAgent = null) + public Task GetTempFile(HttpRequestOptions options) { - ValidateParams(url, cancellationToken); + var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp"); + + return GetTempFile(options, tempFile, 0); + } - if (progress == null) + /// + /// Gets the temp file. + /// + /// The options. + /// The temp file. + /// The resume count. + /// Task{System.String}. + /// progress + /// + private async Task GetTempFile(HttpRequestOptions options, string tempFile, int resumeCount) + { + ValidateParams(options.Url, options.CancellationToken); + + if (options.Progress == null) { throw new ArgumentNullException("progress"); } - cancellationToken.ThrowIfCancellationRequested(); + options.CancellationToken.ThrowIfCancellationRequested(); - var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp"); + var message = new HttpRequestMessage(HttpMethod.Get, options.Url); - var message = new HttpRequestMessage(HttpMethod.Get, url); + if (!string.IsNullOrEmpty(options.UserAgent)) + { + message.Headers.Add("User-Agent", options.UserAgent); + } - if (!string.IsNullOrEmpty(userAgent)) + if (options.ResourcePool != null) { - message.Headers.Add("User-Agent", userAgent); + await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } - if (resourcePool != null) + options.Progress.Report(0); + + _logger.Info("HttpClientManager.GetTempFile url: {0}, temp file: {1}", options.Url, tempFile); + + FileStream tempFileStream; + + if (resumeCount > 0 && File.Exists(tempFile)) { - await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); + tempFileStream = new FileStream(tempFile, FileMode.Open, FileAccess.Write, FileShare.Read, + StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous); + + var startPosition = tempFileStream.Length; + tempFileStream.Seek(startPosition, SeekOrigin.Current); + + message.Headers.Range = new RangeHeaderValue(startPosition, null); + + _logger.Info("Resuming from position {1} for {0}", options.Url, startPosition); + } + else + { + tempFileStream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, + StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous); } - _logger.Info("HttpClientManager.GetTempFile url: {0}, temp file: {1}", url, tempFile); + var serverSupportsRangeRequests = false; + + Exception downloadException = null; try { - cancellationToken.ThrowIfCancellationRequested(); + options.CancellationToken.ThrowIfCancellationRequested(); - using (var response = await GetHttpClient(GetHostFromUrl(url)).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) + using (var response = await GetHttpClient(GetHostFromUrl(options.Url)).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, options.CancellationToken).ConfigureAwait(false)) { EnsureSuccessStatusCode(response); - cancellationToken.ThrowIfCancellationRequested(); + options.CancellationToken.ThrowIfCancellationRequested(); + + var rangeValue = string.Join(" ", response.Headers.AcceptRanges.ToArray()); + serverSupportsRangeRequests = rangeValue.IndexOf("bytes", StringComparison.OrdinalIgnoreCase) != -1 || rangeValue.IndexOf("*", StringComparison.OrdinalIgnoreCase) != -1; + + if (!serverSupportsRangeRequests && resumeCount > 0) + { + _logger.Info("Server does not support range requests for {0}", options.Url); + tempFileStream.Position = 0; + } IEnumerable lengthValues; @@ -256,75 +303,97 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager // We're not able to track progress using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { - using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) - { - await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); - } + await stream.CopyToAsync(tempFileStream, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } else { var length = long.Parse(string.Join(string.Empty, lengthValues.ToArray())); - using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), progress.Report, length)) + using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, length)) { - using (var fs = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous)) - { - await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false); - } + await stream.CopyToAsync(tempFileStream, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } - progress.Report(100); + options.Progress.Report(100); - cancellationToken.ThrowIfCancellationRequested(); + options.CancellationToken.ThrowIfCancellationRequested(); } - - return tempFile; } - catch (OperationCanceledException ex) + catch (Exception ex) { - // Cleanup - if (File.Exists(tempFile)) - { - File.Delete(tempFile); - } - - throw GetCancellationException(url, cancellationToken, ex); + downloadException = ex; } - catch (HttpRequestException ex) + finally { - _logger.ErrorException("Error getting response from " + url, ex); + tempFileStream.Dispose(); - // Cleanup - if (File.Exists(tempFile)) + if (options.ResourcePool != null) { - File.Delete(tempFile); + options.ResourcePool.Release(); } - - throw new HttpException(ex.Message, ex); } - catch (Exception ex) + + if (downloadException != null) { - _logger.ErrorException("Error getting response from " + url, ex); + await HandleTempFileException(downloadException, options, tempFile, serverSupportsRangeRequests, resumeCount).ConfigureAwait(false); + } + + return tempFile; + } + /// + /// Handles the temp file exception. + /// + /// The ex. + /// The options. + /// The temp file. + /// if set to true [server supports range requests]. + /// The resume count. + /// Task. + /// + private Task HandleTempFileException(Exception ex, HttpRequestOptions options, string tempFile, bool serverSupportsRangeRequests, int resumeCount) + { + var operationCanceledException = ex as OperationCanceledException; + + if (operationCanceledException != null) + { // Cleanup if (File.Exists(tempFile)) { File.Delete(tempFile); } - throw; + throw GetCancellationException(options.Url, options.CancellationToken, operationCanceledException); } - finally + + _logger.ErrorException("Error getting response from " + options.Url, ex); + + var httpRequestException = ex as HttpRequestException; + + // Cleanup + if (File.Exists(tempFile)) { - if (resourcePool != null) + // Try to resume + if (httpRequestException != null && serverSupportsRangeRequests && resumeCount < options.MaxResumeCount && new FileInfo(tempFile).Length > 0) { - resourcePool.Release(); + _logger.Info("Attempting to resume download from {0}", options.Url); + + return GetTempFile(options, tempFile, resumeCount + 1); } + + File.Delete(tempFile); + } + + if (httpRequestException != null) + { + throw new HttpException(ex.Message, ex); } - } + throw ex; + } + /// /// Downloads the contents of a given url into a MemoryStream /// @@ -519,19 +588,6 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager return Post(url, postData, null, cancellationToken); } - /// - /// Gets the temp file. - /// - /// The URL. - /// The cancellation token. - /// The progress. - /// The user agent. - /// Task{System.String}. - public Task GetTempFile(string url, CancellationToken cancellationToken, IProgress progress, string userAgent = null) - { - return GetTempFile(url, null, cancellationToken, progress, userAgent); - } - /// /// Gets the memory stream. /// diff --git a/MediaBrowser.Common.Implementations/Updates/PackageManager.cs b/MediaBrowser.Common.Implementations/Updates/PackageManager.cs index f73857da18..594027bcbd 100644 --- a/MediaBrowser.Common.Implementations/Updates/PackageManager.cs +++ b/MediaBrowser.Common.Implementations/Updates/PackageManager.cs @@ -71,7 +71,14 @@ namespace MediaBrowser.Common.Implementations.Updates var target = Path.Combine(isArchive ? _appPaths.TempUpdatePath : _appPaths.PluginsPath, package.targetFilename); // Download to temporary file so that, if interrupted, it won't destroy the existing installation - var tempFile = await _httpClient.GetTempFile(package.sourceUrl, cancellationToken, progress).ConfigureAwait(false); + var tempFile = await _httpClient.GetTempFile(new HttpRequestOptions + { + Url = package.sourceUrl, + CancellationToken = cancellationToken, + Progress = progress, + MaxResumeCount = 3 + + }).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); diff --git a/MediaBrowser.Common/MediaBrowser.Common.csproj b/MediaBrowser.Common/MediaBrowser.Common.csproj index d05ebaf74d..bc0d79e454 100644 --- a/MediaBrowser.Common/MediaBrowser.Common.csproj +++ b/MediaBrowser.Common/MediaBrowser.Common.csproj @@ -61,6 +61,7 @@ + diff --git a/MediaBrowser.Common/Net/HttpRequestOptions.cs b/MediaBrowser.Common/Net/HttpRequestOptions.cs new file mode 100644 index 0000000000..a45acb206e --- /dev/null +++ b/MediaBrowser.Common/Net/HttpRequestOptions.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading; + +namespace MediaBrowser.Common.Net +{ + /// + /// Class HttpRequestOptions + /// + public class HttpRequestOptions + { + /// + /// Gets or sets the URL. + /// + /// The URL. + public string Url { get; set; } + + /// + /// Gets or sets the cancellation token. + /// + /// The cancellation token. + public CancellationToken CancellationToken { get; set; } + + /// + /// Gets or sets the resource pool. + /// + /// The resource pool. + public SemaphoreSlim ResourcePool { get; set; } + + /// + /// Gets or sets the user agent. + /// + /// The user agent. + public string UserAgent { get; set; } + + /// + /// Gets or sets the max resume count. + /// + /// The max resume count. + public int MaxResumeCount { get; set; } + + /// + /// Gets or sets the progress. + /// + /// The progress. + public IProgress Progress { get; set; } + } +} diff --git a/MediaBrowser.Common/Net/IHttpClient.cs b/MediaBrowser.Common/Net/IHttpClient.cs index cec3ccff21..f443341a0e 100644 --- a/MediaBrowser.Common/Net/IHttpClient.cs +++ b/MediaBrowser.Common/Net/IHttpClient.cs @@ -49,29 +49,15 @@ namespace MediaBrowser.Common.Net /// The cancellation token. /// Task{Stream}. Task Post(string url, Dictionary postData, CancellationToken cancellationToken); - + /// /// Downloads the contents of a given url into a temporary location /// - /// The URL. - /// The resource pool. - /// The cancellation token. - /// The progress. - /// The user agent. + /// The options. /// Task{System.String}. /// progress /// - Task GetTempFile(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken, IProgress progress, string userAgent = null); - - /// - /// Gets the temp file. - /// - /// The URL. - /// The cancellation token. - /// The progress. - /// The user agent. - /// Task{System.String}. - Task GetTempFile(string url, CancellationToken cancellationToken, IProgress progress, string userAgent = null); + Task GetTempFile(HttpRequestOptions options); /// /// Downloads the contents of a given url into a MemoryStream diff --git a/MediaBrowser.Controller/Providers/ImageFromMediaLocationProvider.cs b/MediaBrowser.Controller/Providers/ImageFromMediaLocationProvider.cs index 56f4792059..4cd9b20a1a 100644 --- a/MediaBrowser.Controller/Providers/ImageFromMediaLocationProvider.cs +++ b/MediaBrowser.Controller/Providers/ImageFromMediaLocationProvider.cs @@ -100,7 +100,7 @@ namespace MediaBrowser.Controller.Providers { var path = item.Images[image]; - return IsInSameDirectory(item, path) && !item.ResolveArgs.GetMetaFileByPath(path).HasValue; + return IsInMetaLocation(item, path) && !item.ResolveArgs.GetMetaFileByPath(path).HasValue; }).ToList(); // Now remove them from the dictionary @@ -122,7 +122,7 @@ namespace MediaBrowser.Controller.Providers } // Only validate paths from the same directory - need to copy to a list because we are going to potentially modify the collection below - var deletedImages = item.BackdropImagePaths.Where(path => IsInSameDirectory(item, path) && !item.ResolveArgs.GetMetaFileByPath(path).HasValue).ToList(); + var deletedImages = item.BackdropImagePaths.Where(path => IsInMetaLocation(item, path) && !item.ResolveArgs.GetMetaFileByPath(path).HasValue).ToList(); // Now remove them from the dictionary foreach (var path in deletedImages) @@ -137,7 +137,7 @@ namespace MediaBrowser.Controller.Providers /// The item. /// The path. /// true if [is in same directory] [the specified item]; otherwise, false. - private bool IsInSameDirectory(BaseItem item, string path) + private bool IsInMetaLocation(BaseItem item, string path) { return string.Equals(Path.GetDirectoryName(path), item.MetaLocation, StringComparison.OrdinalIgnoreCase); } diff --git a/MediaBrowser.WebDashboard/Html/scripts/site.js b/MediaBrowser.WebDashboard/Html/scripts/site.js index cb4ba3bdd5..75521f9fde 100644 --- a/MediaBrowser.WebDashboard/Html/scripts/site.js +++ b/MediaBrowser.WebDashboard/Html/scripts/site.js @@ -397,23 +397,26 @@ var Dashboard = { if (!info.HasPendingRestart) { Dashboard.reloadPage(); } else { - Dashboard.reloadPageWhenServerAvailable(retryCount); + Dashboard.retryReload(retryCount); } - }).fail(function () { - - setTimeout(function () { + }).fail(function() { + Dashboard.retryReload(retryCount); + }); + }, + + retryReload: function (retryCount) { + setTimeout(function () { - retryCount = retryCount || 0; - retryCount++; + retryCount = retryCount || 0; + retryCount++; - if (retryCount < 10) { - Dashboard.reloadPageWhenServerAvailable(retryCount); - } else { - Dashboard.suppressAjaxErrors = false; - } - }, 500); - }); + if (retryCount < 10) { + Dashboard.reloadPageWhenServerAvailable(retryCount); + } else { + Dashboard.suppressAjaxErrors = false; + } + }, 500); }, getPosterViewHtml: function (options) { diff --git a/Nuget/MediaBrowser.Common.Internal.nuspec b/Nuget/MediaBrowser.Common.Internal.nuspec index 09d26ab819..8986333f07 100644 --- a/Nuget/MediaBrowser.Common.Internal.nuspec +++ b/Nuget/MediaBrowser.Common.Internal.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Common.Internal - 3.0.45 + 3.0.47 MediaBrowser.Common.Internal Luke ebr,Luke,scottisafool @@ -12,7 +12,7 @@ Contains common components shared by Media Browser Theatre and Media Browser Server. Not intended for plugin developer consumption. Copyright © Media Browser 2013 - + diff --git a/Nuget/MediaBrowser.Common.nuspec b/Nuget/MediaBrowser.Common.nuspec index 72eec30d4a..847ced431a 100644 --- a/Nuget/MediaBrowser.Common.nuspec +++ b/Nuget/MediaBrowser.Common.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Common - 3.0.45 + 3.0.47 MediaBrowser.Common Media Browser Team ebr,Luke,scottisafool diff --git a/Nuget/MediaBrowser.Server.Core.nuspec b/Nuget/MediaBrowser.Server.Core.nuspec index ba5263d610..46fcd708ef 100644 --- a/Nuget/MediaBrowser.Server.Core.nuspec +++ b/Nuget/MediaBrowser.Server.Core.nuspec @@ -2,7 +2,7 @@ MediaBrowser.Server.Core - 3.0.45 + 3.0.47 Media Browser.Server.Core Media Browser Team ebr,Luke,scottisafool @@ -12,7 +12,7 @@ Contains core components required to build plugins for Media Browser Server. Copyright © Media Browser 2013 - +