From bac75ac6d965311d477091380091293dd16f2b61 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Thu, 20 Mar 2014 00:08:15 -0700 Subject: [PATCH] New: Failed download handling for Nzbget --- .../Extensions/StreamExtensions.cs | 21 ++++++ src/NzbDrone.Common/NzbDrone.Common.csproj | 1 + .../ParserTests/CrapParserFixture.cs | 1 + .../ParserTests/PathParserFixture.cs | 1 + .../Clients/Nzbget/NzbGetQueueItem.cs | 4 + .../Download/Clients/Nzbget/Nzbget.cs | 70 ++++++++++++++---- ...ueResponse.cs => NzbgetBooleanResponse.cs} | 2 +- .../Clients/Nzbget/NzbgetHistoryItem.cs | 18 +++++ .../{NzbGetQueue.cs => NzbgetListResponse.cs} | 4 +- .../Clients/Nzbget/NzbgetParameter.cs | 9 +++ .../Download/Clients/Nzbget/NzbgetProxy.cs | 73 +++++++++++++++++-- .../Download/Clients/Sabnzbd/SabnzbdProxy.cs | 18 +---- .../Download/DownloadClientBase.cs | 1 - src/NzbDrone.Core/NzbDrone.Core.csproj | 6 +- 14 files changed, 189 insertions(+), 40 deletions(-) create mode 100644 src/NzbDrone.Common/Extensions/StreamExtensions.cs rename src/NzbDrone.Core/Download/Clients/Nzbget/{EnqueueResponse.cs => NzbgetBooleanResponse.cs} (81%) create mode 100644 src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetHistoryItem.cs rename src/NzbDrone.Core/Download/Clients/Nzbget/{NzbGetQueue.cs => NzbgetListResponse.cs} (71%) create mode 100644 src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetParameter.cs diff --git a/src/NzbDrone.Common/Extensions/StreamExtensions.cs b/src/NzbDrone.Common/Extensions/StreamExtensions.cs new file mode 100644 index 000000000..6283f5fc0 --- /dev/null +++ b/src/NzbDrone.Common/Extensions/StreamExtensions.cs @@ -0,0 +1,21 @@ +using System.IO; + +namespace NzbDrone.Common.Extensions +{ + public static class StreamExtensions + { + public static byte[] ToBytes(this Stream input) + { + var buffer = new byte[16 * 1024]; + using (var ms = new MemoryStream()) + { + int read; + while ((read = input.Read(buffer, 0, buffer.Length)) > 0) + { + ms.Write(buffer, 0, read); + } + return ms.ToArray(); + } + } + } +} diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 6ee9aeeb4..9d1e80e38 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -107,6 +107,7 @@ + diff --git a/src/NzbDrone.Core.Test/ParserTests/CrapParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/CrapParserFixture.cs index 83fd99d00..9ed19f36f 100644 --- a/src/NzbDrone.Core.Test/ParserTests/CrapParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/CrapParserFixture.cs @@ -27,6 +27,7 @@ namespace NzbDrone.Core.Test.ParserTests [TestCase("86420f8ee425340d8894bf3bc636b66404b95f18")] [TestCase("ce39afb7da6cf7c04eba3090f0a309f609883862")] [TestCase("THIS SHOULD NEVER PARSE")] + [TestCase("Vh1FvU3bJXw6zs8EEUX4bMo5vbbMdHghxHirc.mkv")] public void should_not_parse_crap(string title) { Parser.Parser.ParseTitle(title).Should().BeNull(); diff --git a/src/NzbDrone.Core.Test/ParserTests/PathParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/PathParserFixture.cs index d9f46806a..2eed10534 100644 --- a/src/NzbDrone.Core.Test/ParserTests/PathParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/PathParserFixture.cs @@ -31,6 +31,7 @@ namespace NzbDrone.Core.Test.ParserTests [TestCase(@"C:\Test\Unsorted\The.Big.Bang.Theory.S01E01.720p.HDTV\tbbt101.avi", 1, 1)] [TestCase(@"C:\Test\Unsorted\Terminator.The.Sarah.Connor.Chronicles.S02E19.720p.BluRay.x264-SiNNERS-RP\ba27283b17c00d01193eacc02a8ba98eeb523a76.mkv", 2, 19)] [TestCase(@"C:\Test\Unsorted\Terminator.The.Sarah.Connor.Chronicles.S02E18.720p.BluRay.x264-SiNNERS-RP\45a55debe3856da318cc35882ad07e43cd32fd15.mkv", 2, 18)] + [TestCase(@"C:\Test\The.Blacklist.S01E16.720p.HDTV.X264-DIMENSION\XRmZciqkBopq4851Ddbipe\Vh1FvU3bJXw6zs8EEUX4bMo5vbbMdHghxHirc.mkv", 1, 16)] public void should_parse_from_path(string path, int season, int episode) { var result = Parser.Parser.ParsePath(path); diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueueItem.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueueItem.cs index 39bc8eb51..38292bb26 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueueItem.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueueItem.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace NzbDrone.Core.Download.Clients.Nzbget { @@ -6,10 +7,13 @@ namespace NzbDrone.Core.Download.Clients.Nzbget { private string _nzbName; public Int32 NzbId { get; set; } + public Int32 FirstId { get; set; } + public Int32 LastId { get; set; } public string NzbName { get; set; } public String Category { get; set; } public Int32 FileSizeMb { get; set; } public Int32 RemainingSizeMb { get; set; } public Int32 PausedSizeMb { get; set; } + public List Parameters { get; set; } } } diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs index 07221549d..46d4cd715 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using NLog; +using NzbDrone.Common; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; @@ -13,14 +14,17 @@ namespace NzbDrone.Core.Download.Clients.Nzbget { private readonly INzbgetProxy _proxy; private readonly IParsingService _parsingService; + private readonly IHttpProvider _httpProvider; private readonly Logger _logger; public Nzbget(INzbgetProxy proxy, IParsingService parsingService, + IHttpProvider httpProvider, Logger logger) { _proxy = proxy; _parsingService = parsingService; + _httpProvider = httpProvider; _logger = logger; } @@ -29,16 +33,18 @@ namespace NzbDrone.Core.Download.Clients.Nzbget var url = remoteEpisode.Release.DownloadUrl; var title = remoteEpisode.Release.Title + ".nzb"; - string cat = Settings.TvCategory; + string category = Settings.TvCategory; int priority = remoteEpisode.IsRecentEpisode() ? Settings.RecentTvPriority : Settings.OlderTvPriority; _logger.Info("Adding report [{0}] to the queue.", title); - var success = _proxy.AddNzb(Settings, title, cat, priority, false, url); - - _logger.Debug("Queue Response: [{0}]", success); + using (var nzb = _httpProvider.DownloadStream(url)) + { + _logger.Info("Adding report [{0}] to the queue.", title); + var response = _proxy.DownloadNzb(nzb, title, category, priority, Settings); - return null; + return response; + } } public override IEnumerable GetQueue() @@ -57,14 +63,16 @@ namespace NzbDrone.Core.Download.Clients.Nzbget var queueItems = new List(); - foreach (var nzbGetQueueItem in queue) + foreach (var item in queue) { + var droneParameter = item.Parameters.SingleOrDefault(p => p.Name == "drone"); + var queueItem = new QueueItem(); - queueItem.Id = nzbGetQueueItem.NzbId.ToString(); - queueItem.Title = nzbGetQueueItem.NzbName; - queueItem.Size = nzbGetQueueItem.FileSizeMb; - queueItem.Sizeleft = nzbGetQueueItem.RemainingSizeMb; - queueItem.Status = nzbGetQueueItem.FileSizeMb == nzbGetQueueItem.PausedSizeMb ? "paused" : "queued"; + queueItem.Id = droneParameter == null ? item.NzbId.ToString() : droneParameter.Value.ToString(); + queueItem.Title = item.NzbName; + queueItem.Size = item.FileSizeMb; + queueItem.Sizeleft = item.RemainingSizeMb; + queueItem.Status = item.FileSizeMb == item.PausedSizeMb ? "paused" : "queued"; var parsedEpisodeInfo = Parser.Parser.ParseTitle(queueItem.Title); if (parsedEpisodeInfo == null) continue; @@ -81,7 +89,43 @@ namespace NzbDrone.Core.Download.Clients.Nzbget public override IEnumerable GetHistory(int start = 0, int limit = 10) { - return new HistoryItem[0]; + List history; + + try + { + history = _proxy.GetHistory(Settings); + } + catch (DownloadClientException ex) + { + _logger.ErrorException(ex.Message, ex); + return Enumerable.Empty(); + } + + var historyItems = new List(); + var successStatues = new[] {"SUCCESS", "NONE"}; + + foreach (var item in history) + { + var droneParameter = item.Parameters.SingleOrDefault(p => p.Name == "drone"); + var status = successStatues.Contains(item.ParStatus) && + successStatues.Contains(item.ScriptStatus) + ? HistoryStatus.Completed + : HistoryStatus.Failed; + + var historyItem = new HistoryItem(); + historyItem.Id = droneParameter == null ? item.Id.ToString() : droneParameter.Value.ToString(); + historyItem.Title = item.Name; + historyItem.Size = item.FileSizeMb.ToString(); //Why is this a string? + historyItem.DownloadTime = 0; + historyItem.Storage = item.DestDir; + historyItem.Category = item.Category; + historyItem.Message = String.Format("PAR Status: {0} - Script Status: {1}", item.ParStatus, item.ScriptStatus); + historyItem.Status = status; + + historyItems.Add(historyItem); + } + + return historyItems; } public override void RemoveFromQueue(string id) @@ -91,7 +135,7 @@ namespace NzbDrone.Core.Download.Clients.Nzbget public override void RemoveFromHistory(string id) { - throw new NotImplementedException(); + _proxy.RemoveFromHistory(id, Settings); } public override void Test() diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/EnqueueResponse.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetBooleanResponse.cs similarity index 81% rename from src/NzbDrone.Core/Download/Clients/Nzbget/EnqueueResponse.cs rename to src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetBooleanResponse.cs index f16799151..6c536ba7d 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/EnqueueResponse.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetBooleanResponse.cs @@ -2,7 +2,7 @@ namespace NzbDrone.Core.Download.Clients.Nzbget { - public class EnqueueResponse + public class NzbgetBooleanResponse { public String Version { get; set; } public Boolean Result { get; set; } diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetHistoryItem.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetHistoryItem.cs new file mode 100644 index 000000000..af90178a8 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetHistoryItem.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; + +namespace NzbDrone.Core.Download.Clients.Nzbget +{ + public class NzbgetHistoryItem + { + private string _nzbName; + public Int32 Id { get; set; } + public String Name { get; set; } + public String Category { get; set; } + public Int32 FileSizeMb { get; set; } + public String ParStatus { get; set; } + public String ScriptStatus { get; set; } + public String DestDir { get; set; } + public List Parameters { get; set; } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueue.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetListResponse.cs similarity index 71% rename from src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueue.cs rename to src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetListResponse.cs index f7ec8a1be..bb51dbcc6 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbGetQueue.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetListResponse.cs @@ -4,11 +4,11 @@ using Newtonsoft.Json; namespace NzbDrone.Core.Download.Clients.Nzbget { - public class NzbgetQueue + public class NzbgetListResponse { public String Version { get; set; } [JsonProperty(PropertyName = "result")] - public List QueueItems { get; set; } + public List QueueItems { get; set; } } } diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetParameter.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetParameter.cs new file mode 100644 index 000000000..d2b728e85 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetParameter.cs @@ -0,0 +1,9 @@ +using System; +namespace NzbDrone.Core.Download.Clients.Nzbget +{ + public class NzbgetParameter + { + public String Name { get; set; } + public object Value { get; set; } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs index 678a63a56..91d4de26d 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using NLog; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Serializer; using NzbDrone.Core.Rest; using RestSharp; @@ -9,9 +12,11 @@ namespace NzbDrone.Core.Download.Clients.Nzbget { public interface INzbgetProxy { - bool AddNzb(NzbgetSettings settings, params object[] parameters); + string DownloadNzb(Stream nzb, string title, string category, int priority, NzbgetSettings settings); List GetQueue(NzbgetSettings settings); + List GetHistory(NzbgetSettings settings); VersionResponse GetVersion(NzbgetSettings settings); + void RemoveFromHistory(string id, NzbgetSettings settings); } public class NzbgetProxy : INzbgetProxy @@ -23,18 +28,50 @@ namespace NzbDrone.Core.Download.Clients.Nzbget _logger = logger; } - public bool AddNzb(NzbgetSettings settings, params object[] parameters) + public string DownloadNzb(Stream nzb, string title, string category, int priority, NzbgetSettings settings) { - var request = BuildRequest(new JsonRequest("appendurl", parameters)); + var parameters = new object[] { title, category, priority, false, Convert.ToBase64String(nzb.ToBytes()) }; + var request = BuildRequest(new JsonRequest("append", parameters)); - return Json.Deserialize(ProcessRequest(request, settings)).Result; + var response = Json.Deserialize(ProcessRequest(request, settings)); + _logger.Debug("Queue Response: [{0}]", response.Result); + + if (!response.Result) + { + return null; + } + + var queue = GetQueue(settings); + var item = queue.FirstOrDefault(q => q.NzbName == title.Substring(0, title.Length - 4)); + + if (item == null) + { + return null; + } + + var droneId = Guid.NewGuid().ToString().Replace("-", ""); + var editResult = EditQueue("GroupSetParameter", 0, "drone=" + droneId, item.LastId, settings); + + if (editResult) + { + _logger.Debug("Nzbget download drone parameter set to: {0}", droneId); + } + + return droneId; } public List GetQueue(NzbgetSettings settings) { var request = BuildRequest(new JsonRequest("listgroups")); - return Json.Deserialize(ProcessRequest(request, settings)).QueueItems; + return Json.Deserialize>(ProcessRequest(request, settings)).QueueItems; + } + + public List GetHistory(NzbgetSettings settings) + { + var request = BuildRequest(new JsonRequest("history")); + + return Json.Deserialize>(ProcessRequest(request, settings)).QueueItems; } public VersionResponse GetVersion(NzbgetSettings settings) @@ -44,6 +81,32 @@ namespace NzbDrone.Core.Download.Clients.Nzbget return Json.Deserialize(ProcessRequest(request, settings)); } + public void RemoveFromHistory(string id, NzbgetSettings settings) + { + var history = GetHistory(settings); + var item = history.SingleOrDefault(h => h.Parameters.SingleOrDefault(p => p.Name == "drone") != null); + + if (item == null) + { + _logger.Warn("Unable to remove item from nzbget's history, Unknown ID: {0}", id); + return; + } + + if (!EditQueue("HistoryDelete", 0, "", item.Id, settings)) + { + _logger.Warn("Failed to remove item from nzbget history, {0} [{1}]", item.Name, item.Id); + } + } + + private bool EditQueue(string command, int offset, string editText, int id, NzbgetSettings settings) + { + var parameters = new object[] { command, offset, editText, id }; + var request = BuildRequest(new JsonRequest("editqueue", parameters)); + var response = Json.Deserialize(ProcessRequest(request, settings)); + + return response.Result; + } + private string ProcessRequest(IRestRequest restRequest, NzbgetSettings settings) { var client = BuildClient(settings); diff --git a/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs b/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs index e3b987e4c..51f18cac5 100644 --- a/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs @@ -3,6 +3,7 @@ using System.IO; using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Serializer; using NzbDrone.Core.Download.Clients.Sabnzbd.Responses; using NzbDrone.Core.Instrumentation.Extensions; @@ -35,7 +36,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd var request = new RestRequest(Method.POST); var action = String.Format("mode=addfile&cat={0}&priority={1}", category, priority); - request.AddFile("name", ReadFully(nzb), title, "application/x-nzb"); + request.AddFile("name", nzb.ToBytes(), title, "application/x-nzb"); SabnzbdAddResponse response; @@ -161,20 +162,5 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd if (result.Failed) throw new DownloadClientException("Error response received from SABnzbd: {0}", result.Error); } - - //TODO: Find a better home for this - private byte[] ReadFully(Stream input) - { - byte[] buffer = new byte[16 * 1024]; - using (MemoryStream ms = new MemoryStream()) - { - int read; - while ((read = input.Read(buffer, 0, buffer.Length)) > 0) - { - ms.Write(buffer, 0, read); - } - return ms.ToArray(); - } - } } } diff --git a/src/NzbDrone.Core/Download/DownloadClientBase.cs b/src/NzbDrone.Core/Download/DownloadClientBase.cs index ceaf945a7..b38131161 100644 --- a/src/NzbDrone.Core/Download/DownloadClientBase.cs +++ b/src/NzbDrone.Core/Download/DownloadClientBase.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using NLog; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 3f75a0767..6010d826c 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -236,6 +236,8 @@ + + @@ -500,10 +502,10 @@ - + - +