From 7548c17007bbd33f3413ab16d95d895e37f24ed5 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sat, 2 Oct 2021 10:55:05 -0500 Subject: [PATCH] New: Aria2 Fixes #2334 Fixes #2385 --- .../Extensions/PathExtensions.cs | 28 ++ .../Download/Clients/Aria2/Aria2.cs | 268 ++++++++++++++++++ .../Download/Clients/Aria2/Aria2Containers.cs | 111 ++++++++ .../Download/Clients/Aria2/Aria2Proxy.cs | 225 +++++++++++++++ .../Download/Clients/Aria2/Aria2Settings.cs | 49 ++++ 5 files changed, 681 insertions(+) create mode 100644 src/NzbDrone.Core/Download/Clients/Aria2/Aria2.cs create mode 100644 src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs create mode 100644 src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs create mode 100644 src/NzbDrone.Core/Download/Clients/Aria2/Aria2Settings.cs diff --git a/src/NzbDrone.Common/Extensions/PathExtensions.cs b/src/NzbDrone.Common/Extensions/PathExtensions.cs index 3b78858ac..3618afd62 100644 --- a/src/NzbDrone.Common/Extensions/PathExtensions.cs +++ b/src/NzbDrone.Common/Extensions/PathExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using NzbDrone.Common.Disk; using NzbDrone.Common.EnsureThat; @@ -238,6 +239,33 @@ namespace NzbDrone.Common.Extensions return null; } + public static string GetLongestCommonPath(this List paths) + { + var firstPath = paths.First(); + var length = firstPath.Length; + + for (int i = 1; i < paths.Count; i++) + { + var path = paths[i]; + + length = Math.Min(length, path.Length); + + for (int characterIndex = 0; characterIndex < length; characterIndex++) + { + if (path[characterIndex] != firstPath[characterIndex]) + { + length = characterIndex; + break; + } + } + } + + var substring = firstPath.Substring(0, length); + var lastSeparatorIndex = substring.LastIndexOfAny(new[] { '/', '\\' }); + + return substring.Substring(0, lastSeparatorIndex); + } + public static string ProcessNameToExe(this string processName, PlatformType runtime) { if (OsInfo.IsWindows || runtime != PlatformType.NetCore) diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2.cs new file mode 100644 index 000000000..89949a257 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2.cs @@ -0,0 +1,268 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using CookComputing.XmlRpc; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.MediaFiles.TorrentInfo; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.RemotePathMappings; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Download.Clients.Aria2 +{ + public class Aria2 : TorrentClientBase + { + private readonly IAria2Proxy _proxy; + + public override string Name => "Aria2"; + + public Aria2(IAria2Proxy proxy, + ITorrentFileInfoReader torrentFileInfoReader, + IHttpClient httpClient, + IConfigService configService, + IDiskProvider diskProvider, + IRemotePathMappingService remotePathMappingService, + Logger logger) + : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger) + { + _proxy = proxy; + } + + protected override string AddFromMagnetLink(RemoteAlbum remoteAlbum, string hash, string magnetLink) + { + var gid = _proxy.AddMagnet(Settings, magnetLink); + + var tries = 10; + var retryDelay = 500; + + // Wait a bit for the magnet to be resolved. + if (!WaitForTorrent(gid, hash, tries, retryDelay)) + { + _logger.Warn($"Aria2 could not add magnent within {tries * retryDelay / 1000} seconds, download may remain stuck: {magnetLink}."); + return hash; + } + + _logger.Debug($"Aria2 AddFromMagnetLink '{hash}' -> '{gid}'"); + + return hash; + } + + protected override string AddFromTorrentFile(RemoteAlbum remoteAlbum, string hash, string filename, byte[] fileContent) + { + var gid = _proxy.AddTorrent(Settings, fileContent); + + var tries = 10; + var retryDelay = 500; + + // Wait a bit for the magnet to be resolved. + if (!WaitForTorrent(gid, hash, tries, retryDelay)) + { + _logger.Warn($"Aria2 could not add torrent within {tries * retryDelay / 1000} seconds, download may remain stuck: {filename}."); + return hash; + } + + return hash; + } + + public override IEnumerable GetItems() + { + var torrents = _proxy.GetTorrents(Settings); + + foreach (var torrent in torrents) + { + var firstFile = torrent.Files?.FirstOrDefault(); + + //skip metadata download + if (firstFile?.Path?.Contains("[METADATA]") == true) + { + continue; + } + + var completedLength = long.Parse(torrent.CompletedLength); + var totalLength = long.Parse(torrent.TotalLength); + var uploadedLength = long.Parse(torrent.UploadLength); + var downloadSpeed = long.Parse(torrent.DownloadSpeed); + + var status = DownloadItemStatus.Failed; + var title = ""; + + if (torrent.Bittorrent?.ContainsKey("info") == true && ((XmlRpcStruct)torrent.Bittorrent["info"]).ContainsKey("name")) + { + title = ((XmlRpcStruct)torrent.Bittorrent["info"])["name"].ToString(); + } + + switch (torrent.Status) + { + case "active": + if (completedLength == totalLength) + { + status = DownloadItemStatus.Completed; + } + else + { + status = DownloadItemStatus.Downloading; + } + + break; + case "waiting": + status = DownloadItemStatus.Queued; + break; + case "paused": + status = DownloadItemStatus.Paused; + break; + case "error": + status = DownloadItemStatus.Failed; + break; + case "complete": + status = DownloadItemStatus.Completed; + break; + case "removed": + status = DownloadItemStatus.Failed; + break; + } + + _logger.Trace($"- aria2 getstatus hash:'{torrent.InfoHash}' gid:'{torrent.Gid}' status:'{status}' total:{totalLength} completed:'{completedLength}'"); + + var outputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(GetOutputPath(torrent))); + + yield return new DownloadClientItem + { + CanMoveFiles = false, + CanBeRemoved = torrent.Status == "complete", + Category = null, + DownloadClientInfo = DownloadClientItemClientInfo.FromDownloadClient(this), + DownloadId = torrent.InfoHash?.ToUpper(), + IsEncrypted = false, + Message = torrent.ErrorMessage, + OutputPath = outputPath, + RemainingSize = totalLength - completedLength, + RemainingTime = downloadSpeed == 0 ? (TimeSpan?)null : new TimeSpan(0, 0, (int)((totalLength - completedLength) / downloadSpeed)), + Removed = torrent.Status == "removed", + SeedRatio = totalLength > 0 ? (double)uploadedLength / totalLength : 0, + Status = status, + Title = title, + TotalSize = totalLength, + }; + } + } + + public override void RemoveItem(string downloadId, bool deleteData) + { + // Aria2 doesn't support file deletion: https://github.com/aria2/aria2/issues/728 + var hash = downloadId.ToLower(); + var aria2Item = _proxy.GetTorrents(Settings).FirstOrDefault(t => t.InfoHash?.ToLower() == hash); + + if (aria2Item == null) + { + _logger.Error($"Aria2 could not find infoHash '{hash}' for deletion."); + return; + } + + _logger.Debug($"Aria2 removing hash:'{hash}' gid:'{aria2Item.Gid}'"); + + if (aria2Item.Status == "complete" || aria2Item.Status == "error" || aria2Item.Status == "removed") + { + if (!_proxy.RemoveCompletedTorrent(Settings, aria2Item.Gid)) + { + _logger.Error($"Aria2 error while deleting {hash}."); + + return; + } + } + else + { + if (!_proxy.RemoveTorrent(Settings, aria2Item.Gid)) + { + _logger.Error($"Aria2 error while deleting {hash}."); + + return; + } + } + + if (deleteData) + { + DeleteItemData(downloadId); + } + } + + public override DownloadClientInfo GetStatus() + { + var destDir = _proxy.GetGlobals(Settings); + + return new DownloadClientInfo + { + IsLocalhost = Settings.Host.Contains("127.0.0.1") || Settings.Host.Contains("localhost"), + OutputRootFolders = new List { _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(destDir["dir"])) } + }; + } + + private bool WaitForTorrent(string gid, string hash, int tries, int retryDelay) + { + for (var i = 0; i < tries; i++) + { + var found = _proxy.GetFromGID(Settings, gid); + + if (found?.InfoHash?.ToLower() == hash?.ToLower()) + { + return true; + } + + Thread.Sleep(retryDelay); + } + + _logger.Debug("Could not find hash {0} in {1} tries at {2} ms intervals.", hash, tries, retryDelay); + + return false; + } + + protected override void Test(List failures) + { + failures.AddIfNotNull(TestConnection()); + + if (failures.HasErrors()) + { + return; + } + } + + private ValidationFailure TestConnection() + { + try + { + var version = _proxy.GetVersion(Settings); + + if (new Version(version) < new Version("1.34.0")) + { + return new ValidationFailure(string.Empty, "Aria2 version should be at least 1.34.0. Version reported is {0}", version); + } + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to test Aria2"); + + return new NzbDroneValidationFailure("Host", "Unable to connect to Aria2") + { + DetailedDescription = ex.Message + }; + } + + return null; + } + + private string GetOutputPath(Aria2Status torrent) + { + if (torrent.Files.Length == 1) + { + return torrent.Files.First().Path; + } + + return torrent.Files.Select(f => f.Path).ToList().GetLongestCommonPath(); + } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs new file mode 100644 index 000000000..d4ab5a49c --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs @@ -0,0 +1,111 @@ +using CookComputing.XmlRpc; + +namespace NzbDrone.Core.Download.Clients.Aria2 +{ + public class Aria2Version + { + [XmlRpcMember("version")] + public string Version; + + [XmlRpcMember("enabledFeatures")] + public string[] EnabledFeatures; + } + + public class Aria2Uri + { + [XmlRpcMember("status")] + public string Status; + + [XmlRpcMember("uri")] + public string Uri; + } + + public class Aria2File + { + [XmlRpcMember("index")] + public string Index; + + [XmlRpcMember("length")] + public string Length; + + [XmlRpcMember("completedLength")] + public string CompletedLength; + + [XmlRpcMember("path")] + public string Path; + + [XmlRpcMember("selected")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string Selected; + + [XmlRpcMember("uris")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public Aria2Uri[] Uris; + } + + public class Aria2Status + { + [XmlRpcMember("bittorrent")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public XmlRpcStruct Bittorrent; + + [XmlRpcMember("bitfield")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string Bitfield; + + [XmlRpcMember("infoHash")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string InfoHash; + + [XmlRpcMember("completedLength")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string CompletedLength; + + [XmlRpcMember("connections")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string Connections; + + [XmlRpcMember("dir")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string Dir; + + [XmlRpcMember("downloadSpeed")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string DownloadSpeed; + + [XmlRpcMember("files")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public Aria2File[] Files; + + [XmlRpcMember("gid")] + public string Gid; + + [XmlRpcMember("numPieces")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string NumPieces; + + [XmlRpcMember("pieceLength")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string PieceLength; + + [XmlRpcMember("status")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string Status; + + [XmlRpcMember("totalLength")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string TotalLength; + + [XmlRpcMember("uploadLength")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string UploadLength; + + [XmlRpcMember("uploadSpeed")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string UploadSpeed; + + [XmlRpcMember("errorMessage")] + [XmlRpcMissingMapping(MappingAction.Ignore)] + public string ErrorMessage; + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs new file mode 100644 index 000000000..3e7b5a6be --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs @@ -0,0 +1,225 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Net; +using CookComputing.XmlRpc; +using NLog; + +namespace NzbDrone.Core.Download.Clients.Aria2 +{ + public interface IAria2Proxy + { + string GetVersion(Aria2Settings settings); + string AddMagnet(Aria2Settings settings, string magnet); + string AddTorrent(Aria2Settings settings, byte[] torrent); + bool RemoveTorrent(Aria2Settings settings, string gid); + bool RemoveCompletedTorrent(Aria2Settings settings, string gid); + Dictionary GetGlobals(Aria2Settings settings); + List GetTorrents(Aria2Settings settings); + Aria2Status GetFromGID(Aria2Settings settings, string gid); + } + + public interface IAria2 : IXmlRpcProxy + { + [XmlRpcMethod("aria2.getVersion")] + Aria2Version GetVersion(string token); + + [XmlRpcMethod("aria2.addUri")] + string AddUri(string token, string[] uri); + + [XmlRpcMethod("aria2.addTorrent")] + string AddTorrent(string token, byte[] torrent); + + [XmlRpcMethod("aria2.forceRemove")] + string Remove(string token, string gid); + + [XmlRpcMethod("aria2.removeDownloadResult")] + string RemoveResult(string token, string gid); + + [XmlRpcMethod("aria2.tellStatus")] + Aria2Status GetFromGid(string token, string gid); + + [XmlRpcMethod("aria2.getGlobalOption")] + XmlRpcStruct GetGlobalOption(string token); + + [XmlRpcMethod("aria2.tellActive")] + Aria2Status[] GetActive(string token); + + [XmlRpcMethod("aria2.tellWaiting")] + Aria2Status[] GetWaiting(string token, int offset, int num); + + [XmlRpcMethod("aria2.tellStopped")] + Aria2Status[] GetStopped(string token, int offset, int num); + } + + public class Aria2Proxy : IAria2Proxy + { + private readonly Logger _logger; + + public Aria2Proxy(Logger logger) + { + _logger = logger; + } + + private string GetToken(Aria2Settings settings) + { + return $"token:{settings?.SecretToken}"; + } + + private string GetURL(Aria2Settings settings) + { + return $"http{(settings.UseSsl ? "s" : "")}://{settings.Host}:{settings.Port}{settings.RpcPath}"; + } + + public string GetVersion(Aria2Settings settings) + { + _logger.Trace("> aria2.getVersion"); + + var client = BuildClient(settings); + var version = ExecuteRequest(() => client.GetVersion(GetToken(settings))); + + _logger.Trace("< aria2.getVersion"); + + return version.Version; + } + + public Aria2Status GetFromGID(Aria2Settings settings, string gid) + { + _logger.Trace("> aria2.tellStatus"); + + var client = BuildClient(settings); + var found = ExecuteRequest(() => client.GetFromGid(GetToken(settings), gid)); + + _logger.Trace("< aria2.tellStatus"); + + return found; + } + + public List GetTorrents(Aria2Settings settings) + { + _logger.Trace("> aria2.tellActive"); + + var client = BuildClient(settings); + + var active = ExecuteRequest(() => client.GetActive(GetToken(settings))); + + _logger.Trace("< aria2.tellActive"); + + _logger.Trace("> aria2.tellWaiting"); + + var waiting = ExecuteRequest(() => client.GetWaiting(GetToken(settings), 0, 10 * 1024)); + + _logger.Trace("< aria2.tellWaiting"); + + _logger.Trace("> aria2.tellStopped"); + + var stopped = ExecuteRequest(() => client.GetStopped(GetToken(settings), 0, 10 * 1024)); + + _logger.Trace("< aria2.tellStopped"); + + var items = new List(); + + items.AddRange(active); + items.AddRange(waiting); + items.AddRange(stopped); + + return items; + } + + public Dictionary GetGlobals(Aria2Settings settings) + { + _logger.Trace("> aria2.getGlobalOption"); + + var client = BuildClient(settings); + var options = ExecuteRequest(() => client.GetGlobalOption(GetToken(settings))); + + _logger.Trace("< aria2.getGlobalOption"); + + var ret = new Dictionary(); + + foreach (DictionaryEntry option in options) + { + ret.Add(option.Key.ToString(), option.Value?.ToString()); + } + + return ret; + } + + public string AddMagnet(Aria2Settings settings, string magnet) + { + _logger.Trace("> aria2.addUri"); + + var client = BuildClient(settings); + var gid = ExecuteRequest(() => client.AddUri(GetToken(settings), new[] { magnet })); + + _logger.Trace("< aria2.addUri"); + + return gid; + } + + public string AddTorrent(Aria2Settings settings, byte[] torrent) + { + _logger.Trace("> aria2.addTorrent"); + + var client = BuildClient(settings); + var gid = ExecuteRequest(() => client.AddTorrent(GetToken(settings), torrent)); + + _logger.Trace("< aria2.addTorrent"); + + return gid; + } + + public bool RemoveTorrent(Aria2Settings settings, string gid) + { + _logger.Trace("> aria2.forceRemove"); + + var client = BuildClient(settings); + var gidres = ExecuteRequest(() => client.Remove(GetToken(settings), gid)); + + _logger.Trace("< aria2.forceRemove"); + + return gid == gidres; + } + + public bool RemoveCompletedTorrent(Aria2Settings settings, string gid) + { + _logger.Trace("> aria2.removeDownloadResult"); + + var client = BuildClient(settings); + var result = ExecuteRequest(() => client.RemoveResult(GetToken(settings), gid)); + + _logger.Trace("< aria2.removeDownloadResult"); + + return result == "OK"; + } + + private IAria2 BuildClient(Aria2Settings settings) + { + var client = XmlRpcProxyGen.Create(); + client.Url = GetURL(settings); + + return client; + } + + private T ExecuteRequest(Func task) + { + try + { + return task(); + } + catch (XmlRpcServerException ex) + { + throw new DownloadClientException("Unable to connect to aria2, please check your settings", ex); + } + catch (WebException ex) + { + if (ex.Status == WebExceptionStatus.TrustFailure) + { + throw new DownloadClientUnavailableException("Unable to connect to aria2, certificate validation failed.", ex); + } + + throw new DownloadClientUnavailableException("Unable to connect to aria2, please check your settings", ex); + } + } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Settings.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Settings.cs new file mode 100644 index 000000000..a78732c52 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Settings.cs @@ -0,0 +1,49 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Download.Clients.Aria2 +{ + public class Aria2SettingsValidator : AbstractValidator + { + public Aria2SettingsValidator() + { + RuleFor(c => c.Host).ValidHost(); + } + } + + public class Aria2Settings : IProviderConfig + { + private static readonly Aria2SettingsValidator Validator = new Aria2SettingsValidator(); + + public Aria2Settings() + { + Host = "localhost"; + Port = 6800; + RpcPath = "/rpc"; + UseSsl = false; + SecretToken = "MySecretToken"; + } + + [FieldDefinition(0, Label = "Host", Type = FieldType.Textbox)] + public string Host { get; set; } + + [FieldDefinition(1, Label = "Port", Type = FieldType.Number)] + public int Port { get; set; } + + [FieldDefinition(2, Label = "RPC Path", Type = FieldType.Textbox)] + public string RpcPath { get; set; } + + [FieldDefinition(3, Label = "Use SSL", Type = FieldType.Checkbox)] + public bool UseSsl { get; set; } + + [FieldDefinition(4, Label = "Secret token", Type = FieldType.Password, Privacy = PrivacyLevel.Password)] + public string SecretToken { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +}