From 455b76c45c233debc92a9cdc66cefa729f23f87f Mon Sep 17 00:00:00 2001 From: Bogdan Date: Sat, 20 May 2023 01:36:25 +0300 Subject: [PATCH] New: Add TorrentRssIndexer Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> --- .../TorrentRss/TorrentRssIndexer.cs | 92 ++++++ .../TorrentRssIndexerParserSettings.cs | 12 + .../TorrentRssIndexerRequestGenerator.cs | 63 ++++ .../TorrentRss/TorrentRssIndexerSettings.cs | 51 +++ .../TorrentRss/TorrentRssParserFactory.cs | 65 ++++ .../TorrentRss/TorrentRssSettingsDetector.cs | 305 ++++++++++++++++++ src/NzbDrone.Core/Indexers/IndexerFactory.cs | 5 +- src/NzbDrone.Core/Indexers/RssParser.cs | 42 +-- .../Indexers/TorrentRssParser.cs | 78 ++++- 9 files changed, 683 insertions(+), 30 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexer.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerParserSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssParserFactory.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssSettingsDetector.cs diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexer.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexer.cs new file mode 100644 index 000000000..a9ccb4f32 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexer.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using NLog; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public class TorrentRssIndexer : TorrentIndexerBase + { + private readonly ITorrentRssParserFactory _torrentRssParserFactory; + + public override string Name => "Torrent RSS Feed"; + public override string[] IndexerUrls => new[] { "" }; + public override string Description => "Generic RSS Feed containing torrents"; + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override IndexerPrivacy Privacy => IndexerPrivacy.Public; + public override int PageSize => 0; + public override IndexerCapabilities Capabilities => SetCapabilities(); + + public TorrentRssIndexer(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger, ITorrentRssParserFactory torrentRssParserFactory) + : base(httpClient, eventAggregator, indexerStatusService, configService, logger) + { + _torrentRssParserFactory = torrentRssParserFactory; + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new TorrentRssIndexerRequestGenerator { Settings = Settings }; + } + + public override IParseIndexerResponse GetParser() + { + return _torrentRssParserFactory.GetParser(Settings); + } + + public override IEnumerable DefaultDefinitions + { + get + { + yield return GetDefinition("showRSS", "showRSS is a service that allows you to keep track of your favorite TV shows", GetSettings("https://showrss.info/other/all.rss", allowZeroSize: true, defaultReleaseSize: 512)); + yield return GetDefinition("Torrent RSS Feed", "Generic RSS Feed containing torrents", GetSettings("")); + } + } + + private IndexerDefinition GetDefinition(string name, string description, TorrentRssIndexerSettings settings) + { + return new IndexerDefinition + { + Enable = true, + Name = name, + Description = description, + Implementation = GetType().Name, + Settings = settings, + Protocol = DownloadProtocol.Torrent, + SupportsRss = SupportsRss, + SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect, + SupportsPagination = SupportsPagination, + Capabilities = Capabilities + }; + } + + private TorrentRssIndexerSettings GetSettings(string url, bool? allowZeroSize = null, double? defaultReleaseSize = null) + { + var settings = new TorrentRssIndexerSettings + { + BaseUrl = url, + AllowZeroSize = allowZeroSize.GetValueOrDefault(false) + }; + + if (defaultReleaseSize.HasValue) + { + settings.DefaultReleaseSize = defaultReleaseSize; + } + + return settings; + } + + private IndexerCapabilities SetCapabilities() + { + var caps = new IndexerCapabilities + { + SearchParams = new List(), + }; + + caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.Other); + + return caps; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerParserSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerParserSettings.cs new file mode 100644 index 000000000..f72a359a0 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerParserSettings.cs @@ -0,0 +1,12 @@ +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public class TorrentRssIndexerParserSettings + { + public bool UseEZTVFormat { get; set; } + public bool ParseSeedersInDescription { get; set; } + public bool UseEnclosureUrl { get; set; } + public bool UseEnclosureLength { get; set; } + public bool ParseSizeInDescription { get; set; } + public string SizeElementName { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerRequestGenerator.cs new file mode 100644 index 000000000..715e1ecf1 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerRequestGenerator.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public class TorrentRssIndexerRequestGenerator : IIndexerRequestGenerator + { + public TorrentRssIndexerSettings Settings { get; set; } + + public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public virtual IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public virtual IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public virtual IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public virtual IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + if (searchCriteria.IsRssSearch) + { + pageableRequests.Add(GetRssRequests()); + } + + return pageableRequests; + } + + private IEnumerable GetRssRequests() + { + var request = new IndexerRequest(Settings.BaseUrl.Trim().TrimEnd('/'), HttpAccept.Rss); + + if (Settings.Cookie.IsNotNullOrWhiteSpace()) + { + foreach (var cookie in HttpHeader.ParseCookies(Settings.Cookie)) + { + request.HttpRequest.Cookies[cookie.Key] = cookie.Value; + } + } + + yield return request; + } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerSettings.cs new file mode 100644 index 000000000..a0525135b --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssIndexerSettings.cs @@ -0,0 +1,51 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public class TorrentRssIndexerSettingsValidator : AbstractValidator + { + public TorrentRssIndexerSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + + RuleFor(c => c.BaseSettings).SetValidator(new IndexerCommonSettingsValidator()); + RuleFor(c => c.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator()); + } + } + + public class TorrentRssIndexerSettings : ITorrentIndexerSettings + { + private static readonly TorrentRssIndexerSettingsValidator Validator = new (); + + public TorrentRssIndexerSettings() + { + BaseUrl = string.Empty; + AllowZeroSize = false; + } + + [FieldDefinition(0, Label = "Full RSS Feed URL", HelpTextWarning = "To sync to your apps you will need to include the 8000(Other) in the Sync Categories", HelpLink = "https://wiki.servarr.com/en/prowlarr/faq#can-i-add-any-generic-torrent-rss-feed")] + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "Cookie", HelpText = "If your site requires a login cookie to access the RSS, you'll have to retrieve it via a browser.")] + public string Cookie { get; set; } + + [FieldDefinition(2, Type = FieldType.Checkbox, Label = "Allow Zero Size", HelpText="Enabling this will allow you to use feeds that don't specify release size, but be careful, size related checks will not be performed.")] + public bool AllowZeroSize { get; set; } + + [FieldDefinition(3, Type = FieldType.Number, Label = "Default Release Size", HelpText="Add a default size for feeds with missing sizes.", Unit = "MB", Advanced = true)] + public double? DefaultReleaseSize { get; set; } + + [FieldDefinition(20)] + public IndexerBaseSettings BaseSettings { get; set; } = new (); + + [FieldDefinition(21)] + public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new (); + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssParserFactory.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssParserFactory.cs new file mode 100644 index 000000000..fd11862b1 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssParserFactory.cs @@ -0,0 +1,65 @@ +using System; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Indexers.Exceptions; + +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public interface ITorrentRssParserFactory + { + TorrentRssParser GetParser(TorrentRssIndexerSettings settings); + } + + public class TorrentRssParserFactory : ITorrentRssParserFactory + { + protected readonly Logger _logger; + + private readonly ICached _settingsCache; + + private readonly ITorrentRssSettingsDetector _torrentRssSettingsDetector; + + public TorrentRssParserFactory(ICacheManager cacheManager, ITorrentRssSettingsDetector torrentRssSettingsDetector, Logger logger) + { + _settingsCache = cacheManager.GetCache(GetType()); + _torrentRssSettingsDetector = torrentRssSettingsDetector; + _logger = logger; + } + + public TorrentRssParser GetParser(TorrentRssIndexerSettings indexerSettings) + { + var key = indexerSettings.ToJson(); + var parserSettings = _settingsCache.Get(key, () => DetectParserSettings(indexerSettings), TimeSpan.FromDays(7)); + + if (parserSettings.UseEZTVFormat) + { + return new EzrssTorrentRssParser(); + } + + return new TorrentRssParser + { + UseGuidInfoUrl = false, + ParseSeedersInDescription = parserSettings.ParseSeedersInDescription, + + UseEnclosureUrl = parserSettings.UseEnclosureUrl, + UseEnclosureLength = parserSettings.UseEnclosureLength, + ParseSizeInDescription = parserSettings.ParseSizeInDescription, + SizeElementName = parserSettings.SizeElementName, + + DefaultReleaseSize = indexerSettings.DefaultReleaseSize + }; + } + + private TorrentRssIndexerParserSettings DetectParserSettings(TorrentRssIndexerSettings indexerSettings) + { + var settings = _torrentRssSettingsDetector.Detect(indexerSettings); + + if (settings == null) + { + throw new UnsupportedFeedException("Could not parse feed from {0}", indexerSettings.BaseUrl); + } + + return settings; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssSettingsDetector.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssSettingsDetector.cs new file mode 100644 index 000000000..919b0c338 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentRss/TorrentRssSettingsDetector.cs @@ -0,0 +1,305 @@ +using System; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.Definitions.TorrentRss +{ + public interface ITorrentRssSettingsDetector + { + TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings); + } + + public class TorrentRssSettingsDetector : ITorrentRssSettingsDetector + { + private const long ValidSizeThreshold = 2 * 1024 * 1024; + + private readonly IHttpClient _httpClient; + private readonly Logger _logger; + + public TorrentRssSettingsDetector(IHttpClient httpClient, Logger logger) + { + _httpClient = httpClient; + _logger = logger; + } + + public TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings) + { + _logger.Debug("Evaluating TorrentRss feed '{0}'", settings.BaseUrl); + + try + { + var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = settings }; + var request = requestGenerator.GetSearchRequests(new BasicSearchCriteria()).GetAllTiers().First().First(); + + HttpResponse httpResponse; + + try + { + httpResponse = _httpClient.Execute(request.HttpRequest); + } + catch (Exception ex) + { + _logger.Warn(ex, "Unable to connect to indexer {0}: {1}", request.Url, ex.Message); + return null; + } + + var indexerResponse = new IndexerResponse(request, httpResponse); + + return GetParserSettings(indexerResponse, settings); + } + catch (Exception ex) + { + ex.WithData("FeedUrl", settings.BaseUrl); + throw; + } + } + + private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings) + { + var settings = GetEzrssParserSettings(response, indexerSettings); + + if (settings != null) + { + return settings; + } + + settings = GetGenericTorrentRssParserSettings(response, indexerSettings); + + if (settings != null) + { + return settings; + } + + return null; + } + + private TorrentRssIndexerParserSettings GetEzrssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings) + { + if (!IsEZTVFeed(response)) + { + return null; + } + + _logger.Trace("Feed has Ezrss schema"); + + var parser = new EzrssTorrentRssParser(); + var releases = ParseResponse(parser, response); + + try + { + ValidateReleases(releases, indexerSettings); + ValidateReleaseSize(releases, indexerSettings); + + _logger.Debug("Feed was parseable by Ezrss Parser"); + return new TorrentRssIndexerParserSettings + { + UseEZTVFormat = true + }; + } + catch (Exception ex) + { + _logger.Trace(ex, "Feed wasn't parsable by Ezrss Parser"); + return null; + } + } + + private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings) + { + var parser = new TorrentRssParser + { + UseEnclosureUrl = true, + UseEnclosureLength = true, + ParseSeedersInDescription = true + }; + + var item = parser.GetItems(response).FirstOrDefault(); + if (item == null) + { + throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable."); + } + + var settings = new TorrentRssIndexerParserSettings() + { + UseEnclosureUrl = true, + UseEnclosureLength = true, + ParseSeedersInDescription = true + }; + + if (item.Element("enclosure") == null) + { + parser.UseEnclosureUrl = settings.UseEnclosureUrl = false; + } + + var releases = ParseResponse(parser, response); + ValidateReleases(releases, indexerSettings); + + if (!releases.Any(v => v.Seeders.HasValue)) + { + _logger.Trace("Feed doesn't have Seeders in Description, disabling option."); + parser.ParseSeedersInDescription = settings.ParseSeedersInDescription = false; + } + + if (!releases.Any(r => r.Size < ValidSizeThreshold)) + { + _logger.Trace("Feed has valid size in enclosure."); + return settings; + } + + parser.UseEnclosureLength = settings.UseEnclosureLength = false; + + foreach (var sizeElementName in new[] { "size", "Size" }) + { + parser.SizeElementName = settings.SizeElementName = sizeElementName; + + releases = ParseResponse(parser, response); + ValidateReleases(releases, indexerSettings); + + if (!releases.Any(r => r.Size < ValidSizeThreshold)) + { + _logger.Trace("Feed has valid size in Size element."); + return settings; + } + } + + parser.SizeElementName = settings.SizeElementName = null; + parser.ParseSizeInDescription = settings.ParseSizeInDescription = true; + + releases = ParseResponse(parser, response); + ValidateReleases(releases, indexerSettings); + + if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Length / 2) + { + if (releases.Any(r => r.Size < ValidSizeThreshold)) + { + _logger.Debug("Feed {0} contains very small releases.", response.Request.Url); + } + + _logger.Trace("Feed has valid size in description."); + return settings; + } + + parser.ParseSizeInDescription = settings.ParseSizeInDescription = false; + + _logger.Debug("Feed doesn't have release size."); + + releases = ParseResponse(parser, response); + ValidateReleases(releases, indexerSettings); + ValidateReleaseSize(releases, indexerSettings); + + return settings; + } + + private bool IsEZTVFeed(IndexerResponse response) + { + var content = XmlCleaner.ReplaceEntities(response.Content); + content = XmlCleaner.ReplaceUnicode(content); + + using var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, ValidationType = ValidationType.None, IgnoreComments = true, XmlResolver = null }); + var document = XDocument.Load(xmlTextReader); + + // Check Namespace + if (document.Root == null) + { + throw new InvalidDataException("Could not parse IndexerResponse into XML."); + } + + var ns = document.Root.GetNamespaceOfPrefix("torrent"); + if (ns == "http://xmlns.ezrss.it/0.1/") + { + _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace"); + return true; + } + + // Check DTD in DocType + if (document.DocumentType != null && document.DocumentType.SystemId == "http://xmlns.ezrss.it/0.1/dtd/") + { + _logger.Trace("Identified feed as EZTV compatible by EZTV DTD"); + return true; + } + + // Check namespaces + if (document.Descendants().Any(v => v.GetDefaultNamespace().NamespaceName == "http://xmlns.ezrss.it/0.1/")) + { + _logger.Trace("Identified feed as EZTV compatible by EZTV Namespace"); + return true; + } + + return false; + } + + private TorrentInfo[] ParseResponse(IParseIndexerResponse parser, IndexerResponse response) + { + try + { + var releases = parser.ParseResponse(response).Cast().ToArray(); + return releases; + } + catch (Exception ex) + { + _logger.Debug(ex, "Unable to parse indexer feed: " + ex.Message); + throw new UnsupportedFeedException("Unable to parse indexer: " + ex.Message); + } + } + + private void ValidateReleases(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings) + { + if (releases == null || releases.Empty()) + { + throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable."); + } + + var torrentInfo = releases.First(); + + _logger.Trace("TorrentInfo: \n{0}", torrentInfo.ToString("L")); + + if (releases.Any(r => r.Title.IsNullOrWhiteSpace())) + { + throw new UnsupportedFeedException("Feed contains releases without title."); + } + + if (releases.Any(r => !IsValidDownloadUrl(r.DownloadUrl))) + { + throw new UnsupportedFeedException("Failed to find a valid download url in the feed."); + } + + var total = releases.Where(v => v.Guid != null).Select(v => v.Guid).ToArray(); + var distinct = total.Distinct().ToArray(); + + if (distinct.Length != total.Length) + { + throw new UnsupportedFeedException("Feed contains releases with same guid, rejecting malformed rss feed."); + } + } + + private void ValidateReleaseSize(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings) + { + if (!indexerSettings.AllowZeroSize && releases.Any(r => r.Size == 0)) + { + throw new UnsupportedFeedException("Feed doesn't contain the release content size."); + } + + if (releases.Any(r => r.Size != 0 && r.Size < ValidSizeThreshold)) + { + throw new UnsupportedFeedException("Size of one or more releases lower than {0}, feed must contain release content size.", ValidSizeThreshold.SizeSuffix()); + } + } + + private static bool IsValidDownloadUrl(string url) + { + if (url.IsNullOrWhiteSpace()) + { + return false; + } + + return url.StartsWith("magnet:") || url.StartsWith("http:") || url.StartsWith("https:"); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 9bb810805..f76b0389f 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -177,9 +177,10 @@ namespace NzbDrone.Core.Indexers } var definitions = provider.DefaultDefinitions - .Where(v => v.Name != null && v.Name != nameof(Cardigann) && v.Name != nameof(Newznab.Newznab) && v.Name != nameof(Torznab.Torznab)); + .Where(v => v.Name != null && v.Name != nameof(Cardigann) && v.Name != nameof(Newznab.Newznab) && v.Name != nameof(Torznab.Torznab)) + .Cast(); - foreach (IndexerDefinition definition in definitions) + foreach (var definition in definitions) { SetProviderCharacteristics(provider, definition); yield return definition; diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 6994bb626..db3231831 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -206,7 +206,7 @@ namespace NzbDrone.Core.Indexers if (dateString.IsNullOrWhiteSpace()) { - throw new UnsupportedFeedException("Rss feed must have a pubDate element with a valid publish date."); + throw new UnsupportedFeedException("Each item in the RSS feed must have a pubDate element with a valid publish date."); } return XElementExtensions.ParseDate(dateString); @@ -273,26 +273,26 @@ namespace NzbDrone.Core.Indexers protected virtual RssEnclosure[] GetEnclosures(XElement item) { var enclosures = item.Elements("enclosure") - .Select(v => - { - try - { - return new RssEnclosure - { - Url = v.Attribute("url").Value, - Type = v.Attribute("type").Value, - Length = (long)v.Attribute("length") - }; - } - catch (Exception e) - { - _logger.Warn(e, "Failed to get enclosure for: {0}", item.Title()); - } - - return null; - }) - .Where(v => v != null) - .ToArray(); + .Select(v => + { + try + { + return new RssEnclosure + { + Url = v.Attribute("url")?.Value, + Type = v.Attribute("type")?.Value, + Length = v.Attribute("length")?.Value?.ParseInt64() ?? 0 + }; + } + catch (Exception e) + { + _logger.Warn(e, "Failed to get enclosure for: {0}", item.Title()); + } + + return null; + }) + .Where(v => v != null) + .ToArray(); return enclosures; } diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index 5efe5fd64..4563c7ab3 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; using MonoTorrent; @@ -9,12 +10,30 @@ namespace NzbDrone.Core.Indexers { public class TorrentRssParser : RssParser { + // Use to sum/calculate Peers as Leechers+Seeders + public bool CalculatePeersAsSum { get; set; } + + // Use the specified element name to determine the Infohash + public string InfoHashElementName { get; set; } + // Parse various seeder/leecher/peers formats in the description element to determine number of seeders. public bool ParseSeedersInDescription { get; set; } - // Use the specified element name to determine the size + // Use the specified element name to determine the Peers + public string PeersElementName { get; set; } + + // Use the specified element name to determine the Seeds + public string SeedsElementName { get; set; } + + // Use the specified element name to determine the Size public string SizeElementName { get; set; } + // Use the specified element name to determine the Magnet link + public string MagnetElementName { get; set; } + + // Default size for when release sizes aren't available + public double? DefaultReleaseSize { get; set; } + public TorrentRssParser() { PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes; @@ -40,14 +59,28 @@ namespace NzbDrone.Core.Indexers result.InfoHash = GetInfoHash(item); result.MagnetUrl = GetMagnetUrl(item); result.Seeders = GetSeeders(item); - result.Peers = GetPeers(item); + + if (CalculatePeersAsSum) + { + result.Peers = GetPeers(item) + result.Seeders; + } + else + { + result.Peers = GetPeers(item); + } return result; } protected virtual string GetInfoHash(XElement item) { + if (InfoHashElementName.IsNotNullOrWhiteSpace()) + { + return item.FindDecendants(InfoHashElementName).FirstOrDefault().Value; + } + var magnetUrl = GetMagnetUrl(item); + if (magnetUrl.IsNotNullOrWhiteSpace()) { try @@ -64,10 +97,21 @@ namespace NzbDrone.Core.Indexers protected virtual string GetMagnetUrl(XElement item) { - var downloadUrl = GetDownloadUrl(item); - if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + if (MagnetElementName.IsNotNullOrWhiteSpace()) { - return downloadUrl; + var magnetURL = item.FindDecendants(MagnetElementName).FirstOrDefault().Value; + if (magnetURL.IsNotNullOrWhiteSpace() && magnetURL.StartsWith("magnet:")) + { + return magnetURL; + } + } + else + { + var downloadUrl = GetDownloadUrl(item); + if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + { + return downloadUrl; + } } return null; @@ -75,6 +119,8 @@ namespace NzbDrone.Core.Indexers protected virtual int? GetSeeders(XElement item) { + // safe to always use the element if it's present (and valid) + // fall back to description if ParseSeedersInDescription is enabled if (ParseSeedersInDescription && item.Element("description") != null) { var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value); @@ -93,6 +139,12 @@ namespace NzbDrone.Core.Indexers } } + var seeds = item.FindDecendants(SeedsElementName).SingleOrDefault(); + if (seeds != null) + { + return (int)seeds; + } + return null; } @@ -116,6 +168,12 @@ namespace NzbDrone.Core.Indexers } } + if (PeersElementName.IsNotNullOrWhiteSpace()) + { + var itempeers = item.FindDecendants(PeersElementName).SingleOrDefault(); + return int.Parse(itempeers.Value); + } + return null; } @@ -124,12 +182,18 @@ namespace NzbDrone.Core.Indexers var size = base.GetSize(item); if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace()) { - if (item.Element(SizeElementName) != null) + var itemsize = item.FindDecendants(SizeElementName).SingleOrDefault(); + if (itemsize != null) { - size = ParseSize(item.Element(SizeElementName).Value, true); + size = ParseSize(itemsize.Value, true); } } + if (size == 0 && DefaultReleaseSize is > 0) + { + return (long)(DefaultReleaseSize * 1024f * 1024f); + } + return size; }