diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 3a7558d17..41957e2e1 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using FluentValidation.Results; using NLog; using NzbDrone.Common.Http; @@ -15,6 +16,8 @@ namespace NzbDrone.Core.Indexers public abstract class IndexerBase : IIndexer where TSettings : IIndexerSettings, new() { + private static readonly Regex MultiRegex = new (@"\b(?multi)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + protected readonly IIndexerStatusService _indexerStatusService; protected readonly IConfigService _configService; protected readonly IParsingService _parsingService; @@ -75,6 +78,7 @@ namespace NzbDrone.Core.Indexers protected virtual IList CleanupReleases(IEnumerable releases) { var result = releases.DistinctBy(v => v.Guid).ToList(); + var settings = Definition.Settings as IIndexerSettings; result.ForEach(c => { diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index bd3c5646d..380673ed1 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -4,6 +4,8 @@ using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Newznab @@ -101,6 +103,30 @@ namespace NzbDrone.Core.Indexers.Newznab return ParseUrl(item.TryGetValue("comments")); } + protected override List GetLanguages(XElement item) + { + var languges = TryGetMultipleNewznabAttributes(item, "language"); + var results = new List(); + + // Try to find elements for some indexers that suck at following the rules. + if (languges.Count == 0) + { + languges = item.Elements("language").Select(e => e.Value).ToList(); + } + + foreach (var language in languges) + { + var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null; + + if (mappedLanguage != null) + { + results.Add(mappedLanguage); + } + } + + return results; + } + protected override long GetSize(XElement item) { long size; @@ -184,5 +210,22 @@ namespace NzbDrone.Core.Indexers.Newznab return defaultValue; } + + protected List TryGetMultipleNewznabAttributes(XElement item, string key) + { + var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + var results = new List(); + + foreach (var element in attrElements) + { + var attrValue = element.Attribute("value"); + if (attrValue != null) + { + results.Add(attrValue.Value); + } + } + + return results; + } } } diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 02d982fdf..c5ff6c2e9 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -12,6 +12,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers @@ -159,6 +160,7 @@ namespace NzbDrone.Core.Indexers releaseInfo.InfoUrl = GetInfoUrl(item); releaseInfo.CommentUrl = GetCommentUrl(item); releaseInfo.Categories = GetCategories(item); + releaseInfo.Languages = GetLanguages(item); try { @@ -230,6 +232,11 @@ namespace NzbDrone.Core.Indexers return new List(); } + protected virtual List GetLanguages(XElement item) + { + return new List(); + } + protected virtual long GetSize(XElement item) { if (UseEnclosureLength) diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs index ed55df05f..08e28221e 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs @@ -4,6 +4,8 @@ using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Torznab @@ -82,6 +84,30 @@ namespace NzbDrone.Core.Indexers.Torznab return ParseUrl(item.TryGetValue("comments")); } + protected override List GetLanguages(XElement item) + { + var languges = TryGetMultipleTorznabAttributes(item, "language"); + var results = new List(); + + // Try to find elements for some indexers that suck at following the rules. + if (languges.Count == 0) + { + languges = item.Elements("language").Select(e => e.Value).ToList(); + } + + foreach (var language in languges) + { + var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null; + + if (mappedLanguage != null) + { + results.Add(mappedLanguage); + } + } + + return results; + } + protected override long GetSize(XElement item) { long size; @@ -184,5 +210,22 @@ namespace NzbDrone.Core.Indexers.Torznab return defaultValue; } + + protected List TryGetMultipleTorznabAttributes(XElement item, string key) + { + var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + var results = new List(); + + foreach (var element in attrElements) + { + var attrValue = element.Attribute("value"); + if (attrValue != null) + { + results.Add(attrValue.Value); + } + } + + return results; + } } } diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index eafa3a932..71c4d5caa 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -4,11 +4,17 @@ using System.Text; using Newtonsoft.Json; using NzbDrone.Core.Download.Pending; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Languages; namespace NzbDrone.Core.Parser.Model { public class ReleaseInfo { + public ReleaseInfo() + { + Languages = new List(); + } + public string Guid { get; set; } public string Title { get; set; } public long Size { get; set; } @@ -29,6 +35,8 @@ namespace NzbDrone.Core.Parser.Model public string Codec { get; set; } public List Categories { get; set; } + public List Languages { get; set; } + // Used to track pending releases that are being reprocessed [JsonIgnore] public PendingReleaseReason? PendingReleaseReason { get; set; }