New: Use languages from Torznab/Newznab attributes if given

src/NzbDrone.Core.Test/Download/Aggregation/Aggregators/AggregateLanguagesFixture.cs

@@ -32,6 +32,7 @@ namespace NzbDrone.Core.Test.Download.Aggregation.Aggregators
             _remoteMovie = Builder<RemoteMovie>.CreateNew()
                                                  .With(l => l.ParsedMovieInfo = null)
                                                  .With(l => l.Movie = _movie)
+                                                 .With(l => l.Release = new ReleaseInfo())
                                                  .Build();
         }
 

src/NzbDrone.Core/Download/Aggregation/Aggregators/AggregateLanguages.cs

@@ -20,6 +20,7 @@ namespace NzbDrone.Core.Download.Aggregation.Aggregators
         public RemoteMovie Aggregate(RemoteMovie remoteMovie)
         {
             var parsedMovieInfo = remoteMovie.ParsedMovieInfo;
+            var releaseInfo = remoteMovie.Release;
             var languages = parsedMovieInfo.Languages;
             var movie = remoteMovie.Movie;
             var releaseTokens = parsedMovieInfo.SimpleReleaseTitle ?? parsedMovieInfo.ReleaseTitle;
@@ -30,30 +31,40 @@ namespace NzbDrone.Core.Download.Aggregation.Aggregators
             {
                 _logger.Debug("Unable to aggregate languages, using parsed values: {0}", string.Join(", ", languages.ToList()));
 
-                remoteMovie.Languages = languages;
+                remoteMovie.Languages = releaseInfo.Languages.Any() ? releaseInfo.Languages : languages;
 
                 return remoteMovie;
             }
 
-            var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title);
+            if (releaseInfo != null && releaseInfo.Languages.Any())
+            {
+                _logger.Debug("Languages provided by indexer, using release values: {0}", string.Join(", ", releaseInfo.Languages));
 
-            if (!movieTitleLanguage.Contains(Language.Unknown))
+                // Use languages from release (given by indexer or user) if available
+                languages = releaseInfo.Languages;
+            }
+            else
             {
-                var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title);
-                var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase);
+                var movieTitleLanguage = LanguageParser.ParseLanguages(movie.Title);
 
-                if (movieTitleIndex >= 0)
+                if (!movieTitleLanguage.Contains(Language.Unknown))
                 {
-                    releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length);
-                    languagesToRemove.AddRange(movieTitleLanguage);
+                    var normalizedEpisodeTitle = Parser.Parser.NormalizeEpisodeTitle(movie.Title);
+                    var movieTitleIndex = normalizedReleaseTokens.IndexOf(normalizedEpisodeTitle, StringComparison.CurrentCultureIgnoreCase);
+
+                    if (movieTitleIndex >= 0)
+                    {
+                        releaseTokens = releaseTokens.Remove(movieTitleIndex, normalizedEpisodeTitle.Length);
+                        languagesToRemove.AddRange(movieTitleLanguage);
+                    }
                 }
-            }
 
-            // Remove any languages still in the title that would normally be removed
-            languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList();
+                // Remove any languages still in the title that would normally be removed
+                languagesToRemove = languagesToRemove.Except(LanguageParser.ParseLanguages(releaseTokens)).ToList();
 
-            // Remove all languages that aren't part of the updated releaseTokens
-            languages = languages.Except(languagesToRemove).ToList();
+                // Remove all languages that aren't part of the updated releaseTokens
+                languages = languages.Except(languagesToRemove).ToList();
+            }
 
             // Use movie language as fallback if we couldn't parse a language
             if (languages.Count == 0 || (languages.Count == 1 && languages.First() == Language.Unknown))

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -1,11 +1,14 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.RegularExpressions;
 using FluentValidation.Results;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Languages;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
@@ -15,6 +18,8 @@ namespace NzbDrone.Core.Indexers
     public abstract class IndexerBase<TSettings> : IIndexer
         where TSettings : IIndexerSettings, new()
     {
+        private static readonly Regex MultiRegex = new (@"\b(?<multi>multi)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
         protected readonly IIndexerStatusService _indexerStatusService;
         protected readonly IConfigService _configService;
         protected readonly IParsingService _parsingService;
@@ -73,9 +78,16 @@ namespace NzbDrone.Core.Indexers
         protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases)
         {
             var result = releases.DistinctBy(v => v.Guid).ToList();
+            var settings = Definition.Settings as IIndexerSettings;
 
             result.ForEach(c =>
             {
+                // Use multi languages from setting if ReleaseInfo languages is empty
+                if (c.Languages.Empty() && MultiRegex.IsMatch(c.Title) && settings.MultiLanguages.Any())
+                {
+                    c.Languages = settings.MultiLanguages.Select(i => (Language)i).ToList();
+                }
+
                 c.IndexerId = Definition.Id;
                 c.Indexer = Definition.Name;
                 c.DownloadProtocol = Protocol;

src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs

@@ -5,6 +5,8 @@ using System.Linq;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Languages;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Newznab
@@ -97,6 +99,30 @@ namespace NzbDrone.Core.Indexers.Newznab
             return ParseUrl(item.TryGetValue("comments"));
         }
 
+        protected override List<Language> GetLanguages(XElement item)
+        {
+            var languges = TryGetMultipleNewznabAttributes(item, "language");
+            var results = new List<Language>();
+
+            // Try to find <language> elements for some indexers that suck at following the rules.
+            if (languges.Count == 0)
+            {
+                languges = item.Elements("language").Select(e => e.Value).ToList();
+            }
+
+            foreach (var language in languges)
+            {
+                var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null;
+
+                if (mappedLanguage != null)
+                {
+                    results.Add(mappedLanguage);
+                }
+            }
+
+            return results;
+        }
+
         protected override long GetSize(XElement item)
         {
             long size;
@@ -188,5 +214,22 @@ namespace NzbDrone.Core.Indexers.Newznab
 
             return defaultValue;
         }
+
+        protected List<string> TryGetMultipleNewznabAttributes(XElement item, string key)
+        {
+            var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
+            var results = new List<string>();
+
+            foreach (var element in attrElements)
+            {
+                var attrValue = element.Attribute("value");
+                if (attrValue != null)
+                {
+                    results.Add(attrValue.Value);
+                }
+            }
+
+            return results;
+        }
     }
 }

src/NzbDrone.Core/Indexers/RssParser.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -12,6 +12,7 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Instrumentation;
 using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Languages;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers
@@ -160,6 +161,7 @@ namespace NzbDrone.Core.Indexers
             releaseInfo.DownloadUrl = GetDownloadUrl(item);
             releaseInfo.InfoUrl = GetInfoUrl(item);
             releaseInfo.CommentUrl = GetCommentUrl(item);
+            releaseInfo.Languages = GetLanguages(item);
 
             try
             {
@@ -226,6 +228,11 @@ namespace NzbDrone.Core.Indexers
             return ParseUrl((string)item.Element("comments"));
         }
 
+        protected virtual List<Language> GetLanguages(XElement item)
+        {
+            return new List<Language>();
+        }
+
         protected virtual long GetSize(XElement item)
         {
             if (UseEnclosureLength)

src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Languages;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Torznab
@@ -93,6 +95,30 @@ namespace NzbDrone.Core.Indexers.Torznab
             return ParseUrl(item.TryGetValue("comments"));
         }
 
+        protected override List<Language> GetLanguages(XElement item)
+        {
+            var languges = TryGetMultipleTorznabAttributes(item, "language");
+            var results = new List<Language>();
+
+            // Try to find <language> elements for some indexers that suck at following the rules.
+            if (languges.Count == 0)
+            {
+                languges = item.Elements("language").Select(e => e.Value).ToList();
+            }
+
+            foreach (var language in languges)
+            {
+                var mappedLanguage = IsoLanguages.FindByName(language)?.Language ?? null;
+
+                if (mappedLanguage != null)
+                {
+                    results.Add(mappedLanguage);
+                }
+            }
+
+            return results;
+        }
+
         protected override long GetSize(XElement item)
         {
             long size;
@@ -224,5 +250,22 @@ namespace NzbDrone.Core.Indexers.Torznab
 
             return defaultValue;
         }
+
+        protected List<string> TryGetMultipleTorznabAttributes(XElement item, string key)
+        {
+            var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
+            var results = new List<string>();
+
+            foreach (var element in attrElements)
+            {
+                var attrValue = element.Attribute("value");
+                if (attrValue != null)
+                {
+                    results.Add(attrValue.Value);
+                }
+            }
+
+            return results;
+        }
     }
 }

src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs

@@ -1,11 +1,18 @@
 using System;
+using System.Collections.Generic;
 using System.Text;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Languages;
 
 namespace NzbDrone.Core.Parser.Model
 {
     public class ReleaseInfo
     {
+        public ReleaseInfo()
+        {
+            Languages = new List<Language>();
+        }
+
         public string Guid { get; set; }
         public string Title { get; set; }
         public long Size { get; set; }
@@ -27,6 +34,7 @@ namespace NzbDrone.Core.Parser.Model
         public string Resolution { get; set; }
 
         public IndexerFlags IndexerFlags { get; set; }
+        public List<Language> Languages { get; set; }
 
         public int Age
         {