Add Multiple Languages

Closes #6385
Closes #6564
Closes #6694
Closes #6463

Co-Authored-By: siankatabg <siankatabg@users.noreply.github.com>
Co-Authored-By: tandy1000 <24867509+tandy-1000@users.noreply.github.com>
Co-Authored-By: Kristof Mattei <864376+kristof-mattei@users.noreply.github.com>
Co-Authored-By: Oleksandr Hulyi <4095184+pamidur@users.noreply.github.com>

src/NzbDrone.Core.Test/Languages/LanguageFixture.cs

@@ -43,7 +43,10 @@ namespace NzbDrone.Core.Test.Languages
                 new object[] { 28, Language.Thai },
                 new object[] { 29, Language.Bulgarian },
                 new object[] { 30, Language.PortugueseBR },
-                new object[] { 31, Language.Arabic }
+                new object[] { 31, Language.Arabic },
+                new object[] { 32, Language.Ukrainian },
+                new object[] { 33, Language.Persian },
+                new object[] { 34, Language.Bengali },
             };
 
         public static object[] ToIntCases =
@@ -81,7 +84,10 @@ namespace NzbDrone.Core.Test.Languages
                 new object[] { Language.Thai, 28 },
                 new object[] { Language.Bulgarian, 29 },
                 new object[] { Language.PortugueseBR, 30 },
-                new object[] { Language.Arabic, 31 }
+                new object[] { Language.Arabic, 31 },
+                new object[] { Language.Ukrainian, 32 },
+                new object[] { Language.Persian, 33 },
+                new object[] { Language.Bengali, 34 },
             };
 
         [Test]

src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs

@@ -153,6 +153,7 @@ namespace NzbDrone.Core.Test.ParserTests
 
         [TestCase("Movie.Title.1994.Bulgarian.1080p.XviD-LOL")]
         [TestCase("Movie.Title.1994.BGAUDIO.1080p.XviD-LOL")]
+        [TestCase("Movie.Title.1994.BG.AUDIO.1080p.XviD-LOL")]
         public void should_parse_language_bulgarian(string postTitle)
         {
             var result = Parser.Parser.ParseMovieTitle(postTitle, true);
@@ -306,6 +307,40 @@ namespace NzbDrone.Core.Test.ParserTests
             result.Languages.Should().BeEquivalentTo(Language.Arabic);
         }
 
+        [TestCase("Movie.Title [1989, BDRip] MVO + DVO + UKR (MVO) + Sub")]
+        [TestCase("Movie.Title (2006) BDRemux 1080p 2xUkr | Sub Ukr")]
+        [TestCase("Movie.Title [1984, BDRip 720p] MVO + MVO + Dub + AVO + 3xUkr")]
+        [TestCase("Movie.Title.2019.UKRAINIAN.WEBRip.x264-VXT")]
+        public void should_parse_language_ukrainian(string postTitle)
+        {
+            var result = Parser.Parser.ParseMovieTitle(postTitle, true);
+
+            result.Languages.Should().BeEquivalentTo(Language.Ukrainian);
+        }
+
+        [TestCase("Movie.Title [1937, BDRip 1080p] Dub UKR/Eng + Sub rus")]
+        [TestCase("Movie.Title.[2003.BDRemux.1080p].Dub.MVO.(2xUkr/Fra).Sub.(Rus/Fra)")]
+        public void should_parse_language_ukrainian_multi(string postTitle)
+        {
+            var result = Parser.Parser.ParseMovieTitle(postTitle, true);
+
+            result.Languages.Should().Contain(Language.Ukrainian);
+        }
+
+        [TestCase("Movie.Title.2019.PERSIAN.WEBRip.x264-VXT")]
+        public void should_parse_language_persian(string postTitle)
+        {
+            var result = Parser.Parser.ParseMovieTitle(postTitle);
+            result.Languages.Should().BeEquivalentTo(Language.Persian);
+        }
+
+        [TestCase("Movie.Title.2019.BENGALI.WEBRip.x264-VXT")]
+        public void should_parse_language_bengali(string postTitle)
+        {
+            var result = Parser.Parser.ParseMovieTitle(postTitle);
+            result.Languages.Should().BeEquivalentTo(Language.Bengali);
+        }
+
         [TestCase("Movie.Title.en.sub")]
         [TestCase("Movie Title.eng.sub")]
         [TestCase("Movie.Title.eng.forced.sub")]

src/NzbDrone.Core/Languages/Language.cs

@@ -102,6 +102,9 @@ namespace NzbDrone.Core.Languages
         public static Language Bulgarian => new Language(29, "Bulgarian");
         public static Language PortugueseBR => new Language(30, "Portuguese (Brazil)");
         public static Language Arabic => new Language(31, "Arabic");
+        public static Language Ukrainian => new Language(32, "Unkrainian");
+        public static Language Persian => new Language(33, "Persian");
+        public static Language Bengali => new Language(34, "Bengali");
         public static Language Any => new Language(-1, "Any");
         public static Language Original => new Language(-2, "Original");
 

src/NzbDrone.Core/Parser/IsoLanguages.cs

@@ -39,7 +39,10 @@ namespace NzbDrone.Core.Parser
                                                                new IsoLanguage("bg", "", "bul", "Bulgarian", Language.Bulgarian),
                                                                new IsoLanguage("ro", "", "ron", "Romanian", Language.Romanian),
                                                                new IsoLanguage("pt", "br", "", "Portuguese (Brazil)", Language.PortugueseBR),
-                                                               new IsoLanguage("ar", "", "ara", "Arabic", Language.Arabic)
+                                                               new IsoLanguage("ar", "", "ara", "Arabic", Language.Arabic),
+                                                               new IsoLanguage("uk", "", "uar", "Ukrainian", Language.Ukrainian),
+                                                               new IsoLanguage("fa", "", "fas", "Persian", Language.Persian),
+                                                               new IsoLanguage("be", "", "ben", "Bengali", Language.Bengali)
                                                            };
 
         public static IsoLanguage Find(string isoCode)

src/NzbDrone.Core/Parser/LanguageParser.cs

@@ -14,11 +14,27 @@ namespace NzbDrone.Core.Parser
     {
         private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(LanguageParser));
 
-        private static readonly Regex LanguageRegex = new Regex(@"(?:\W|_|^)(?<italian>\b(?:ita|italian)\b)|(?<german>german\b|videomann|ger[. ]dub)|(?<flemish>flemish)|(?<bulgarian>bgaudio)|(?<brazilian>dublado)|(?<greek>greek)|(?<french>\b(?:FR|VO|VFF|VFQ|VFI|VF2|TRUEFRENCH|FRE|FRA)\b)|(?<russian>\brus\b)|(?<english>\beng\b)|(?<hungarian>\b(?:HUNDUB|HUN)\b)|(?<hebrew>\bHebDub\b)|(?<polish>\b(?:PL\W?DUB|DUB\W?PL|LEK\W?PL|PL\W?LEK)\b)|(?<chinese>\[(?:CH[ST]|BIG5|GB)\]|简|繁|字幕)",
+        private static readonly Regex LanguageRegex = new Regex(@"(?:\W|_|^)(?<italian>\b(?:ita|italian)\b)|
-                                                                RegexOptions.IgnoreCase | RegexOptions.Compiled);
+                                                                            (?<german>german\b|videomann|ger[. ]dub)|
-
+                                                                            (?<flemish>flemish)|
-        private static readonly Regex CaseSensitiveLanguageRegex = new Regex(@"(?:(?i)(?<!SUB[\W|_|^]))(?:(?<lithuanian>\bLT\b)|(?<czech>\bCZ\b)|(?<polish>\bPL\b))(?:(?i)(?![\W|_|^]SUB))",
+                                                                            (?<bulgarian>bgaudio)|
-                                                                RegexOptions.Compiled);
+                                                                            (?<brazilian>dublado)|
+                                                                            (?<greek>greek)|
+                                                                            (?<french>\b(?:FR|VO|VFF|VFQ|VFI|VF2|TRUEFRENCH|FRE|FRA)\b)|
+                                                                            (?<russian>\brus\b)|
+                                                                            (?<english>\beng\b)|
+                                                                            (?<hungarian>\b(?:HUNDUB|HUN)\b)|
+                                                                            (?<hebrew>\bHebDub\b)|
+                                                                            (?<polish>\b(?:PL\W?DUB|DUB\W?PL|LEK\W?PL|PL\W?LEK)\b)|
+                                                                            (?<chinese>\[(?:CH[ST]|BIG5|GB)\]|简|繁|字幕)|
+                                                                            (?<ukrainian>(?:(?:\dx)?UKR))",
+                                                                RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
+
+        private static readonly Regex CaseSensitiveLanguageRegex = new Regex(@"(?:(?i)(?<!SUB[\W|_|^]))(?:(?<lithuanian>\bLT\b)|
+                                                                                                          (?<czech>\bCZ\b)|
+                                                                                                          (?<polish>\bPL\b)|
+                                                                                                          (?<bulgarian>\bBG\b))(?:(?i)(?![\W|_|^]SUB))",
+                                                                RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
 
         private static readonly Regex SubtitleLanguageRegex = new Regex(".+?[-_. ](?<iso_code>[a-z]{2,3})(?:[-_. ]forced)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
 
@@ -147,6 +163,21 @@ namespace NzbDrone.Core.Parser
                 languages.Add(Language.Hebrew);
             }
 
+            if (lowerTitle.Contains("ukrainian"))
+            {
+                languages.Add(Language.Ukrainian);
+            }
+
+            if (lowerTitle.Contains("persian"))
+            {
+                languages.Add(Language.Persian);
+            }
+
+            if (lowerTitle.Contains("bengali"))
+            {
+                languages.Add(Language.Bengali);
+            }
+
             // Case sensitive
             var caseSensitiveMatch = CaseSensitiveLanguageRegex.Match(title);
 
@@ -165,6 +196,11 @@ namespace NzbDrone.Core.Parser
                 languages.Add(Language.Polish);
             }
 
+            if (caseSensitiveMatch.Groups["bulgarian"].Captures.Cast<Capture>().Any())
+            {
+                languages.Add(Language.Bulgarian);
+            }
+
             var matches = LanguageRegex.Matches(title);
 
             foreach (Match match in matches)
@@ -238,6 +274,11 @@ namespace NzbDrone.Core.Parser
                 {
                     languages.Add(Language.Chinese);
                 }
+
+                if (match.Groups["ukrainian"].Success)
+                {
+                    languages.Add(Language.Ukrainian);
+                }
             }
 
             if (title.ToLower().Contains("multi"))