Fixed external subtitles indexation with accented characters that resulted in download in loop. #1961

libs/subliminal_patch/core.py

@@ -1,6 +1,5 @@
 # coding=utf-8
-from __future__ import absolute_import
-import codecs
+import six
 import json
 import re
 import os
@@ -11,37 +10,27 @@ import traceback
 import time
 import operator
 import unicodedata
-
 import itertools
-from six.moves.http_client import ResponseNotReady
-
 import rarfile
 import requests
 
+from os import scandir
 from collections import defaultdict
 from bs4 import UnicodeDammit
 from babelfish import LanguageReverseError
 from guessit.jsonutils import GuessitEncoder
-from subliminal import ProviderError, refiner_manager
+from subliminal import refiner_manager
 from concurrent.futures import as_completed
 
 from .extensions import provider_registry
 from .exceptions import MustGetBlacklisted
 from .score import compute_score as default_compute_score
-from subliminal.exceptions import ServiceUnavailable, DownloadLimitExceeded
 from subliminal.utils import hash_napiprojekt, hash_opensubtitles, hash_shooter, hash_thesubdb
 from subliminal.video import VIDEO_EXTENSIONS, Video, Episode, Movie
 from subliminal.core import guessit, ProviderPool, io, is_windows_special_path, \
     ThreadPoolExecutor, check_video
-from subliminal_patch.exceptions import TooManyRequests, APIThrottled
 
 from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE, FULL_LANGUAGE_LIST
-try:
-    from os import scandir
-    _scandir_generic = scandir
-except ImportError:
-    from scandir import scandir, scandir_generic as _scandir_generic
-import six
 
 logger = logging.getLogger(__name__)
 
@@ -534,8 +523,8 @@ class SZProviderPool(ProviderPool):
                     continue
 
             # download
-            logger.debug("%r: Trying to download subtitle with matches %s, score: %s; release(s): %s", subtitle, matches,
-                         score, subtitle.release_info)
+            logger.debug("%r: Trying to download subtitle with matches %s, score: %s; release(s): %s", subtitle,
+                         matches, score, subtitle.release_info)
             if self.download_subtitle(subtitle):
                 subtitle.score = score
                 downloaded_subtitles.append(subtitle)
@@ -613,6 +602,7 @@ class SZAsyncProviderPool(SZProviderPool):
         to the number of :attr:`~ProviderPool.providers`.
 
     """
+
     def __init__(self, max_workers=None, *args, **kwargs):
         super(SZAsyncProviderPool, self).__init__(*args, **kwargs)
 
@@ -810,18 +800,14 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski
     return video
 
 
-def _search_external_subtitles(path, languages=None, only_one=False, scandir_generic=False, match_strictness="strict"):
+def _search_external_subtitles(path, languages=None, only_one=False, match_strictness="strict"):
     dirpath, filename = os.path.split(path)
     dirpath = dirpath or '.'
     fn_no_ext, fileext = os.path.splitext(filename)
-    fn_no_ext_lower = fn_no_ext.lower()
+    fn_no_ext_lower = unicodedata.normalize('NFC', fn_no_ext.lower())
     subtitles = {}
-    _scandir = _scandir_generic if scandir_generic else scandir
 
-    for entry in _scandir(dirpath):
-        if (not entry.name or entry.name in ('\x0c', '$', ',', '\x7f')) and not scandir_generic:
-            logger.debug('Could not determine the name of the file, retrying with scandir_generic')
-            return _search_external_subtitles(path, languages, only_one, True)
+    for entry in scandir(dirpath):
         if not entry.is_file(follow_symlinks=False):
             continue
 
@@ -860,9 +846,11 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen
             hi_tag = ["hi", "cc", "sdh"]
             hi = any(i for i in hi_tag if i in adv_tag)
 
-        #add simplified/traditional chinese detection
-        simplified_chinese = ["chs", "sc", "zhs", "hans","zh-hans", "gb", "简", "简中", "简体", "简体中文", "中英双语", "中日双语","中法双语","简体&英文"]
-        traditional_chinese = ["cht", "tc", "zht", "hant","zh-hant", "big5", "繁", "繁中", "繁体", "繁體","繁体中文", "繁體中文", "正體中文", "中英雙語", "中日雙語","中法雙語","繁体&英文"]
+        # add simplified/traditional chinese detection
+        simplified_chinese = ["chs", "sc", "zhs", "hans", "zh-hans", "gb", "简", "简中", "简体", "简体中文", "中英双语",
+                              "中日双语", "中法双语", "简体&英文"]
+        traditional_chinese = ["cht", "tc", "zht", "hant", "zh-hant", "big5", "繁", "繁中", "繁体", "繁體", "繁体中文",
+                               "繁體中文", "正體中文", "中英雙語", "中日雙語", "中法雙語", "繁体&英文"]
         p_root = p_root.replace('zh-TW', 'zht')
 
         # remove possible language code for matching
@@ -888,7 +876,7 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen
                 language.forced = forced
                 language.hi = hi
             except (ValueError, LanguageReverseError):
-                #add simplified/traditional chinese detection
+                # add simplified/traditional chinese detection
                 if any(ext in str(language_code) for ext in simplified_chinese):
                     language = Language.fromietf('zh')
                     language.forced = forced
@@ -932,13 +920,8 @@ def search_external_subtitles(path, languages=None, only_one=False, match_strict
         logger.debug("external subs: scanning path %s", abspath)
 
         if os.path.isdir(os.path.dirname(abspath)):
-            try:
-                subtitles.update(_search_external_subtitles(abspath, languages=languages,
-                                                            only_one=only_one, match_strictness=match_strictness))
-            except OSError:
-                subtitles.update(_search_external_subtitles(abspath, languages=languages,
-                                                            only_one=only_one, match_strictness=match_strictness,
-                                                            scandir_generic=True))
+            subtitles.update(_search_external_subtitles(abspath, languages=languages, only_one=only_one,
+                                                        match_strictness=match_strictness))
     logger.debug("external subs: found %s", subtitles)
     return subtitles