Improved search speed by reusing providers pools

bazarr/api/episodes/episodes_subtitles.py

@@ -10,7 +10,7 @@ from database import TableEpisodes, get_audio_profile_languages, get_profile_id
 from ..utils import authenticate
 from helper import path_mappings
 from get_providers import get_providers, get_providers_auth
-from get_subtitle import download_subtitle, manual_upload_subtitle
+from get_subtitle import generate_subtitles, manual_upload_subtitle
 from utils import history_log, delete_subtitles
 from notifier import send_notifications
 from list_subtitles import store_subtitles
@@ -44,9 +44,6 @@ class EpisodesSubtitles(Resource):
         hi = request.form.get('hi').capitalize()
         forced = request.form.get('forced').capitalize()
 
-        providers_list = get_providers()
-        providers_auth = get_providers_auth()
-
         audio_language_list = get_audio_profile_languages(episode_id=sonarrEpisodeId)
         if len(audio_language_list) > 0:
             audio_language = audio_language_list[0]['name']
@@ -54,10 +51,10 @@ class EpisodesSubtitles(Resource):
             audio_language = None
 
         try:
-            result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list,
-                                       providers_auth, sceneName, title, 'series',
-                                       profile_id=get_profile_id(episode_id=sonarrEpisodeId))
-            if result is not None:
+            result = list(generate_subtitles(episodePath, [(language, hi, forced)], audio_language, sceneName,
+                                             title, 'series', profile_id=get_profile_id(episode_id=sonarrEpisodeId)))
+            if result:
+                result = result[0]
                 message = result[0]
                 path = result[1]
                 forced = result[5]

bazarr/api/movies/movies_subtitles.py

@@ -10,7 +10,7 @@ from database import TableMovies, get_audio_profile_languages, get_profile_id
 from ..utils import authenticate
 from helper import path_mappings
 from get_providers import get_providers, get_providers_auth
-from get_subtitle import download_subtitle, manual_upload_subtitle
+from get_subtitle import manual_upload_subtitle, generate_subtitles
 from utils import history_log_movie, delete_subtitles
 from notifier import send_notifications_movie
 from list_subtitles import store_subtitles_movie
@@ -56,10 +56,10 @@ class MoviesSubtitles(Resource):
             audio_language = None
 
         try:
-            result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list,
-                                       providers_auth, sceneName, title, 'movie',
-                                       profile_id=get_profile_id(movie_id=radarrId))
-            if result is not None:
+            result = list(generate_subtitles(moviePath, [(language, hi, forced)], audio_language,
+                                             sceneName, title, 'movie', profile_id=get_profile_id(movie_id=radarrId)))
+            if result:
+                result = result[0]
                 message = result[0]
                 path = result[1]
                 forced = result[5]

bazarr/get_subtitle.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+# fmt: off
 
 import os
 import sys
@@ -12,15 +13,19 @@ import re
 import subliminal
 import copy
 import operator
+import time
+
 from functools import reduce
+from inspect import getfullargspec
 from peewee import fn
 from datetime import datetime, timedelta
 from subzero.language import Language
 from subzero.video import parse_video
 from subliminal import region, score as subliminal_scores, \
     list_subtitles, Episode, Movie
-from subliminal_patch.core import SZAsyncProviderPool, download_best_subtitles, save_subtitles, download_subtitles, \
-    list_all_subtitles, get_subtitle_path
+from subliminal_patch.core import SZAsyncProviderPool, save_subtitles, get_subtitle_path
+
+from subliminal_patch.core_persistent import download_best_subtitles, list_all_subtitles, download_subtitles
 from subliminal_patch.score import compute_score
 from subliminal_patch.subtitle import Subtitle
 from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2, \
@@ -44,7 +49,6 @@ from analytics import track_event
 from locale import getpreferredencoding
 from score import movie_score, series_score
 
-
 def get_video(path, title, sceneName, providers=None, media_type="movie"):
     """
     Construct `Video` instance
@@ -83,43 +87,122 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"):
         logging.exception("BAZARR Error trying to get video information for this file: " + original_path)
 
 
-def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title,
-                      media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None):
-    # fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
-    #  one query
+# fmt: on
+def _init_pool(media_type, profile_id=None, providers=None):
+    pool = provider_pool()
+    return pool(
+        providers=providers or get_providers(),
+        provider_configs=get_providers_auth(),
+        blacklist=get_blacklist(media_type),
+        throttle_callback=provider_throttle,
+        ban_list=get_ban_list(profile_id),
+        language_hook=None,
+    )
+
+
+_pools = {}
+
+
+def _get_pool(media_type, profile_id=None):
+    try:
+        return _pools[f'{media_type}_{profile_id or ""}']
+    except KeyError:
+        _update_pool(media_type, profile_id)
+
+        return _pools[f'{media_type}_{profile_id or ""}']
+
+
+def _update_pool(media_type, profile_id=None):
+    pool_key = f'{media_type}_{profile_id or ""}'
+    logging.debug("BAZARR updating pool: %s", pool_key)
+
+    # Init a new pool if not present
+    if pool_key not in _pools:
+        logging.debug("BAZARR pool not initialized: %s. Initializing", pool_key)
+        _pools[pool_key] = _init_pool(media_type, profile_id)
+
+    pool = _pools[pool_key]
+    if pool is None:
+        return False
+
+    return pool.update(
+        get_providers(),
+        get_providers_auth(),
+        get_blacklist(media_type),
+        get_ban_list(profile_id),
+    )
+
+
+def update_pools(f):
+    """Decorator that ensures all pools are updated on each function run.
+    It will detect any config changes in Bazarr"""
+
+    def decorated(*args, **kwargs):
+        logging.debug("BAZARR updating pools: %s", _pools)
+
+        start = time.time()
+        args_spec = getfullargspec(f).args
+
+        try:
+            profile_id = args[args_spec.index("profile_id")]
+        except (IndexError, ValueError):
+            profile_id = None
+
+        updated = _update_pool(args[args_spec.index("media_type")], profile_id)
+
+        if updated:
+            logging.info("BAZARR pools update elapsed time: %s", time.time() - start)
+
+        return f(*args, **kwargs)
+
+    return decorated
+
+
+# fmt: off
+
+@update_pools
+def generate_subtitles(path, languages, audio_language, sceneName, title, media_type,
+                       forced_minimum_score=None, is_upgrade=False, profile_id=None):
+    if not languages:
+        return None
 
     if settings.general.getboolean('utf8_encode'):
         os.environ["SZ_KEEP_ENCODING"] = ""
     else:
         os.environ["SZ_KEEP_ENCODING"] = "True"
 
-    logging.debug('BAZARR Searching subtitles for this file: ' + path)
-    if hi == "True":
-        hi = "force HI"
-    else:
-        hi = "force non-HI"
+    language_set = set()
 
-    if forced == "True":
-        providers_auth['podnapisi']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
-        providers_auth['subscene']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
-        providers_auth['opensubtitles']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
-    else:
-        providers_auth['podnapisi']['only_foreign'] = False
-        providers_auth['subscene']['only_foreign'] = False
-        providers_auth['opensubtitles']['only_foreign'] = False
+    if not isinstance(languages, (set, list)):
+        languages = [languages]
 
-    language_set = set()
+    pool = _get_pool(media_type, profile_id)
+    providers = pool.providers
 
-    if not isinstance(language, list):
-        language = [language]
+    for l in languages:
+        l, hi_item, forced_item = l
+        logging.debug('BAZARR Searching subtitles for this file: ' + path)
+        if hi_item == "True":
+            hi = "force HI"
+        else:
+            hi = "force non-HI"
+
+        # Fixme: This block should be updated elsewhere
+        if forced_item == "True":
+            pool.provider_configs['podnapisi']['only_foreign'] = True
+            pool.provider_configs['subscene']['only_foreign'] = True
+            pool.provider_configs['opensubtitles']['only_foreign'] = True
+        else:
+            pool.provider_configs['podnapisi']['only_foreign'] = False
+            pool.provider_configs['subscene']['only_foreign'] = False
+            pool.provider_configs['opensubtitles']['only_foreign'] = False
 
-    for l in language:
         # Always use alpha2 in API Request
         l = alpha3_from_alpha2(l)
 
         lang_obj = _get_lang_obj(l)
 
-        if forced == "True":
+        if forced_item == "True":
             lang_obj = Language.rebuild(lang_obj, forced=True)
         if hi == "force HI":
             lang_obj = Language.rebuild(lang_obj, hi=True)
@@ -144,6 +227,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
     """
     video = get_video(force_unicode(path), title, sceneName, providers=providers,
                       media_type=media_type)
+
     if video:
         handler = series_score if media_type == "series" else movie_score
         min_score, max_score, scores = _get_scores(media_type, minimum_score_movie, minimum_score)
@@ -151,19 +235,11 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
         if providers:
             if forced_minimum_score:
                 min_score = int(forced_minimum_score) + 1
-            downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
-                                                           providers=providers,
-                                                           provider_configs=providers_auth,
-                                                           pool_class=provider_pool(),
+            downloaded_subtitles = download_best_subtitles({video}, language_set, pool,
+                                                           int(min_score), hi,
                                                            compute_score=compute_score,
                                                            throttle_time=None,  # fixme
-                                                           blacklist=get_blacklist(media_type=media_type),
-                                                           ban_list=get_ban_list(profile_id),
-                                                           throttle_callback=provider_throttle,
-                                                           score_obj=handler,
-                                                           pre_download_hook=None,  # fixme
-                                                           post_download_hook=None,  # fixme
-                                                           language_hook=None)  # fixme
+                                                           score_obj=handler)
         else:
             downloaded_subtitles = None
             logging.info("BAZARR All providers are throttled")
@@ -287,7 +363,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
 
                         track_event(category=downloaded_provider, action=action, label=downloaded_language)
 
-                        return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
+                        yield message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
                                subtitle.language.forced, subtitle.id, reversed_subtitles_path, subtitle.language.hi
 
         if not saved_any:
@@ -299,7 +375,8 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
     logging.debug('BAZARR Ended searching Subtitles for file: ' + path)
 
 
-def manual_search(path, profileId, providers, providers_auth, sceneName, title, media_type):
+@update_pools
+def manual_search(path, profile_id, providers, providers_auth, sceneName, title, media_type):
     logging.debug('BAZARR Manually searching subtitles for this file: ' + path)
 
     final_subtitles = []
@@ -308,7 +385,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
     language_set = set()
 
     # where [3] is items list of dict(id, lang, forced, hi)
-    language_items = get_profiles_list(profile_id=int(profileId))['items']
+    language_items = get_profiles_list(profile_id=int(profile_id))['items']
+    pool = _get_pool(media_type, profile_id)
 
     for language in language_items:
         forced = language['forced']
@@ -323,8 +401,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
         if forced == "True":
             lang_obj = Language.rebuild(lang_obj, forced=True)
 
-            providers_auth['podnapisi']['also_foreign'] = True
-            providers_auth['opensubtitles']['also_foreign'] = True
+            pool.provider_configs['podnapisi']['also_foreign'] = True
+            pool.provider_configs['opensubtitles']['also_foreign'] = True
 
         if hi == "True":
             lang_obj = Language.rebuild(lang_obj, hi=True)
@@ -358,29 +436,20 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
 
         try:
             if providers:
-                subtitles = list_all_subtitles([video], language_set,
-                                               providers=providers,
-                                               provider_configs=providers_auth,
-                                               blacklist=get_blacklist(media_type=media_type),
-                                               ban_list=get_ban_list(profileId),
-                                               throttle_callback=provider_throttle,
-                                               language_hook=None)  # fixme
+                subtitles = list_all_subtitles([video], language_set, pool)
 
                 if 'subscene' in providers:
+                    s_pool = _init_pool("movie", profile_id, {"subscene"})
+
                     subscene_language_set = set()
                     for language in language_set:
                         if language.forced:
                             subscene_language_set.add(language)
                     if len(subscene_language_set):
-                        providers_auth['subscene']['only_foreign'] = True
-                        subtitles_subscene = list_all_subtitles([video], subscene_language_set,
-                                                                providers=['subscene'],
-                                                                provider_configs=providers_auth,
-                                                                blacklist=get_blacklist(media_type=media_type),
-                                                                ban_list=get_ban_list(profileId),
-                                                                throttle_callback=provider_throttle,
-                                                                language_hook=None)  # fixme
-                        providers_auth['subscene']['only_foreign'] = False
+                        s_pool.provider_configs['subscene'] = {}
+                        s_pool.provider_configs['subscene']['only_foreign'] = True
+                        subtitles_subscene = list_all_subtitles([video], subscene_language_set, s_pool)
+                        s_pool.provider_configs['subscene']['only_foreign'] = False
                         subtitles[video] += subtitles_subscene[video]
             else:
                 subtitles = []
@@ -407,6 +476,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
                         logging.debug(u"BAZARR Skipping %s, because it doesn't match our series/episode", s)
                         continue
 
+                initial_hi = None
                 initial_hi_match = False
                 for language in initial_language_set:
                     if s.language.basename == language.basename and \
@@ -468,6 +538,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
     return final_subtitles
 
 
+@update_pools
 def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName,
                              title, media_type, profile_id):
     logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path)
@@ -496,13 +567,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl
         min_score, max_score, scores = _get_scores(media_type)
         try:
             if provider:
-                download_subtitles([subtitle],
-                                   providers={provider},
-                                   provider_configs=providers_auth,
-                                   pool_class=provider_pool(),
-                                   blacklist=get_blacklist(media_type=media_type),
-                                   ban_list=get_ban_list(profile_id),
-                                   throttle_callback=provider_throttle)
+                download_subtitles([subtitle], _get_pool(media_type, profile_id))
                 logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
             else:
                 logging.info("BAZARR All providers are throttled")
@@ -765,8 +830,6 @@ def series_download_subtitles(no):
                       "ignored because of monitored status, series type or series tags: {}".format(no))
         return
 
-    providers_auth = get_providers_auth()
-
     count_episodes_details = len(episodes_details)
 
     for i, episode in enumerate(episodes_details):
@@ -782,6 +845,13 @@ def series_download_subtitles(no):
                           value=i,
                           count=count_episodes_details)
 
+            audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
+            if len(audio_language_list) > 0:
+                audio_language = audio_language_list[0]['name']
+            else:
+                audio_language = 'None'
+            
+            languages = []
             for language in ast.literal_eval(episode['missing_subtitles']):
                 # confirm if language is still missing or if cutoff have been reached
                 confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
@@ -792,40 +862,36 @@ def series_download_subtitles(no):
                     continue
 
                 if language is not None:
-                    audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
-                    if len(audio_language_list) > 0:
-                        audio_language = audio_language_list[0]['name']
+                    hi_ = "True" if language.endswith(':hi') else "False"
+                    forced_ ="True" if language.endswith(':forced') else "False"
+                    languages.append((language.split(":")[0], hi_, forced_))
+
+            if not languages:
+                continue
+
+            for result in generate_subtitles(path_mappings.path_replace(episode['path']),
+                                           languages,
+                                           audio_language,
+                                           str(episode['scene_name']),
+                                           episode['title'], 'series'):
+                if result:
+                    message = result[0]
+                    path = result[1]
+                    forced = result[5]
+                    if result[8]:
+                        language_code = result[2] + ":hi"
+                    elif forced:
+                        language_code = result[2] + ":forced"
                     else:
-                        audio_language = 'None'
-
-                    result = download_subtitle(path_mappings.path_replace(episode['path']),
-                                               language.split(':')[0],
-                                               audio_language,
-                                               "True" if language.endswith(':hi') else "False",
-                                               "True" if language.endswith(':forced') else "False",
-                                               providers_list,
-                                               providers_auth,
-                                               str(episode['scene_name']),
-                                               episode['title'],
-                                               'series')
-                    if result is not None:
-                        message = result[0]
-                        path = result[1]
-                        forced = result[5]
-                        if result[8]:
-                            language_code = result[2] + ":hi"
-                        elif forced:
-                            language_code = result[2] + ":forced"
-                        else:
-                            language_code = result[2]
-                        provider = result[3]
-                        score = result[4]
-                        subs_id = result[6]
-                        subs_path = result[7]
-                        store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
-                        history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score,
-                                    subs_id, subs_path)
-                        send_notifications(no, episode['sonarrEpisodeId'], message)
+                        language_code = result[2]
+                    provider = result[3]
+                    score = result[4]
+                    subs_id = result[6]
+                    subs_path = result[7]
+                    store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
+                    history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score,
+                                subs_id, subs_path)
+                    send_notifications(no, episode['sonarrEpisodeId'], message)
         else:
             logging.info("BAZARR All providers are throttled")
             break
@@ -871,6 +937,14 @@ def episode_download_subtitles(no, send_progress=False):
                                                                          episode['episodeTitle']),
                               value=0,
                               count=1)
+
+            audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
+            if len(audio_language_list) > 0:
+                audio_language = audio_language_list[0]['name']
+            else:
+                audio_language = 'None'
+
+            languages = []
             for language in ast.literal_eval(episode['missing_subtitles']):
                 # confirm if language is still missing or if cutoff have been reached
                 confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
@@ -881,40 +955,38 @@ def episode_download_subtitles(no, send_progress=False):
                     continue
 
                 if language is not None:
-                    audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
-                    if len(audio_language_list) > 0:
-                        audio_language = audio_language_list[0]['name']
+                    hi_ = "True" if language.endswith(':hi') else "False"
+                    forced_ ="True" if language.endswith(':forced') else "False"
+                    languages.append((language.split(":")[0], hi_, forced_))
+
+            if not languages:
+                continue
+
+            for result in generate_subtitles(path_mappings.path_replace(episode['path']),
+                                       languages,
+                                       audio_language,
+                                       str(episode['scene_name']),
+                                       episode['title'],
+                                       'series'):
+                if result:
+                    message = result[0]
+                    path = result[1]
+                    forced = result[5]
+                    if result[8]:
+                        language_code = result[2] + ":hi"
+                    elif forced:
+                        language_code = result[2] + ":forced"
                     else:
-                        audio_language = 'None'
-
-                    result = download_subtitle(path_mappings.path_replace(episode['path']),
-                                               language.split(':')[0],
-                                               audio_language,
-                                               "True" if language.endswith(':hi') else "False",
-                                               "True" if language.endswith(':forced') else "False",
-                                               providers_list,
-                                               providers_auth,
-                                               str(episode['scene_name']),
-                                               episode['title'],
-                                               'series')
-                    if result is not None:
-                        message = result[0]
-                        path = result[1]
-                        forced = result[5]
-                        if result[8]:
-                            language_code = result[2] + ":hi"
-                        elif forced:
-                            language_code = result[2] + ":forced"
-                        else:
-                            language_code = result[2]
-                        provider = result[3]
-                        score = result[4]
-                        subs_id = result[6]
-                        subs_path = result[7]
-                        store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
-                        history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
-                                    language_code, provider, score, subs_id, subs_path)
-                        send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
+                        language_code = result[2]
+                    provider = result[3]
+                    score = result[4]
+                    subs_id = result[6]
+                    subs_path = result[7]
+                    store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
+                    history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
+                                language_code, provider, score, subs_id, subs_path)
+                    send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
+
             if send_progress:
                 hide_progress(id='episode_search_progress_{}'.format(no))
         else:
@@ -941,16 +1013,28 @@ def movies_download_subtitles(no):
     else:
         movie = movies[0]
 
-    providers_auth = get_providers_auth()
-
     if ast.literal_eval(movie['missing_subtitles']):
         count_movie = len(ast.literal_eval(movie['missing_subtitles']))
     else:
         count_movie = 0
 
+    audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
+    if len(audio_language_list) > 0:
+        audio_language = audio_language_list[0]['name']
+    else:
+        audio_language = 'None'
+
+    languages = []
+    providers_list = None
+
     for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])):
         providers_list = get_providers()
 
+        if language is not None:
+            hi_ = "True" if language.endswith(':hi') else "False"
+            forced_ ="True" if language.endswith(':forced') else "False"
+            languages.append((language.split(":")[0], hi_, forced_))
+
         if providers_list:
             # confirm if language is still missing or if cutoff have been reached
             confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
@@ -966,47 +1050,100 @@ def movies_download_subtitles(no):
                           value=i,
                           count=count_movie)
 
-            if language is not None:
-                audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
-                if len(audio_language_list) > 0:
-                    audio_language = audio_language_list[0]['name']
-                else:
-                    audio_language = 'None'
+    if providers_list:
+        for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
+                                       languages,
+                                       audio_language,
+                                       str(movie['sceneName']),
+                                       movie['title'],
+                                       'movie'):
 
-                result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
-                                           language.split(':')[0],
-                                           audio_language,
-                                           "True" if language.endswith(':hi') else "False",
-                                           "True" if language.endswith(':forced') else "False",
-                                           providers_list,
-                                           providers_auth,
-                                           str(movie['sceneName']),
-                                           movie['title'],
-                                           'movie')
-                if result is not None:
-                    message = result[0]
-                    path = result[1]
-                    forced = result[5]
-                    if result[8]:
-                        language_code = result[2] + ":hi"
-                    elif forced:
-                        language_code = result[2] + ":forced"
-                    else:
-                        language_code = result[2]
-                    provider = result[3]
-                    score = result[4]
-                    subs_id = result[6]
-                    subs_path = result[7]
-                    store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
-                    history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path)
-                    send_notifications_movie(no, message)
-        else:
-            logging.info("BAZARR All providers are throttled")
-            break
+            if result:
+                message = result[0]
+                path = result[1]
+                forced = result[5]
+                if result[8]:
+                    language_code = result[2] + ":hi"
+                elif forced:
+                    language_code = result[2] + ":forced"
+                else:
+                    language_code = result[2]
+                provider = result[3]
+                score = result[4]
+                subs_id = result[6]
+                subs_path = result[7]
+                store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
+                history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path)
+                send_notifications_movie(no, message)
+    else:
+        logging.info("BAZARR All providers are throttled")
 
     hide_progress(id='movie_search_progress_{}'.format(no))
 
 
+def _wanted_episode(episode):
+    audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
+    if len(audio_language_list) > 0:
+        audio_language = audio_language_list[0]['name']
+    else:
+        audio_language = 'None'
+
+    languages = []
+    for language in ast.literal_eval(episode['missing_subtitles']):
+
+        # confirm if language is still missing or if cutoff have been reached
+        confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
+            .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
+            .dicts() \
+            .get()
+        if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
+            continue
+
+        if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']):
+            TableEpisodes.update({TableEpisodes.failedAttempts:
+                                  updateFailedAttempts(desired_language=language,
+                                                       attempt_string=episode['failedAttempts'])}) \
+                .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
+                .execute()
+
+
+            hi_ = "True" if language.endswith(':hi') else "False"
+            forced_ ="True" if language.endswith(':forced') else "False"
+            languages.append((language.split(":")[0], hi_, forced_))
+
+        else:
+            logging.debug(
+                f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
+                f"language: {language}")
+
+    for result in generate_subtitles(path_mappings.path_replace(episode['path']),
+                                   languages,
+                                   audio_language,
+                                   str(episode['scene_name']),
+                                   episode['title'],
+                                   'series'):
+        if result:
+            message = result[0]
+            path = result[1]
+            forced = result[5]
+            if result[8]:
+                language_code = result[2] + ":hi"
+            elif forced:
+                language_code = result[2] + ":forced"
+            else:
+                language_code = result[2]
+            provider = result[3]
+            score = result[4]
+            subs_id = result[6]
+            subs_path = result[7]
+            store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
+            history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
+                        language_code, provider, score, subs_id, subs_path)
+            event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
+            event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
+            send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
+
+
 def wanted_download_subtitles(sonarr_episode_id):
     episodes_details = TableEpisodes.select(TableEpisodes.path,
                                             TableEpisodes.missing_subtitles,
@@ -1021,73 +1158,76 @@ def wanted_download_subtitles(sonarr_episode_id):
         .dicts()
     episodes_details = list(episodes_details)
 
-    providers_auth = get_providers_auth()
-
     for episode in episodes_details:
         providers_list = get_providers()
 
         if providers_list:
-            for language in ast.literal_eval(episode['missing_subtitles']):
-                # confirm if language is still missing or if cutoff have been reached
-                confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
-                    .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
-                    .dicts() \
-                    .get()
-                if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
-                    continue
-
-                if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']):
-                    TableEpisodes.update({TableEpisodes.failedAttempts:
-                                          updateFailedAttempts(desired_language=language,
-                                                               attempt_string=episode['failedAttempts'])}) \
-                        .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
-                        .execute()
-
-                    audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
-                    if len(audio_language_list) > 0:
-                        audio_language = audio_language_list[0]['name']
-                    else:
-                        audio_language = 'None'
-
-                    result = download_subtitle(path_mappings.path_replace(episode['path']),
-                                               language.split(':')[0],
-                                               audio_language,
-                                               "True" if language.endswith(':hi') else "False",
-                                               "True" if language.endswith(':forced') else "False",
-                                               providers_list,
-                                               providers_auth,
-                                               str(episode['scene_name']),
-                                               episode['title'],
-                                               'series')
-                    if result is not None:
-                        message = result[0]
-                        path = result[1]
-                        forced = result[5]
-                        if result[8]:
-                            language_code = result[2] + ":hi"
-                        elif forced:
-                            language_code = result[2] + ":forced"
-                        else:
-                            language_code = result[2]
-                        provider = result[3]
-                        score = result[4]
-                        subs_id = result[6]
-                        subs_path = result[7]
-                        store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
-                        history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
-                                    language_code, provider, score, subs_id, subs_path)
-                        event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
-                        event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
-                        send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
-                else:
-                    logging.debug(
-                        f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
-                        f"language: {language}")
+            _wanted_episode(episode)
         else:
             logging.info("BAZARR All providers are throttled")
             break
 
 
+def _wanted_movie(movie):
+    audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
+    if len(audio_language_list) > 0:
+        audio_language = audio_language_list[0]['name']
+    else:
+        audio_language = 'None'
+
+    languages = []
+
+    for language in ast.literal_eval(movie['missing_subtitles']):
+        # confirm if language is still missing or if cutoff have been reached
+        confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
+            .where(TableMovies.radarrId == movie['radarrId']) \
+            .dicts() \
+            .get()
+        if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
+            continue
+
+        if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']):
+            TableMovies.update({TableMovies.failedAttempts:
+                                updateFailedAttempts(desired_language=language,
+                                                     attempt_string=movie['failedAttempts'])}) \
+                .where(TableMovies.radarrId == movie['radarrId']) \
+                .execute()
+
+            hi_ = "True" if language.endswith(':hi') else "False"
+            forced_ ="True" if language.endswith(':forced') else "False"
+            languages.append((language.split(":")[0], hi_, forced_))
+
+        else:
+            logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
+                         f"language: {language}")
+
+    for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
+                                languages,
+                                audio_language,
+                                str(movie['sceneName']),
+                                movie['title'], 'movie'):
+
+        if result:
+            message = result[0]
+            path = result[1]
+            forced = result[5]
+            if result[8]:
+                language_code = result[2] + ":hi"
+            elif forced:
+                language_code = result[2] + ":forced"
+            else:
+                language_code = result[2]
+            provider = result[3]
+            score = result[4]
+            subs_id = result[6]
+            subs_path = result[7]
+            store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
+            history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
+                              subs_id, subs_path)
+            event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
+            send_notifications_movie(movie['radarrId'], message)
+
+
 def wanted_download_subtitles_movie(radarr_id):
     movies_details = TableMovies.select(TableMovies.path,
                                         TableMovies.missing_subtitles,
@@ -1100,66 +1240,11 @@ def wanted_download_subtitles_movie(radarr_id):
         .dicts()
     movies_details = list(movies_details)
 
-    providers_auth = get_providers_auth()
-
     for movie in movies_details:
         providers_list = get_providers()
 
         if providers_list:
-            for language in ast.literal_eval(movie['missing_subtitles']):
-                # confirm if language is still missing or if cutoff have been reached
-                confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
-                    .where(TableMovies.radarrId == movie['radarrId']) \
-                    .dicts() \
-                    .get()
-                if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
-                    continue
-
-                if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']):
-                    TableMovies.update({TableMovies.failedAttempts:
-                                        updateFailedAttempts(desired_language=language,
-                                                             attempt_string=movie['failedAttempts'])}) \
-                        .where(TableMovies.radarrId == movie['radarrId']) \
-                        .execute()
-
-                    audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
-                    if len(audio_language_list) > 0:
-                        audio_language = audio_language_list[0]['name']
-                    else:
-                        audio_language = 'None'
-
-                    result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
-                                               language.split(':')[0],
-                                               audio_language,
-                                               "True" if language.endswith(':hi') else "False",
-                                               "True" if language.endswith(':forced') else "False",
-                                               providers_list,
-                                               providers_auth,
-                                               str(movie['sceneName']),
-                                               movie['title'],
-                                               'movie')
-                    if result is not None:
-                        message = result[0]
-                        path = result[1]
-                        forced = result[5]
-                        if result[8]:
-                            language_code = result[2] + ":hi"
-                        elif forced:
-                            language_code = result[2] + ":forced"
-                        else:
-                            language_code = result[2]
-                        provider = result[3]
-                        score = result[4]
-                        subs_id = result[6]
-                        subs_path = result[7]
-                        store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
-                        history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
-                                          subs_id, subs_path)
-                        event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
-                        send_notifications_movie(movie['radarrId'], message)
-                else:
-                    logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
-                                 f"language: {language}")
+            _wanted_movie(movie)
         else:
             logging.info("BAZARR All providers are throttled")
             break
@@ -1471,8 +1556,6 @@ def upgrade_subtitles():
 
         count_movie_to_upgrade = len(movies_to_upgrade)
 
-    providers_auth = get_providers_auth()
-
     if settings.general.getboolean('use_sonarr'):
         for i, episode in enumerate(episodes_to_upgrade):
             providers_list = get_providers()
@@ -1508,19 +1591,17 @@ def upgrade_subtitles():
             else:
                 audio_language = 'None'
 
-            result = download_subtitle(path_mappings.path_replace(episode['video_path']),
-                                       language,
+            result = list(generate_subtitles(path_mappings.path_replace(episode['video_path']),
+                                       [(language, is_hi, is_forced)],
                                        audio_language,
-                                       is_hi,
-                                       is_forced,
-                                       providers_list,
-                                       providers_auth,
                                        str(episode['scene_name']),
                                        episode['title'],
                                        'series',
                                        forced_minimum_score=int(episode['score']),
-                                       is_upgrade=True)
-            if result is not None:
+                                       is_upgrade=True))
+
+            if result:
+                result = result[0]
                 message = result[0]
                 path = result[1]
                 forced = result[5]
@@ -1573,19 +1654,16 @@ def upgrade_subtitles():
             else:
                 audio_language = 'None'
 
-            result = download_subtitle(path_mappings.path_replace_movie(movie['video_path']),
-                                       language,
+            result = list(generate_subtitles(path_mappings.path_replace_movie(movie['video_path']),
+                                       [(language, is_hi, is_forced)],
                                        audio_language,
-                                       is_hi,
-                                       is_forced,
-                                       providers_list,
-                                       providers_auth,
                                        str(movie['sceneName']),
                                        movie['title'],
                                        'movie',
                                        forced_minimum_score=int(movie['score']),
-                                       is_upgrade=True)
-            if result is not None:
+                                       is_upgrade=True))
+            if result:
+                result = result[0]
                 message = result[0]
                 path = result[1]
                 forced = result[5]

libs/subliminal/providers/__init__.py

@@ -161,5 +161,8 @@ class Provider(object):
         """
         raise NotImplementedError
 
+    def ping(self):
+        return True
+
     def __repr__(self):
         return '<%s [%r]>' % (self.__class__.__name__, self.video_types)

libs/subliminal_patch/core.py

@@ -5,6 +5,7 @@ import json
 import re
 import os
 import logging
+import datetime
 import socket
 import traceback
 import time
@@ -55,6 +56,8 @@ REMOVE_CRAP_FROM_FILENAME = re.compile(r"(?i)(?:([\s_-]+(?:obfuscated|scrambled|
 
 SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '.vtt')
 
+_POOL_LIFETIME = datetime.timedelta(hours=12)
+
 
 def remove_crap_from_fn(fn):
     # in case of the second regex part, the legit release group name will be in group(2), if it's followed by [string]
@@ -69,7 +72,7 @@ class SZProviderPool(ProviderPool):
     def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
                  pre_download_hook=None, post_download_hook=None, language_hook=None):
         #: Name of providers to use
-        self.providers = providers
+        self.providers = set(providers or [])
 
         #: Provider configuration
         self.provider_configs = provider_configs or {}
@@ -91,9 +94,69 @@ class SZProviderPool(ProviderPool):
         self.post_download_hook = post_download_hook
         self.language_hook = language_hook
 
+        self._born = time.time()
+
         if not self.throttle_callback:
             self.throttle_callback = lambda x, y: x
 
+    def update(self, providers, provider_configs, blacklist, ban_list):
+        # Check if the pool was initialized enough hours ago
+        self._check_lifetime()
+
+        # Check if any new provider has been added
+        updated = set(providers) != self.providers or ban_list != self.ban_list
+        removed_providers = list(sorted(self.providers - set(providers)))
+        new_providers = list(sorted(set(providers) - self.providers))
+
+        # Terminate and delete removed providers from instance
+        for removed in removed_providers:
+            try:
+                del self[removed]
+                # If the user has updated the providers but hasn't made any
+                # subtitle searches yet, the removed provider won't be in the
+                # self dictionary
+            except KeyError:
+                pass
+
+        if updated:
+            logger.debug("Removed providers: %s", removed_providers)
+            logger.debug("New providers: %s", new_providers)
+
+            self.discarded_providers.difference_update(new_providers)
+            self.providers.difference_update(removed_providers)
+            self.providers.update(list(providers))
+
+        self.blacklist = blacklist
+
+        # Restart providers with new configs
+        for key, val in provider_configs.items():
+            # key: provider's name; val: config dict
+            old_val = self.provider_configs.get(key)
+
+            if old_val == val:
+                continue
+
+            logger.debug("Restarting provider: %s", key)
+            try:
+                provider = provider_registry[key](**val)
+                provider.initialize()
+            except Exception as error:
+                self.throttle_callback(key, error)
+            else:
+                self.initialized_providers[key] = provider
+                updated = True
+
+        self.provider_configs = provider_configs
+
+        return updated
+
+    def _check_lifetime(self):
+        # This method is used to avoid possible memory leaks
+        if abs(self._born - time.time()) > _POOL_LIFETIME.seconds:
+            logger.info("%s elapsed. Terminating providers", _POOL_LIFETIME)
+            self._born = time.time()
+            self.terminate()
+
     def __enter__(self):
         return self
 
@@ -170,17 +233,7 @@ class SZProviderPool(ProviderPool):
         logger.info('Listing subtitles with provider %r and languages %r', provider, provider_languages)
         results = []
         try:
-            try:
-                results = self[provider].list_subtitles(video, provider_languages)
-            except ResponseNotReady:
-                logger.error('Provider %r response error, reinitializing', provider)
-                try:
-                    self[provider].terminate()
-                    self[provider].initialize()
-                    results = self[provider].list_subtitles(video, provider_languages)
-                except:
-                    logger.error('Provider %r reinitialization error: %s', provider, traceback.format_exc())
-
+            results = self[provider].list_subtitles(video, provider_languages)
             seen = []
             out = []
             for s in results:
@@ -198,16 +251,13 @@ class SZProviderPool(ProviderPool):
                         continue
                 if s.id in seen:
                     continue
+
                 s.plex_media_fps = float(video.fps) if video.fps else None
                 out.append(s)
                 seen.append(s.id)
 
             return out
 
-        except (requests.Timeout, socket.timeout) as e:
-            logger.error('Provider %r timed out', provider)
-            self.throttle_callback(provider, e)
-
         except Exception as e:
             logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
             self.throttle_callback(provider, e)
@@ -289,16 +339,6 @@ class SZProviderPool(ProviderPool):
                 logger.error('Provider %r connection error', subtitle.provider_name)
                 self.throttle_callback(subtitle.provider_name, e)
 
-            except ResponseNotReady as e:
-                logger.error('Provider %r response error, reinitializing', subtitle.provider_name)
-                try:
-                    self[subtitle.provider_name].terminate()
-                    self[subtitle.provider_name].initialize()
-                except:
-                    logger.error('Provider %r reinitialization error: %s', subtitle.provider_name,
-                                 traceback.format_exc())
-                    self.throttle_callback(subtitle.provider_name, e)
-
             except rarfile.BadRarFile:
                 logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)
                 logger.debug("RAR Traceback: %s", traceback.format_exc())

libs/subliminal_patch/core_persistent.py

@@ -0,0 +1,92 @@
+# coding=utf-8
+from __future__ import absolute_import
+
+from collections import defaultdict
+import logging
+import time
+
+from subliminal.core import check_video
+
+logger = logging.getLogger(__name__)
+
+# list_all_subtitles, list_supported_languages, list_supported_video_types, download_subtitles, download_best_subtitles
+def list_all_subtitles(videos, languages, pool_instance):
+    listed_subtitles = defaultdict(list)
+
+    # return immediatly if no video passed the checks
+    if not videos:
+        return listed_subtitles
+
+    for video in videos:
+        logger.info("Listing subtitles for %r", video)
+        subtitles = pool_instance.list_subtitles(
+            video, languages - video.subtitle_languages
+        )
+        listed_subtitles[video].extend(subtitles)
+        logger.info("Found %d subtitle(s)", len(subtitles))
+
+    return listed_subtitles
+
+
+def list_supported_languages(pool_instance):
+    return pool_instance.list_supported_languages()
+
+
+def list_supported_video_types(pool_instance):
+    return pool_instance.list_supported_video_types()
+
+
+def download_subtitles(subtitles, pool_instance):
+    for subtitle in subtitles:
+        logger.info("Downloading subtitle %r with score %s", subtitle, subtitle.score)
+        pool_instance.download_subtitle(subtitle)
+
+
+def download_best_subtitles(
+    videos,
+    languages,
+    pool_instance,
+    min_score=0,
+    hearing_impaired=False,
+    only_one=False,
+    compute_score=None,
+    throttle_time=0,
+    score_obj=None,
+):
+    downloaded_subtitles = defaultdict(list)
+
+    # check videos
+    checked_videos = []
+    for video in videos:
+        if not check_video(video, languages=languages, undefined=only_one):
+            logger.info("Skipping video %r", video)
+            continue
+        checked_videos.append(video)
+
+    # return immediately if no video passed the checks
+    if not checked_videos:
+        return downloaded_subtitles
+
+    got_multiple = len(checked_videos) > 1
+
+    # download best subtitles
+    for video in checked_videos:
+        logger.info("Downloading best subtitles for %r", video)
+        subtitles = pool_instance.download_best_subtitles(
+            pool_instance.list_subtitles(video, languages - video.subtitle_languages),
+            video,
+            languages,
+            min_score=min_score,
+            hearing_impaired=hearing_impaired,
+            only_one=only_one,
+            compute_score=compute_score,
+            score_obj=score_obj,
+        )
+        logger.info("Downloaded %d subtitle(s)", len(subtitles))
+        downloaded_subtitles[video].extend(subtitles)
+
+        if got_multiple and throttle_time:
+            logger.debug("Waiting %ss before continuing ...", throttle_time)
+            time.sleep(throttle_time)
+
+    return downloaded_subtitles

libs/subliminal_patch/providers/__init__.py

@@ -1,8 +1,11 @@
 # coding=utf-8
 
 from __future__ import absolute_import
+
+import functools
 import importlib
 import os
+import logging
 import subliminal
 from subliminal.providers import Provider as _Provider
 from subliminal.subtitle import Subtitle as _Subtitle
@@ -14,11 +17,50 @@ from subzero.lib.io import get_viable_encoding
 import six
 
 
+logger = logging.getLogger(__name__)
+
+
 class Provider(_Provider):
     hash_verifiable = False
     hearing_impaired_verifiable = False
     skip_wrong_fps = True
 
+    def ping(self):
+        """Check if the provider is alive."""
+        return True
+
+
+def reinitialize_on_error(exceptions: tuple, attempts=1):
+    """Method decorator for Provider class. It will reinitialize the instance
+    and re-run the method in case of exceptions.
+
+    :param exceptions: tuple of expected exceptions
+    :param attempts: number of attempts to call the method
+    """
+
+    def real_decorator(method):
+        @functools.wraps(method)
+        def wrapper(self, *args, **kwargs):
+            inc = 1
+            while True:
+                try:
+                    return method(self, *args, **kwargs)
+                except exceptions as error:
+                    if inc > attempts:
+                        raise
+
+                    logger.exception(error)
+                    logger.debug("Reinitializing %s instance (%s attempt)", self, inc)
+
+                    self.terminate()
+                    self.initialize()
+
+                    inc += 1
+
+        return wrapper
+
+    return real_decorator
+
 
 # register providers
 # fixme: this is bad

libs/subliminal_patch/providers/addic7ed.py

@@ -11,12 +11,14 @@ from urllib.parse import quote_plus
 import babelfish
 from dogpile.cache.api import NO_VALUE
 from requests import Session
+from requests.exceptions import RequestException
 from subliminal.cache import region
 from subliminal.video import Episode, Movie
 from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
 from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
     Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
 from subliminal.subtitle import fix_line_ending
+from subliminal_patch.providers import reinitialize_on_error
 from subliminal_patch.utils import sanitize
 from subliminal_patch.exceptions import TooManyRequests
 from subliminal_patch.pitcher import pitchers, load_verification, store_verification
@@ -550,6 +552,7 @@ class Addic7edProvider(_Addic7edProvider):
 
         return subtitles
 
+    @reinitialize_on_error((RequestException,), attempts=1)
     def list_subtitles(self, video, languages):
         if isinstance(video, Episode):
             # lookup show_id
@@ -586,6 +589,7 @@ class Addic7edProvider(_Addic7edProvider):
 
         return []
 
+    @reinitialize_on_error((RequestException,), attempts=1)
     def download_subtitle(self, subtitle):
         last_dls = region.get("addic7ed_dls")
         now = datetime.datetime.now()

libs/subliminal_patch/providers/legendasdivx.py

@@ -10,6 +10,7 @@ from requests.exceptions import HTTPError
 import rarfile
 
 from guessit import guessit
+from requests.exceptions import RequestException
 from subliminal.cache import region
 from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded
 from subliminal.providers import ParserBeautifulSoup
@@ -18,7 +19,7 @@ from subliminal.utils import sanitize, sanitize_release_group
 from subliminal.video import Episode, Movie
 from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked
 from subliminal_patch.http import RetryingCFSession
-from subliminal_patch.providers import Provider
+from subliminal_patch.providers import Provider, reinitialize_on_error
 from subliminal_patch.score import get_scores, framerate_equal
 from subliminal_patch.subtitle import Subtitle, guess_matches
 from subzero.language import Language
@@ -260,6 +261,7 @@ class LegendasdivxProvider(Provider):
             )
         return subtitles
 
+    @reinitialize_on_error((RequestException,), attempts=1)
     def query(self, video, languages):
 
         _searchurl = self.searchurl
@@ -362,7 +364,8 @@ class LegendasdivxProvider(Provider):
 
     def list_subtitles(self, video, languages):
         return self.query(video, languages)
-
+    
+    @reinitialize_on_error((RequestException,), attempts=1)
     def download_subtitle(self, subtitle):
 
         try:

libs/subliminal_patch/providers/legendastv.py

@@ -8,8 +8,10 @@ from subliminal.exceptions import ConfigurationError
 from subliminal.providers.legendastv import LegendasTVSubtitle as _LegendasTVSubtitle, \
     LegendasTVProvider as _LegendasTVProvider, Episode, Movie, guessit, sanitize, region, type_map, \
     raise_for_status, json, SHOW_EXPIRATION_TIME, title_re, season_re, datetime, pytz, NO_VALUE, releases_key, \
-    SUBTITLE_EXTENSIONS, language_converters
+    SUBTITLE_EXTENSIONS, language_converters, ServiceUnavailable
 
+from requests.exceptions import RequestException
+from subliminal_patch.providers import reinitialize_on_error
 from subliminal_patch.subtitle import guess_matches
 from subzero.language import Language
 
@@ -184,6 +186,7 @@ class LegendasTVProvider(_LegendasTVProvider):
 
         return titles_found
 
+    @reinitialize_on_error((RequestException, ServiceUnavailable), attempts=1)
     def query(self, language, titles, season=None, episode=None, year=None, imdb_id=None):
         # search for titles
         titles_found = self.search_titles(titles, season, year, imdb_id)

libs/subliminal_patch/providers/opensubtitles.py

@@ -18,6 +18,7 @@ from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSub
     DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError
 from .mixins import ProviderRetryMixin
 from subliminal.subtitle import fix_line_ending
+from subliminal_patch.providers import reinitialize_on_error
 from subliminal_patch.http import SubZeroRequestsTransport
 from subliminal_patch.utils import sanitize, fix_inconsistent_naming
 from subliminal.cache import region
@@ -236,7 +237,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
     def terminate(self):
         self.server = None
         self.token = None
-
+    
     def list_subtitles(self, video, languages):
         """
         :param video:
@@ -272,6 +273,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
                           use_tag_search=self.use_tag_search, only_foreign=self.only_foreign,
                           also_foreign=self.also_foreign)
 
+    @reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
     def query(self, video, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None,
               tag=None, use_tag_search=False, only_foreign=False, also_foreign=False):
         # fill the search criteria
@@ -377,6 +379,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
 
         return subtitles
 
+    @reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
     def download_subtitle(self, subtitle):
         logger.info('Downloading subtitle %r', subtitle)
         response = self.use_token_or_login(

libs/subliminal_patch/providers/opensubtitlescom.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 import os
+import time
 import datetime
 
 from requests import Session, ConnectionError, Timeout, ReadTimeout
@@ -147,15 +148,18 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider):
         self.password = password
         self.video = None
         self.use_hash = use_hash
+        self._started = None
 
     def initialize(self):
-        self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME)
-        if self.token is NO_VALUE:
-            self.login()
+        self._started = time.time()
+        self.login()
 
     def terminate(self):
         self.session.close()
 
+    def ping(self):
+        return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME
+
     def login(self):
         try:
             r = self.session.post(self.server_url + 'login',

libs/subliminal_patch/providers/subscene.py

@@ -20,13 +20,14 @@ import rarfile
 from babelfish import language_converters
 from guessit import guessit
 from dogpile.cache.api import NO_VALUE
+from requests.exceptions import RequestException
 from subliminal import Episode, ProviderError
 from subliminal.video import Episode, Movie
 from subliminal.exceptions import ConfigurationError, ServiceUnavailable
 from subliminal.utils import sanitize_release_group
 from subliminal.cache import region
 from subliminal_patch.http import RetryingCFSession
-from subliminal_patch.providers import Provider
+from subliminal_patch.providers import Provider, reinitialize_on_error
 from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
 from subliminal_patch.subtitle import Subtitle, guess_matches
 from subliminal_patch.converters.subscene import language_ids, supported_languages
@@ -315,7 +316,9 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
             return search(*args, **kwargs)
         except requests.HTTPError:
             region.delete("subscene_cookies2")
+            raise
 
+    @reinitialize_on_error((RequestException,), attempts=1)
     def query(self, video):
         subtitles = []
         if isinstance(video, Episode):

libs/subliminal_patch/providers/xsubs.py

@@ -6,6 +6,7 @@ import re
 from subzero.language import Language
 from guessit import guessit
 from requests import Session
+from requests.exceptions import RequestException
 
 from subliminal.providers import ParserBeautifulSoup, Provider
 from subliminal import __short_version__
@@ -16,6 +17,7 @@ from subliminal.subtitle import Subtitle, fix_line_ending
 from subliminal.utils import sanitize, sanitize_release_group
 from subliminal.video import Episode
 from subliminal_patch.subtitle import guess_matches
+from subliminal_patch.providers import reinitialize_on_error
 
 logger = logging.getLogger(__name__)
 article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$')
@@ -189,7 +191,8 @@ class XSubsProvider(Provider):
                 break
 
         return int(show_id) if show_id else None
-
+    
+    @reinitialize_on_error((RequestException,), attempts=1)
     def query(self, show_id, series, season, year=None, country=None):
         # get the season list of the show
         logger.info('Getting the season list of show id %d', show_id)
@@ -291,6 +294,7 @@ class XSubsProvider(Provider):
 
         return []
 
+    @reinitialize_on_error((RequestException,), attempts=1)
     def download_subtitle(self, subtitle):
         if isinstance(subtitle, XSubsSubtitle):
             # download the subtitle