From d8f14560e3db044dce044cb1feba3855bd458ecc Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sat, 1 Jan 2022 10:21:19 -0500 Subject: [PATCH] Improved search speed by reusing providers pools --- bazarr/api/episodes/episodes_subtitles.py | 13 +- bazarr/api/movies/movies_subtitles.py | 10 +- bazarr/get_subtitle.py | 682 ++++++++++-------- libs/subliminal/providers/__init__.py | 3 + libs/subliminal_patch/core.py | 92 ++- libs/subliminal_patch/core_persistent.py | 92 +++ libs/subliminal_patch/providers/__init__.py | 42 ++ libs/subliminal_patch/providers/addic7ed.py | 4 + .../providers/legendasdivx.py | 7 +- libs/subliminal_patch/providers/legendastv.py | 5 +- .../providers/opensubtitles.py | 5 +- .../providers/opensubtitlescom.py | 10 +- libs/subliminal_patch/providers/subscene.py | 5 +- libs/subliminal_patch/providers/xsubs.py | 6 +- 14 files changed, 626 insertions(+), 350 deletions(-) create mode 100644 libs/subliminal_patch/core_persistent.py diff --git a/bazarr/api/episodes/episodes_subtitles.py b/bazarr/api/episodes/episodes_subtitles.py index 70f59857c..b1c98a823 100644 --- a/bazarr/api/episodes/episodes_subtitles.py +++ b/bazarr/api/episodes/episodes_subtitles.py @@ -10,7 +10,7 @@ from database import TableEpisodes, get_audio_profile_languages, get_profile_id from ..utils import authenticate from helper import path_mappings from get_providers import get_providers, get_providers_auth -from get_subtitle import download_subtitle, manual_upload_subtitle +from get_subtitle import generate_subtitles, manual_upload_subtitle from utils import history_log, delete_subtitles from notifier import send_notifications from list_subtitles import store_subtitles @@ -44,9 +44,6 @@ class EpisodesSubtitles(Resource): hi = request.form.get('hi').capitalize() forced = request.form.get('forced').capitalize() - providers_list = get_providers() - providers_auth = get_providers_auth() - audio_language_list = get_audio_profile_languages(episode_id=sonarrEpisodeId) if len(audio_language_list) > 0: audio_language = audio_language_list[0]['name'] @@ -54,10 +51,10 @@ class EpisodesSubtitles(Resource): audio_language = None try: - result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list, - providers_auth, sceneName, title, 'series', - profile_id=get_profile_id(episode_id=sonarrEpisodeId)) - if result is not None: + result = list(generate_subtitles(episodePath, [(language, hi, forced)], audio_language, sceneName, + title, 'series', profile_id=get_profile_id(episode_id=sonarrEpisodeId))) + if result: + result = result[0] message = result[0] path = result[1] forced = result[5] diff --git a/bazarr/api/movies/movies_subtitles.py b/bazarr/api/movies/movies_subtitles.py index c4f1a1e2a..19a2d695c 100644 --- a/bazarr/api/movies/movies_subtitles.py +++ b/bazarr/api/movies/movies_subtitles.py @@ -10,7 +10,7 @@ from database import TableMovies, get_audio_profile_languages, get_profile_id from ..utils import authenticate from helper import path_mappings from get_providers import get_providers, get_providers_auth -from get_subtitle import download_subtitle, manual_upload_subtitle +from get_subtitle import manual_upload_subtitle, generate_subtitles from utils import history_log_movie, delete_subtitles from notifier import send_notifications_movie from list_subtitles import store_subtitles_movie @@ -56,10 +56,10 @@ class MoviesSubtitles(Resource): audio_language = None try: - result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list, - providers_auth, sceneName, title, 'movie', - profile_id=get_profile_id(movie_id=radarrId)) - if result is not None: + result = list(generate_subtitles(moviePath, [(language, hi, forced)], audio_language, + sceneName, title, 'movie', profile_id=get_profile_id(movie_id=radarrId))) + if result: + result = result[0] message = result[0] path = result[1] forced = result[5] diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index fcf69bc14..25336c44b 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -1,4 +1,5 @@ # coding=utf-8 +# fmt: off import os import sys @@ -12,15 +13,19 @@ import re import subliminal import copy import operator +import time + from functools import reduce +from inspect import getfullargspec from peewee import fn from datetime import datetime, timedelta from subzero.language import Language from subzero.video import parse_video from subliminal import region, score as subliminal_scores, \ list_subtitles, Episode, Movie -from subliminal_patch.core import SZAsyncProviderPool, download_best_subtitles, save_subtitles, download_subtitles, \ - list_all_subtitles, get_subtitle_path +from subliminal_patch.core import SZAsyncProviderPool, save_subtitles, get_subtitle_path + +from subliminal_patch.core_persistent import download_best_subtitles, list_all_subtitles, download_subtitles from subliminal_patch.score import compute_score from subliminal_patch.subtitle import Subtitle from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2, \ @@ -44,7 +49,6 @@ from analytics import track_event from locale import getpreferredencoding from score import movie_score, series_score - def get_video(path, title, sceneName, providers=None, media_type="movie"): """ Construct `Video` instance @@ -83,43 +87,122 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"): logging.exception("BAZARR Error trying to get video information for this file: " + original_path) -def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title, - media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None): - # fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in - # one query +# fmt: on +def _init_pool(media_type, profile_id=None, providers=None): + pool = provider_pool() + return pool( + providers=providers or get_providers(), + provider_configs=get_providers_auth(), + blacklist=get_blacklist(media_type), + throttle_callback=provider_throttle, + ban_list=get_ban_list(profile_id), + language_hook=None, + ) + + +_pools = {} + + +def _get_pool(media_type, profile_id=None): + try: + return _pools[f'{media_type}_{profile_id or ""}'] + except KeyError: + _update_pool(media_type, profile_id) + + return _pools[f'{media_type}_{profile_id or ""}'] + + +def _update_pool(media_type, profile_id=None): + pool_key = f'{media_type}_{profile_id or ""}' + logging.debug("BAZARR updating pool: %s", pool_key) + + # Init a new pool if not present + if pool_key not in _pools: + logging.debug("BAZARR pool not initialized: %s. Initializing", pool_key) + _pools[pool_key] = _init_pool(media_type, profile_id) + + pool = _pools[pool_key] + if pool is None: + return False + + return pool.update( + get_providers(), + get_providers_auth(), + get_blacklist(media_type), + get_ban_list(profile_id), + ) + + +def update_pools(f): + """Decorator that ensures all pools are updated on each function run. + It will detect any config changes in Bazarr""" + + def decorated(*args, **kwargs): + logging.debug("BAZARR updating pools: %s", _pools) + + start = time.time() + args_spec = getfullargspec(f).args + + try: + profile_id = args[args_spec.index("profile_id")] + except (IndexError, ValueError): + profile_id = None + + updated = _update_pool(args[args_spec.index("media_type")], profile_id) + + if updated: + logging.info("BAZARR pools update elapsed time: %s", time.time() - start) + + return f(*args, **kwargs) + + return decorated + + +# fmt: off + +@update_pools +def generate_subtitles(path, languages, audio_language, sceneName, title, media_type, + forced_minimum_score=None, is_upgrade=False, profile_id=None): + if not languages: + return None if settings.general.getboolean('utf8_encode'): os.environ["SZ_KEEP_ENCODING"] = "" else: os.environ["SZ_KEEP_ENCODING"] = "True" - logging.debug('BAZARR Searching subtitles for this file: ' + path) - if hi == "True": - hi = "force HI" - else: - hi = "force non-HI" + language_set = set() - if forced == "True": - providers_auth['podnapisi']['only_foreign'] = True ## fixme: This is also in get_providers_auth() - providers_auth['subscene']['only_foreign'] = True ## fixme: This is also in get_providers_auth() - providers_auth['opensubtitles']['only_foreign'] = True ## fixme: This is also in get_providers_auth() - else: - providers_auth['podnapisi']['only_foreign'] = False - providers_auth['subscene']['only_foreign'] = False - providers_auth['opensubtitles']['only_foreign'] = False + if not isinstance(languages, (set, list)): + languages = [languages] - language_set = set() + pool = _get_pool(media_type, profile_id) + providers = pool.providers - if not isinstance(language, list): - language = [language] + for l in languages: + l, hi_item, forced_item = l + logging.debug('BAZARR Searching subtitles for this file: ' + path) + if hi_item == "True": + hi = "force HI" + else: + hi = "force non-HI" + + # Fixme: This block should be updated elsewhere + if forced_item == "True": + pool.provider_configs['podnapisi']['only_foreign'] = True + pool.provider_configs['subscene']['only_foreign'] = True + pool.provider_configs['opensubtitles']['only_foreign'] = True + else: + pool.provider_configs['podnapisi']['only_foreign'] = False + pool.provider_configs['subscene']['only_foreign'] = False + pool.provider_configs['opensubtitles']['only_foreign'] = False - for l in language: # Always use alpha2 in API Request l = alpha3_from_alpha2(l) lang_obj = _get_lang_obj(l) - if forced == "True": + if forced_item == "True": lang_obj = Language.rebuild(lang_obj, forced=True) if hi == "force HI": lang_obj = Language.rebuild(lang_obj, hi=True) @@ -144,6 +227,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro """ video = get_video(force_unicode(path), title, sceneName, providers=providers, media_type=media_type) + if video: handler = series_score if media_type == "series" else movie_score min_score, max_score, scores = _get_scores(media_type, minimum_score_movie, minimum_score) @@ -151,19 +235,11 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro if providers: if forced_minimum_score: min_score = int(forced_minimum_score) + 1 - downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi, - providers=providers, - provider_configs=providers_auth, - pool_class=provider_pool(), + downloaded_subtitles = download_best_subtitles({video}, language_set, pool, + int(min_score), hi, compute_score=compute_score, throttle_time=None, # fixme - blacklist=get_blacklist(media_type=media_type), - ban_list=get_ban_list(profile_id), - throttle_callback=provider_throttle, - score_obj=handler, - pre_download_hook=None, # fixme - post_download_hook=None, # fixme - language_hook=None) # fixme + score_obj=handler) else: downloaded_subtitles = None logging.info("BAZARR All providers are throttled") @@ -287,7 +363,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro track_event(category=downloaded_provider, action=action, label=downloaded_language) - return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \ + yield message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \ subtitle.language.forced, subtitle.id, reversed_subtitles_path, subtitle.language.hi if not saved_any: @@ -299,7 +375,8 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro logging.debug('BAZARR Ended searching Subtitles for file: ' + path) -def manual_search(path, profileId, providers, providers_auth, sceneName, title, media_type): +@update_pools +def manual_search(path, profile_id, providers, providers_auth, sceneName, title, media_type): logging.debug('BAZARR Manually searching subtitles for this file: ' + path) final_subtitles = [] @@ -308,7 +385,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, language_set = set() # where [3] is items list of dict(id, lang, forced, hi) - language_items = get_profiles_list(profile_id=int(profileId))['items'] + language_items = get_profiles_list(profile_id=int(profile_id))['items'] + pool = _get_pool(media_type, profile_id) for language in language_items: forced = language['forced'] @@ -323,8 +401,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, if forced == "True": lang_obj = Language.rebuild(lang_obj, forced=True) - providers_auth['podnapisi']['also_foreign'] = True - providers_auth['opensubtitles']['also_foreign'] = True + pool.provider_configs['podnapisi']['also_foreign'] = True + pool.provider_configs['opensubtitles']['also_foreign'] = True if hi == "True": lang_obj = Language.rebuild(lang_obj, hi=True) @@ -358,29 +436,20 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, try: if providers: - subtitles = list_all_subtitles([video], language_set, - providers=providers, - provider_configs=providers_auth, - blacklist=get_blacklist(media_type=media_type), - ban_list=get_ban_list(profileId), - throttle_callback=provider_throttle, - language_hook=None) # fixme + subtitles = list_all_subtitles([video], language_set, pool) if 'subscene' in providers: + s_pool = _init_pool("movie", profile_id, {"subscene"}) + subscene_language_set = set() for language in language_set: if language.forced: subscene_language_set.add(language) if len(subscene_language_set): - providers_auth['subscene']['only_foreign'] = True - subtitles_subscene = list_all_subtitles([video], subscene_language_set, - providers=['subscene'], - provider_configs=providers_auth, - blacklist=get_blacklist(media_type=media_type), - ban_list=get_ban_list(profileId), - throttle_callback=provider_throttle, - language_hook=None) # fixme - providers_auth['subscene']['only_foreign'] = False + s_pool.provider_configs['subscene'] = {} + s_pool.provider_configs['subscene']['only_foreign'] = True + subtitles_subscene = list_all_subtitles([video], subscene_language_set, s_pool) + s_pool.provider_configs['subscene']['only_foreign'] = False subtitles[video] += subtitles_subscene[video] else: subtitles = [] @@ -407,6 +476,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, logging.debug(u"BAZARR Skipping %s, because it doesn't match our series/episode", s) continue + initial_hi = None initial_hi_match = False for language in initial_language_set: if s.language.basename == language.basename and \ @@ -468,6 +538,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, return final_subtitles +@update_pools def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName, title, media_type, profile_id): logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path) @@ -496,13 +567,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl min_score, max_score, scores = _get_scores(media_type) try: if provider: - download_subtitles([subtitle], - providers={provider}, - provider_configs=providers_auth, - pool_class=provider_pool(), - blacklist=get_blacklist(media_type=media_type), - ban_list=get_ban_list(profile_id), - throttle_callback=provider_throttle) + download_subtitles([subtitle], _get_pool(media_type, profile_id)) logging.debug('BAZARR Subtitles file downloaded for this file:' + path) else: logging.info("BAZARR All providers are throttled") @@ -765,8 +830,6 @@ def series_download_subtitles(no): "ignored because of monitored status, series type or series tags: {}".format(no)) return - providers_auth = get_providers_auth() - count_episodes_details = len(episodes_details) for i, episode in enumerate(episodes_details): @@ -782,6 +845,13 @@ def series_download_subtitles(no): value=i, count=count_episodes_details) + audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] for language in ast.literal_eval(episode['missing_subtitles']): # confirm if language is still missing or if cutoff have been reached confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \ @@ -792,40 +862,36 @@ def series_download_subtitles(no): continue if language is not None: - audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) - if len(audio_language_list) > 0: - audio_language = audio_language_list[0]['name'] + hi_ = "True" if language.endswith(':hi') else "False" + forced_ ="True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + + if not languages: + continue + + for result in generate_subtitles(path_mappings.path_replace(episode['path']), + languages, + audio_language, + str(episode['scene_name']), + episode['title'], 'series'): + if result: + message = result[0] + path = result[1] + forced = result[5] + if result[8]: + language_code = result[2] + ":hi" + elif forced: + language_code = result[2] + ":forced" else: - audio_language = 'None' - - result = download_subtitle(path_mappings.path_replace(episode['path']), - language.split(':')[0], - audio_language, - "True" if language.endswith(':hi') else "False", - "True" if language.endswith(':forced') else "False", - providers_list, - providers_auth, - str(episode['scene_name']), - episode['title'], - 'series') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - if result[8]: - language_code = result[2] + ":hi" - elif forced: - language_code = result[2] + ":forced" - else: - language_code = result[2] - provider = result[3] - score = result[4] - subs_id = result[6] - subs_path = result[7] - store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) - history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score, - subs_id, subs_path) - send_notifications(no, episode['sonarrEpisodeId'], message) + language_code = result[2] + provider = result[3] + score = result[4] + subs_id = result[6] + subs_path = result[7] + store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) + history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score, + subs_id, subs_path) + send_notifications(no, episode['sonarrEpisodeId'], message) else: logging.info("BAZARR All providers are throttled") break @@ -871,6 +937,14 @@ def episode_download_subtitles(no, send_progress=False): episode['episodeTitle']), value=0, count=1) + + audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] for language in ast.literal_eval(episode['missing_subtitles']): # confirm if language is still missing or if cutoff have been reached confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \ @@ -881,40 +955,38 @@ def episode_download_subtitles(no, send_progress=False): continue if language is not None: - audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) - if len(audio_language_list) > 0: - audio_language = audio_language_list[0]['name'] + hi_ = "True" if language.endswith(':hi') else "False" + forced_ ="True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + + if not languages: + continue + + for result in generate_subtitles(path_mappings.path_replace(episode['path']), + languages, + audio_language, + str(episode['scene_name']), + episode['title'], + 'series'): + if result: + message = result[0] + path = result[1] + forced = result[5] + if result[8]: + language_code = result[2] + ":hi" + elif forced: + language_code = result[2] + ":forced" else: - audio_language = 'None' - - result = download_subtitle(path_mappings.path_replace(episode['path']), - language.split(':')[0], - audio_language, - "True" if language.endswith(':hi') else "False", - "True" if language.endswith(':forced') else "False", - providers_list, - providers_auth, - str(episode['scene_name']), - episode['title'], - 'series') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - if result[8]: - language_code = result[2] + ":hi" - elif forced: - language_code = result[2] + ":forced" - else: - language_code = result[2] - provider = result[3] - score = result[4] - subs_id = result[6] - subs_path = result[7] - store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) - history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, - language_code, provider, score, subs_id, subs_path) - send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) + language_code = result[2] + provider = result[3] + score = result[4] + subs_id = result[6] + subs_path = result[7] + store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) + history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, + language_code, provider, score, subs_id, subs_path) + send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) + if send_progress: hide_progress(id='episode_search_progress_{}'.format(no)) else: @@ -941,16 +1013,28 @@ def movies_download_subtitles(no): else: movie = movies[0] - providers_auth = get_providers_auth() - if ast.literal_eval(movie['missing_subtitles']): count_movie = len(ast.literal_eval(movie['missing_subtitles'])) else: count_movie = 0 + audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId']) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] + providers_list = None + for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])): providers_list = get_providers() + if language is not None: + hi_ = "True" if language.endswith(':hi') else "False" + forced_ ="True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + if providers_list: # confirm if language is still missing or if cutoff have been reached confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \ @@ -966,47 +1050,100 @@ def movies_download_subtitles(no): value=i, count=count_movie) - if language is not None: - audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId']) - if len(audio_language_list) > 0: - audio_language = audio_language_list[0]['name'] - else: - audio_language = 'None' + if providers_list: + for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']), + languages, + audio_language, + str(movie['sceneName']), + movie['title'], + 'movie'): - result = download_subtitle(path_mappings.path_replace_movie(movie['path']), - language.split(':')[0], - audio_language, - "True" if language.endswith(':hi') else "False", - "True" if language.endswith(':forced') else "False", - providers_list, - providers_auth, - str(movie['sceneName']), - movie['title'], - 'movie') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - if result[8]: - language_code = result[2] + ":hi" - elif forced: - language_code = result[2] + ":forced" - else: - language_code = result[2] - provider = result[3] - score = result[4] - subs_id = result[6] - subs_path = result[7] - store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) - history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path) - send_notifications_movie(no, message) - else: - logging.info("BAZARR All providers are throttled") - break + if result: + message = result[0] + path = result[1] + forced = result[5] + if result[8]: + language_code = result[2] + ":hi" + elif forced: + language_code = result[2] + ":forced" + else: + language_code = result[2] + provider = result[3] + score = result[4] + subs_id = result[6] + subs_path = result[7] + store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) + history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path) + send_notifications_movie(no, message) + else: + logging.info("BAZARR All providers are throttled") hide_progress(id='movie_search_progress_{}'.format(no)) +def _wanted_episode(episode): + audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] + for language in ast.literal_eval(episode['missing_subtitles']): + + # confirm if language is still missing or if cutoff have been reached + confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \ + .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \ + .dicts() \ + .get() + if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']): + continue + + if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']): + TableEpisodes.update({TableEpisodes.failedAttempts: + updateFailedAttempts(desired_language=language, + attempt_string=episode['failedAttempts'])}) \ + .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \ + .execute() + + + hi_ = "True" if language.endswith(':hi') else "False" + forced_ ="True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + + else: + logging.debug( + f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and " + f"language: {language}") + + for result in generate_subtitles(path_mappings.path_replace(episode['path']), + languages, + audio_language, + str(episode['scene_name']), + episode['title'], + 'series'): + if result: + message = result[0] + path = result[1] + forced = result[5] + if result[8]: + language_code = result[2] + ":hi" + elif forced: + language_code = result[2] + ":forced" + else: + language_code = result[2] + provider = result[3] + score = result[4] + subs_id = result[6] + subs_path = result[7] + store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) + history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, + language_code, provider, score, subs_id, subs_path) + event_stream(type='series', action='update', payload=episode['sonarrSeriesId']) + event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId']) + send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) + + def wanted_download_subtitles(sonarr_episode_id): episodes_details = TableEpisodes.select(TableEpisodes.path, TableEpisodes.missing_subtitles, @@ -1021,73 +1158,76 @@ def wanted_download_subtitles(sonarr_episode_id): .dicts() episodes_details = list(episodes_details) - providers_auth = get_providers_auth() - for episode in episodes_details: providers_list = get_providers() if providers_list: - for language in ast.literal_eval(episode['missing_subtitles']): - # confirm if language is still missing or if cutoff have been reached - confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \ - .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \ - .dicts() \ - .get() - if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']): - continue - - if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']): - TableEpisodes.update({TableEpisodes.failedAttempts: - updateFailedAttempts(desired_language=language, - attempt_string=episode['failedAttempts'])}) \ - .where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \ - .execute() - - audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId']) - if len(audio_language_list) > 0: - audio_language = audio_language_list[0]['name'] - else: - audio_language = 'None' - - result = download_subtitle(path_mappings.path_replace(episode['path']), - language.split(':')[0], - audio_language, - "True" if language.endswith(':hi') else "False", - "True" if language.endswith(':forced') else "False", - providers_list, - providers_auth, - str(episode['scene_name']), - episode['title'], - 'series') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - if result[8]: - language_code = result[2] + ":hi" - elif forced: - language_code = result[2] + ":forced" - else: - language_code = result[2] - provider = result[3] - score = result[4] - subs_id = result[6] - subs_path = result[7] - store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) - history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, - language_code, provider, score, subs_id, subs_path) - event_stream(type='series', action='update', payload=episode['sonarrSeriesId']) - event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId']) - send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) - else: - logging.debug( - f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and " - f"language: {language}") + _wanted_episode(episode) else: logging.info("BAZARR All providers are throttled") break +def _wanted_movie(movie): + audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId']) + if len(audio_language_list) > 0: + audio_language = audio_language_list[0]['name'] + else: + audio_language = 'None' + + languages = [] + + for language in ast.literal_eval(movie['missing_subtitles']): + # confirm if language is still missing or if cutoff have been reached + confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \ + .where(TableMovies.radarrId == movie['radarrId']) \ + .dicts() \ + .get() + if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']): + continue + + if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']): + TableMovies.update({TableMovies.failedAttempts: + updateFailedAttempts(desired_language=language, + attempt_string=movie['failedAttempts'])}) \ + .where(TableMovies.radarrId == movie['radarrId']) \ + .execute() + + hi_ = "True" if language.endswith(':hi') else "False" + forced_ ="True" if language.endswith(':forced') else "False" + languages.append((language.split(":")[0], hi_, forced_)) + + else: + logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and " + f"language: {language}") + + for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']), + languages, + audio_language, + str(movie['sceneName']), + movie['title'], 'movie'): + + if result: + message = result[0] + path = result[1] + forced = result[5] + if result[8]: + language_code = result[2] + ":hi" + elif forced: + language_code = result[2] + ":forced" + else: + language_code = result[2] + provider = result[3] + score = result[4] + subs_id = result[6] + subs_path = result[7] + store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) + history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score, + subs_id, subs_path) + event_stream(type='movie-wanted', action='delete', payload=movie['radarrId']) + send_notifications_movie(movie['radarrId'], message) + + def wanted_download_subtitles_movie(radarr_id): movies_details = TableMovies.select(TableMovies.path, TableMovies.missing_subtitles, @@ -1100,66 +1240,11 @@ def wanted_download_subtitles_movie(radarr_id): .dicts() movies_details = list(movies_details) - providers_auth = get_providers_auth() - for movie in movies_details: providers_list = get_providers() if providers_list: - for language in ast.literal_eval(movie['missing_subtitles']): - # confirm if language is still missing or if cutoff have been reached - confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \ - .where(TableMovies.radarrId == movie['radarrId']) \ - .dicts() \ - .get() - if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']): - continue - - if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']): - TableMovies.update({TableMovies.failedAttempts: - updateFailedAttempts(desired_language=language, - attempt_string=movie['failedAttempts'])}) \ - .where(TableMovies.radarrId == movie['radarrId']) \ - .execute() - - audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId']) - if len(audio_language_list) > 0: - audio_language = audio_language_list[0]['name'] - else: - audio_language = 'None' - - result = download_subtitle(path_mappings.path_replace_movie(movie['path']), - language.split(':')[0], - audio_language, - "True" if language.endswith(':hi') else "False", - "True" if language.endswith(':forced') else "False", - providers_list, - providers_auth, - str(movie['sceneName']), - movie['title'], - 'movie') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - if result[8]: - language_code = result[2] + ":hi" - elif forced: - language_code = result[2] + ":forced" - else: - language_code = result[2] - provider = result[3] - score = result[4] - subs_id = result[6] - subs_path = result[7] - store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) - history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score, - subs_id, subs_path) - event_stream(type='movie-wanted', action='delete', payload=movie['radarrId']) - send_notifications_movie(movie['radarrId'], message) - else: - logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and " - f"language: {language}") + _wanted_movie(movie) else: logging.info("BAZARR All providers are throttled") break @@ -1471,8 +1556,6 @@ def upgrade_subtitles(): count_movie_to_upgrade = len(movies_to_upgrade) - providers_auth = get_providers_auth() - if settings.general.getboolean('use_sonarr'): for i, episode in enumerate(episodes_to_upgrade): providers_list = get_providers() @@ -1508,19 +1591,17 @@ def upgrade_subtitles(): else: audio_language = 'None' - result = download_subtitle(path_mappings.path_replace(episode['video_path']), - language, + result = list(generate_subtitles(path_mappings.path_replace(episode['video_path']), + [(language, is_hi, is_forced)], audio_language, - is_hi, - is_forced, - providers_list, - providers_auth, str(episode['scene_name']), episode['title'], 'series', forced_minimum_score=int(episode['score']), - is_upgrade=True) - if result is not None: + is_upgrade=True)) + + if result: + result = result[0] message = result[0] path = result[1] forced = result[5] @@ -1573,19 +1654,16 @@ def upgrade_subtitles(): else: audio_language = 'None' - result = download_subtitle(path_mappings.path_replace_movie(movie['video_path']), - language, + result = list(generate_subtitles(path_mappings.path_replace_movie(movie['video_path']), + [(language, is_hi, is_forced)], audio_language, - is_hi, - is_forced, - providers_list, - providers_auth, str(movie['sceneName']), movie['title'], 'movie', forced_minimum_score=int(movie['score']), - is_upgrade=True) - if result is not None: + is_upgrade=True)) + if result: + result = result[0] message = result[0] path = result[1] forced = result[5] diff --git a/libs/subliminal/providers/__init__.py b/libs/subliminal/providers/__init__.py index 882236ca3..1de7b53c6 100644 --- a/libs/subliminal/providers/__init__.py +++ b/libs/subliminal/providers/__init__.py @@ -161,5 +161,8 @@ class Provider(object): """ raise NotImplementedError + def ping(self): + return True + def __repr__(self): return '<%s [%r]>' % (self.__class__.__name__, self.video_types) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index f4605b1eb..9ed698c09 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -5,6 +5,7 @@ import json import re import os import logging +import datetime import socket import traceback import time @@ -55,6 +56,8 @@ REMOVE_CRAP_FROM_FILENAME = re.compile(r"(?i)(?:([\s_-]+(?:obfuscated|scrambled| SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '.vtt') +_POOL_LIFETIME = datetime.timedelta(hours=12) + def remove_crap_from_fn(fn): # in case of the second regex part, the legit release group name will be in group(2), if it's followed by [string] @@ -69,7 +72,7 @@ class SZProviderPool(ProviderPool): def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None, pre_download_hook=None, post_download_hook=None, language_hook=None): #: Name of providers to use - self.providers = providers + self.providers = set(providers or []) #: Provider configuration self.provider_configs = provider_configs or {} @@ -91,9 +94,69 @@ class SZProviderPool(ProviderPool): self.post_download_hook = post_download_hook self.language_hook = language_hook + self._born = time.time() + if not self.throttle_callback: self.throttle_callback = lambda x, y: x + def update(self, providers, provider_configs, blacklist, ban_list): + # Check if the pool was initialized enough hours ago + self._check_lifetime() + + # Check if any new provider has been added + updated = set(providers) != self.providers or ban_list != self.ban_list + removed_providers = list(sorted(self.providers - set(providers))) + new_providers = list(sorted(set(providers) - self.providers)) + + # Terminate and delete removed providers from instance + for removed in removed_providers: + try: + del self[removed] + # If the user has updated the providers but hasn't made any + # subtitle searches yet, the removed provider won't be in the + # self dictionary + except KeyError: + pass + + if updated: + logger.debug("Removed providers: %s", removed_providers) + logger.debug("New providers: %s", new_providers) + + self.discarded_providers.difference_update(new_providers) + self.providers.difference_update(removed_providers) + self.providers.update(list(providers)) + + self.blacklist = blacklist + + # Restart providers with new configs + for key, val in provider_configs.items(): + # key: provider's name; val: config dict + old_val = self.provider_configs.get(key) + + if old_val == val: + continue + + logger.debug("Restarting provider: %s", key) + try: + provider = provider_registry[key](**val) + provider.initialize() + except Exception as error: + self.throttle_callback(key, error) + else: + self.initialized_providers[key] = provider + updated = True + + self.provider_configs = provider_configs + + return updated + + def _check_lifetime(self): + # This method is used to avoid possible memory leaks + if abs(self._born - time.time()) > _POOL_LIFETIME.seconds: + logger.info("%s elapsed. Terminating providers", _POOL_LIFETIME) + self._born = time.time() + self.terminate() + def __enter__(self): return self @@ -170,17 +233,7 @@ class SZProviderPool(ProviderPool): logger.info('Listing subtitles with provider %r and languages %r', provider, provider_languages) results = [] try: - try: - results = self[provider].list_subtitles(video, provider_languages) - except ResponseNotReady: - logger.error('Provider %r response error, reinitializing', provider) - try: - self[provider].terminate() - self[provider].initialize() - results = self[provider].list_subtitles(video, provider_languages) - except: - logger.error('Provider %r reinitialization error: %s', provider, traceback.format_exc()) - + results = self[provider].list_subtitles(video, provider_languages) seen = [] out = [] for s in results: @@ -198,16 +251,13 @@ class SZProviderPool(ProviderPool): continue if s.id in seen: continue + s.plex_media_fps = float(video.fps) if video.fps else None out.append(s) seen.append(s.id) return out - except (requests.Timeout, socket.timeout) as e: - logger.error('Provider %r timed out', provider) - self.throttle_callback(provider, e) - except Exception as e: logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc()) self.throttle_callback(provider, e) @@ -289,16 +339,6 @@ class SZProviderPool(ProviderPool): logger.error('Provider %r connection error', subtitle.provider_name) self.throttle_callback(subtitle.provider_name, e) - except ResponseNotReady as e: - logger.error('Provider %r response error, reinitializing', subtitle.provider_name) - try: - self[subtitle.provider_name].terminate() - self[subtitle.provider_name].initialize() - except: - logger.error('Provider %r reinitialization error: %s', subtitle.provider_name, - traceback.format_exc()) - self.throttle_callback(subtitle.provider_name, e) - except rarfile.BadRarFile: logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name) logger.debug("RAR Traceback: %s", traceback.format_exc()) diff --git a/libs/subliminal_patch/core_persistent.py b/libs/subliminal_patch/core_persistent.py new file mode 100644 index 000000000..ee55b12a8 --- /dev/null +++ b/libs/subliminal_patch/core_persistent.py @@ -0,0 +1,92 @@ +# coding=utf-8 +from __future__ import absolute_import + +from collections import defaultdict +import logging +import time + +from subliminal.core import check_video + +logger = logging.getLogger(__name__) + +# list_all_subtitles, list_supported_languages, list_supported_video_types, download_subtitles, download_best_subtitles +def list_all_subtitles(videos, languages, pool_instance): + listed_subtitles = defaultdict(list) + + # return immediatly if no video passed the checks + if not videos: + return listed_subtitles + + for video in videos: + logger.info("Listing subtitles for %r", video) + subtitles = pool_instance.list_subtitles( + video, languages - video.subtitle_languages + ) + listed_subtitles[video].extend(subtitles) + logger.info("Found %d subtitle(s)", len(subtitles)) + + return listed_subtitles + + +def list_supported_languages(pool_instance): + return pool_instance.list_supported_languages() + + +def list_supported_video_types(pool_instance): + return pool_instance.list_supported_video_types() + + +def download_subtitles(subtitles, pool_instance): + for subtitle in subtitles: + logger.info("Downloading subtitle %r with score %s", subtitle, subtitle.score) + pool_instance.download_subtitle(subtitle) + + +def download_best_subtitles( + videos, + languages, + pool_instance, + min_score=0, + hearing_impaired=False, + only_one=False, + compute_score=None, + throttle_time=0, + score_obj=None, +): + downloaded_subtitles = defaultdict(list) + + # check videos + checked_videos = [] + for video in videos: + if not check_video(video, languages=languages, undefined=only_one): + logger.info("Skipping video %r", video) + continue + checked_videos.append(video) + + # return immediately if no video passed the checks + if not checked_videos: + return downloaded_subtitles + + got_multiple = len(checked_videos) > 1 + + # download best subtitles + for video in checked_videos: + logger.info("Downloading best subtitles for %r", video) + subtitles = pool_instance.download_best_subtitles( + pool_instance.list_subtitles(video, languages - video.subtitle_languages), + video, + languages, + min_score=min_score, + hearing_impaired=hearing_impaired, + only_one=only_one, + compute_score=compute_score, + score_obj=score_obj, + ) + logger.info("Downloaded %d subtitle(s)", len(subtitles)) + downloaded_subtitles[video].extend(subtitles) + + if got_multiple and throttle_time: + logger.debug("Waiting %ss before continuing ...", throttle_time) + time.sleep(throttle_time) + + return downloaded_subtitles diff --git a/libs/subliminal_patch/providers/__init__.py b/libs/subliminal_patch/providers/__init__.py index ced4694f3..b8a850b58 100644 --- a/libs/subliminal_patch/providers/__init__.py +++ b/libs/subliminal_patch/providers/__init__.py @@ -1,8 +1,11 @@ # coding=utf-8 from __future__ import absolute_import + +import functools import importlib import os +import logging import subliminal from subliminal.providers import Provider as _Provider from subliminal.subtitle import Subtitle as _Subtitle @@ -14,11 +17,50 @@ from subzero.lib.io import get_viable_encoding import six +logger = logging.getLogger(__name__) + + class Provider(_Provider): hash_verifiable = False hearing_impaired_verifiable = False skip_wrong_fps = True + def ping(self): + """Check if the provider is alive.""" + return True + + +def reinitialize_on_error(exceptions: tuple, attempts=1): + """Method decorator for Provider class. It will reinitialize the instance + and re-run the method in case of exceptions. + + :param exceptions: tuple of expected exceptions + :param attempts: number of attempts to call the method + """ + + def real_decorator(method): + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + inc = 1 + while True: + try: + return method(self, *args, **kwargs) + except exceptions as error: + if inc > attempts: + raise + + logger.exception(error) + logger.debug("Reinitializing %s instance (%s attempt)", self, inc) + + self.terminate() + self.initialize() + + inc += 1 + + return wrapper + + return real_decorator + # register providers # fixme: this is bad diff --git a/libs/subliminal_patch/providers/addic7ed.py b/libs/subliminal_patch/providers/addic7ed.py index ebdd7ae2b..e7ff5e2c4 100644 --- a/libs/subliminal_patch/providers/addic7ed.py +++ b/libs/subliminal_patch/providers/addic7ed.py @@ -11,12 +11,14 @@ from urllib.parse import quote_plus import babelfish from dogpile.cache.api import NO_VALUE from requests import Session +from requests.exceptions import RequestException from subliminal.cache import region from subliminal.video import Episode, Movie from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \ Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup from subliminal.subtitle import fix_line_ending +from subliminal_patch.providers import reinitialize_on_error from subliminal_patch.utils import sanitize from subliminal_patch.exceptions import TooManyRequests from subliminal_patch.pitcher import pitchers, load_verification, store_verification @@ -550,6 +552,7 @@ class Addic7edProvider(_Addic7edProvider): return subtitles + @reinitialize_on_error((RequestException,), attempts=1) def list_subtitles(self, video, languages): if isinstance(video, Episode): # lookup show_id @@ -586,6 +589,7 @@ class Addic7edProvider(_Addic7edProvider): return [] + @reinitialize_on_error((RequestException,), attempts=1) def download_subtitle(self, subtitle): last_dls = region.get("addic7ed_dls") now = datetime.datetime.now() diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 2f7ac60e7..e4a5ab292 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -10,6 +10,7 @@ from requests.exceptions import HTTPError import rarfile from guessit import guessit +from requests.exceptions import RequestException from subliminal.cache import region from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded from subliminal.providers import ParserBeautifulSoup @@ -18,7 +19,7 @@ from subliminal.utils import sanitize, sanitize_release_group from subliminal.video import Episode, Movie from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked from subliminal_patch.http import RetryingCFSession -from subliminal_patch.providers import Provider +from subliminal_patch.providers import Provider, reinitialize_on_error from subliminal_patch.score import get_scores, framerate_equal from subliminal_patch.subtitle import Subtitle, guess_matches from subzero.language import Language @@ -260,6 +261,7 @@ class LegendasdivxProvider(Provider): ) return subtitles + @reinitialize_on_error((RequestException,), attempts=1) def query(self, video, languages): _searchurl = self.searchurl @@ -362,7 +364,8 @@ class LegendasdivxProvider(Provider): def list_subtitles(self, video, languages): return self.query(video, languages) - + + @reinitialize_on_error((RequestException,), attempts=1) def download_subtitle(self, subtitle): try: diff --git a/libs/subliminal_patch/providers/legendastv.py b/libs/subliminal_patch/providers/legendastv.py index 638f332fb..6ee07ee11 100644 --- a/libs/subliminal_patch/providers/legendastv.py +++ b/libs/subliminal_patch/providers/legendastv.py @@ -8,8 +8,10 @@ from subliminal.exceptions import ConfigurationError from subliminal.providers.legendastv import LegendasTVSubtitle as _LegendasTVSubtitle, \ LegendasTVProvider as _LegendasTVProvider, Episode, Movie, guessit, sanitize, region, type_map, \ raise_for_status, json, SHOW_EXPIRATION_TIME, title_re, season_re, datetime, pytz, NO_VALUE, releases_key, \ - SUBTITLE_EXTENSIONS, language_converters + SUBTITLE_EXTENSIONS, language_converters, ServiceUnavailable +from requests.exceptions import RequestException +from subliminal_patch.providers import reinitialize_on_error from subliminal_patch.subtitle import guess_matches from subzero.language import Language @@ -184,6 +186,7 @@ class LegendasTVProvider(_LegendasTVProvider): return titles_found + @reinitialize_on_error((RequestException, ServiceUnavailable), attempts=1) def query(self, language, titles, season=None, episode=None, year=None, imdb_id=None): # search for titles titles_found = self.search_titles(titles, season, year, imdb_id) diff --git a/libs/subliminal_patch/providers/opensubtitles.py b/libs/subliminal_patch/providers/opensubtitles.py index cfa144670..2918fd6ce 100644 --- a/libs/subliminal_patch/providers/opensubtitles.py +++ b/libs/subliminal_patch/providers/opensubtitles.py @@ -18,6 +18,7 @@ from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSub DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError from .mixins import ProviderRetryMixin from subliminal.subtitle import fix_line_ending +from subliminal_patch.providers import reinitialize_on_error from subliminal_patch.http import SubZeroRequestsTransport from subliminal_patch.utils import sanitize, fix_inconsistent_naming from subliminal.cache import region @@ -236,7 +237,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): def terminate(self): self.server = None self.token = None - + def list_subtitles(self, video, languages): """ :param video: @@ -272,6 +273,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): use_tag_search=self.use_tag_search, only_foreign=self.only_foreign, also_foreign=self.also_foreign) + @reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1) def query(self, video, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None, tag=None, use_tag_search=False, only_foreign=False, also_foreign=False): # fill the search criteria @@ -377,6 +379,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): return subtitles + @reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1) def download_subtitle(self, subtitle): logger.info('Downloading subtitle %r', subtitle) response = self.use_token_or_login( diff --git a/libs/subliminal_patch/providers/opensubtitlescom.py b/libs/subliminal_patch/providers/opensubtitlescom.py index 94e041ab7..e2a86664c 100644 --- a/libs/subliminal_patch/providers/opensubtitlescom.py +++ b/libs/subliminal_patch/providers/opensubtitlescom.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os +import time import datetime from requests import Session, ConnectionError, Timeout, ReadTimeout @@ -147,15 +148,18 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): self.password = password self.video = None self.use_hash = use_hash + self._started = None def initialize(self): - self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME) - if self.token is NO_VALUE: - self.login() + self._started = time.time() + self.login() def terminate(self): self.session.close() + def ping(self): + return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME + def login(self): try: r = self.session.post(self.server_url + 'login', diff --git a/libs/subliminal_patch/providers/subscene.py b/libs/subliminal_patch/providers/subscene.py index 42f0221b0..66329a779 100644 --- a/libs/subliminal_patch/providers/subscene.py +++ b/libs/subliminal_patch/providers/subscene.py @@ -20,13 +20,14 @@ import rarfile from babelfish import language_converters from guessit import guessit from dogpile.cache.api import NO_VALUE +from requests.exceptions import RequestException from subliminal import Episode, ProviderError from subliminal.video import Episode, Movie from subliminal.exceptions import ConfigurationError, ServiceUnavailable from subliminal.utils import sanitize_release_group from subliminal.cache import region from subliminal_patch.http import RetryingCFSession -from subliminal_patch.providers import Provider +from subliminal_patch.providers import Provider, reinitialize_on_error from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin from subliminal_patch.subtitle import Subtitle, guess_matches from subliminal_patch.converters.subscene import language_ids, supported_languages @@ -315,7 +316,9 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin): return search(*args, **kwargs) except requests.HTTPError: region.delete("subscene_cookies2") + raise + @reinitialize_on_error((RequestException,), attempts=1) def query(self, video): subtitles = [] if isinstance(video, Episode): diff --git a/libs/subliminal_patch/providers/xsubs.py b/libs/subliminal_patch/providers/xsubs.py index c7f166390..c23328582 100644 --- a/libs/subliminal_patch/providers/xsubs.py +++ b/libs/subliminal_patch/providers/xsubs.py @@ -6,6 +6,7 @@ import re from subzero.language import Language from guessit import guessit from requests import Session +from requests.exceptions import RequestException from subliminal.providers import ParserBeautifulSoup, Provider from subliminal import __short_version__ @@ -16,6 +17,7 @@ from subliminal.subtitle import Subtitle, fix_line_ending from subliminal.utils import sanitize, sanitize_release_group from subliminal.video import Episode from subliminal_patch.subtitle import guess_matches +from subliminal_patch.providers import reinitialize_on_error logger = logging.getLogger(__name__) article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$') @@ -189,7 +191,8 @@ class XSubsProvider(Provider): break return int(show_id) if show_id else None - + + @reinitialize_on_error((RequestException,), attempts=1) def query(self, show_id, series, season, year=None, country=None): # get the season list of the show logger.info('Getting the season list of show id %d', show_id) @@ -291,6 +294,7 @@ class XSubsProvider(Provider): return [] + @reinitialize_on_error((RequestException,), attempts=1) def download_subtitle(self, subtitle): if isinstance(subtitle, XSubsSubtitle): # download the subtitle