From 8303cb94dc27e0399e9399e8c9a2bdde1394c338 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 28 Jul 2019 21:49:19 +0200 Subject: [PATCH] Optimize provider throttling. --- bazarr/get_providers.py | 62 ++++++++--- bazarr/get_subtitle.py | 240 +++++++++++++++++++++++----------------- 2 files changed, 184 insertions(+), 118 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index 8a033be49..fbfb04ded 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -4,6 +4,7 @@ import datetime import logging import subliminal_patch import pretty +import time from get_args import args from config import settings @@ -33,6 +34,8 @@ PROVIDER_THROTTLE_MAP = { PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska", "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"] +throttle_count = {} + if not settings.general.throtteled_providers: tp = {} else: @@ -44,7 +47,7 @@ def provider_pool(): return subliminal_patch.core.SZAsyncProviderPool return subliminal_patch.core.SZProviderPool - + def get_providers(): changed = False providers_list = [] @@ -52,23 +55,23 @@ def get_providers(): for provider in settings.general.enabled_providers.lower().split(','): reason, until, throttle_desc = tp.get(provider, (None, None, None)) providers_list.append(provider) - + if reason: now = datetime.datetime.now() if now < until: logging.debug("Not using %s until %s, because of: %s", provider, - until.strftime("%y/%m/%d %H:%M"), reason) + until.strftime("%y/%m/%d %H:%M"), reason) providers_list.remove(provider) else: logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason) del tp[provider] settings.general.throtteled_providers = str(tp) changed = True - + if changed: with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: settings.write(handle) - + # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles # for provider in providers_list: # if provider in PROVIDERS_FORCED_OFF: @@ -109,7 +112,7 @@ def get_providers_auth(): }, 'xsubs': {'username': settings.xsubs.username, 'password': settings.xsubs.password, - }, + }, 'assrt': {'token': settings.assrt.token, }, 'napisy24': {'username': settings.napisy24.username, 'password': settings.napisy24.password, @@ -137,21 +140,52 @@ def provider_throttle(name, exception): throttle_delta, throttle_description = throttle_data throttle_until = datetime.datetime.now() + throttle_delta - tp[name] = (cls_name, throttle_until, throttle_description) + if throttled_count(name): + tp[name] = (cls_name, throttle_until, throttle_description) + settings.general.throtteled_providers = str(tp) + with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: + settings.write(handle) + + logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description, + throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message) + + +def throttled_count(name): + global throttle_count + if name in throttle_count.keys(): + if 'count' in throttle_count[name].keys(): + for key, value in throttle_count[name].items(): + if key == 'count': + value += 1 + throttle_count[name]['count'] = value + else: + throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} - settings.general.throtteled_providers = str(tp) - with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: - settings.write(handle) + else: + throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} - logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description, - throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message) + if throttle_count[name]['count'] < 5: + if throttle_count[name]['time'] > datetime.datetime.now(): + logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name, + throttle_count[name]['count']) + time.sleep(5) + return False + else: + throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))} + logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name, + throttle_count[name]['count']) + time.sleep(5) + return False + else: + return True + def update_throttled_provider(): changed = False if settings.general.enabled_providers: for provider in settings.general.enabled_providers.lower().split(','): reason, until, throttle_desc = tp.get(provider, (None, None, None)) - + if reason: now = datetime.datetime.now() if now < until: @@ -161,7 +195,7 @@ def update_throttled_provider(): del tp[provider] settings.general.throtteled_providers = str(tp) changed = True - + if changed: with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: settings.write(handle) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 2a88970de..6c1e1f46f 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -5,7 +5,6 @@ import sys import sqlite3 import ast import logging -import operator import subprocess import time import cPickle as pickle @@ -13,7 +12,6 @@ import codecs import types import re import subliminal -import subliminal_patch from datetime import datetime, timedelta from subzero.language import Language from subzero.video import parse_video @@ -58,26 +56,23 @@ def get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=No path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1]) used_scene_name = True hash_from = original_path - if providers: - try: - video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name, - hash_from=hash_from) - video.used_scene_name = used_scene_name - video.original_name = original_name - video.original_path = original_path - refine_from_db(original_path, video) - - if use_mediainfo: - refine_from_mediainfo(original_path, video) - - logging.debug('BAZARR is using those video object properties: %s', vars(video)) - return video + + try: + video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name, + hash_from=hash_from) + video.used_scene_name = used_scene_name + video.original_name = original_name + video.original_path = original_path + refine_from_db(original_path, video) - except: - logging.exception("BAZARR Error trying to get video information for this file: " + path) - else: - logging.info("BAZARR All providers are throttled") - return None + if use_mediainfo: + refine_from_mediainfo(original_path, video) + + logging.debug('BAZARR is using those video object properties: %s', vars(video)) + return video + + except: + logging.exception("BAZARR Error trying to get video information for this file: " + path) def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_score_series_perc=240 * 100 / 360.0, @@ -125,9 +120,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce language = [language] if forced == "True": - providers_auth['podnapisi']['only_foreign'] = True - providers_auth['subscene']['only_foreign'] = True - providers_auth['opensubtitles']['only_foreign'] = True + providers_auth['podnapisi']['only_foreign'] = True ## fixme: This is also in get_providers_auth() + providers_auth['subscene']['only_foreign'] = True ## fixme: This is also in get_providers_auth() + providers_auth['opensubtitles']['only_foreign'] = True ## fixme: This is also in get_providers_auth() else: providers_auth['podnapisi']['only_foreign'] = False providers_auth['subscene']['only_foreign'] = False @@ -161,8 +156,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce post_download_hook=None, language_hook=None """ - - video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type) + video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, + media_type=media_type) if video: min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie), min_score_series_perc=int(minimum_score)) @@ -184,6 +179,7 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce else: downloaded_subtitles = None logging.info("BAZARR All providers are throttled") + return None saved_any = False if downloaded_subtitles: @@ -273,9 +269,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce if not saved_any: logging.debug('BAZARR No subtitles were found for this file: ' + path) return None - + subliminal.region.backend.sync() - + logging.debug('BAZARR Ended searching subtitles for file: ' + path) @@ -317,8 +313,12 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa minimum_score_movie = settings.general.minimum_score_movie use_postprocessing = settings.general.getboolean('use_postprocessing') postprocessing_cmd = settings.general.postprocessing_cmd - - video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type) + if providers: + video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, + media_type=media_type) + else: + logging.info("BAZARR All providers are throttled") + return None if video: min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie), min_score_series_perc=int(minimum_score)) @@ -333,6 +333,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa else: subtitles = [] logging.info("BAZARR All providers are throttled") + return None except Exception as e: logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path) else: @@ -368,9 +369,9 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa final_subtitles = sorted(subtitles_list, key=lambda x: x['score'], reverse=True) logging.debug('BAZARR ' + str(len(final_subtitles)) + " subtitles have been found for this file: " + path) logging.debug('BAZARR Ended searching subtitles for this file: ' + path) - + subliminal.region.backend.sync() - + return final_subtitles @@ -389,7 +390,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro use_postprocessing = settings.general.getboolean('use_postprocessing') postprocessing_cmd = settings.general.postprocessing_cmd single = settings.general.getboolean('single_language') - video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider}, media_type=media_type) + video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider}, + media_type=media_type) if video: min_score, max_score, scores = get_scores(video, media_type) try: @@ -483,9 +485,9 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str( subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.") return None - + subliminal.region.backend.sync() - + logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path) @@ -510,23 +512,29 @@ def series_download_subtitles(no): count_episodes_details = len(episodes_details) for i, episode in enumerate(episodes_details, 1): - for language in ast.literal_eval(episode[1]): - if language is not None: - notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i, - length=count_episodes_details) - result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), - series_details[0], series_details[2], providers_list, - providers_auth, str(episode[3]), series_details[1], 'series') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - language_code = result[2] + ":forced" if forced else result[2] - provider = result[3] - score = result[4] - store_subtitles(path_replace(episode[0])) - history_log(1, no, episode[2], message, path, language_code, provider, score) - send_notifications(no, episode[2], message) + if providers_list: + for language in ast.literal_eval(episode[1]): + if language is not None: + notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i, + length=count_episodes_details) + result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), + series_details[0], series_details[2], providers_list, + providers_auth, str(episode[3]), series_details[1], + 'series') + if result is not None: + message = result[0] + path = result[1] + forced = result[5] + language_code = result[2] + ":forced" if forced else result[2] + provider = result[3] + score = result[4] + store_subtitles(path_replace(episode[0])) + history_log(1, no, episode[2], message, path, language_code, provider, score) + send_notifications(no, episode[2], message) + else: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + break list_missing_subtitles(no) if count_episodes_details: @@ -551,26 +559,30 @@ def episode_download_subtitles(no): providers_auth = get_providers_auth() for episode in episodes_details: - for language in ast.literal_eval(episode[1]): - if language is not None: - notifications.write(msg='Searching for ' + str( - language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]), - queue='get_subtitle') - result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), - episode[4], episode[7], providers_list, providers_auth, str(episode[3]), - episode[5], 'series') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - language_code = result[2] + ":forced" if forced else result[2] - provider = result[3] - score = result[4] - store_subtitles(path_replace(episode[0])) - history_log(1, episode[6], episode[2], message, path, language_code, provider, score) - send_notifications(episode[6], episode[2], message) - - list_missing_subtitles(episode[6]) + if providers_list: + for language in ast.literal_eval(episode[1]): + if language is not None: + notifications.write(msg='Searching for ' + str( + language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]), + queue='get_subtitle') + result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), + episode[4], episode[7], providers_list, providers_auth, str(episode[3]), + episode[5], 'series') + if result is not None: + message = result[0] + path = result[1] + forced = result[5] + language_code = result[2] + ":forced" if forced else result[2] + provider = result[3] + score = result[4] + store_subtitles(path_replace(episode[0])) + history_log(1, episode[6], episode[2], message, path, language_code, provider, score) + send_notifications(episode[6], episode[2], message) + else: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + list_missing_subtitles(episode[6]) + break def movies_download_subtitles(no): @@ -587,20 +599,26 @@ def movies_download_subtitles(no): count_movie = len(ast.literal_eval(movie[1])) for i, language in enumerate(ast.literal_eval(movie[1]), 1): - if language is not None: - notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i, length=count_movie) - result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], - movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie') - if result is not None: - message = result[0] - path = result[1] - forced = result[5] - language_code = result[2] + ":forced" if forced else result[2] - provider = result[3] - score = result[4] - store_subtitles_movie(path_replace_movie(movie[0])) - history_log_movie(1, no, message, path, language_code, provider, score) - send_notifications_movie(no, message) + if providers_list: + if language is not None: + notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i, + length=count_movie) + result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], + movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie') + if result is not None: + message = result[0] + path = result[1] + forced = result[5] + language_code = result[2] + ":forced" if forced else result[2] + provider = result[3] + score = result[4] + store_subtitles_movie(path_replace_movie(movie[0])) + history_log_movie(1, no, message, path, language_code, provider, score) + send_notifications_movie(no, message) + else: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + break list_missing_subtitles_movies(no) if count_movie: @@ -744,24 +762,28 @@ def wanted_search_missing_subtitles(): movies = c.fetchall() c.close() - providers = get_providers() if settings.general.getboolean('use_sonarr'): - if providers: - count_episodes = len(episodes) - for i, episode in enumerate(episodes, 1): + + count_episodes = len(episodes) + for i, episode in enumerate(episodes, 1): + providers = get_providers() + if providers: wanted_download_subtitles(episode[0], i, count_episodes) - else: - notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') - logging.info("BAZARR All providers are throttled") + else: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + return if settings.general.getboolean('use_radarr'): - if providers: - count_movies = len(movies) - for i, movie in enumerate(movies, 1): + count_movies = len(movies) + for i, movie in enumerate(movies, 1): + providers = get_providers() + if providers: wanted_download_subtitles_movie(movie[0], i, count_movies) - else: - notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') - logging.info("BAZARR All providers are throttled") + else: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + return logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.') @@ -841,23 +863,23 @@ def refine_from_db(path, video): def refine_from_mediainfo(path, video): if video.fps: return - + exe = get_binary('mediainfo') if not exe: logging.debug('BAZARR MediaInfo library not found!') return else: logging.debug('BAZARR MediaInfo library used is %s', exe) - + media_info = MediaInfo.parse(path, library_file=exe) - + video_track = next((t for t in media_info.tracks if t.track_type == 'Video'), None) if not video_track: logging.debug('BAZARR MediaInfo was unable to find video tracks in the file!') return - + logging.debug('MediaInfo found: %s', video_track.to_data()) - + if not video.fps: if video_track.frame_rate: video.fps = float(video_track.frame_rate) @@ -920,6 +942,11 @@ def upgrade_subtitles(): if settings.general.getboolean('use_sonarr'): for i, episode in enumerate(episodes_to_upgrade, 1): + providers = get_providers() + if not providers: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + return if episode[9] != "None": desired_languages = ast.literal_eval(str(episode[9])) if episode[10] == "True": @@ -957,6 +984,11 @@ def upgrade_subtitles(): if settings.general.getboolean('use_radarr'): for i, movie in enumerate(movies_to_upgrade, 1): + providers = get_providers() + if not providers: + notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long') + logging.info("BAZARR All providers are throttled") + return if movie[8] != "None": desired_languages = ast.literal_eval(str(movie[8])) if movie[9] == "True":