From acbbf6e58bf83246bcd786649641b3e5a0493537 Mon Sep 17 00:00:00 2001 From: Halali Date: Thu, 24 Jan 2019 15:00:03 +0100 Subject: [PATCH] Add chmod, provider throttle, multithread options --- bazarr/config.py | 13 +-- bazarr/get_providers.py | 91 +++++++++++++++++++- bazarr/get_subtitle.py | 178 +++++++++++++++------------------------- bazarr/main.py | 8 ++ views/settings.tpl | 54 ++++++++++++ 5 files changed, 225 insertions(+), 119 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index 895488068..e911c9417 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -32,7 +32,10 @@ defaults = { 'minimum_score_movie': '70', 'use_embedded_subs': 'True', 'adaptive_searching': 'False', - 'enabled_providers': '' + 'enabled_providers': '', + 'throtteled_providers': '', + 'multithreading': 'True', + 'chmod': '0640' }, 'auth': { 'type': 'None', @@ -69,16 +72,16 @@ defaults = { 'username': '', 'password': '', 'use_tag_search': 'False', - 'vip': 'False', - 'ssl': 'False', + 'vip': 'True', + 'ssl': 'True', 'timeout': '15', 'skip_wrong_fps': 'False' }, 'addic7ed': { 'username': '', 'password': '', - 'random_agents': 'False' -}, + 'random_agents': 'True' + }, 'legendastv': { 'username': '', 'password': '' diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index aa8a5f8b0..32bf6dba6 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -1,17 +1,79 @@ # coding=utf-8 +import os import datetime +import logging +import subliminal_patch +from get_args import args from config import settings from subliminal_patch.exceptions import TooManyRequests, APIThrottled from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable +VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled) +PROVIDER_THROTTLE_MAP = { + "default": { + TooManyRequests: (datetime.timedelta(hours=1), "1 hour"), + DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), + ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), + APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), + }, + "opensubtitles": { + TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), + DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), + APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"), + }, + "addic7ed": { + DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), + TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), + } +} + +PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska", + "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"] + +if not settings.general.throtteled_providers: + tp = {} +else: + tp = eval(str(settings.general.throtteled_providers)) + + +def provider_pool(): + if settings.general.getboolean('multithreading'): + return subliminal_patch.core.SZAsyncProviderPool + return subliminal_patch.core.SZProviderPool + + def get_providers(): + changed = False providers_list = [] if settings.general.enabled_providers: for provider in settings.general.enabled_providers.lower().split(','): + reason, until, throttle_desc = tp.get(provider, (None, None, None)) providers_list.append(provider) - else: + + if reason: + now = datetime.datetime.now() + if now < until: + logging.info("Not using %s until %s, because of: %s", provider, + until.strftime("%y/%m/%d %H:%M"), reason) + providers_list.remove(provider) + else: + logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason) + del tp[provider] + settings.general.throtteled_providers = str(tp) + changed = True + + if changed: + with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: + settings.write(handle) + + # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles + # for provider in providers_list: + # if provider in PROVIDERS_FORCED_OFF: + # providers_list.remove(provider) + + if not providers_list: providers_list = None return providers_list @@ -47,3 +109,30 @@ def get_providers_auth(): } return providers_auth + + +def provider_throttle(name, exception): + cls = getattr(exception, "__class__") + cls_name = getattr(cls, "__name__") + if cls not in VALID_THROTTLE_EXCEPTIONS: + for valid_cls in VALID_THROTTLE_EXCEPTIONS: + if isinstance(cls, valid_cls): + cls = valid_cls + + throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \ + PROVIDER_THROTTLE_MAP["default"].get(cls, None) + + if not throttle_data: + return + + throttle_delta, throttle_description = throttle_data + throttle_until = datetime.datetime.now() + throttle_delta + + tp[name] = (cls_name, throttle_until, throttle_description) + + settings.general.throtteled_providers = str(tp) + with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: + settings.write(handle) + + logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description, + throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 9c9ba2ed4..040505a9e 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -28,9 +28,11 @@ from helper import path_replace, path_replace_movie, path_replace_reverse, \ from list_subtitles import store_subtitles, list_missing_subtitles, store_subtitles_movie, list_missing_subtitles_movies from utils import history_log, history_log_movie from notifier import send_notifications, send_notifications_movie -from get_providers import get_providers, get_providers_auth +from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool from get_args import args from queueconfig import q4ws +from subliminal_patch.exceptions import TooManyRequests, APIThrottled +from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable # configure the cache @@ -57,13 +59,17 @@ def get_video(path, title, sceneName, use_scenename, providers=None, media_type= # use the sceneName but keep the folder structure for better guessing path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1]) dont_use_actual_file = True - try: - video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file) - video.used_scene_name = dont_use_actual_file - video.original_name = original_name - video.original_path = original_path - return video + if providers: + video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file) + video.used_scene_name = dont_use_actual_file + video.original_name = original_name + video.original_path = original_path + return video + else: + logging.info("BAZARR All providers are throttled") + return None + except: logging.exception("BAZARR Error trying to get video information for this file: " + path) @@ -144,102 +150,31 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName, AsyncProviderPool: implement: blacklist=None, - throttle_callback=None, pre_download_hook=None, post_download_hook=None, language_hook=None """ - """ - throttle_callback: - - VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled) - - PROVIDER_THROTTLE_MAP = { - "default": { - TooManyRequests: (datetime.timedelta(hours=1), "1 hour"), - DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), - ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), - APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), - }, - "opensubtitles": { - TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), - DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), - APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"), - }, - "addic7ed": { - DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), - TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), - } - } - - throttle_callback gist: - def provider_throttle(self, name, exception): - cls = getattr(exception, "__class__") - cls_name = getattr(cls, "__name__") - if cls not in VALID_THROTTLE_EXCEPTIONS: - for valid_cls in VALID_THROTTLE_EXCEPTIONS: - if isinstance(cls, valid_cls): - cls = valid_cls - - throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \ - PROVIDER_THROTTLE_MAP["default"].get(cls, None) - - if not throttle_data: - return - - throttle_delta, throttle_description = throttle_data - throttle_until = datetime.datetime.now() + throttle_delta - - # save throttle_until together with provider name somewhere, then implement dynamic provider_list based on - # that - - provider_configs= - {'addic7ed': {'username': Prefs['provider.addic7ed.username'], - 'password': Prefs['provider.addic7ed.password'], - 'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents1']), - }, - 'opensubtitles': {'username': Prefs['provider.opensubtitles.username'], - 'password': Prefs['provider.opensubtitles.password'], - 'use_tag_search': self.exact_filenames, - 'only_foreign': self.forced_only, - 'also_foreign': self.forced_also, - 'is_vip': cast_bool(Prefs['provider.opensubtitles.is_vip']), - 'use_ssl': os_use_https, - 'timeout': self.advanced.providers.opensubtitles.timeout or 15, - 'skip_wrong_fps': os_skip_wrong_fps, - }, - 'podnapisi': { - 'only_foreign': self.forced_only, - 'also_foreign': self.forced_also, - }, - 'subscene': { - 'only_foreign': self.forced_only, - }, - 'legendastv': {'username': Prefs['provider.legendastv.username'], - 'password': Prefs['provider.legendastv.password'], - }, - 'assrt': {'token': Prefs['provider.assrt.token'], } - } - - """ - video = get_video(path, title, sceneName, use_scenename, providers=providers, media_type=media_type) if video: min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie), min_score_series_perc=int(minimum_score)) - - downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi, - providers=providers, - provider_configs=providers_auth, - pool_class=SZAsyncProviderPool, - compute_score=compute_score, - throttle_time=None, # fixme - blacklist=None, # fixme - throttle_callback=None, # fixme - pre_download_hook=None, # fixme - post_download_hook=None, # fixme - language_hook=None) # fixme + + if providers: + downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi, + providers=providers, + provider_configs=providers_auth, + pool_class=provider_pool(), + compute_score=compute_score, + throttle_time=None, # fixme + blacklist=None, # fixme + throttle_callback=provider_throttle, + pre_download_hook=None, # fixme + post_download_hook=None, # fixme + language_hook=None) # fixme + else: + downloaded_subtitles = None + logging.info("BAZARR All providers are throttled") saved_any = False if downloaded_subtitles: @@ -251,7 +186,7 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName, saved_subtitles = save_subtitles(video.original_path, subtitles, single=single, tags=None, # fixme directory=None, # fixme - chmod=None, # fixme + chmod=int(settings.general.chmod), # fixme # formats=("srt", "vtt") path_decoder=force_unicode ) @@ -344,12 +279,16 @@ def manual_search(path, language, hi, providers, providers_auth, sceneName, titl min_score_series_perc=int(minimum_score)) try: - subtitles = list_subtitles([video], language_set, - providers=providers, - provider_configs=providers_auth, - pool_class=SZAsyncProviderPool, # fixme: make async optional - throttle_callback=None, # fixme - language_hook=None) # fixme + if providers: + subtitles = list_subtitles([video], language_set, + providers=providers, + provider_configs=providers_auth, + pool_class=provider_pool(), + throttle_callback=provider_throttle, + language_hook=None) # fixme + else: + subtitles = [] + logging.info("BAZARR All providers are throttled") except Exception as e: logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path) else: @@ -402,12 +341,16 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a if video: min_score, max_score, scores = get_scores(video, media_type) try: - download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth, - pool_class=SZAsyncProviderPool, - throttle_callback=None) # fixme - logging.debug('BAZARR Subtitles file downloaded for this file:' + path) + if provider: + download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth, + pool_class=provider_pool(), + throttle_callback=provider_throttle) + logging.debug('BAZARR Subtitles file downloaded for this file:' + path) + else: + logging.info("BAZARR All providers are throttled") + return None except Exception as e: - logging.exception('BAZARR Error downloading subtitles for this file ' + path) + logging.exception('BAZARR Error downloading subtitles for this file ' + path + e) return None else: if not subtitle.is_valid(): @@ -468,7 +411,8 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a return message else: logging.error( - "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.") + "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str( + subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.") return None logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path) @@ -559,7 +503,7 @@ def wanted_download_subtitles(path): for i in range(len(attempt)): if attempt[i][0] == language: - if search_active(attempt[i][1]) is True: + if search_active(attempt[i][1]): q4ws.append( 'Searching ' + str(language_from_alpha2(language)) + ' subtitles for this file: ' + path) message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), @@ -648,14 +592,22 @@ def wanted_search_missing_subtitles(): movies = c.fetchall() c.close() - + providers = get_providers() if settings.general.getboolean('use_sonarr'): - for episode in episodes: - wanted_download_subtitles(episode[0]) + if providers: + for episode in episodes: + wanted_download_subtitles(episode[0]) + else: + q4ws.append('BAZARR All providers are throttled') + logging.info("BAZARR All providers are throttled") if settings.general.getboolean('use_radarr'): - for movie in movies: - wanted_download_subtitles_movie(movie[0]) + if providers: + for movie in movies: + wanted_download_subtitles_movie(movie[0]) + else: + q4ws.append('BAZARR All providers are throttled') + logging.info("BAZARR All providers are throttled") logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.') diff --git a/bazarr/main.py b/bazarr/main.py index 72545446c..f7c7870a7 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1103,6 +1103,7 @@ def save_settings(): settings_general_debug = 'False' else: settings_general_debug = 'True' + settings_general_chmod = request.forms.get('settings_general_chmod') settings_general_sourcepath = request.forms.getall('settings_general_sourcepath') settings_general_destpath = request.forms.getall('settings_general_destpath') settings_general_pathmapping = [] @@ -1138,6 +1139,11 @@ def save_settings(): settings_general_adaptive_searching = 'False' else: settings_general_adaptive_searching = 'True' + settings_general_multithreading = request.forms.get('settings_general_multithreading') + if settings_general_multithreading is None: + settings_general_multithreading = 'False' + else: + settings_general_multithreading = 'True' settings_general_minimum_score = request.forms.get('settings_general_minimum_score') settings_general_minimum_score_movies = request.forms.get('settings_general_minimum_score_movies') settings_general_use_postprocessing = request.forms.get('settings_general_use_postprocessing') @@ -1170,6 +1176,7 @@ def save_settings(): settings.general.base_url = text_type(settings_general_baseurl) settings.general.path_mappings = text_type(settings_general_pathmapping) settings.general.debug = text_type(settings_general_debug) + settings.general.chmod = text_type(settings_general_chmod) settings.general.branch = text_type(settings_general_branch) settings.general.auto_update = text_type(settings_general_automatic) settings.general.single_language = text_type(settings_general_single_language) @@ -1184,6 +1191,7 @@ def save_settings(): settings.general.minimum_score_movie = text_type(settings_general_minimum_score_movies) settings.general.use_embedded_subs = text_type(settings_general_embedded) settings.general.adaptive_searching = text_type(settings_general_adaptive_searching) + settings.general.multithreading = text_type(settings_general_multithreading) if after != before: configured() diff --git a/views/settings.tpl b/views/settings.tpl index df6659ca7..84e025d87 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -153,6 +153,26 @@ +
+
+ +
+
+
+
+ + +
+
+
+ +
+
@@ -1076,6 +1096,26 @@
+ +
+
+ +
+
+
+ + +
+
+ +
Subtitles providers
@@ -1760,6 +1800,12 @@ $("#settings_adaptive_searching").checkbox('uncheck'); } + if ($('#settings_multithreading').data("multithreading") === "True") { + $("#settings_multithreading").checkbox('check'); + } else { + $("#settings_multithreading").checkbox('uncheck'); + } + if ($('#settings_addic7ed_random_agents').data("randomagents") === "True") { $("#settings_addic7ed_random_agents").checkbox('check'); } else { @@ -2057,6 +2103,14 @@ } ] }, + settings_general_chmod: { + rules: [ + { + type: 'regExp[^(0[0-7]{3})$]', + prompt: 'Please use only 4-digit integers with leading 0 (e.g.: 775)' + } + ] + }, settings_auth_password : { depends: 'settings_auth_username', rules : [