open-source-mirrors
/
bazarr
mirror of https://github.com/morpheus65535/bazarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add chmod, provider throttle, multithread options

Browse Source
pull/489/head
Halali 5 years ago
parent 8472a48e28
commit acbbf6e58b
5 changed files with 225 additions and 119 deletions
Show Stats Download Patch File Download Diff File

13
bazarr/config.py
Unescape View File

@ -32,7 +32,10 @@ defaults = {
'minimum_score_movie': '70',
'use_embedded_subs': 'True',
'adaptive_searching': 'False',
'enabled_providers': ''
'enabled_providers': '',
'throtteled_providers': '',
'multithreading': 'True',
'chmod': '0640'
},
'auth': {
'type': 'None',
@ -69,16 +72,16 @@ defaults = {
'username': '',
'password': '',
'use_tag_search': 'False',
'vip': 'False',
'ssl': 'False',
'vip': 'True',
'ssl': 'True',
'timeout': '15',
'skip_wrong_fps': 'False'
},
'addic7ed': {
'username': '',
'password': '',
'random_agents': 'False'
},
'random_agents': 'True'
},
'legendastv': {
'username': '',
'password': ''

91
bazarr/get_providers.py
Unescape View File

@ -1,17 +1,79 @@
# coding=utf-8
import os
import datetime
import logging
import subliminal_patch
from get_args import args
from config import settings
from subliminal_patch.exceptions import TooManyRequests, APIThrottled
from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled)
PROVIDER_THROTTLE_MAP = {
"default": {
TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
},
"opensubtitles": {
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
},
"addic7ed": {
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
}
}
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska",
"supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
if not settings.general.throtteled_providers:
tp = {}
else:
tp = eval(str(settings.general.throtteled_providers))
def provider_pool():
if settings.general.getboolean('multithreading'):
return subliminal_patch.core.SZAsyncProviderPool
return subliminal_patch.core.SZProviderPool
def get_providers():
changed = False
providers_list = []
if settings.general.enabled_providers:
for provider in settings.general.enabled_providers.lower().split(','):
reason, until, throttle_desc = tp.get(provider, (None, None, None))
providers_list.append(provider)
else:
if reason:
now = datetime.datetime.now()
if now < until:
logging.info("Not using %s until %s, because of: %s", provider,
until.strftime("%y/%m/%d %H:%M"), reason)
providers_list.remove(provider)
else:
logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
del tp[provider]
settings.general.throtteled_providers = str(tp)
changed = True
if changed:
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)
# if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles
# for provider in providers_list:
# if provider in PROVIDERS_FORCED_OFF:
# providers_list.remove(provider)
if not providers_list:
providers_list = None
return providers_list
@ -47,3 +109,30 @@ def get_providers_auth():
}
return providers_auth
def provider_throttle(name, exception):
cls = getattr(exception, "__class__")
cls_name = getattr(cls, "__name__")
if cls not in VALID_THROTTLE_EXCEPTIONS:
for valid_cls in VALID_THROTTLE_EXCEPTIONS:
if isinstance(cls, valid_cls):
cls = valid_cls
throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
PROVIDER_THROTTLE_MAP["default"].get(cls, None)
if not throttle_data:
return
throttle_delta, throttle_description = throttle_data
throttle_until = datetime.datetime.now() + throttle_delta
tp[name] = (cls_name, throttle_until, throttle_description)
settings.general.throtteled_providers = str(tp)
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)
logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)

178
bazarr/get_subtitle.py
Unescape View File

@ -28,9 +28,11 @@ from helper import path_replace, path_replace_movie, path_replace_reverse, \
from list_subtitles import store_subtitles, list_missing_subtitles, store_subtitles_movie, list_missing_subtitles_movies
from utils import history_log, history_log_movie
from notifier import send_notifications, send_notifications_movie
from get_providers import get_providers, get_providers_auth
from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool
from get_args import args
from queueconfig import q4ws
from subliminal_patch.exceptions import TooManyRequests, APIThrottled
from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
# configure the cache
@ -57,13 +59,17 @@ def get_video(path, title, sceneName, use_scenename, providers=None, media_type=
# use the sceneName but keep the folder structure for better guessing
path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1])
dont_use_actual_file = True
try:
video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file)
video.used_scene_name = dont_use_actual_file
video.original_name = original_name
video.original_path = original_path
return video
if providers:
video = parse_video(path, hints=hints, providers=providers, dry_run=dont_use_actual_file)
video.used_scene_name = dont_use_actual_file
video.original_name = original_name
video.original_path = original_path
return video
else:
logging.info("BAZARR All providers are throttled")
return None
except:
logging.exception("BAZARR Error trying to get video information for this file: " + path)
@ -144,102 +150,31 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
AsyncProviderPool:
implement:
blacklist=None,
throttle_callback=None,
pre_download_hook=None,
post_download_hook=None,
language_hook=None
"""
"""
throttle_callback:
VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled)
PROVIDER_THROTTLE_MAP = {
"default": {
TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
},
"opensubtitles": {
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
},
"addic7ed": {
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
}
}
throttle_callback gist:
def provider_throttle(self, name, exception):
cls = getattr(exception, "__class__")
cls_name = getattr(cls, "__name__")
if cls not in VALID_THROTTLE_EXCEPTIONS:
for valid_cls in VALID_THROTTLE_EXCEPTIONS:
if isinstance(cls, valid_cls):
cls = valid_cls
throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
PROVIDER_THROTTLE_MAP["default"].get(cls, None)
if not throttle_data:
return
throttle_delta, throttle_description = throttle_data
throttle_until = datetime.datetime.now() + throttle_delta
# save throttle_until together with provider name somewhere, then implement dynamic provider_list based on
# that
provider_configs=
{'addic7ed': {'username': Prefs['provider.addic7ed.username'],
'password': Prefs['provider.addic7ed.password'],
'use_random_agents': cast_bool(Prefs['provider.addic7ed.use_random_agents1']),
},
'opensubtitles': {'username': Prefs['provider.opensubtitles.username'],
'password': Prefs['provider.opensubtitles.password'],
'use_tag_search': self.exact_filenames,
'only_foreign': self.forced_only,
'also_foreign': self.forced_also,
'is_vip': cast_bool(Prefs['provider.opensubtitles.is_vip']),
'use_ssl': os_use_https,
'timeout': self.advanced.providers.opensubtitles.timeout or 15,
'skip_wrong_fps': os_skip_wrong_fps,
},
'podnapisi': {
'only_foreign': self.forced_only,
'also_foreign': self.forced_also,
},
'subscene': {
'only_foreign': self.forced_only,
},
'legendastv': {'username': Prefs['provider.legendastv.username'],
'password': Prefs['provider.legendastv.password'],
},
'assrt': {'token': Prefs['provider.assrt.token'], }
}
"""
video = get_video(path, title, sceneName, use_scenename, providers=providers, media_type=media_type)
if video:
min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
min_score_series_perc=int(minimum_score))
downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
providers=providers,
provider_configs=providers_auth,
pool_class=SZAsyncProviderPool,
compute_score=compute_score,
throttle_time=None, # fixme
blacklist=None, # fixme
throttle_callback=None, # fixme
pre_download_hook=None, # fixme
post_download_hook=None, # fixme
language_hook=None) # fixme
if providers:
downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
providers=providers,
provider_configs=providers_auth,
pool_class=provider_pool(),
compute_score=compute_score,
throttle_time=None, # fixme
blacklist=None, # fixme
throttle_callback=provider_throttle,
pre_download_hook=None, # fixme
post_download_hook=None, # fixme
language_hook=None) # fixme
else:
downloaded_subtitles = None
logging.info("BAZARR All providers are throttled")
saved_any = False
if downloaded_subtitles:
@ -251,7 +186,7 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
saved_subtitles = save_subtitles(video.original_path, subtitles, single=single,
tags=None, # fixme
directory=None, # fixme
chmod=None, # fixme
chmod=int(settings.general.chmod), # fixme
# formats=("srt", "vtt")
path_decoder=force_unicode
)
@ -344,12 +279,16 @@ def manual_search(path, language, hi, providers, providers_auth, sceneName, titl
min_score_series_perc=int(minimum_score))
try:
subtitles = list_subtitles([video], language_set,
providers=providers,
provider_configs=providers_auth,
pool_class=SZAsyncProviderPool, # fixme: make async optional
throttle_callback=None, # fixme
language_hook=None) # fixme
if providers:
subtitles = list_subtitles([video], language_set,
providers=providers,
provider_configs=providers_auth,
pool_class=provider_pool(),
throttle_callback=provider_throttle,
language_hook=None) # fixme
else:
subtitles = []
logging.info("BAZARR All providers are throttled")
except Exception as e:
logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
else:
@ -402,12 +341,16 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
if video:
min_score, max_score, scores = get_scores(video, media_type)
try:
download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth,
pool_class=SZAsyncProviderPool,
throttle_callback=None) # fixme
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
if provider:
download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth,
pool_class=provider_pool(),
throttle_callback=provider_throttle)
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
else:
logging.info("BAZARR All providers are throttled")
return None
except Exception as e:
logging.exception('BAZARR Error downloading subtitles for this file ' + path)
logging.exception('BAZARR Error downloading subtitles for this file ' + path + e)
return None
else:
if not subtitle.is_valid():
@ -468,7 +411,8 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
return message
else:
logging.error(
"BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
"BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(
subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
return None
logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path)
@ -559,7 +503,7 @@ def wanted_download_subtitles(path):
for i in range(len(attempt)):
if attempt[i][0] == language:
if search_active(attempt[i][1]) is True:
if search_active(attempt[i][1]):
q4ws.append(
'Searching ' + str(language_from_alpha2(language)) + ' subtitles for this file: ' + path)
message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
@ -648,14 +592,22 @@ def wanted_search_missing_subtitles():
movies = c.fetchall()
c.close()
providers = get_providers()
if settings.general.getboolean('use_sonarr'):
for episode in episodes:
wanted_download_subtitles(episode[0])
if providers:
for episode in episodes:
wanted_download_subtitles(episode[0])
else:
q4ws.append('BAZARR All providers are throttled')
logging.info("BAZARR All providers are throttled")
if settings.general.getboolean('use_radarr'):
for movie in movies:
wanted_download_subtitles_movie(movie[0])
if providers:
for movie in movies:
wanted_download_subtitles_movie(movie[0])
else:
q4ws.append('BAZARR All providers are throttled')
logging.info("BAZARR All providers are throttled")
logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.')

8
bazarr/main.py
Unescape View File

@ -1103,6 +1103,7 @@ def save_settings():
settings_general_debug = 'False'
else:
settings_general_debug = 'True'
settings_general_chmod = request.forms.get('settings_general_chmod')
settings_general_sourcepath = request.forms.getall('settings_general_sourcepath')
settings_general_destpath = request.forms.getall('settings_general_destpath')
settings_general_pathmapping = []
@ -1138,6 +1139,11 @@ def save_settings():
settings_general_adaptive_searching = 'False'
else:
settings_general_adaptive_searching = 'True'
settings_general_multithreading = request.forms.get('settings_general_multithreading')
if settings_general_multithreading is None:
settings_general_multithreading = 'False'
else:
settings_general_multithreading = 'True'
settings_general_minimum_score = request.forms.get('settings_general_minimum_score')
settings_general_minimum_score_movies = request.forms.get('settings_general_minimum_score_movies')
settings_general_use_postprocessing = request.forms.get('settings_general_use_postprocessing')
@ -1170,6 +1176,7 @@ def save_settings():
settings.general.base_url = text_type(settings_general_baseurl)
settings.general.path_mappings = text_type(settings_general_pathmapping)
settings.general.debug = text_type(settings_general_debug)
settings.general.chmod = text_type(settings_general_chmod)
settings.general.branch = text_type(settings_general_branch)
settings.general.auto_update = text_type(settings_general_automatic)
settings.general.single_language = text_type(settings_general_single_language)
@ -1184,6 +1191,7 @@ def save_settings():
settings.general.minimum_score_movie = text_type(settings_general_minimum_score_movies)
settings.general.use_embedded_subs = text_type(settings_general_embedded)
settings.general.adaptive_searching = text_type(settings_general_adaptive_searching)
settings.general.multithreading = text_type(settings_general_multithreading)
if after != before:
configured()

54
views/settings.tpl
Unescape View File

@ -153,6 +153,26 @@
</div>
</div>
<div class="middle aligned row">
<div class="right aligned four wide column">
<label>Set subtitle file permissions to</label>
</div>
<div class="five wide column">
<div class='field'>
<div id="settings_chmod" class="ui fluid input">
<input name="settings_general_chmod" type="text"
value={{ settings.general.chmod }}>
<label></label>
</div>
</div>
</div>
<div class="collapsed center aligned column">
<div class="ui basic icon" data-tooltip="Integer, e.g.: 0775" data-inverted="">
<i class="help circle large icon"></i>
</div>
</div>
</div>
<div class="middle aligned row">
<div class="right aligned four wide column">
<label>Page size</label>
@ -1076,6 +1096,26 @@
</div>
</div>
</div>
<div class="middle aligned row">
<div class="right aligned four wide column">
<label>Search enabled providers simultaneously</label>
</div>
<div class="one wide column">
<div id="settings_multithreading" class="ui toggle checkbox"
data-multithreading={{ settings.general.getboolean('multithreading') }}>
<input name="settings_general_multithreading" type="checkbox">
<label></label>
</div>
</div>
<div class="collapsed column">
<div class="collapsed center aligned column">
<div class="ui basic icon" data-tooltip="Multithreading" data-inverted="">
<i class="help circle large icon"></i>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="ui dividing header">Subtitles providers</div>
@ -1760,6 +1800,12 @@
$("#settings_adaptive_searching").checkbox('uncheck');
}
if ($('#settings_multithreading').data("multithreading") === "True") {
$("#settings_multithreading").checkbox('check');
} else {
$("#settings_multithreading").checkbox('uncheck');
}
if ($('#settings_addic7ed_random_agents').data("randomagents") === "True") {
$("#settings_addic7ed_random_agents").checkbox('check');
} else {
@ -2057,6 +2103,14 @@
}
]
},
settings_general_chmod: {
rules: [
{
type: 'regExp[^(0[0-7]{3})$]',
prompt: 'Please use only 4-digit integers with leading 0 (e.g.: 775)'
}
]
},
settings_auth_password : {
depends: 'settings_auth_username',
rules : [

Write Preview
Loading…
Cancel
Save