open-source-mirrors
/
bazarr
mirror of https://github.com/morpheus65535/bazarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize provider throttling.

Browse Source
pull/511/head
Halali 5 years ago
parent e023e82e44
commit 8303cb94dc
2 changed files with 184 additions and 118 deletions
Show Stats Download Patch File Download Diff File

62
bazarr/get_providers.py
Unescape View File

@ -4,6 +4,7 @@ import datetime
import logging
import subliminal_patch
import pretty
import time
from get_args import args
from config import settings
@ -33,6 +34,8 @@ PROVIDER_THROTTLE_MAP = {
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska",
"supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
throttle_count = {}
if not settings.general.throtteled_providers:
tp = {}
else:
@ -44,7 +47,7 @@ def provider_pool():
return subliminal_patch.core.SZAsyncProviderPool
return subliminal_patch.core.SZProviderPool
def get_providers():
changed = False
providers_list = []
@ -52,23 +55,23 @@ def get_providers():
for provider in settings.general.enabled_providers.lower().split(','):
reason, until, throttle_desc = tp.get(provider, (None, None, None))
providers_list.append(provider)
if reason:
now = datetime.datetime.now()
if now < until:
logging.debug("Not using %s until %s, because of: %s", provider,
until.strftime("%y/%m/%d %H:%M"), reason)
until.strftime("%y/%m/%d %H:%M"), reason)
providers_list.remove(provider)
else:
logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
del tp[provider]
settings.general.throtteled_providers = str(tp)
changed = True
if changed:
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)
# if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles
# for provider in providers_list:
# if provider in PROVIDERS_FORCED_OFF:
@ -109,7 +112,7 @@ def get_providers_auth():
},
'xsubs': {'username': settings.xsubs.username,
'password': settings.xsubs.password,
},
},
'assrt': {'token': settings.assrt.token, },
'napisy24': {'username': settings.napisy24.username,
'password': settings.napisy24.password,
@ -137,21 +140,52 @@ def provider_throttle(name, exception):
throttle_delta, throttle_description = throttle_data
throttle_until = datetime.datetime.now() + throttle_delta
tp[name] = (cls_name, throttle_until, throttle_description)
if throttled_count(name):
tp[name] = (cls_name, throttle_until, throttle_description)
settings.general.throtteled_providers = str(tp)
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)
logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
def throttled_count(name):
global throttle_count
if name in throttle_count.keys():
if 'count' in throttle_count[name].keys():
for key, value in throttle_count[name].items():
if key == 'count':
value += 1
throttle_count[name]['count'] = value
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
settings.general.throtteled_providers = str(tp)
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
if throttle_count[name]['count'] < 5:
if throttle_count[name]['time'] > datetime.datetime.now():
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
throttle_count[name]['count'])
time.sleep(5)
return False
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
throttle_count[name]['count'])
time.sleep(5)
return False
else:
return True
def update_throttled_provider():
changed = False
if settings.general.enabled_providers:
for provider in settings.general.enabled_providers.lower().split(','):
reason, until, throttle_desc = tp.get(provider, (None, None, None))
if reason:
now = datetime.datetime.now()
if now < until:
@ -161,7 +195,7 @@ def update_throttled_provider():
del tp[provider]
settings.general.throtteled_providers = str(tp)
changed = True
if changed:
with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
settings.write(handle)

240
bazarr/get_subtitle.py
Unescape View File

@ -5,7 +5,6 @@ import sys
import sqlite3
import ast
import logging
import operator
import subprocess
import time
import cPickle as pickle
@ -13,7 +12,6 @@ import codecs
import types
import re
import subliminal
import subliminal_patch
from datetime import datetime, timedelta
from subzero.language import Language
from subzero.video import parse_video
@ -58,26 +56,23 @@ def get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=No
path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1])
used_scene_name = True
hash_from = original_path
if providers:
try:
video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name,
hash_from=hash_from)
video.used_scene_name = used_scene_name
video.original_name = original_name
video.original_path = original_path
refine_from_db(original_path, video)
if use_mediainfo:
refine_from_mediainfo(original_path, video)
logging.debug('BAZARR is using those video object properties: %s', vars(video))
return video
try:
video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name,
hash_from=hash_from)
video.used_scene_name = used_scene_name
video.original_name = original_name
video.original_path = original_path
refine_from_db(original_path, video)
except:
logging.exception("BAZARR Error trying to get video information for this file: " + path)
else:
logging.info("BAZARR All providers are throttled")
return None
if use_mediainfo:
refine_from_mediainfo(original_path, video)
logging.debug('BAZARR is using those video object properties: %s', vars(video))
return video
except:
logging.exception("BAZARR Error trying to get video information for this file: " + path)
def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_score_series_perc=240 * 100 / 360.0,
@ -125,9 +120,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
language = [language]
if forced == "True":
providers_auth['podnapisi']['only_foreign'] = True
providers_auth['subscene']['only_foreign'] = True
providers_auth['opensubtitles']['only_foreign'] = True
providers_auth['podnapisi']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
providers_auth['subscene']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
providers_auth['opensubtitles']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
else:
providers_auth['podnapisi']['only_foreign'] = False
providers_auth['subscene']['only_foreign'] = False
@ -161,8 +156,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
post_download_hook=None,
language_hook=None
"""
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
media_type=media_type)
if video:
min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
min_score_series_perc=int(minimum_score))
@ -184,6 +179,7 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
else:
downloaded_subtitles = None
logging.info("BAZARR All providers are throttled")
return None
saved_any = False
if downloaded_subtitles:
@ -273,9 +269,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
if not saved_any:
logging.debug('BAZARR No subtitles were found for this file: ' + path)
return None
subliminal.region.backend.sync()
logging.debug('BAZARR Ended searching subtitles for file: ' + path)
@ -317,8 +313,12 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
minimum_score_movie = settings.general.minimum_score_movie
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
if providers:
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
media_type=media_type)
else:
logging.info("BAZARR All providers are throttled")
return None
if video:
min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
min_score_series_perc=int(minimum_score))
@ -333,6 +333,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
else:
subtitles = []
logging.info("BAZARR All providers are throttled")
return None
except Exception as e:
logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
else:
@ -368,9 +369,9 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
final_subtitles = sorted(subtitles_list, key=lambda x: x['score'], reverse=True)
logging.debug('BAZARR ' + str(len(final_subtitles)) + " subtitles have been found for this file: " + path)
logging.debug('BAZARR Ended searching subtitles for this file: ' + path)
subliminal.region.backend.sync()
return final_subtitles
@ -389,7 +390,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
single = settings.general.getboolean('single_language')
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider}, media_type=media_type)
video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider},
media_type=media_type)
if video:
min_score, max_score, scores = get_scores(video, media_type)
try:
@ -483,9 +485,9 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
"BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(
subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
return None
subliminal.region.backend.sync()
logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path)
@ -510,23 +512,29 @@ def series_download_subtitles(no):
count_episodes_details = len(episodes_details)
for i, episode in enumerate(episodes_details, 1):
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i,
length=count_episodes_details)
result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
series_details[0], series_details[2], providers_list,
providers_auth, str(episode[3]), series_details[1], 'series')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, no, episode[2], message, path, language_code, provider, score)
send_notifications(no, episode[2], message)
if providers_list:
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i,
length=count_episodes_details)
result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
series_details[0], series_details[2], providers_list,
providers_auth, str(episode[3]), series_details[1],
'series')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, no, episode[2], message, path, language_code, provider, score)
send_notifications(no, episode[2], message)
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
break
list_missing_subtitles(no)
if count_episodes_details:
@ -551,26 +559,30 @@ def episode_download_subtitles(no):
providers_auth = get_providers_auth()
for episode in episodes_details:
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for ' + str(
language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]),
queue='get_subtitle')
result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
episode[4], episode[7], providers_list, providers_auth, str(episode[3]),
episode[5], 'series')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, episode[6], episode[2], message, path, language_code, provider, score)
send_notifications(episode[6], episode[2], message)
list_missing_subtitles(episode[6])
if providers_list:
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for ' + str(
language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]),
queue='get_subtitle')
result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
episode[4], episode[7], providers_list, providers_auth, str(episode[3]),
episode[5], 'series')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, episode[6], episode[2], message, path, language_code, provider, score)
send_notifications(episode[6], episode[2], message)
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
list_missing_subtitles(episode[6])
break
def movies_download_subtitles(no):
@ -587,20 +599,26 @@ def movies_download_subtitles(no):
count_movie = len(ast.literal_eval(movie[1]))
for i, language in enumerate(ast.literal_eval(movie[1]), 1):
if language is not None:
notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i, length=count_movie)
result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4],
movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles_movie(path_replace_movie(movie[0]))
history_log_movie(1, no, message, path, language_code, provider, score)
send_notifications_movie(no, message)
if providers_list:
if language is not None:
notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i,
length=count_movie)
result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4],
movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie')
if result is not None:
message = result[0]
path = result[1]
forced = result[5]
language_code = result[2] + ":forced" if forced else result[2]
provider = result[3]
score = result[4]
store_subtitles_movie(path_replace_movie(movie[0]))
history_log_movie(1, no, message, path, language_code, provider, score)
send_notifications_movie(no, message)
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
break
list_missing_subtitles_movies(no)
if count_movie:
@ -744,24 +762,28 @@ def wanted_search_missing_subtitles():
movies = c.fetchall()
c.close()
providers = get_providers()
if settings.general.getboolean('use_sonarr'):
if providers:
count_episodes = len(episodes)
for i, episode in enumerate(episodes, 1):
count_episodes = len(episodes)
for i, episode in enumerate(episodes, 1):
providers = get_providers()
if providers:
wanted_download_subtitles(episode[0], i, count_episodes)
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
return
if settings.general.getboolean('use_radarr'):
if providers:
count_movies = len(movies)
for i, movie in enumerate(movies, 1):
count_movies = len(movies)
for i, movie in enumerate(movies, 1):
providers = get_providers()
if providers:
wanted_download_subtitles_movie(movie[0], i, count_movies)
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
else:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
return
logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.')
@ -841,23 +863,23 @@ def refine_from_db(path, video):
def refine_from_mediainfo(path, video):
if video.fps:
return
exe = get_binary('mediainfo')
if not exe:
logging.debug('BAZARR MediaInfo library not found!')
return
else:
logging.debug('BAZARR MediaInfo library used is %s', exe)
media_info = MediaInfo.parse(path, library_file=exe)
video_track = next((t for t in media_info.tracks if t.track_type == 'Video'), None)
if not video_track:
logging.debug('BAZARR MediaInfo was unable to find video tracks in the file!')
return
logging.debug('MediaInfo found: %s', video_track.to_data())
if not video.fps:
if video_track.frame_rate:
video.fps = float(video_track.frame_rate)
@ -920,6 +942,11 @@ def upgrade_subtitles():
if settings.general.getboolean('use_sonarr'):
for i, episode in enumerate(episodes_to_upgrade, 1):
providers = get_providers()
if not providers:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
return
if episode[9] != "None":
desired_languages = ast.literal_eval(str(episode[9]))
if episode[10] == "True":
@ -957,6 +984,11 @@ def upgrade_subtitles():
if settings.general.getboolean('use_radarr'):
for i, movie in enumerate(movies_to_upgrade, 1):
providers = get_providers()
if not providers:
notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
logging.info("BAZARR All providers are throttled")
return
if movie[8] != "None":
desired_languages = ast.literal_eval(str(movie[8]))
if movie[9] == "True":

Write Preview
Loading…
Cancel
Save