open-source-mirrors
/
bazarr
mirror of https://github.com/morpheus65535/bazarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved search speed by reusing providers pools

Browse Source
pull/1658/head
morpheus65535 3 years ago committed by GitHub
parent 01e1723325
commit d8f14560e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 626 additions and 350 deletions
Show Stats Download Patch File Download Diff File

13
bazarr/api/episodes/episodes_subtitles.py
Unescape View File

@ -10,7 +10,7 @@ from database import TableEpisodes, get_audio_profile_languages, get_profile_id
from ..utils import authenticate
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
from get_subtitle import download_subtitle, manual_upload_subtitle
from get_subtitle import generate_subtitles, manual_upload_subtitle
from utils import history_log, delete_subtitles
from notifier import send_notifications
from list_subtitles import store_subtitles
@ -44,9 +44,6 @@ class EpisodesSubtitles(Resource):
hi = request.form.get('hi').capitalize()
forced = request.form.get('forced').capitalize()
providers_list = get_providers()
providers_auth = get_providers_auth()
audio_language_list = get_audio_profile_languages(episode_id=sonarrEpisodeId)
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
@ -54,10 +51,10 @@ class EpisodesSubtitles(Resource):
audio_language = None
try:
result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list,
providers_auth, sceneName, title, 'series',
profile_id=get_profile_id(episode_id=sonarrEpisodeId))
if result is not None:
result = list(generate_subtitles(episodePath, [(language, hi, forced)], audio_language, sceneName,
title, 'series', profile_id=get_profile_id(episode_id=sonarrEpisodeId)))
if result:
result = result[0]
message = result[0]
path = result[1]
forced = result[5]

10
bazarr/api/movies/movies_subtitles.py
Unescape View File

@ -10,7 +10,7 @@ from database import TableMovies, get_audio_profile_languages, get_profile_id
from ..utils import authenticate
from helper import path_mappings
from get_providers import get_providers, get_providers_auth
from get_subtitle import download_subtitle, manual_upload_subtitle
from get_subtitle import manual_upload_subtitle, generate_subtitles
from utils import history_log_movie, delete_subtitles
from notifier import send_notifications_movie
from list_subtitles import store_subtitles_movie
@ -56,10 +56,10 @@ class MoviesSubtitles(Resource):
audio_language = None
try:
result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list,
providers_auth, sceneName, title, 'movie',
profile_id=get_profile_id(movie_id=radarrId))
if result is not None:
result = list(generate_subtitles(moviePath, [(language, hi, forced)], audio_language,
sceneName, title, 'movie', profile_id=get_profile_id(movie_id=radarrId)))
if result:
result = result[0]
message = result[0]
path = result[1]
forced = result[5]

464
bazarr/get_subtitle.py
Unescape View File

@ -1,4 +1,5 @@
# coding=utf-8
# fmt: off
import os
import sys
@ -12,15 +13,19 @@ import re
import subliminal
import copy
import operator
import time
from functools import reduce
from inspect import getfullargspec
from peewee import fn
from datetime import datetime, timedelta
from subzero.language import Language
from subzero.video import parse_video
from subliminal import region, score as subliminal_scores, \
list_subtitles, Episode, Movie
from subliminal_patch.core import SZAsyncProviderPool, download_best_subtitles, save_subtitles, download_subtitles, \
list_all_subtitles, get_subtitle_path
from subliminal_patch.core import SZAsyncProviderPool, save_subtitles, get_subtitle_path
from subliminal_patch.core_persistent import download_best_subtitles, list_all_subtitles, download_subtitles
from subliminal_patch.score import compute_score
from subliminal_patch.subtitle import Subtitle
from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2, \
@ -44,7 +49,6 @@ from analytics import track_event
from locale import getpreferredencoding
from score import movie_score, series_score
def get_video(path, title, sceneName, providers=None, media_type="movie"):
"""
Construct `Video` instance
@ -83,43 +87,122 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"):
logging.exception("BAZARR Error trying to get video information for this file: " + original_path)
def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title,
media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None):
# fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
# one query
# fmt: on
def _init_pool(media_type, profile_id=None, providers=None):
pool = provider_pool()
return pool(
providers=providers or get_providers(),
provider_configs=get_providers_auth(),
blacklist=get_blacklist(media_type),
throttle_callback=provider_throttle,
ban_list=get_ban_list(profile_id),
language_hook=None,
)
_pools = {}
def _get_pool(media_type, profile_id=None):
try:
return _pools[f'{media_type}_{profile_id or ""}']
except KeyError:
_update_pool(media_type, profile_id)
return _pools[f'{media_type}_{profile_id or ""}']
def _update_pool(media_type, profile_id=None):
pool_key = f'{media_type}_{profile_id or ""}'
logging.debug("BAZARR updating pool: %s", pool_key)
# Init a new pool if not present
if pool_key not in _pools:
logging.debug("BAZARR pool not initialized: %s. Initializing", pool_key)
_pools[pool_key] = _init_pool(media_type, profile_id)
pool = _pools[pool_key]
if pool is None:
return False
return pool.update(
get_providers(),
get_providers_auth(),
get_blacklist(media_type),
get_ban_list(profile_id),
)
def update_pools(f):
"""Decorator that ensures all pools are updated on each function run.
It will detect any config changes in Bazarr"""
def decorated(*args, **kwargs):
logging.debug("BAZARR updating pools: %s", _pools)
start = time.time()
args_spec = getfullargspec(f).args
try:
profile_id = args[args_spec.index("profile_id")]
except (IndexError, ValueError):
profile_id = None
updated = _update_pool(args[args_spec.index("media_type")], profile_id)
if updated:
logging.info("BAZARR pools update elapsed time: %s", time.time() - start)
return f(*args, **kwargs)
return decorated
# fmt: off
@update_pools
def generate_subtitles(path, languages, audio_language, sceneName, title, media_type,
forced_minimum_score=None, is_upgrade=False, profile_id=None):
if not languages:
return None
if settings.general.getboolean('utf8_encode'):
os.environ["SZ_KEEP_ENCODING"] = ""
else:
os.environ["SZ_KEEP_ENCODING"] = "True"
language_set = set()
if not isinstance(languages, (set, list)):
languages = [languages]
pool = _get_pool(media_type, profile_id)
providers = pool.providers
for l in languages:
l, hi_item, forced_item = l
logging.debug('BAZARR Searching subtitles for this file: ' + path)
if hi == "True":
if hi_item == "True":
hi = "force HI"
else:
hi = "force non-HI"
if forced == "True":
providers_auth['podnapisi']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
providers_auth['subscene']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
providers_auth['opensubtitles']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
# Fixme: This block should be updated elsewhere
if forced_item == "True":
pool.provider_configs['podnapisi']['only_foreign'] = True
pool.provider_configs['subscene']['only_foreign'] = True
pool.provider_configs['opensubtitles']['only_foreign'] = True
else:
providers_auth['podnapisi']['only_foreign'] = False
providers_auth['subscene']['only_foreign'] = False
providers_auth['opensubtitles']['only_foreign'] = False
pool.provider_configs['podnapisi']['only_foreign'] = False
pool.provider_configs['subscene']['only_foreign'] = False
pool.provider_configs['opensubtitles']['only_foreign'] = False
language_set = set()
if not isinstance(language, list):
language = [language]
for l in language:
# Always use alpha2 in API Request
l = alpha3_from_alpha2(l)
lang_obj = _get_lang_obj(l)
if forced == "True":
if forced_item == "True":
lang_obj = Language.rebuild(lang_obj, forced=True)
if hi == "force HI":
lang_obj = Language.rebuild(lang_obj, hi=True)
@ -144,6 +227,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
"""
video = get_video(force_unicode(path), title, sceneName, providers=providers,
media_type=media_type)
if video:
handler = series_score if media_type == "series" else movie_score
min_score, max_score, scores = _get_scores(media_type, minimum_score_movie, minimum_score)
@ -151,19 +235,11 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
if providers:
if forced_minimum_score:
min_score = int(forced_minimum_score) + 1
downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
providers=providers,
provider_configs=providers_auth,
pool_class=provider_pool(),
downloaded_subtitles = download_best_subtitles({video}, language_set, pool,
int(min_score), hi,
compute_score=compute_score,
throttle_time=None, # fixme
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profile_id),
throttle_callback=provider_throttle,
score_obj=handler,
pre_download_hook=None, # fixme
post_download_hook=None, # fixme
language_hook=None) # fixme
score_obj=handler)
else:
downloaded_subtitles = None
logging.info("BAZARR All providers are throttled")
@ -287,7 +363,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
track_event(category=downloaded_provider, action=action, label=downloaded_language)
return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
yield message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
subtitle.language.forced, subtitle.id, reversed_subtitles_path, subtitle.language.hi
if not saved_any:
@ -299,7 +375,8 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
logging.debug('BAZARR Ended searching Subtitles for file: ' + path)
def manual_search(path, profileId, providers, providers_auth, sceneName, title, media_type):
@update_pools
def manual_search(path, profile_id, providers, providers_auth, sceneName, title, media_type):
logging.debug('BAZARR Manually searching subtitles for this file: ' + path)
final_subtitles = []
@ -308,7 +385,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
language_set = set()
# where [3] is items list of dict(id, lang, forced, hi)
language_items = get_profiles_list(profile_id=int(profileId))['items']
language_items = get_profiles_list(profile_id=int(profile_id))['items']
pool = _get_pool(media_type, profile_id)
for language in language_items:
forced = language['forced']
@ -323,8 +401,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
if forced == "True":
lang_obj = Language.rebuild(lang_obj, forced=True)
providers_auth['podnapisi']['also_foreign'] = True
providers_auth['opensubtitles']['also_foreign'] = True
pool.provider_configs['podnapisi']['also_foreign'] = True
pool.provider_configs['opensubtitles']['also_foreign'] = True
if hi == "True":
lang_obj = Language.rebuild(lang_obj, hi=True)
@ -358,29 +436,20 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
try:
if providers:
subtitles = list_all_subtitles([video], language_set,
providers=providers,
provider_configs=providers_auth,
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profileId),
throttle_callback=provider_throttle,
language_hook=None) # fixme
subtitles = list_all_subtitles([video], language_set, pool)
if 'subscene' in providers:
s_pool = _init_pool("movie", profile_id, {"subscene"})
subscene_language_set = set()
for language in language_set:
if language.forced:
subscene_language_set.add(language)
if len(subscene_language_set):
providers_auth['subscene']['only_foreign'] = True
subtitles_subscene = list_all_subtitles([video], subscene_language_set,
providers=['subscene'],
provider_configs=providers_auth,
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profileId),
throttle_callback=provider_throttle,
language_hook=None) # fixme
providers_auth['subscene']['only_foreign'] = False
s_pool.provider_configs['subscene'] = {}
s_pool.provider_configs['subscene']['only_foreign'] = True
subtitles_subscene = list_all_subtitles([video], subscene_language_set, s_pool)
s_pool.provider_configs['subscene']['only_foreign'] = False
subtitles[video] += subtitles_subscene[video]
else:
subtitles = []
@ -407,6 +476,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
logging.debug(u"BAZARR Skipping %s, because it doesn't match our series/episode", s)
continue
initial_hi = None
initial_hi_match = False
for language in initial_language_set:
if s.language.basename == language.basename and \
@ -468,6 +538,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
return final_subtitles
@update_pools
def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName,
title, media_type, profile_id):
logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path)
@ -496,13 +567,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl
min_score, max_score, scores = _get_scores(media_type)
try:
if provider:
download_subtitles([subtitle],
providers={provider},
provider_configs=providers_auth,
pool_class=provider_pool(),
blacklist=get_blacklist(media_type=media_type),
ban_list=get_ban_list(profile_id),
throttle_callback=provider_throttle)
download_subtitles([subtitle], _get_pool(media_type, profile_id))
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
else:
logging.info("BAZARR All providers are throttled")
@ -765,8 +830,6 @@ def series_download_subtitles(no):
"ignored because of monitored status, series type or series tags: {}".format(no))
return
providers_auth = get_providers_auth()
count_episodes_details = len(episodes_details)
for i, episode in enumerate(episodes_details):
@ -782,6 +845,13 @@ def series_download_subtitles(no):
value=i,
count=count_episodes_details)
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
languages = []
for language in ast.literal_eval(episode['missing_subtitles']):
# confirm if language is still missing or if cutoff have been reached
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
@ -792,23 +862,19 @@ def series_download_subtitles(no):
continue
if language is not None:
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
hi_ = "True" if language.endswith(':hi') else "False"
forced_ ="True" if language.endswith(':forced') else "False"
languages.append((language.split(":")[0], hi_, forced_))
if not languages:
continue
result = download_subtitle(path_mappings.path_replace(episode['path']),
language.split(':')[0],
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
languages,
audio_language,
"True" if language.endswith(':hi') else "False",
"True" if language.endswith(':forced') else "False",
providers_list,
providers_auth,
str(episode['scene_name']),
episode['title'],
'series')
if result is not None:
episode['title'], 'series'):
if result:
message = result[0]
path = result[1]
forced = result[5]
@ -871,6 +937,14 @@ def episode_download_subtitles(no, send_progress=False):
episode['episodeTitle']),
value=0,
count=1)
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
languages = []
for language in ast.literal_eval(episode['missing_subtitles']):
# confirm if language is still missing or if cutoff have been reached
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
@ -881,23 +955,20 @@ def episode_download_subtitles(no, send_progress=False):
continue
if language is not None:
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
hi_ = "True" if language.endswith(':hi') else "False"
forced_ ="True" if language.endswith(':forced') else "False"
languages.append((language.split(":")[0], hi_, forced_))
result = download_subtitle(path_mappings.path_replace(episode['path']),
language.split(':')[0],
if not languages:
continue
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
languages,
audio_language,
"True" if language.endswith(':hi') else "False",
"True" if language.endswith(':forced') else "False",
providers_list,
providers_auth,
str(episode['scene_name']),
episode['title'],
'series')
if result is not None:
'series'):
if result:
message = result[0]
path = result[1]
forced = result[5]
@ -915,6 +986,7 @@ def episode_download_subtitles(no, send_progress=False):
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
language_code, provider, score, subs_id, subs_path)
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
if send_progress:
hide_progress(id='episode_search_progress_{}'.format(no))
else:
@ -941,16 +1013,28 @@ def movies_download_subtitles(no):
else:
movie = movies[0]
providers_auth = get_providers_auth()
if ast.literal_eval(movie['missing_subtitles']):
count_movie = len(ast.literal_eval(movie['missing_subtitles']))
else:
count_movie = 0
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
languages = []
providers_list = None
for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])):
providers_list = get_providers()
if language is not None:
hi_ = "True" if language.endswith(':hi') else "False"
forced_ ="True" if language.endswith(':forced') else "False"
languages.append((language.split(":")[0], hi_, forced_))
if providers_list:
# confirm if language is still missing or if cutoff have been reached
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
@ -966,24 +1050,15 @@ def movies_download_subtitles(no):
value=i,
count=count_movie)
if language is not None:
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
language.split(':')[0],
if providers_list:
for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
languages,
audio_language,
"True" if language.endswith(':hi') else "False",
"True" if language.endswith(':forced') else "False",
providers_list,
providers_auth,
str(movie['sceneName']),
movie['title'],
'movie')
if result is not None:
'movie'):
if result:
message = result[0]
path = result[1]
forced = result[5]
@ -1002,32 +1077,20 @@ def movies_download_subtitles(no):
send_notifications_movie(no, message)
else:
logging.info("BAZARR All providers are throttled")
break
hide_progress(id='movie_search_progress_{}'.format(no))
def wanted_download_subtitles(sonarr_episode_id):
episodes_details = TableEpisodes.select(TableEpisodes.path,
TableEpisodes.missing_subtitles,
TableEpisodes.sonarrEpisodeId,
TableEpisodes.sonarrSeriesId,
TableEpisodes.audio_language,
TableEpisodes.scene_name,
TableEpisodes.failedAttempts,
TableShows.title)\
.join(TableShows, on=(TableEpisodes.sonarrSeriesId == TableShows.sonarrSeriesId))\
.where((TableEpisodes.sonarrEpisodeId == sonarr_episode_id))\
.dicts()
episodes_details = list(episodes_details)
providers_auth = get_providers_auth()
for episode in episodes_details:
providers_list = get_providers()
def _wanted_episode(episode):
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
if providers_list:
languages = []
for language in ast.literal_eval(episode['missing_subtitles']):
# confirm if language is still missing or if cutoff have been reached
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
@ -1043,23 +1106,23 @@ def wanted_download_subtitles(sonarr_episode_id):
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
.execute()
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
hi_ = "True" if language.endswith(':hi') else "False"
forced_ ="True" if language.endswith(':forced') else "False"
languages.append((language.split(":")[0], hi_, forced_))
else:
audio_language = 'None'
logging.debug(
f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
f"language: {language}")
result = download_subtitle(path_mappings.path_replace(episode['path']),
language.split(':')[0],
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
languages,
audio_language,
"True" if language.endswith(':hi') else "False",
"True" if language.endswith(':forced') else "False",
providers_list,
providers_auth,
str(episode['scene_name']),
episode['title'],
'series')
if result is not None:
'series'):
if result:
message = result[0]
path = result[1]
forced = result[5]
@ -1079,33 +1142,41 @@ def wanted_download_subtitles(sonarr_episode_id):
event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
else:
logging.debug(
f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
f"language: {language}")
else:
logging.info("BAZARR All providers are throttled")
break
def wanted_download_subtitles_movie(radarr_id):
movies_details = TableMovies.select(TableMovies.path,
TableMovies.missing_subtitles,
TableMovies.radarrId,
TableMovies.audio_language,
TableMovies.sceneName,
TableMovies.failedAttempts,
TableMovies.title)\
.where((TableMovies.radarrId == radarr_id))\
def wanted_download_subtitles(sonarr_episode_id):
episodes_details = TableEpisodes.select(TableEpisodes.path,
TableEpisodes.missing_subtitles,
TableEpisodes.sonarrEpisodeId,
TableEpisodes.sonarrSeriesId,
TableEpisodes.audio_language,
TableEpisodes.scene_name,
TableEpisodes.failedAttempts,
TableShows.title)\
.join(TableShows, on=(TableEpisodes.sonarrSeriesId == TableShows.sonarrSeriesId))\
.where((TableEpisodes.sonarrEpisodeId == sonarr_episode_id))\
.dicts()
movies_details = list(movies_details)
providers_auth = get_providers_auth()
episodes_details = list(episodes_details)
for movie in movies_details:
for episode in episodes_details:
providers_list = get_providers()
if providers_list:
_wanted_episode(episode)
else:
logging.info("BAZARR All providers are throttled")
break
def _wanted_movie(movie):
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
else:
audio_language = 'None'
languages = []
for language in ast.literal_eval(movie['missing_subtitles']):
# confirm if language is still missing or if cutoff have been reached
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
@ -1122,23 +1193,21 @@ def wanted_download_subtitles_movie(radarr_id):
.where(TableMovies.radarrId == movie['radarrId']) \
.execute()
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
if len(audio_language_list) > 0:
audio_language = audio_language_list[0]['name']
hi_ = "True" if language.endswith(':hi') else "False"
forced_ ="True" if language.endswith(':forced') else "False"
languages.append((language.split(":")[0], hi_, forced_))
else:
audio_language = 'None'
logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
f"language: {language}")
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
language.split(':')[0],
for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
languages,
audio_language,
"True" if language.endswith(':hi') else "False",
"True" if language.endswith(':forced') else "False",
providers_list,
providers_auth,
str(movie['sceneName']),
movie['title'],
'movie')
if result is not None:
movie['title'], 'movie'):
if result:
message = result[0]
path = result[1]
forced = result[5]
@ -1157,9 +1226,25 @@ def wanted_download_subtitles_movie(radarr_id):
subs_id, subs_path)
event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
send_notifications_movie(movie['radarrId'], message)
else:
logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
f"language: {language}")
def wanted_download_subtitles_movie(radarr_id):
movies_details = TableMovies.select(TableMovies.path,
TableMovies.missing_subtitles,
TableMovies.radarrId,
TableMovies.audio_language,
TableMovies.sceneName,
TableMovies.failedAttempts,
TableMovies.title)\
.where((TableMovies.radarrId == radarr_id))\
.dicts()
movies_details = list(movies_details)
for movie in movies_details:
providers_list = get_providers()
if providers_list:
_wanted_movie(movie)
else:
logging.info("BAZARR All providers are throttled")
break
@ -1471,8 +1556,6 @@ def upgrade_subtitles():
count_movie_to_upgrade = len(movies_to_upgrade)
providers_auth = get_providers_auth()
if settings.general.getboolean('use_sonarr'):
for i, episode in enumerate(episodes_to_upgrade):
providers_list = get_providers()
@ -1508,19 +1591,17 @@ def upgrade_subtitles():
else:
audio_language = 'None'
result = download_subtitle(path_mappings.path_replace(episode['video_path']),
language,
result = list(generate_subtitles(path_mappings.path_replace(episode['video_path']),
[(language, is_hi, is_forced)],
audio_language,
is_hi,
is_forced,
providers_list,
providers_auth,
str(episode['scene_name']),
episode['title'],
'series',
forced_minimum_score=int(episode['score']),
is_upgrade=True)
if result is not None:
is_upgrade=True))
if result:
result = result[0]
message = result[0]
path = result[1]
forced = result[5]
@ -1573,19 +1654,16 @@ def upgrade_subtitles():
else:
audio_language = 'None'
result = download_subtitle(path_mappings.path_replace_movie(movie['video_path']),
language,
result = list(generate_subtitles(path_mappings.path_replace_movie(movie['video_path']),
[(language, is_hi, is_forced)],
audio_language,
is_hi,
is_forced,
providers_list,
providers_auth,
str(movie['sceneName']),
movie['title'],
'movie',
forced_minimum_score=int(movie['score']),
is_upgrade=True)
if result is not None:
is_upgrade=True))
if result:
result = result[0]
message = result[0]
path = result[1]
forced = result[5]

3
libs/subliminal/providers/__init__.py
Unescape View File

@ -161,5 +161,8 @@ class Provider(object):
"""
raise NotImplementedError
def ping(self):
return True
def __repr__(self):
return '<%s [%r]>' % (self.__class__.__name__, self.video_types)

90
libs/subliminal_patch/core.py
Unescape View File

@ -5,6 +5,7 @@ import json
import re
import os
import logging
import datetime
import socket
import traceback
import time
@ -55,6 +56,8 @@ REMOVE_CRAP_FROM_FILENAME = re.compile(r"(?i)(?:([\s_-]+(?:obfuscated|scrambled|
SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '.vtt')
_POOL_LIFETIME = datetime.timedelta(hours=12)
def remove_crap_from_fn(fn):
# in case of the second regex part, the legit release group name will be in group(2), if it's followed by [string]
@ -69,7 +72,7 @@ class SZProviderPool(ProviderPool):
def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
pre_download_hook=None, post_download_hook=None, language_hook=None):
#: Name of providers to use
self.providers = providers
self.providers = set(providers or [])
#: Provider configuration
self.provider_configs = provider_configs or {}
@ -91,9 +94,69 @@ class SZProviderPool(ProviderPool):
self.post_download_hook = post_download_hook
self.language_hook = language_hook
self._born = time.time()
if not self.throttle_callback:
self.throttle_callback = lambda x, y: x
def update(self, providers, provider_configs, blacklist, ban_list):
# Check if the pool was initialized enough hours ago
self._check_lifetime()
# Check if any new provider has been added
updated = set(providers) != self.providers or ban_list != self.ban_list
removed_providers = list(sorted(self.providers - set(providers)))
new_providers = list(sorted(set(providers) - self.providers))
# Terminate and delete removed providers from instance
for removed in removed_providers:
try:
del self[removed]
# If the user has updated the providers but hasn't made any
# subtitle searches yet, the removed provider won't be in the
# self dictionary
except KeyError:
pass
if updated:
logger.debug("Removed providers: %s", removed_providers)
logger.debug("New providers: %s", new_providers)
self.discarded_providers.difference_update(new_providers)
self.providers.difference_update(removed_providers)
self.providers.update(list(providers))
self.blacklist = blacklist
# Restart providers with new configs
for key, val in provider_configs.items():
# key: provider's name; val: config dict
old_val = self.provider_configs.get(key)
if old_val == val:
continue
logger.debug("Restarting provider: %s", key)
try:
provider = provider_registry[key](**val)
provider.initialize()
except Exception as error:
self.throttle_callback(key, error)
else:
self.initialized_providers[key] = provider
updated = True
self.provider_configs = provider_configs
return updated
def _check_lifetime(self):
# This method is used to avoid possible memory leaks
if abs(self._born - time.time()) > _POOL_LIFETIME.seconds:
logger.info("%s elapsed. Terminating providers", _POOL_LIFETIME)
self._born = time.time()
self.terminate()
def __enter__(self):
return self
@ -169,18 +232,8 @@ class SZProviderPool(ProviderPool):
# list subtitles
logger.info('Listing subtitles with provider %r and languages %r', provider, provider_languages)
results = []
try:
try:
results = self[provider].list_subtitles(video, provider_languages)
except ResponseNotReady:
logger.error('Provider %r response error, reinitializing', provider)
try:
self[provider].terminate()
self[provider].initialize()
results = self[provider].list_subtitles(video, provider_languages)
except:
logger.error('Provider %r reinitialization error: %s', provider, traceback.format_exc())
seen = []
out = []
for s in results:
@ -198,16 +251,13 @@ class SZProviderPool(ProviderPool):
continue
if s.id in seen:
continue
s.plex_media_fps = float(video.fps) if video.fps else None
out.append(s)
seen.append(s.id)
return out
except (requests.Timeout, socket.timeout) as e:
logger.error('Provider %r timed out', provider)
self.throttle_callback(provider, e)
except Exception as e:
logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
self.throttle_callback(provider, e)
@ -289,16 +339,6 @@ class SZProviderPool(ProviderPool):
logger.error('Provider %r connection error', subtitle.provider_name)
self.throttle_callback(subtitle.provider_name, e)
except ResponseNotReady as e:
logger.error('Provider %r response error, reinitializing', subtitle.provider_name)
try:
self[subtitle.provider_name].terminate()
self[subtitle.provider_name].initialize()
except:
logger.error('Provider %r reinitialization error: %s', subtitle.provider_name,
traceback.format_exc())
self.throttle_callback(subtitle.provider_name, e)
except rarfile.BadRarFile:
logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)
logger.debug("RAR Traceback: %s", traceback.format_exc())

92
libs/subliminal_patch/core_persistent.py
Unescape View File

@ -0,0 +1,92 @@
# coding=utf-8
from __future__ import absolute_import
from collections import defaultdict
import logging
import time
from subliminal.core import check_video
logger = logging.getLogger(__name__)
# list_all_subtitles, list_supported_languages, list_supported_video_types, download_subtitles, download_best_subtitles
def list_all_subtitles(videos, languages, pool_instance):
listed_subtitles = defaultdict(list)
# return immediatly if no video passed the checks
if not videos:
return listed_subtitles
for video in videos:
logger.info("Listing subtitles for %r", video)
subtitles = pool_instance.list_subtitles(
video, languages - video.subtitle_languages
)
listed_subtitles[video].extend(subtitles)
logger.info("Found %d subtitle(s)", len(subtitles))
return listed_subtitles
def list_supported_languages(pool_instance):
return pool_instance.list_supported_languages()
def list_supported_video_types(pool_instance):
return pool_instance.list_supported_video_types()
def download_subtitles(subtitles, pool_instance):
for subtitle in subtitles:
logger.info("Downloading subtitle %r with score %s", subtitle, subtitle.score)
pool_instance.download_subtitle(subtitle)
def download_best_subtitles(
videos,
languages,
pool_instance,
min_score=0,
hearing_impaired=False,
only_one=False,
compute_score=None,
throttle_time=0,
score_obj=None,
):
downloaded_subtitles = defaultdict(list)
# check videos
checked_videos = []
for video in videos:
if not check_video(video, languages=languages, undefined=only_one):
logger.info("Skipping video %r", video)
continue
checked_videos.append(video)
# return immediately if no video passed the checks
if not checked_videos:
return downloaded_subtitles
got_multiple = len(checked_videos) > 1
# download best subtitles
for video in checked_videos:
logger.info("Downloading best subtitles for %r", video)
subtitles = pool_instance.download_best_subtitles(
pool_instance.list_subtitles(video, languages - video.subtitle_languages),
video,
languages,
min_score=min_score,
hearing_impaired=hearing_impaired,
only_one=only_one,
compute_score=compute_score,
score_obj=score_obj,
)
logger.info("Downloaded %d subtitle(s)", len(subtitles))
downloaded_subtitles[video].extend(subtitles)
if got_multiple and throttle_time:
logger.debug("Waiting %ss before continuing ...", throttle_time)
time.sleep(throttle_time)
return downloaded_subtitles

42
libs/subliminal_patch/providers/__init__.py
Unescape View File

@ -1,8 +1,11 @@
# coding=utf-8
from __future__ import absolute_import
import functools
import importlib
import os
import logging
import subliminal
from subliminal.providers import Provider as _Provider
from subliminal.subtitle import Subtitle as _Subtitle
@ -14,11 +17,50 @@ from subzero.lib.io import get_viable_encoding
import six
logger = logging.getLogger(__name__)
class Provider(_Provider):
hash_verifiable = False
hearing_impaired_verifiable = False
skip_wrong_fps = True
def ping(self):
"""Check if the provider is alive."""
return True
def reinitialize_on_error(exceptions: tuple, attempts=1):
"""Method decorator for Provider class. It will reinitialize the instance
and re-run the method in case of exceptions.
:param exceptions: tuple of expected exceptions
:param attempts: number of attempts to call the method
"""
def real_decorator(method):
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
inc = 1
while True:
try:
return method(self, *args, **kwargs)
except exceptions as error:
if inc > attempts:
raise
logger.exception(error)
logger.debug("Reinitializing %s instance (%s attempt)", self, inc)
self.terminate()
self.initialize()
inc += 1
return wrapper
return real_decorator
# register providers
# fixme: this is bad

4
libs/subliminal_patch/providers/addic7ed.py
Unescape View File

@ -11,12 +11,14 @@ from urllib.parse import quote_plus
import babelfish
from dogpile.cache.api import NO_VALUE
from requests import Session
from requests.exceptions import RequestException
from subliminal.cache import region
from subliminal.video import Episode, Movie
from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
from subliminal.subtitle import fix_line_ending
from subliminal_patch.providers import reinitialize_on_error
from subliminal_patch.utils import sanitize
from subliminal_patch.exceptions import TooManyRequests
from subliminal_patch.pitcher import pitchers, load_verification, store_verification
@ -550,6 +552,7 @@ class Addic7edProvider(_Addic7edProvider):
return subtitles
@reinitialize_on_error((RequestException,), attempts=1)
def list_subtitles(self, video, languages):
if isinstance(video, Episode):
# lookup show_id
@ -586,6 +589,7 @@ class Addic7edProvider(_Addic7edProvider):
return []
@reinitialize_on_error((RequestException,), attempts=1)
def download_subtitle(self, subtitle):
last_dls = region.get("addic7ed_dls")
now = datetime.datetime.now()

5
libs/subliminal_patch/providers/legendasdivx.py
Unescape View File

@ -10,6 +10,7 @@ from requests.exceptions import HTTPError
import rarfile
from guessit import guessit
from requests.exceptions import RequestException
from subliminal.cache import region
from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded
from subliminal.providers import ParserBeautifulSoup
@ -18,7 +19,7 @@ from subliminal.utils import sanitize, sanitize_release_group
from subliminal.video import Episode, Movie
from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked
from subliminal_patch.http import RetryingCFSession
from subliminal_patch.providers import Provider
from subliminal_patch.providers import Provider, reinitialize_on_error
from subliminal_patch.score import get_scores, framerate_equal
from subliminal_patch.subtitle import Subtitle, guess_matches
from subzero.language import Language
@ -260,6 +261,7 @@ class LegendasdivxProvider(Provider):
)
return subtitles
@reinitialize_on_error((RequestException,), attempts=1)
def query(self, video, languages):
_searchurl = self.searchurl
@ -363,6 +365,7 @@ class LegendasdivxProvider(Provider):
def list_subtitles(self, video, languages):
return self.query(video, languages)
@reinitialize_on_error((RequestException,), attempts=1)
def download_subtitle(self, subtitle):
try:

5
libs/subliminal_patch/providers/legendastv.py
Unescape View File

@ -8,8 +8,10 @@ from subliminal.exceptions import ConfigurationError
from subliminal.providers.legendastv import LegendasTVSubtitle as _LegendasTVSubtitle, \
LegendasTVProvider as _LegendasTVProvider, Episode, Movie, guessit, sanitize, region, type_map, \
raise_for_status, json, SHOW_EXPIRATION_TIME, title_re, season_re, datetime, pytz, NO_VALUE, releases_key, \
SUBTITLE_EXTENSIONS, language_converters
SUBTITLE_EXTENSIONS, language_converters, ServiceUnavailable
from requests.exceptions import RequestException
from subliminal_patch.providers import reinitialize_on_error
from subliminal_patch.subtitle import guess_matches
from subzero.language import Language
@ -184,6 +186,7 @@ class LegendasTVProvider(_LegendasTVProvider):
return titles_found
@reinitialize_on_error((RequestException, ServiceUnavailable), attempts=1)
def query(self, language, titles, season=None, episode=None, year=None, imdb_id=None):
# search for titles
titles_found = self.search_titles(titles, season, year, imdb_id)

3
libs/subliminal_patch/providers/opensubtitles.py
Unescape View File

@ -18,6 +18,7 @@ from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSub
DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError
from .mixins import ProviderRetryMixin
from subliminal.subtitle import fix_line_ending
from subliminal_patch.providers import reinitialize_on_error
from subliminal_patch.http import SubZeroRequestsTransport
from subliminal_patch.utils import sanitize, fix_inconsistent_naming
from subliminal.cache import region
@ -272,6 +273,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
use_tag_search=self.use_tag_search, only_foreign=self.only_foreign,
also_foreign=self.also_foreign)
@reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
def query(self, video, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None,
tag=None, use_tag_search=False, only_foreign=False, also_foreign=False):
# fill the search criteria
@ -377,6 +379,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
return subtitles
@reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
def download_subtitle(self, subtitle):
logger.info('Downloading subtitle %r', subtitle)
response = self.use_token_or_login(

8
libs/subliminal_patch/providers/opensubtitlescom.py
Unescape View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import logging
import os
import time
import datetime
from requests import Session, ConnectionError, Timeout, ReadTimeout
@ -147,15 +148,18 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider):
self.password = password
self.video = None
self.use_hash = use_hash
self._started = None
def initialize(self):
self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME)
if self.token is NO_VALUE:
self._started = time.time()
self.login()
def terminate(self):
self.session.close()
def ping(self):
return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME
def login(self):
try:
r = self.session.post(self.server_url + 'login',

5
libs/subliminal_patch/providers/subscene.py
Unescape View File

@ -20,13 +20,14 @@ import rarfile
from babelfish import language_converters
from guessit import guessit
from dogpile.cache.api import NO_VALUE
from requests.exceptions import RequestException
from subliminal import Episode, ProviderError
from subliminal.video import Episode, Movie
from subliminal.exceptions import ConfigurationError, ServiceUnavailable
from subliminal.utils import sanitize_release_group
from subliminal.cache import region
from subliminal_patch.http import RetryingCFSession
from subliminal_patch.providers import Provider
from subliminal_patch.providers import Provider, reinitialize_on_error
from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
from subliminal_patch.subtitle import Subtitle, guess_matches
from subliminal_patch.converters.subscene import language_ids, supported_languages
@ -315,7 +316,9 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
return search(*args, **kwargs)
except requests.HTTPError:
region.delete("subscene_cookies2")
raise
@reinitialize_on_error((RequestException,), attempts=1)
def query(self, video):
subtitles = []
if isinstance(video, Episode):

4
libs/subliminal_patch/providers/xsubs.py
Unescape View File

@ -6,6 +6,7 @@ import re
from subzero.language import Language
from guessit import guessit
from requests import Session
from requests.exceptions import RequestException
from subliminal.providers import ParserBeautifulSoup, Provider
from subliminal import __short_version__
@ -16,6 +17,7 @@ from subliminal.subtitle import Subtitle, fix_line_ending
from subliminal.utils import sanitize, sanitize_release_group
from subliminal.video import Episode
from subliminal_patch.subtitle import guess_matches
from subliminal_patch.providers import reinitialize_on_error
logger = logging.getLogger(__name__)
article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$')
@ -190,6 +192,7 @@ class XSubsProvider(Provider):
return int(show_id) if show_id else None
@reinitialize_on_error((RequestException,), attempts=1)
def query(self, show_id, series, season, year=None, country=None):
# get the season list of the show
logger.info('Getting the season list of show id %d', show_id)
@ -291,6 +294,7 @@ class XSubsProvider(Provider):
return []
@reinitialize_on_error((RequestException,), attempts=1)
def download_subtitle(self, subtitle):
if isinstance(subtitle, XSubsSubtitle):
# download the subtitle

Write Preview
Loading…
Cancel
Save