Your ROOT_URL in app.ini is https://git.cloudchain.link/ but you are visiting https://dash.bss.nz/open-source-mirrors/bazarr/commit/f93af2f3e02c77f827cb369bb8be29b0022f8241 You should set ROOT_URL correctly, otherwise the web may not work correctly.
open-source-mirrors
/
bazarr
mirror of https://github.com/morpheus65535/bazarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/development' into development

Browse Source 
# Conflicts:
#	bazarr/get_providers.py
pull/1236/head
morpheus65535 4 years ago
parent 17577b35be c605d6fa38
commit f93af2f3e0
1 changed files with 107 additions and 94 deletions

201
bazarr/get_providers.py
Unescape View File

@ -16,6 +16,7 @@ from subliminal.providers.opensubtitles import DownloadLimitReached
from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
from subliminal import region as subliminal_cache_region
def time_until_end_of_day(dt=None):
# type: (datetime.datetime) -> datetime.timedelta
"""
@ -26,6 +27,7 @@ def time_until_end_of_day(dt=None):
tomorrow = dt + datetime.timedelta(days=1)
return datetime.datetime.combine(tomorrow, datetime.time.min) - dt
hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1
VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled,
@ -34,44 +36,48 @@ VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled
requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, socket.timeout)
PROVIDER_THROTTLE_MAP = {
"default": {
TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
ParseResponseError: (datetime.timedelta(hours=6), "6 hours"),
requests.exceptions.Timeout: (datetime.timedelta(hours=1), "1 hour"),
socket.timeout: (datetime.timedelta(hours=1), "1 hour"),
"default" : {
TooManyRequests : (datetime.timedelta(hours=1), "1 hour"),
DownloadLimitExceeded : (datetime.timedelta(hours=3), "3 hours"),
ServiceUnavailable : (datetime.timedelta(minutes=20), "20 minutes"),
APIThrottled : (datetime.timedelta(minutes=10), "10 minutes"),
ParseResponseError : (datetime.timedelta(hours=6), "6 hours"),
requests.exceptions.Timeout : (datetime.timedelta(hours=1), "1 hour"),
socket.timeout : (datetime.timedelta(hours=1), "1 hour"),
requests.exceptions.ConnectTimeout: (datetime.timedelta(hours=1), "1 hour"),
requests.exceptions.ReadTimeout: (datetime.timedelta(hours=1), "1 hour"),
requests.exceptions.ReadTimeout : (datetime.timedelta(hours=1), "1 hour"),
},
"opensubtitles": {
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
"opensubtitles" : {
TooManyRequests : (datetime.timedelta(hours=3), "3 hours"),
DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
DownloadLimitReached: (datetime.timedelta(hours=6), "6 hours"),
APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
DownloadLimitReached : (datetime.timedelta(hours=6), "6 hours"),
APIThrottled : (datetime.timedelta(seconds=15), "15 seconds"),
},
"opensubtitlescom": {
TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"),
DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
TooManyRequests : (datetime.timedelta(minutes=1), "1 minute"),
DownloadLimitExceeded: (
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
},
"addic7ed": {
"addic7ed" : {
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"),
TooManyRequests : (datetime.timedelta(minutes=5), "5 minutes"),
IPAddressBlocked : (datetime.timedelta(hours=1), "1 hours"),
},
"titulky": {
DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours")
"titulky" : {
DownloadLimitExceeded: (
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day)))
},
"legendasdivx": {
TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
DownloadLimitExceeded: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
"legendasdivx" : {
TooManyRequests : (datetime.timedelta(hours=3), "3 hours"),
DownloadLimitExceeded: (
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
IPAddressBlocked : (
datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))),
}
}
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", "hosszupuska",
"supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter",
"hosszupuska", "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
throttle_count = {}
@ -88,7 +94,7 @@ def get_providers():
for provider in settings.general.enabled_providers.lower().split(','):
reason, until, throttle_desc = tp.get(provider, (None, None, None))
providers_list.append(provider)
if reason:
now = datetime.datetime.now()
if now < until:
@ -104,65 +110,80 @@ def get_providers():
# for provider in providers_list:
# if provider in PROVIDERS_FORCED_OFF:
# providers_list.remove(provider)
if not providers_list:
providers_list = None
return providers_list
def get_providers_auth():
providers_auth = {
'addic7ed': {'username': settings.addic7ed.username,
'password': settings.addic7ed.password,
},
'opensubtitles': {'username': settings.opensubtitles.username,
'password': settings.opensubtitles.password,
'use_tag_search': settings.opensubtitles.getboolean('use_tag_search'),
'only_foreign': False, # fixme
'also_foreign': False, # fixme
'is_vip': settings.opensubtitles.getboolean('vip'),
'use_ssl': settings.opensubtitles.getboolean('ssl'),
'timeout': int(settings.opensubtitles.timeout) or 15,
'skip_wrong_fps': settings.opensubtitles.getboolean('skip_wrong_fps'),
},
return {
'addic7ed' : {
'username': settings.addic7ed.username,
'password': settings.addic7ed.password,
},
'opensubtitles' : {
'username' : settings.opensubtitles.username,
'password' : settings.opensubtitles.password,
'use_tag_search': settings.opensubtitles.getboolean(
'use_tag_search'
),
'only_foreign' : False, # fixme
'also_foreign' : False, # fixme
'is_vip' : settings.opensubtitles.getboolean('vip'),
'use_ssl' : settings.opensubtitles.getboolean('ssl'),
'timeout' : int(settings.opensubtitles.timeout) or 15,
'skip_wrong_fps': settings.opensubtitles.getboolean(
'skip_wrong_fps'
),
},
'opensubtitlescom': {'username': settings.opensubtitlescom.username,
'password': settings.opensubtitlescom.password,
'use_hash': settings.opensubtitlescom.getboolean('use_hash'),
'api_key': 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1'
'api_key' : 's38zmzVlW7IlYruWi7mHwDYl2SfMQoC1'
},
'podnapisi': {
'podnapisi' : {
'only_foreign': False, # fixme
'also_foreign': False, # fixme
},
'subscene': {'username': settings.subscene.username,
'password': settings.subscene.password,
'only_foreign': False, # fixme
},
'legendasdivx': {'username': settings.legendasdivx.username,
'password': settings.legendasdivx.password,
'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'),
},
'legendastv': {'username': settings.legendastv.username,
'password': settings.legendastv.password,
},
'xsubs': {'username': settings.xsubs.username,
'password': settings.xsubs.password,
},
'assrt': {'token': settings.assrt.token, },
'napisy24': {'username': settings.napisy24.username,
'password': settings.napisy24.password,
},
'betaseries': {'token': settings.betaseries.token},
'titulky': {'username': settings.titulky.username,
'password': settings.titulky.password,
},
'titlovi': {'username': settings.titlovi.username,
'password': settings.titlovi.password,
},
'subscene' : {
'username' : settings.subscene.username,
'password' : settings.subscene.password,
'only_foreign': False, # fixme
},
'legendasdivx' : {
'username' : settings.legendasdivx.username,
'password' : settings.legendasdivx.password,
'skip_wrong_fps': settings.legendasdivx.getboolean(
'skip_wrong_fps'
),
},
'legendastv' : {
'username': settings.legendastv.username,
'password': settings.legendastv.password,
},
'xsubs' : {
'username': settings.xsubs.username,
'password': settings.xsubs.password,
},
'assrt' : {
'token': settings.assrt.token,
},
'napisy24' : {
'username': settings.napisy24.username,
'password': settings.napisy24.password,
},
'betaseries' : {'token': settings.betaseries.token},
'titulky' : {
'username': settings.titulky.username,
'password': settings.titulky.password,
},
'titlovi' : {
'username': settings.titlovi.username,
'password': settings.titlovi.password,
},
}
return providers_auth
def provider_throttle(name, exception):
@ -172,17 +193,17 @@ def provider_throttle(name, exception):
for valid_cls in VALID_THROTTLE_EXCEPTIONS:
if isinstance(cls, valid_cls):
cls = valid_cls
throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
PROVIDER_THROTTLE_MAP["default"].get(cls, None)
if throttle_data:
throttle_delta, throttle_description = throttle_data
else:
throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes"
throttle_until = datetime.datetime.now() + throttle_delta
if cls_name not in VALID_COUNT_EXCEPTIONS or throttled_count(name):
if cls_name == 'ValueError' and exception.args[0].startswith('unsupported pickle protocol'):
for fn in subliminal_cache_region.backend.all_filenames:
@ -210,24 +231,18 @@ def throttled_count(name):
throttle_count[name]['count'] = value
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
if throttle_count[name]['count'] < 5:
if throttle_count[name]['time'] > datetime.datetime.now():
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
throttle_count[name]['count'])
time.sleep(5)
return False
else:
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
throttle_count[name]['count'])
time.sleep(5)
return False
else:
if throttle_count[name]['count'] >= 5:
return True
if throttle_count[name]['time'] <= datetime.datetime.now():
throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
throttle_count[name]['count'])
time.sleep(5)
return False
def update_throttled_provider():
@ -242,9 +257,7 @@ def update_throttled_provider():
if reason:
now = datetime.datetime.now()
if now < until:
pass
else:
if now >= until:
logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
del tp[provider]
set_throttled_providers(str(tp))

Write Preview
Loading…
Cancel
Save