From abcf03e3892072b4ca27a15e1ae2f5f2e8ee35b9 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Thu, 7 May 2020 12:16:25 +0100 Subject: [PATCH 01/13] fixed retries to get download link to avoid throttling --- .../providers/legendasdivx.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index e9f505bdf..4fff72552 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -170,7 +170,7 @@ class LegendasdivxProvider(Provider): logger.error("Couldn't retrieve session ID, check your credentials") raise AuthenticationError("Please check your credentials.") except Exception as e: - if 'bloqueado' in res.text.lower(): # blocked IP address + if (res and 'bloqueado' in res.text.lower()): # blocked IP address logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) @@ -193,9 +193,9 @@ class LegendasdivxProvider(Provider): for _subbox in _allsubs: hits = 0 for th in _subbox.findAll("th", {"class": "color2"}): - if th.string == 'Hits:': - hits = int(th.parent.find("td").string) - if th.string == 'Idioma:': + if th.text == 'Hits:': + hits = int(th.parent.find("td").text) + if th.text == 'Idioma:': lang = th.parent.find("td").find("img").get('src') if 'brazil' in lang.lower(): lang = Language.fromopensubtitles('pob') @@ -209,13 +209,12 @@ class LegendasdivxProvider(Provider): download = _subbox.find("a", {"class": "sub_download"}) # sometimes BSoup can't find 'a' tag and returns None. - i = 0 - while not (download): # must get it... trying again... - download = _subbox.find("a", {"class": "sub_download"}) - i=+1 - logger.debug("Try number {0} try!".format(str(i))) - dl = download.get('href') - logger.debug("Found subtitle on: %s" % self.download_link.format(link=dl)) + try: + dl = download.get('href') + logger.debug("Found subtitle link on: {0}").format(self.download_link.format(link=dl)) + except: + logger.debug("Couldn't find download link. Trying next...") + continue # get subtitle uploader sub_header = _subbox.find("div", {"class" :"sub_header"}) @@ -268,7 +267,7 @@ class LegendasdivxProvider(Provider): self.session.headers.update(self.headers.items()) res = self.session.get(_searchurl.format(query=querytext)) - if "A legenda não foi encontrada" in res.text: + if (res and "A legenda não foi encontrada" in res.text): logger.warning('%s not found', querytext) return [] @@ -281,12 +280,13 @@ class LegendasdivxProvider(Provider): #get number of pages bases on results found page_header = bsoup.find("div", {"class": "pager_bar"}) - results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) + results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) if page_header else 0 + logger.debug("Found %s subtitles" % str(results_found)) num_pages = (int(results_found) // 10) + 1 num_pages = min(MAX_PAGES, num_pages) if num_pages > 1: - for num_page in range(2, num_pages+2): + for num_page in range(2, num_pages+1): _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page)) logger.debug("Moving to next page: %s" % _search_next) res = self.session.get(_search_next) @@ -305,6 +305,8 @@ class LegendasdivxProvider(Provider): if res: if res.status_code in ['500', '503']: raise ServiceUnavailable("Legendasdivx.pt :: 503 - Service Unavailable") + elif res.status_code == '403': + raise ParseResponseError("Legendasdivx.pt :: 403 - Forbidden") elif 'limite' in res.text.lower(): # daily downloads limit reached raise DownloadLimitReached("Legendasdivx.pt :: Download limit reached") elif 'bloqueado' in res.text.lower(): # blocked IP address From 8da77a72eb0e22ab187a5df4dee4ac3681a1637c Mon Sep 17 00:00:00 2001 From: Bazarr Date: Thu, 7 May 2020 12:53:26 +0100 Subject: [PATCH 02/13] simplig --- libs/subliminal_patch/providers/legendasdivx.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 4fff72552..77e1cd22a 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -8,6 +8,7 @@ import rarfile import zipfile from requests import Session +from requests.exceptions import HTTPError from guessit import guessit from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded from subliminal_patch.providers import Provider @@ -303,11 +304,7 @@ class LegendasdivxProvider(Provider): res = self.session.get(subtitle.page_link) res.raise_for_status() if res: - if res.status_code in ['500', '503']: - raise ServiceUnavailable("Legendasdivx.pt :: 503 - Service Unavailable") - elif res.status_code == '403': - raise ParseResponseError("Legendasdivx.pt :: 403 - Forbidden") - elif 'limite' in res.text.lower(): # daily downloads limit reached + if 'limite' in res.text.lower(): # daily downloads limit reached raise DownloadLimitReached("Legendasdivx.pt :: Download limit reached") elif 'bloqueado' in res.text.lower(): # blocked IP address raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) From 1cdf07884f6652e974caa4f9eebd231568306ff9 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Thu, 7 May 2020 13:07:34 +0100 Subject: [PATCH 03/13] simplified raise exceptions (raise_for_status() already raises exceptions --- libs/subliminal_patch/providers/legendasdivx.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 77e1cd22a..fda404622 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -163,6 +163,11 @@ class LegendasdivxProvider(Provider): res = self.session.post(self.loginpage, data) res.raise_for_status() + if (res and 'bloqueado' in res.text.lower()): # blocked IP address + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) + + #make sure we're logged in try: logger.debug('Logged in successfully: PHPSESSID: %s' % self.session.cookies.get_dict()['PHPSESSID']) @@ -171,9 +176,6 @@ class LegendasdivxProvider(Provider): logger.error("Couldn't retrieve session ID, check your credentials") raise AuthenticationError("Please check your credentials.") except Exception as e: - if (res and 'bloqueado' in res.text.lower()): # blocked IP address - logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) @@ -305,8 +307,10 @@ class LegendasdivxProvider(Provider): res.raise_for_status() if res: if 'limite' in res.text.lower(): # daily downloads limit reached - raise DownloadLimitReached("Legendasdivx.pt :: Download limit reached") + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitReached("Legendasdivx.pt :: Daily download limit reached!") elif 'bloqueado' in res.text.lower(): # blocked IP address + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) archive = self._get_archive(res.content) From 83d81a6946155ab3ad73afc9ecffec24fe9ca517 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Thu, 7 May 2020 13:09:50 +0100 Subject: [PATCH 04/13] no need to import HTTPerror exception. Not treated --- libs/subliminal_patch/providers/legendasdivx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index fda404622..54e755a0c 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -8,7 +8,6 @@ import rarfile import zipfile from requests import Session -from requests.exceptions import HTTPError from guessit import guessit from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded from subliminal_patch.providers import Provider From 075f053f17033fbcc20acc5edbfa94bec32535d5 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Thu, 7 May 2020 13:47:00 +0100 Subject: [PATCH 05/13] treat exceptions when HTTPError is returned --- bazarr/get_providers.py | 2 +- .../providers/legendasdivx.py | 33 +++++++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index 577518c6e..b630d6a62 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -37,7 +37,7 @@ PROVIDER_THROTTLE_MAP = { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours") }, "legendasdivx": { - TooManyRequests: (datetime.timedelta(hours=2), "2 hours"), + TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), ParseResponseError: (datetime.timedelta(hours=1), "1 hours"), } diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 54e755a0c..0be22757d 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -8,8 +8,10 @@ import rarfile import zipfile from requests import Session +from requests.exceptions import HTTPError from guessit import guessit from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded +from subliminal_patch.exceptions import TooManyRequests from subliminal_patch.providers import Provider from subliminal.providers import ParserBeautifulSoup from subliminal_patch.subtitle import Subtitle @@ -160,20 +162,23 @@ class LegendasdivxProvider(Provider): data['password'] = self.password res = self.session.post(self.loginpage, data) - res.raise_for_status() if (res and 'bloqueado' in res.text.lower()): # blocked IP address logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) + raise TooManyRequests("Legendasdivx.pt :: Your IP is blocked on this server.") #make sure we're logged in try: + res.raise_for_status() logger.debug('Logged in successfully: PHPSESSID: %s' % self.session.cookies.get_dict()['PHPSESSID']) - self.logged_in = True + self.logged_in = True except KeyError: logger.error("Couldn't retrieve session ID, check your credentials") raise AuthenticationError("Please check your credentials.") + except HTTPError as e: + logger.error("Legendasdivx.pt :: HTTP Error %s" % e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) except Exception as e: logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) @@ -303,14 +308,22 @@ class LegendasdivxProvider(Provider): def download_subtitle(self, subtitle): res = self.session.get(subtitle.page_link) - res.raise_for_status() + if res: - if 'limite' in res.text.lower(): # daily downloads limit reached - logger.error("LegendasDivx.pt :: Daily download limit reached!") - raise DownloadLimitReached("Legendasdivx.pt :: Daily download limit reached!") - elif 'bloqueado' in res.text.lower(): # blocked IP address - logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise ParseResponseError("Legendasdivx.pt :: %r" % res.text) + try: + res.raise_for_status() + if 'limite' in res.text.lower(): # daily downloads limit reached + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitReached("Legendasdivx.pt :: Daily download limit reached!") + elif 'bloqueado' in res.text.lower(): # blocked IP address + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise TooManyRequests("LegendasDivx.pt :: Your IP is blocked on this server.") + except HTTPError as e: + logger.error("Legendasdivx.pt :: HTTP Error %s" % e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) + except Exception as e: + logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) archive = self._get_archive(res.content) # extract the subtitle From d9982b36d9538f1882932c8fc28df5892a2b34ca Mon Sep 17 00:00:00 2001 From: Bazarr Date: Fri, 8 May 2020 10:32:12 +0100 Subject: [PATCH 06/13] added new IPAddressBlocked exception and cache Legendasdivx login cookies to not throttle site --- bazarr/get_providers.py | 16 ++- libs/subliminal_patch/exceptions.py | 6 +- .../providers/legendasdivx.py | 130 ++++++++++-------- 3 files changed, 91 insertions(+), 61 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index b630d6a62..ebe1df293 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -8,10 +8,22 @@ import time from get_args import args from config import settings -from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError +from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError, IPAddressBlocked from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable from subliminal import region as subliminal_cache_region +def time_until_end_of_day(dt=None): + # type: (datetime.datetime) -> datetime.timedelta + """ + Get timedelta until end of day on the datetime passed, or current time. + """ + if dt is None: + dt = datetime.datetime.now() + tomorrow = dt + datetime.timedelta(days=1) + return datetime.datetime.combine(tomorrow, datetime.time.min) - dt + +hours_until_end_of_day = time_until_end_of_day().seconds // 3600 + 1 + VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, ParseResponseError) VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled') @@ -39,7 +51,7 @@ PROVIDER_THROTTLE_MAP = { "legendasdivx": { TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), - ParseResponseError: (datetime.timedelta(hours=1), "1 hours"), + IPAddressBlocked: (datetime.timedelta(hours=hours_until_end_of_day), "{} hours".format(str(hours_until_end_of_day))), } } diff --git a/libs/subliminal_patch/exceptions.py b/libs/subliminal_patch/exceptions.py index 82f33ade8..6cd73e769 100644 --- a/libs/subliminal_patch/exceptions.py +++ b/libs/subliminal_patch/exceptions.py @@ -7,11 +7,13 @@ class TooManyRequests(ProviderError): """Exception raised by providers when too many requests are made.""" pass - class APIThrottled(ProviderError): pass - class ParseResponseError(ProviderError): """Exception raised by providers when they are not able to parse the response.""" pass + +class IPAddressBlocked(ProviderError): + """Exception raised when providers block requests from IP Address.""" + pass \ No newline at end of file diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 0be22757d..e652ae860 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -11,7 +11,7 @@ from requests import Session from requests.exceptions import HTTPError from guessit import guessit from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded -from subliminal_patch.exceptions import TooManyRequests +from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked from subliminal_patch.providers import Provider from subliminal.providers import ParserBeautifulSoup from subliminal_patch.subtitle import Subtitle @@ -20,6 +20,9 @@ from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending, guess_matc from subzero.language import Language from subliminal_patch.score import get_scores from subliminal.utils import sanitize, sanitize_release_group +from dogpile.cache.api import NO_VALUE +from subliminal.cache import region +from subliminal_patch.http import RetryingCFSession logger = logging.getLogger(__name__) @@ -132,64 +135,73 @@ class LegendasdivxProvider(Provider): def __init__(self, username, password): # make sure login credentials are configured. if any((username, password)) and not all((username, password)): - raise ConfigurationError('Username and password must be specified') + raise ConfigurationError('Legendasdivx.pt :: Username and password must be specified') self.username = username self.password = password - self.logged_in = False def initialize(self): - self.session = Session() - self.session.headers.update(self.headers) - self.login() + logger.info("Legendasdivx.pt :: Creating session for requests") + #self.session = Session() + self.session = RetryingCFSession() + # re-use PHP Session if present + prev_cookies = region.get("legendasdivx_cookies2") + if prev_cookies != NO_VALUE: + logger.debug("Legendasdivx.pt :: Re-using previous legendasdivx cookies: %s", prev_cookies) + self.session.cookies.update(prev_cookies) + # Login if session has expired + else: + logger.debug("Legendasdivx.pt :: Session cookies not found!") + self.session.headers.update(self.headers) + self.login() def terminate(self): - self.logout() + # session close self.session.close() def login(self): - logger.info('Logging in') - - res = self.session.get(self.loginpage) - bsoup = ParserBeautifulSoup(res.content, ['lxml']) - - _allinputs = bsoup.findAll('input') - data = {} - # necessary to set 'sid' for POST request - for field in _allinputs: - data[field.get('name')] = field.get('value') - - data['username'] = self.username - data['password'] = self.password - - res = self.session.post(self.loginpage, data) - - if (res and 'bloqueado' in res.text.lower()): # blocked IP address - logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise TooManyRequests("Legendasdivx.pt :: Your IP is blocked on this server.") - - #make sure we're logged in + logger.info('Legendasdivx.pt :: Logging in') try: + res = self.session.get(self.loginpage) + res.raise_for_status() + bsoup = ParserBeautifulSoup(res.content, ['lxml']) + + _allinputs = bsoup.findAll('input') + data = {} + # necessary to set 'sid' for POST request + for field in _allinputs: + data[field.get('name')] = field.get('value') + + data['username'] = self.username + data['password'] = self.password + + res = self.session.post(self.loginpage, data) res.raise_for_status() - logger.debug('Logged in successfully: PHPSESSID: %s' % + #make sure we're logged in + logger.debug('Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s' % self.session.cookies.get_dict()['PHPSESSID']) - self.logged_in = True + cj = self.session.cookies.copy() + store_cks = ("PHPSESSID", "phpbb3_2z8zs_sid", "phpbb3_2z8zs_k", "phpbb3_2z8zs_u", "lang") + for cn in iter(self.session.cookies.keys()): + if cn not in store_cks: + del cj[cn] + #store session cookies on cache + logger.debug("Legendasdivx.pt :: Storing legendasdivx session cookies: %r", cj) + region.set("legendasdivx_cookies2", cj) + except KeyError: - logger.error("Couldn't retrieve session ID, check your credentials") - raise AuthenticationError("Please check your credentials.") + logger.error("Legendasdivx.pt :: Couldn't get session ID, check your credentials") + raise AuthenticationError("Legendasdivx.pt :: Couldn't get session ID, check your credentials") except HTTPError as e: - logger.error("Legendasdivx.pt :: HTTP Error %s" % e) - raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) + if res.status_code == "403": + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + else: + logger.error("Legendasdivx.pt :: HTTP Error %s" % e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) except Exception as e: logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) - def logout(self): - if self.logged_in: - logger.info('Legendasdivx:: Logging out') - r = self.session.get(self.logoutpage, timeout=10) - r.raise_for_status() - logger.debug('Legendasdivx :: Logged out') - self.logged_in = False def _process_page(self, video, bsoup, video_filename): @@ -217,10 +229,10 @@ class LegendasdivxProvider(Provider): # sometimes BSoup can't find 'a' tag and returns None. try: - dl = download.get('href') - logger.debug("Found subtitle link on: {0}").format(self.download_link.format(link=dl)) + download_link = self.download_link.format(link=download.get('href')) + logger.debug("Legendasdivx.pt :: Found subtitle link on: %s " % download_link) except: - logger.debug("Couldn't find download link. Trying next...") + logger.debug("Legendasdivx.pt :: Couldn't find download link. Trying next...") continue # get subtitle uploader @@ -230,7 +242,7 @@ class LegendasdivxProvider(Provider): exact_match = False if video.name.lower() in description.lower(): exact_match = True - data = {'link': self.site + '/modules.php' + download.get('href'), + data = {'link': download_link, 'exact_match': exact_match, 'hits': hits, 'uploader': uploader, @@ -275,27 +287,30 @@ class LegendasdivxProvider(Provider): res = self.session.get(_searchurl.format(query=querytext)) if (res and "A legenda não foi encontrada" in res.text): - logger.warning('%s not found', querytext) + logger.warning('Legendasdivx.pt :: %s not found', querytext) return [] bsoup = ParserBeautifulSoup(res.content, ['html.parser']) - subtitles = self._process_page(video, bsoup, video_filename) # search for more than 10 results (legendasdivx uses pagination) # don't throttle - maximum results = 6 * 10 MAX_PAGES = 6 - + #get number of pages bases on results found page_header = bsoup.find("div", {"class": "pager_bar"}) results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) if page_header else 0 - logger.debug("Found %s subtitles" % str(results_found)) + logger.debug("Legendasdivx.pt :: Found %s subtitles" % str(results_found)) num_pages = (int(results_found) // 10) + 1 num_pages = min(MAX_PAGES, num_pages) + # process first page + subtitles = self._process_page(video, bsoup, video_filename) + + # more pages? if num_pages > 1: for num_page in range(2, num_pages+1): _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page)) - logger.debug("Moving to next page: %s" % _search_next) + logger.debug("Legendasdivx.pt :: Moving on to next page: %s" % _search_next) res = self.session.get(_search_next) next_page = ParserBeautifulSoup(res.content, ['html.parser']) subs = self._process_page(video, next_page, video_filename) @@ -315,15 +330,16 @@ class LegendasdivxProvider(Provider): if 'limite' in res.text.lower(): # daily downloads limit reached logger.error("LegendasDivx.pt :: Daily download limit reached!") raise DownloadLimitReached("Legendasdivx.pt :: Daily download limit reached!") - elif 'bloqueado' in res.text.lower(): # blocked IP address - logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise TooManyRequests("LegendasDivx.pt :: Your IP is blocked on this server.") except HTTPError as e: - logger.error("Legendasdivx.pt :: HTTP Error %s" % e) - raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) + if res.status_code == "403": + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + else: + logger.error("Legendasdivx.pt :: HTTP Error %s" % e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) except Exception as e: - logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) - raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) + logger.error("LegendasDivx.pt :: Uncaught error: %r" % e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % e) archive = self._get_archive(res.content) # extract the subtitle From 41a66abe688316b14fa1ef8c7d1f757ca58dfb63 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Fri, 8 May 2020 12:28:06 +0100 Subject: [PATCH 07/13] Throttle connections to Addic7ed provider throwing IPAddressBlocked excpetion --- bazarr/get_providers.py | 2 ++ libs/subliminal_patch/providers/addic7ed.py | 25 ++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index ebe1df293..f45c4ae2d 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -44,6 +44,8 @@ PROVIDER_THROTTLE_MAP = { "addic7ed": { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), + IPAddressBlocked: (datetime.timedelta(hours=1), "1 hours"), + }, "titulky": { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours") diff --git a/libs/subliminal_patch/providers/addic7ed.py b/libs/subliminal_patch/providers/addic7ed.py index 880a4729c..af6813b4c 100644 --- a/libs/subliminal_patch/providers/addic7ed.py +++ b/libs/subliminal_patch/providers/addic7ed.py @@ -9,13 +9,14 @@ from random import randint from dogpile.cache.api import NO_VALUE from requests import Session +from requests.exceptions import ConnectionError from subliminal.cache import region from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \ Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup from subliminal.subtitle import fix_line_ending from subliminal_patch.utils import sanitize -from subliminal_patch.exceptions import TooManyRequests +from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked from subliminal_patch.pitcher import pitchers, load_verification, store_verification from subzero.language import Language @@ -91,15 +92,19 @@ class Addic7edProvider(_Addic7edProvider): # login if self.username and self.password: def check_verification(cache_region): - rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10, - headers={"Referer": self.server_url}) - if rr.status_code == 302: - logger.info('Addic7ed: Login expired') - cache_region.delete("addic7ed_data") - else: - logger.info('Addic7ed: Re-using old login') - self.logged_in = True - return True + try: + rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10, + headers={"Referer": self.server_url}) + if rr.status_code == 302: + logger.info('Addic7ed: Login expired') + cache_region.delete("addic7ed_data") + else: + logger.info('Addic7ed: Re-using old login') + self.logged_in = True + return True + except ConnectionError as e: + logger.debug("Addic7ed: There was a problem reaching the server: %s." % e) + raise IPAddressBlocked("Addic7ed: Your IP is temporarily blocked.") if load_verification("addic7ed", self.session, callback=check_verification): return From f470448bf9e3e52348e48d9793157dd516c71f30 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Fri, 8 May 2020 15:15:52 +0100 Subject: [PATCH 08/13] Improved Exceptions and code more Python3 complying --- .../providers/legendasdivx.py | 148 +++++++++--------- 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index e652ae860..3a4970eae 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -2,27 +2,26 @@ from __future__ import absolute_import import logging import io -import re import os -import rarfile +import re import zipfile - -from requests import Session from requests.exceptions import HTTPError +import rarfile + from guessit import guessit +from subliminal.cache import region from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded +from subliminal.providers import ParserBeautifulSoup +from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending, guess_matches +from subliminal.utils import sanitize, sanitize_release_group +from subliminal.video import Episode, Movie from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked +from subliminal_patch.http import RetryingCFSession from subliminal_patch.providers import Provider -from subliminal.providers import ParserBeautifulSoup +from subliminal_patch.score import get_scores from subliminal_patch.subtitle import Subtitle -from subliminal.video import Episode, Movie -from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending, guess_matches from subzero.language import Language -from subliminal_patch.score import get_scores -from subliminal.utils import sanitize, sanitize_release_group from dogpile.cache.api import NO_VALUE -from subliminal.cache import region -from subliminal_patch.http import RetryingCFSession logger = logging.getLogger(__name__) @@ -141,7 +140,6 @@ class LegendasdivxProvider(Provider): def initialize(self): logger.info("Legendasdivx.pt :: Creating session for requests") - #self.session = Session() self.session = RetryingCFSession() # re-use PHP Session if present prev_cookies = region.get("legendasdivx_cookies2") @@ -164,21 +162,20 @@ class LegendasdivxProvider(Provider): res = self.session.get(self.loginpage) res.raise_for_status() bsoup = ParserBeautifulSoup(res.content, ['lxml']) - + _allinputs = bsoup.findAll('input') data = {} # necessary to set 'sid' for POST request for field in _allinputs: data[field.get('name')] = field.get('value') - + data['username'] = self.username data['password'] = self.password - + res = self.session.post(self.loginpage, data) res.raise_for_status() #make sure we're logged in - logger.debug('Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s' % - self.session.cookies.get_dict()['PHPSESSID']) + logger.debug('Legendasdivx.pt :: Logged in successfully: PHPSESSID: %s', self.session.cookies.get_dict()['PHPSESSID']) cj = self.session.cookies.copy() store_cks = ("PHPSESSID", "phpbb3_2z8zs_sid", "phpbb3_2z8zs_k", "phpbb3_2z8zs_u", "lang") for cn in iter(self.session.cookies.keys()): @@ -192,15 +189,17 @@ class LegendasdivxProvider(Provider): logger.error("Legendasdivx.pt :: Couldn't get session ID, check your credentials") raise AuthenticationError("Legendasdivx.pt :: Couldn't get session ID, check your credentials") except HTTPError as e: - if res.status_code == "403": + if "bloqueado" in res.text.lower(): # ip blocked on server logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") - else: - logger.error("Legendasdivx.pt :: HTTP Error %s" % e) - raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) + if 'limite' in res.text.lower(): # daily downloads limit reached + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitExceeded("Legendasdivx.pt :: Daily download limit reached!") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) except Exception as e: - logger.error("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) - raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % repr(e)) + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) def _process_page(self, video, bsoup, video_filename): @@ -226,17 +225,17 @@ class LegendasdivxProvider(Provider): description = _subbox.find("td", {"class": "td_desc brd_up"}).get_text() #get subtitle link download = _subbox.find("a", {"class": "sub_download"}) - - # sometimes BSoup can't find 'a' tag and returns None. + + # sometimes BSoup can't find 'a' tag and returns None. try: download_link = self.download_link.format(link=download.get('href')) - logger.debug("Legendasdivx.pt :: Found subtitle link on: %s " % download_link) + logger.debug("Legendasdivx.pt :: Found subtitle link on: %s ", download_link) except: logger.debug("Legendasdivx.pt :: Couldn't find download link. Trying next...") continue # get subtitle uploader - sub_header = _subbox.find("div", {"class" :"sub_header"}) + sub_header = _subbox.find("div", {"class" :"sub_header"}) uploader = sub_header.find("a").text if sub_header else 'anonymous' exact_match = False @@ -286,9 +285,23 @@ class LegendasdivxProvider(Provider): self.session.headers.update(self.headers.items()) res = self.session.get(_searchurl.format(query=querytext)) - if (res and "A legenda não foi encontrada" in res.text): - logger.warning('Legendasdivx.pt :: %s not found', querytext) - return [] + try: + res.raise_for_status() + if (res and "A legenda não foi encontrada" in res.text): + logger.warning('Legendasdivx.pt :: %s not found', querytext) + return [] + except HTTPError as e: + if "bloqueado" in res.text.lower(): # ip blocked on server + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + if 'limite' in res.text.lower(): # daily downloads limit reached + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitExceeded("Legendasdivx.pt :: Daily download limit reached!") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) + except Exception as e: + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) bsoup = ParserBeautifulSoup(res.content, ['html.parser']) @@ -296,10 +309,10 @@ class LegendasdivxProvider(Provider): # don't throttle - maximum results = 6 * 10 MAX_PAGES = 6 - #get number of pages bases on results found + # get number of pages bases on results found page_header = bsoup.find("div", {"class": "pager_bar"}) results_found = re.search(r'\((.*?) encontradas\)', page_header.text).group(1) if page_header else 0 - logger.debug("Legendasdivx.pt :: Found %s subtitles" % str(results_found)) + logger.debug("Legendasdivx.pt :: Found %s subtitles", str(results_found)) num_pages = (int(results_found) // 10) + 1 num_pages = min(MAX_PAGES, num_pages) @@ -310,7 +323,7 @@ class LegendasdivxProvider(Provider): if num_pages > 1: for num_page in range(2, num_pages+1): _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page)) - logger.debug("Legendasdivx.pt :: Moving on to next page: %s" % _search_next) + logger.debug("Legendasdivx.pt :: Moving on to next page: %s", _search_next) res = self.session.get(_search_next) next_page = ParserBeautifulSoup(res.content, ['html.parser']) subs = self._process_page(video, next_page, video_filename) @@ -323,34 +336,29 @@ class LegendasdivxProvider(Provider): def download_subtitle(self, subtitle): res = self.session.get(subtitle.page_link) - - if res: - try: - res.raise_for_status() - if 'limite' in res.text.lower(): # daily downloads limit reached - logger.error("LegendasDivx.pt :: Daily download limit reached!") - raise DownloadLimitReached("Legendasdivx.pt :: Daily download limit reached!") - except HTTPError as e: - if res.status_code == "403": - logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") - raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") - else: - logger.error("Legendasdivx.pt :: HTTP Error %s" % e) - raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s" % e) - except Exception as e: - logger.error("LegendasDivx.pt :: Uncaught error: %r" % e) - raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r" % e) - - archive = self._get_archive(res.content) - # extract the subtitle - subtitle_content = self._get_subtitle_from_archive(archive, subtitle) - subtitle.content = fix_line_ending(subtitle_content) - subtitle.normalize() - - return subtitle - - logger.error("Legendasdivx.pt :: there was a problem retrieving subtitle (status %s)" % res.status_code) - return + + try: + res.raise_for_status() + except HTTPError as e: + if "bloqueado" in res.text.lower(): # ip blocked on server + logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") + raise IPAddressBlocked("LegendasDivx.pt :: Your IP is blocked on this server.") + if 'limite' in res.text.lower(): # daily downloads limit reached + logger.error("LegendasDivx.pt :: Daily download limit reached!") + raise DownloadLimitExceeded("Legendasdivx.pt :: Daily download limit reached!") + logger.error("Legendasdivx.pt :: HTTP Error %s", e) + raise TooManyRequests("Legendasdivx.pt :: HTTP Error %s", e) + except Exception as e: + logger.error("LegendasDivx.pt :: Uncaught error: %r", e) + raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) + + archive = self._get_archive(res.content) + # extract the subtitle + subtitle_content = self._get_subtitle_from_archive(archive, subtitle) + subtitle.content = fix_line_ending(subtitle_content) + subtitle.normalize() + + return subtitle def _get_archive(self, content): # open the archive @@ -384,26 +392,26 @@ class LegendasdivxProvider(Provider): if not name.lower().endswith(_subtitle_extensions): continue - _guess = guessit (name) + _guess = guessit(name) if isinstance(subtitle.video, Episode): - logger.debug ("guessing %s" % name) - logger.debug("subtitle S{}E{} video S{}E{}".format(_guess['season'],_guess['episode'],subtitle.video.season,subtitle.video.episode)) + logger.debug("guessing %s", name) + logger.debug("subtitle S%sE%s video S%sE%s", _guess['season'], _guess['episode'], subtitle.video.season, subtitle.video.episode) if subtitle.video.episode != _guess['episode'] or subtitle.video.season != _guess['season']: logger.debug('subtitle does not match video, skipping') continue matches = set() - matches |= guess_matches (subtitle.video, _guess) - logger.debug('srt matches: %s' % matches) - _score = sum ((_scores.get (match, 0) for match in matches)) + matches |= guess_matches(subtitle.video, _guess) + logger.debug('srt matches: %s', matches) + _score = sum((_scores.get(match, 0) for match in matches)) if _score > _max_score: _max_name = name _max_score = _score - logger.debug("new max: {} {}".format(name, _score)) + logger.debug("new max: %s %s", name, _score) if _max_score > 0: - logger.debug("returning from archive: {} scored {}".format(_max_name, _max_score)) + logger.debug("returning from archive: %s scored %s", _max_name, _max_score) return archive.read(_max_name) - raise ValueError("No subtitle found on compressed file. Max score was 0") \ No newline at end of file + raise ValueError("No subtitle found on compressed file. Max score was 0") From 81642e3fe2f4a0a3037a368ccce85963f19677fb Mon Sep 17 00:00:00 2001 From: Bazarr Date: Sat, 9 May 2020 18:38:27 +0100 Subject: [PATCH 09/13] added skip_wrong_fps match and login again if redirect due to expired session cookies --- bazarr/config.py | 3 +- bazarr/get_providers.py | 1 + bazarr/main.py | 14 ++ .../providers/legendasdivx.py | 67 ++++++--- views/providers.tpl | 139 ++++++------------ 5 files changed, 114 insertions(+), 110 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index 64faf369f..b0bb193f6 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -104,7 +104,8 @@ defaults = { }, 'legendasdivx': { 'username': '', - 'password': '' + 'password': '', + 'skip_wrong_fps': 'False' }, 'legendastv': { 'username': '', diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index f45c4ae2d..e4e653b7e 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -135,6 +135,7 @@ def get_providers_auth(): }, 'legendasdivx': {'username': settings.legendasdivx.username, 'password': settings.legendasdivx.password, + 'skip_wrong_fps': settings.legendasdivx.getboolean('skip_wrong_fps'), }, 'legendastv': {'username': settings.legendastv.username, 'password': settings.legendastv.password, diff --git a/bazarr/main.py b/bazarr/main.py index 226600506..12e8c213d 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -401,12 +401,19 @@ def save_wizard(): else: settings_opensubtitles_skip_wrong_fps = 'True' + settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps') + if settings_legendasdivx_skip_wrong_fps is None: + settings_legendasdivx_skip_wrong_fps = 'False' + else: + settings_legendasdivx_skip_wrong_fps = 'True' + settings.addic7ed.username = request.forms.get('settings_addic7ed_username') settings.addic7ed.password = request.forms.get('settings_addic7ed_password') settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents) settings.assrt.token = request.forms.get('settings_assrt_token') settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username') settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password') + settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps) settings.legendastv.username = request.forms.get('settings_legendastv_username') settings.legendastv.password = request.forms.get('settings_legendastv_password') settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username') @@ -1543,12 +1550,19 @@ def save_settings(): region.delete("os_token") region.delete("os_server_url") + settings_legendasdivx_skip_wrong_fps = request.forms.get('settings_legendasdivx_skip_wrong_fps') + if settings_legendasdivx_skip_wrong_fps is None: + settings_legendasdivx_skip_wrong_fps = 'False' + else: + settings_legendasdivx_skip_wrong_fps = 'True' + settings.addic7ed.username = request.forms.get('settings_addic7ed_username') settings.addic7ed.password = request.forms.get('settings_addic7ed_password') settings.addic7ed.random_agents = text_type(settings_addic7ed_random_agents) settings.assrt.token = request.forms.get('settings_assrt_token') settings.legendasdivx.username = request.forms.get('settings_legendasdivx_username') settings.legendasdivx.password = request.forms.get('settings_legendasdivx_password') + settings.legendasdivx.skip_wrong_fps = text_type(settings_legendasdivx_skip_wrong_fps) settings.legendastv.username = request.forms.get('settings_legendastv_username') settings.legendastv.password = request.forms.get('settings_legendastv_password') settings.opensubtitles.username = request.forms.get('settings_opensubtitles_username') diff --git a/libs/subliminal_patch/providers/legendasdivx.py b/libs/subliminal_patch/providers/legendasdivx.py index 3a4970eae..02ad95cce 100644 --- a/libs/subliminal_patch/providers/legendasdivx.py +++ b/libs/subliminal_patch/providers/legendasdivx.py @@ -5,6 +5,7 @@ import io import os import re import zipfile +from time import sleep from requests.exceptions import HTTPError import rarfile @@ -18,7 +19,7 @@ from subliminal.video import Episode, Movie from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked from subliminal_patch.http import RetryingCFSession from subliminal_patch.providers import Provider -from subliminal_patch.score import get_scores +from subliminal_patch.score import get_scores, framerate_equal from subliminal_patch.subtitle import Subtitle from subzero.language import Language from dogpile.cache.api import NO_VALUE @@ -29,7 +30,7 @@ class LegendasdivxSubtitle(Subtitle): """Legendasdivx Subtitle.""" provider_name = 'legendasdivx' - def __init__(self, language, video, data): + def __init__(self, language, video, data, skip_wrong_fps=True): super(LegendasdivxSubtitle, self).__init__(language) self.language = language self.page_link = data['link'] @@ -37,8 +38,11 @@ class LegendasdivxSubtitle(Subtitle): self.exact_match = data['exact_match'] self.description = data['description'] self.video = video + self.sub_frame_rate = data['frame_rate'] self.video_filename = data['video_filename'] self.uploader = data['uploader'] + self.wrong_fps = False + self.skip_wrong_fps = skip_wrong_fps @property def id(self): @@ -51,6 +55,23 @@ class LegendasdivxSubtitle(Subtitle): def get_matches(self, video): matches = set() + # if skip_wrong_fps = True no point to continue if they don't match + subtitle_fps = None + try: + subtitle_fps = float(self.sub_frame_rate) + except ValueError: + pass + + # check fps match and skip based on configuration + if video.fps and subtitle_fps and not framerate_equal(video.fps, subtitle_fps): + self.wrong_fps = True + + if self.skip_wrong_fps: + logger.debug("Legendasdivx :: Skipping subtitle due to FPS mismatch (expected: %s, got: %s)", video.fps, self.sub_frame_rate) + # not a single match :) + return set() + logger.debug("Legendasdivx :: Frame rate mismatch (expected: %s, got: %s, but continuing...)", video.fps, self.sub_frame_rate) + description = sanitize(self.description) if sanitize(self.video_filename) in description: @@ -109,8 +130,6 @@ class LegendasdivxSubtitle(Subtitle): matches.update(['video_codec']) break - # running guessit on a huge description may break guessit - # matches |= guess_matches(video, guessit(self.description)) return matches class LegendasdivxProvider(Provider): @@ -122,21 +141,19 @@ class LegendasdivxProvider(Provider): 'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2"), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Origin': 'https://www.legendasdivx.pt', - 'Referer': 'https://www.legendasdivx.pt', - 'Pragma': 'no-cache', - 'Cache-Control': 'no-cache' + 'Referer': 'https://www.legendasdivx.pt' } loginpage = site + '/forum/ucp.php?mode=login' - logoutpage = site + '/sair.php' searchurl = site + '/modules.php?name=Downloads&file=jz&d_op=search&op=_jz00&query={query}' download_link = site + '/modules.php{link}' - def __init__(self, username, password): + def __init__(self, username, password, skip_wrong_fps=True): # make sure login credentials are configured. if any((username, password)) and not all((username, password)): raise ConfigurationError('Legendasdivx.pt :: Username and password must be specified') self.username = username self.password = password + self.skip_wrong_fps = skip_wrong_fps def initialize(self): logger.info("Legendasdivx.pt :: Creating session for requests") @@ -201,7 +218,6 @@ class LegendasdivxProvider(Provider): logger.error("LegendasDivx.pt :: Uncaught error: %r", e) raise ServiceUnavailable("LegendasDivx.pt :: Uncaught error: %r", e) - def _process_page(self, video, bsoup, video_filename): subtitles = [] @@ -210,17 +226,20 @@ class LegendasdivxProvider(Provider): for _subbox in _allsubs: hits = 0 - for th in _subbox.findAll("th", {"class": "color2"}): + for th in _subbox.findAll("th"): if th.text == 'Hits:': - hits = int(th.parent.find("td").text) + hits = int(th.find_next("td").text) if th.text == 'Idioma:': - lang = th.parent.find("td").find("img").get('src') + lang = th.find_next("td").find("img").get('src') if 'brazil' in lang.lower(): lang = Language.fromopensubtitles('pob') elif 'portugal' in lang.lower(): lang = Language.fromopensubtitles('por') else: continue + if th.text == "Frame Rate:": + frame_rate = th.find_next("td").text.strip() + # get description for matches description = _subbox.find("td", {"class": "td_desc brd_up"}).get_text() #get subtitle link @@ -245,11 +264,12 @@ class LegendasdivxProvider(Provider): 'exact_match': exact_match, 'hits': hits, 'uploader': uploader, + 'frame_rate': frame_rate, 'video_filename': video_filename, 'description': description } subtitles.append( - LegendasdivxSubtitle(lang, video, data) + LegendasdivxSubtitle(lang, video, data, skip_wrong_fps=self.skip_wrong_fps) ) return subtitles @@ -281,15 +301,23 @@ class LegendasdivxProvider(Provider): querytext = querytext + lang_filter if lang_filter else querytext - self.headers['Referer'] = self.site + '/index.php' - self.session.headers.update(self.headers.items()) - res = self.session.get(_searchurl.format(query=querytext)) - try: + # sleep for a 1 second before another request + sleep(1) + self.headers['Referer'] = self.site + '/index.php' + self.session.headers.update(self.headers) + res = self.session.get(_searchurl.format(query=querytext), allow_redirects=False) res.raise_for_status() - if (res and "A legenda não foi encontrada" in res.text): + if (res.status_code == 200 and "A legenda não foi encontrada" in res.text): logger.warning('Legendasdivx.pt :: %s not found', querytext) return [] + if res.status_code == 302: # got redirected to login page. + # Seems that our session cookies are no longer valid... clean them from cache + region.delete("legendasdivx_cookies2") + logger.debug("Legendasdivx.pt :: Logging in again. Cookies have expired!") + self.login() # login and try again + res = self.session.get(_searchurl.format(query=querytext)) + res.raise_for_status() except HTTPError as e: if "bloqueado" in res.text.lower(): # ip blocked on server logger.error("LegendasDivx.pt :: Your IP is blocked on this server.") @@ -322,6 +350,7 @@ class LegendasdivxProvider(Provider): # more pages? if num_pages > 1: for num_page in range(2, num_pages+1): + sleep(1) # another 1 sec before requesting... _search_next = self.searchurl.format(query=querytext) + "&page={0}".format(str(num_page)) logger.debug("Legendasdivx.pt :: Moving on to next page: %s", _search_next) res = self.session.get(_search_next) diff --git a/views/providers.tpl b/views/providers.tpl index 99c88e637..64349da14 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -265,6 +265,24 @@ +
+
+ +
+
+
+ + +
+
+ +
@@ -481,27 +499,6 @@
-
-
- -
-
-
- - -
-
- -
-
- -
@@ -684,6 +681,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -733,7 +752,7 @@
-
+
@@ -755,28 +774,6 @@
-
-
- -
-
-
- - -
-
- -
-
- -
-
@@ -792,28 +789,6 @@
-
-
- -
-
-
- - -
-
- -
-
- -
-
@@ -833,7 +808,7 @@
-
+
@@ -855,28 +830,6 @@
-
-
- -
-
-
- - -
-
- -
-
- -
-
@@ -944,6 +897,12 @@ $("#settings_opensubtitles_skip_wrong_fps").checkbox('uncheck'); } + if ($('#settings_legendasdivx_skip_wrong_fps').data("ldfps") === "True") { + $("#settings_legendasdivx_skip_wrong_fps").checkbox('check'); + } else { + $("#settings_legendasdivx_skip_wrong_fps").checkbox('uncheck'); + } + $('#settings_providers').dropdown('clear'); $('#settings_providers').dropdown('set selected',{{!enabled_providers}}); $('#settings_providers').dropdown(); @@ -971,4 +930,4 @@ $('#'+$(this).parent().attr('id')+'_option').hide(); } }); - + \ No newline at end of file From 001f947b9bb5caac008e7d689089c69d9218c436 Mon Sep 17 00:00:00 2001 From: Bazarr Date: Sat, 9 May 2020 18:46:15 +0100 Subject: [PATCH 10/13] missing RegieLive from views/providers.tpl --- views/providers.tpl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/views/providers.tpl b/views/providers.tpl index 64349da14..52bc37949 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -499,6 +499,26 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+
From 5f48051e116a0159d50a6b9118130ae3ff2ebd0d Mon Sep 17 00:00:00 2001 From: Bazarr Date: Sat, 9 May 2020 18:47:48 +0100 Subject: [PATCH 11/13] missing closing tag --- views/providers.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/views/providers.tpl b/views/providers.tpl index 52bc37949..b7f180bc3 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -518,6 +518,8 @@
+ +
From 6bbe33b7f43c7b5d74af551c7d8197e18306caab Mon Sep 17 00:00:00 2001 From: Bazarr Date: Sat, 9 May 2020 18:54:57 +0100 Subject: [PATCH 12/13] missing providers --- views/providers.tpl | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/views/providers.tpl b/views/providers.tpl index b7f180bc3..b64ce01fa 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -796,6 +796,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -811,6 +833,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
@@ -852,6 +896,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
From b3550cda2090b7ec8339de02c4ef903fb253056a Mon Sep 17 00:00:00 2001 From: Bazarr Date: Sat, 9 May 2020 19:04:12 +0100 Subject: [PATCH 13/13] removed subz from providers.tpl --- views/providers.tpl | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/views/providers.tpl b/views/providers.tpl index b64ce01fa..3d93e5e9c 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -703,28 +703,6 @@
-
-
- -
-
-
- - -
-
- -
-
- -
-