From 0afad23e91f4828076eba8aade8dbb43ee261a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 2 Oct 2019 21:46:05 -0400 Subject: [PATCH 1/9] Fix for get_***arr_version() when Sonarr and Radarr isn't configured. --- bazarr/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bazarr/utils.py b/bazarr/utils.py index e431d996e..1409f495f 100644 --- a/bazarr/utils.py +++ b/bazarr/utils.py @@ -100,9 +100,10 @@ def get_sonarr_version(): use_sonarr = settings.general.getboolean('use_sonarr') apikey_sonarr = settings.sonarr.apikey sv = url_sonarr + "/api/system/status?apikey=" + apikey_sonarr + sonarr_version = '' if use_sonarr: try: - sonarr_version = requests.get(sv, timeout=30, verify=False).json()['version'] + sonarr_version = requests.get(sv, timeout=60, verify=False).json()['version'] except Exception as e: logging.DEBUG('BAZARR cannot get Sonarr version') @@ -113,9 +114,10 @@ def get_radarr_version(): use_radarr = settings.general.getboolean('use_radarr') apikey_radarr = settings.radarr.apikey rv = url_radarr + "/api/system/status?apikey=" + apikey_radarr + radarr_version = '' if use_radarr: try: - radarr_version = requests.get(rv, timeout=30, verify=False).json()['version'] + radarr_version = requests.get(rv, timeout=60, verify=False).json()['version'] except Exception as e: logging.DEBUG('BAZARR cannot get Radarr version') From c96fb2066306ce9e18fed4d4f47ca5178d11f668 Mon Sep 17 00:00:00 2001 From: ngosang Date: Sun, 22 Sep 2019 17:33:04 +0200 Subject: [PATCH 2/9] Improve provider throttle / status to handle all exceptions --- bazarr/get_providers.py | 7 ++++--- libs/subliminal_patch/core.py | 7 ++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index 461a21615..a2f12559f 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -141,10 +141,11 @@ def provider_throttle(name, exception): throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \ PROVIDER_THROTTLE_MAP["default"].get(cls, None) - if not throttle_data: - return + if throttle_data: + throttle_delta, throttle_description = throttle_data + else: + throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes" - throttle_delta, throttle_description = throttle_data throttle_until = datetime.datetime.now() + throttle_delta if cls_name not in VALID_COUNT_EXCEPTIONS or throttled_count(name): diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index 363477e1f..171d85528 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -186,12 +186,9 @@ class SZProviderPool(ProviderPool): except (requests.Timeout, socket.timeout): logger.error('Provider %r timed out', provider) - except (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled), e: - self.throttle_callback(provider, e) - return - - except: + except Exception as e: logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc()) + self.throttle_callback(provider, e) def list_subtitles(self, video, languages): """List subtitles. From 232eaa19818bf34672c8d996359794eefe9f3163 Mon Sep 17 00:00:00 2001 From: ngosang Date: Sun, 22 Sep 2019 17:24:02 +0200 Subject: [PATCH 3/9] Improve Subdivx provider, handle more exceptions --- bazarr/get_providers.py | 3 +- libs/subliminal_patch/core.py | 4 +- libs/subliminal_patch/exceptions.py | 5 + libs/subliminal_patch/providers/subdivx.py | 155 +++++++++++---------- 4 files changed, 90 insertions(+), 77 deletions(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index a2f12559f..45b0769a7 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -8,7 +8,7 @@ import time from get_args import args from config import settings -from subliminal_patch.exceptions import TooManyRequests, APIThrottled +from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled) @@ -20,6 +20,7 @@ PROVIDER_THROTTLE_MAP = { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"), APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"), + ParseResponseError: (datetime.timedelta(hours=6), "6 hours"), }, "opensubtitles": { TooManyRequests: (datetime.timedelta(hours=3), "3 hours"), diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index 171d85528..bb9b3752b 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -28,7 +28,7 @@ from subliminal.utils import hash_napiprojekt, hash_opensubtitles, hash_shooter, from subliminal.video import VIDEO_EXTENSIONS, Video, Episode, Movie from subliminal.core import guessit, ProviderPool, io, is_windows_special_path, \ ThreadPoolExecutor, check_video -from subliminal_patch.exceptions import TooManyRequests, APIThrottled +from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError from subzero.language import Language from scandir import scandir, scandir_generic as _scandir_generic @@ -280,7 +280,7 @@ class SZProviderPool(ProviderPool): logger.debug("RAR Traceback: %s", traceback.format_exc()) return False - except (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled), e: + except (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, ParseResponseError) as e: self.throttle_callback(subtitle.provider_name, e) self.discarded_providers.add(subtitle.provider_name) return False diff --git a/libs/subliminal_patch/exceptions.py b/libs/subliminal_patch/exceptions.py index e336a10af..9b166a29a 100644 --- a/libs/subliminal_patch/exceptions.py +++ b/libs/subliminal_patch/exceptions.py @@ -9,3 +9,8 @@ class TooManyRequests(ProviderError): class APIThrottled(ProviderError): pass + + +class ParseResponseError(ProviderError): + """Exception raised by providers when they are not able to parse the response.""" + pass diff --git a/libs/subliminal_patch/providers/subdivx.py b/libs/subliminal_patch/providers/subdivx.py index c3ac4b1f7..8fd2b58cf 100644 --- a/libs/subliminal_patch/providers/subdivx.py +++ b/libs/subliminal_patch/providers/subdivx.py @@ -7,13 +7,14 @@ import zipfile import rarfile from subzero.language import Language -from guessit import guessit from requests import Session from subliminal import __short_version__ +from subliminal.exceptions import ServiceUnavailable from subliminal.providers import ParserBeautifulSoup, Provider from subliminal.subtitle import SUBTITLE_EXTENSIONS, Subtitle, fix_line_ending,guess_matches from subliminal.video import Episode, Movie +from subliminal_patch.exceptions import ParseResponseError logger = logging.getLogger(__name__) @@ -119,35 +120,17 @@ class SubdivxSubtitlesProvider(Provider): language = self.language_list[0] search_link = self.server_url + 'index.php' while True: - r = self.session.get(search_link, params=params, timeout=10) - r.raise_for_status() + response = self.session.get(search_link, params=params, timeout=10) + self._check_response(response) - if not r.content: - logger.debug('No data returned from provider') - return [] + try: + page_subtitles = self._parse_subtitles_page(response, language) + except Exception as e: + raise ParseResponseError('Error parsing subtitles list: ' + str(e)) - page_soup = ParserBeautifulSoup(r.content.decode('iso-8859-1', 'ignore'), ['lxml', 'html.parser']) - title_soups = page_soup.find_all("div", {'id': 'menu_detalle_buscador'}) - body_soups = page_soup.find_all("div", {'id': 'buscador_detalle'}) - if len(title_soups) != len(body_soups): - logger.debug('Error in provider data') - return [] - for subtitle in range(0, len(title_soups)): - title_soup, body_soup = title_soups[subtitle], body_soups[subtitle] + subtitles += page_subtitles - # title - title = title_soup.find("a").text.replace("Subtitulo de ", "") - page_link = title_soup.find("a")["href"].replace('http://', 'https://') - - # body - description = body_soup.find("div", {'id': 'buscador_detalle_sub'}).text - - subtitle = self.subtitle_class(language, page_link, description, title) - - logger.debug('Found subtitle %r', subtitle) - subtitles.append(subtitle) - - if len(title_soups) >= 20: + if len(page_subtitles) >= 20: params['pg'] += 1 # search next page time.sleep(self.multi_result_throttle) else: @@ -175,67 +158,91 @@ class SubdivxSubtitlesProvider(Provider): return subtitles - def get_download_link(self, subtitle): - r = self.session.get(subtitle.page_link, timeout=10) - r.raise_for_status() - - if r.content: - page_soup = ParserBeautifulSoup(r.content.decode('iso-8859-1', 'ignore'), ['lxml', 'html.parser']) - links_soup = page_soup.find_all("a", {'class': 'detalle_link'}) - for link_soup in links_soup: - if link_soup['href'].startswith('bajar'): - return self.server_url + link_soup['href'] - - logger.debug('No data returned from provider') - return None - def download_subtitle(self, subtitle): if isinstance(subtitle, SubdivxSubtitle): # download the subtitle logger.info('Downloading subtitle %r', subtitle) # get download link - download_link = self.get_download_link(subtitle) - r = self.session.get(download_link, headers={'Referer': subtitle.page_link}, timeout=30) - r.raise_for_status() + download_link = self._get_download_link(subtitle) - if not r.content: - logger.debug('Unable to download subtitle. No data returned from provider') - return + # download zip / rar file with the subtitle + response = self.session.get(download_link, headers={'Referer': subtitle.page_link}, timeout=30) + self._check_response(response) - archive = _get_archive(r.content) + # open the compressed archive + archive = self._get_archive(response.content) - subtitle_content = _get_subtitle_from_archive(archive) - if subtitle_content: - subtitle.content = fix_line_ending(subtitle_content) - else: - logger.debug('Could not extract subtitle from %r', archive) + # extract the subtitle + subtitle_content = self._get_subtitle_from_archive(archive) + subtitle.content = fix_line_ending(subtitle_content) + + def _check_response(self, response): + if response.status_code != 200: + raise ServiceUnavailable('Bad status code: ' + str(response.status_code)) + def _parse_subtitles_page(self, response, language): + subtitles = [] + + page_soup = ParserBeautifulSoup(response.content.decode('iso-8859-1', 'ignore'), ['lxml', 'html.parser']) + title_soups = page_soup.find_all("div", {'id': 'menu_detalle_buscador'}) + body_soups = page_soup.find_all("div", {'id': 'buscador_detalle'}) + + for subtitle in range(0, len(title_soups)): + title_soup, body_soup = title_soups[subtitle], body_soups[subtitle] + + # title + title = title_soup.find("a").text.replace("Subtitulo de ", "") + page_link = title_soup.find("a")["href"].replace('http://', 'https://') -def _get_archive(content): - # open the archive - archive_stream = io.BytesIO(content) - archive = None - if rarfile.is_rarfile(archive_stream): - logger.debug('Identified rar archive') - archive = rarfile.RarFile(archive_stream) - elif zipfile.is_zipfile(archive_stream): - logger.debug('Identified zip archive') - archive = zipfile.ZipFile(archive_stream) + # body + description = body_soup.find("div", {'id': 'buscador_detalle_sub'}).text - return archive + subtitle = self.subtitle_class(language, page_link, description, title) + + logger.debug('Found subtitle %r', subtitle) + subtitles.append(subtitle) + + return subtitles + + def _get_download_link(self, subtitle): + response = self.session.get(subtitle.page_link, timeout=10) + self._check_response(response) + try: + page_soup = ParserBeautifulSoup(response.content.decode('iso-8859-1', 'ignore'), ['lxml', 'html.parser']) + links_soup = page_soup.find_all("a", {'class': 'detalle_link'}) + for link_soup in links_soup: + if link_soup['href'].startswith('bajar'): + return self.server_url + link_soup['href'] + except Exception as e: + raise ParseResponseError('Error parsing download link: ' + str(e)) + + raise ParseResponseError('Download link not found') + + def _get_archive(self, content): + # open the archive + archive_stream = io.BytesIO(content) + if rarfile.is_rarfile(archive_stream): + logger.debug('Identified rar archive') + archive = rarfile.RarFile(archive_stream) + elif zipfile.is_zipfile(archive_stream): + logger.debug('Identified zip archive') + archive = zipfile.ZipFile(archive_stream) + else: + raise ParseResponseError('Unsupported compressed format') + return archive -def _get_subtitle_from_archive(archive): - for name in archive.namelist(): - # discard hidden files - if os.path.split(name)[-1].startswith('.'): - continue + def _get_subtitle_from_archive(self, archive): + for name in archive.namelist(): + # discard hidden files + if os.path.split(name)[-1].startswith('.'): + continue - # discard non-subtitle files - if not name.lower().endswith(SUBTITLE_EXTENSIONS): - continue + # discard non-subtitle files + if not name.lower().endswith(SUBTITLE_EXTENSIONS): + continue - return archive.read(name) + return archive.read(name) - return None + raise ParseResponseError('Can not find the subtitle in the compressed file') From 69f23c65a8d89f37c6f15e5dc782963087583e4a Mon Sep 17 00:00:00 2001 From: ngosang Date: Sat, 5 Oct 2019 15:29:53 +0200 Subject: [PATCH 4/9] requested changes --- bazarr/get_providers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index 45b0769a7..258a1c1db 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -11,7 +11,8 @@ from config import settings from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable -VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled) +VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled, + ParseResponseError) VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled') PROVIDER_THROTTLE_MAP = { From b3c6def6bc72e85d8ad016cec8347e445b9fad8b Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 5 Oct 2019 22:55:35 +0200 Subject: [PATCH 5/9] Updated subliminal_path and subzero to latest dev version --- libs/subliminal_patch/converters/titlovi.py | 14 - libs/subliminal_patch/core.py | 33 ++- libs/subliminal_patch/providers/titlovi.py | 271 ++++++++++---------- libs/subzero/language.py | 14 + libs/subzero/lib/dict.py | 6 + libs/subzero/video.py | 6 +- 6 files changed, 178 insertions(+), 166 deletions(-) diff --git a/libs/subliminal_patch/converters/titlovi.py b/libs/subliminal_patch/converters/titlovi.py index 940507d4f..761cf79a6 100644 --- a/libs/subliminal_patch/converters/titlovi.py +++ b/libs/subliminal_patch/converters/titlovi.py @@ -27,16 +27,6 @@ class TitloviConverter(LanguageReverseConverter): } self.codes = set(self.from_titlovi.keys()) - # temporary fix, should be removed as soon as API is used - self.lang_from_countrycode = {'ba': ('bos',), - 'en': ('eng',), - 'hr': ('hrv',), - 'mk': ('mkd',), - 'rs': ('srp',), - 'rsc': ('srp', None, 'Cyrl'), - 'si': ('slv',) - } - def convert(self, alpha3, country=None, script=None): if (alpha3, country, script) in self.to_titlovi: return self.to_titlovi[(alpha3, country, script)] @@ -49,9 +39,5 @@ class TitloviConverter(LanguageReverseConverter): if titlovi in self.from_titlovi: return self.from_titlovi[titlovi] - # temporary fix, should be removed as soon as API is used - if titlovi in self.lang_from_countrycode: - return self.lang_from_countrycode[titlovi] - raise ConfigurationError('Unsupported language number for titlovi: %s' % titlovi) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index bb9b3752b..9e96754dd 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -30,7 +30,7 @@ from subliminal.core import guessit, ProviderPool, io, is_windows_special_path, ThreadPoolExecutor, check_video from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError -from subzero.language import Language +from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE from scandir import scandir, scandir_generic as _scandir_generic logger = logging.getLogger(__name__) @@ -571,12 +571,14 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski return video -def _search_external_subtitles(path, languages=None, only_one=False, scandir_generic=False): +def _search_external_subtitles(path, languages=None, only_one=False, scandir_generic=False, match_strictness="strict"): dirpath, filename = os.path.split(path) dirpath = dirpath or '.' - fileroot, fileext = os.path.splitext(filename) + fn_no_ext, fileext = os.path.splitext(filename) + fn_no_ext_lower = fn_no_ext.lower() subtitles = {} _scandir = _scandir_generic if scandir_generic else scandir + for entry in _scandir(dirpath): if (not entry.name or entry.name in ('\x0c', '$', ',', '\x7f')) and not scandir_generic: logger.debug('Could not determine the name of the file, retrying with scandir_generic') @@ -587,9 +589,11 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen p = entry.name # keep only valid subtitle filenames - if not p.lower().startswith(fileroot.lower()) or not p.lower().endswith(SUBTITLE_EXTENSIONS): + if not p.lower().endswith(SUBTITLE_EXTENSIONS): continue + # not p.lower().startswith(fileroot.lower()) or not + p_root, p_ext = os.path.splitext(p) if not INCLUDE_EXOTIC_SUBS and p_ext not in (".srt", ".ass", ".ssa", ".vtt"): continue @@ -608,7 +612,19 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen forced = "forced" in adv_tag # extract the potential language code - language_code = p_root[len(fileroot):].replace('_', '-')[1:] + language_code = p_root.rsplit(".", 1)[1].replace('_', '-') + + # remove possible language code for matching + p_root_bare = ENDSWITH_LANGUAGECODE_RE.sub("", p_root) + + p_root_lower = p_root_bare.lower() + + filename_matches = p_root_lower == fn_no_ext_lower + filename_contains = p_root_lower in fn_no_ext_lower + + if not filename_matches: + if match_strictness == "strict" or (match_strictness == "loose" and not filename_contains): + continue # default language is undefined language = Language('und') @@ -632,7 +648,7 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen return subtitles -def search_external_subtitles(path, languages=None, only_one=False): +def search_external_subtitles(path, languages=None, only_one=False, match_strictness="strict"): """ wrap original search_external_subtitles function to search multiple paths for one given video # todo: cleanup and merge with _search_external_subtitles @@ -653,10 +669,11 @@ def search_external_subtitles(path, languages=None, only_one=False): if os.path.isdir(os.path.dirname(abspath)): try: subtitles.update(_search_external_subtitles(abspath, languages=languages, - only_one=only_one)) + only_one=only_one, match_strictness=match_strictness)) except OSError: subtitles.update(_search_external_subtitles(abspath, languages=languages, - only_one=only_one, scandir_generic=True)) + only_one=only_one, match_strictness=match_strictness, + scandir_generic=True)) logger.debug("external subs: found %s", subtitles) return subtitles diff --git a/libs/subliminal_patch/providers/titlovi.py b/libs/subliminal_patch/providers/titlovi.py index c0a1ffa11..9be0a92f6 100644 --- a/libs/subliminal_patch/providers/titlovi.py +++ b/libs/subliminal_patch/providers/titlovi.py @@ -2,42 +2,35 @@ import io import logging -import math import re -import time +from datetime import datetime +import dateutil.parser import rarfile -from bs4 import BeautifulSoup from zipfile import ZipFile, is_zipfile from rarfile import RarFile, is_rarfile from babelfish import language_converters, Script -from requests import RequestException +from requests import RequestException, codes as request_codes from guessit import guessit from subliminal_patch.http import RetryingCFSession from subliminal_patch.providers import Provider from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin from subliminal_patch.subtitle import Subtitle from subliminal_patch.utils import sanitize, fix_inconsistent_naming as _fix_inconsistent_naming -from subliminal.exceptions import ProviderError +from subliminal.exceptions import ProviderError, AuthenticationError, ConfigurationError from subliminal.score import get_equivalent_release_groups from subliminal.utils import sanitize_release_group from subliminal.subtitle import guess_matches from subliminal.video import Episode, Movie from subliminal.subtitle import fix_line_ending -from subliminal_patch.pitcher import pitchers, load_verification, store_verification -from subzero.language import Language -from random import randint -from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST +from subzero.language import Language +from dogpile.cache.api import NO_VALUE +from subliminal.cache import region # parsing regex definitions title_re = re.compile(r'(?P(?:.+(?= [Aa][Kk][Aa] ))|.+)(?:(?:.+)(?P<altitle>(?<= [Aa][Kk][Aa] ).+))?') -lang_re = re.compile(r'(?<=flags/)(?P<lang>.{2})(?:.)(?P<script>c?)(?:.+)') -season_re = re.compile(r'Sezona (?P<season>\d+)') -episode_re = re.compile(r'Epizoda (?P<episode>\d+)') -year_re = re.compile(r'(?P<year>\d+)') -fps_re = re.compile(r'fps: (?P<fps>.+)') def fix_inconsistent_naming(title): @@ -51,6 +44,7 @@ def fix_inconsistent_naming(title): return _fix_inconsistent_naming(title, {"DC's Legends of Tomorrow": "Legends of Tomorrow", "Marvel's Jessica Jones": "Jessica Jones"}) + logger = logging.getLogger(__name__) # Configure :mod:`rarfile` to use the same path separator as :mod:`zipfile` @@ -62,9 +56,9 @@ language_converters.register('titlovi = subliminal_patch.converters.titlovi:Titl class TitloviSubtitle(Subtitle): provider_name = 'titlovi' - def __init__(self, language, page_link, download_link, sid, releases, title, alt_title=None, season=None, - episode=None, year=None, fps=None, asked_for_release_group=None, asked_for_episode=None): - super(TitloviSubtitle, self).__init__(language, page_link=page_link) + def __init__(self, language, download_link, sid, releases, title, alt_title=None, season=None, + episode=None, year=None, rating=None, download_count=None, asked_for_release_group=None, asked_for_episode=None): + super(TitloviSubtitle, self).__init__(language) self.sid = sid self.releases = self.release_info = releases self.title = title @@ -73,11 +67,21 @@ class TitloviSubtitle(Subtitle): self.episode = episode self.year = year self.download_link = download_link - self.fps = fps + self.rating = rating + self.download_count = download_count self.matches = None self.asked_for_release_group = asked_for_release_group self.asked_for_episode = asked_for_episode + def __repr__(self): + if self.season and self.episode: + return '<%s "%s (%r)" s%.2de%.2d [%s:%s] ID:%r R:%.2f D:%r>' % ( + self.__class__.__name__, self.title, self.year, self.season, self.episode, self.language, self._guessed_encoding, self.sid, + self.rating, self.download_count) + else: + return '<%s "%s (%r)" [%s:%s] ID:%r R:%.2f D:%r>' % ( + self.__class__.__name__, self.title, self.year, self.language, self._guessed_encoding, self.sid, self.rating, self.download_count) + @property def id(self): return self.sid @@ -134,20 +138,62 @@ class TitloviSubtitle(Subtitle): class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): subtitle_class = TitloviSubtitle languages = {Language.fromtitlovi(l) for l in language_converters['titlovi'].codes} | {Language.fromietf('sr-Latn')} - server_url = 'https://titlovi.com' - search_url = server_url + '/titlovi/?' - download_url = server_url + '/download/?type=1&mediaid=' + api_url = 'https://kodi.titlovi.com/api/subtitles' + api_gettoken_url = api_url + '/gettoken' + api_search_url = api_url + '/search' + + def __init__(self, username=None, password=None): + if not all((username, password)): + raise ConfigurationError('Username and password must be specified') + + self.username = username + self.password = password + + self.session = None + + self.user_id = None + self.login_token = None + self.token_exp = None def initialize(self): self.session = RetryingCFSession() #load_verification("titlovi", self.session) + token = region.get("titlovi_token") + if token is not NO_VALUE: + self.user_id, self.login_token, self.token_exp = token + if datetime.now() > self.token_exp: + logger.debug('Token expired') + self.log_in() + else: + logger.debug('Use cached token') + else: + logger.debug('Token not found in cache') + self.log_in() + + def log_in(self): + login_params = dict(username=self.username, password=self.password, json=True) + try: + response = self.session.post(self.api_gettoken_url, params=login_params) + if response.status_code == request_codes.ok: + resp_json = response.json() + self.login_token = resp_json.get('Token') + self.user_id = resp_json.get('UserId') + self.token_exp = dateutil.parser.parse(resp_json.get('ExpirationDate')) + + region.set("titlovi_token", [self.user_id, self.login_token, self.token_exp]) + logger.debug('New token obtained') + + elif response.status_code == request_codes.unauthorized: + raise AuthenticationError('Login failed') + + except RequestException as e: + logger.error(e) def terminate(self): self.session.close() - def query(self, languages, title, season=None, episode=None, year=None, video=None): - items_per_page = 10 - current_page = 1 + def query(self, languages, title, season=None, episode=None, year=None, imdb_id=None, video=None): + search_params = dict() used_languages = languages lang_strings = [str(lang) for lang in used_languages] @@ -162,135 +208,73 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): langs = '|'.join(map(str, [l.titlovi for l in used_languages])) # set query params - params = {'prijevod': title, 'jezik': langs} + search_params['query'] = title + search_params['lang'] = langs is_episode = False if season and episode: is_episode = True - params['s'] = season - params['e'] = episode - if year: - params['g'] = year + search_params['season'] = season + search_params['episode'] = episode + #if year: + # search_params['year'] = year + if imdb_id: + search_params['imdbID'] = imdb_id # loop through paginated results - logger.info('Searching subtitles %r', params) + logger.info('Searching subtitles %r', search_params) subtitles = [] + query_results = [] - while True: - # query the server - try: - r = self.session.get(self.search_url, params=params, timeout=10) - r.raise_for_status() - except RequestException as e: - logger.exception('RequestException %s', e) - break + try: + search_params['token'] = self.login_token + search_params['userid'] = self.user_id + search_params['json'] = True + + response = self.session.get(self.api_search_url, params=search_params) + resp_json = response.json() + if resp_json['SubtitleResults']: + query_results.extend(resp_json['SubtitleResults']) + + + except Exception as e: + logger.error(e) + + for sub in query_results: + + # title and alternate title + match = title_re.search(sub.get('Title')) + if match: + _title = match.group('title') + alt_title = match.group('altitle') else: - try: - soup = BeautifulSoup(r.content, 'lxml') - - # number of results - result_count = int(soup.select_one('.results_count b').string) - except: - result_count = None - - # exit if no results - if not result_count: - if not subtitles: - logger.debug('No subtitles found') - else: - logger.debug("No more subtitles found") - break - - # number of pages with results - pages = int(math.ceil(result_count / float(items_per_page))) - - # get current page - if 'pg' in params: - current_page = int(params['pg']) - - try: - sublist = soup.select('section.titlovi > ul.titlovi > li.subtitleContainer.canEdit') - for sub in sublist: - # subtitle id - sid = sub.find(attrs={'data-id': True}).attrs['data-id'] - # get download link - download_link = self.download_url + sid - # title and alternate title - match = title_re.search(sub.a.string) - if match: - _title = match.group('title') - alt_title = match.group('altitle') - else: - continue - - # page link - page_link = self.server_url + sub.a.attrs['href'] - # subtitle language - _lang = sub.select_one('.lang') - match = lang_re.search(_lang.attrs.get('src', _lang.attrs.get('data-cfsrc', ''))) - if match: - try: - # decode language - lang = Language.fromtitlovi(match.group('lang')+match.group('script')) - except ValueError: - continue - - # relase year or series start year - match = year_re.search(sub.find(attrs={'data-id': True}).parent.i.string) - if match: - r_year = int(match.group('year')) - # fps - match = fps_re.search(sub.select_one('.fps').string) - if match: - fps = match.group('fps') - # releases - releases = str(sub.select_one('.fps').parent.contents[0].string) - - # handle movies and series separately - if is_episode: - # season and episode info - sxe = sub.select_one('.s0xe0y').string - r_season = None - r_episode = None - if sxe: - match = season_re.search(sxe) - if match: - r_season = int(match.group('season')) - match = episode_re.search(sxe) - if match: - r_episode = int(match.group('episode')) - - subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title, - alt_title=alt_title, season=r_season, episode=r_episode, - year=r_year, fps=fps, - asked_for_release_group=video.release_group, - asked_for_episode=episode) - else: - subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title, - alt_title=alt_title, year=r_year, fps=fps, - asked_for_release_group=video.release_group) - logger.debug('Found subtitle %r', subtitle) - - # prime our matches so we can use the values later - subtitle.get_matches(video) - - # add found subtitles - subtitles.append(subtitle) - - finally: - soup.decompose() - - # stop on last page - if current_page >= pages: - break - - # increment current page - params['pg'] = current_page + 1 - logger.debug('Getting page %d', params['pg']) + continue + + # handle movies and series separately + if is_episode: + subtitle = self.subtitle_class(Language.fromtitlovi(sub.get('Lang')), sub.get('Link'), sub.get('Id'), sub.get('Release'), _title, + alt_title=alt_title, season=sub.get('Season'), episode=sub.get('Episode'), + year=sub.get('Year'), rating=sub.get('Rating'), + download_count=sub.get('DownloadCount'), + asked_for_release_group=video.release_group, + asked_for_episode=episode) + else: + subtitle = self.subtitle_class(Language.fromtitlovi(sub.get('Lang')), sub.get('Link'), sub.get('Id'), sub.get('Release'), _title, + alt_title=alt_title, year=sub.get('Year'), rating=sub.get('Rating'), + download_count=sub.get('DownloadCount'), + asked_for_release_group=video.release_group) + logger.debug('Found subtitle %r', subtitle) + + # prime our matches so we can use the values later + subtitle.get_matches(video) + + # add found subtitles + subtitles.append(subtitle) return subtitles def list_subtitles(self, video, languages): season = episode = None + if isinstance(video, Episode): title = video.series season = video.season @@ -300,6 +284,7 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): return [s for s in self.query(languages, fix_inconsistent_naming(title), season=season, episode=episode, year=video.year, + imdb_id=video.imdb_id, video=video)] def download_subtitle(self, subtitle): @@ -337,10 +322,12 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin): sub_to_extract = None for sub_name in subs_in_archive: - if not ('.cyr' in sub_name or '.cir' in sub_name): + _sub_name = sub_name.lower() + + if not ('.cyr' in _sub_name or '.cir' in _sub_name or 'cyr)' in _sub_name): sr_lat_subs.append(sub_name) - if ('.cyr' in sub_name or '.cir' in sub_name) and not '.lat' in sub_name: + if ('.cyr' in sub_name or '.cir' in _sub_name) and not '.lat' in _sub_name.lower(): sr_cyr_subs.append(sub_name) if subtitle.language == 'sr': diff --git a/libs/subzero/language.py b/libs/subzero/language.py index 0a3a5e775..a13bab160 100644 --- a/libs/subzero/language.py +++ b/libs/subzero/language.py @@ -1,5 +1,6 @@ # coding=utf-8 import types +import re from babelfish.exceptions import LanguageError from babelfish import Language as Language_, basestr @@ -134,3 +135,16 @@ class Language(Language_): return Language(*Language_.fromietf(s).__getstate__()) return Language(*Language_.fromalpha3b(s).__getstate__()) + + +IETF_MATCH = ".+\.([^-.]+)(?:-[A-Za-z]+)?$" +ENDSWITH_LANGUAGECODE_RE = re.compile("\.([^-.]{2,3})(?:-[A-Za-z]{2,})?$") + + +def match_ietf_language(s, ietf=False): + language_match = re.match(".+\.([^\.]+)$" if not ietf + else IETF_MATCH, s) + if language_match and len(language_match.groups()) == 1: + language = language_match.groups()[0] + return language + return s diff --git a/libs/subzero/lib/dict.py b/libs/subzero/lib/dict.py index 3f327dcf4..929a9a642 100644 --- a/libs/subzero/lib/dict.py +++ b/libs/subzero/lib/dict.py @@ -107,6 +107,12 @@ class Dicked(object): for key, value in entries.iteritems(): self.__dict__[key] = (Dicked(**value) if isinstance(value, dict) else value) + def has(self, key): + return self._entries is not None and key in self._entries + + def get(self, key, default=None): + return self._entries.get(key, default) if self._entries else default + def __repr__(self): return str(self) diff --git a/libs/subzero/video.py b/libs/subzero/video.py index 13db33ddf..160e1afec 100644 --- a/libs/subzero/video.py +++ b/libs/subzero/video.py @@ -17,7 +17,8 @@ def has_external_subtitle(part_id, stored_subs, language): def set_existing_languages(video, video_info, external_subtitles=False, embedded_subtitles=False, known_embedded=None, - stored_subs=None, languages=None, only_one=False, known_metadata_subs=None): + stored_subs=None, languages=None, only_one=False, known_metadata_subs=None, + match_strictness="strict"): logger.debug(u"Determining existing subtitles for %s", video.name) external_langs_found = set() @@ -27,7 +28,8 @@ def set_existing_languages(video, video_info, external_subtitles=False, embedded external_langs_found = known_metadata_subs external_langs_found.update(set(search_external_subtitles(video.name, languages=languages, - only_one=only_one).values())) + only_one=only_one, + match_strictness=match_strictness).values())) # found external subtitles should be considered? if external_subtitles: From 5ad479f6a782ec3b02d6e7eecb030650522c4c17 Mon Sep 17 00:00:00 2001 From: Halali <moravlvk@gmail.com> Date: Sat, 5 Oct 2019 22:59:11 +0200 Subject: [PATCH 6/9] Remove anti-captcha requirement from titlovi provider --- views/providers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/providers.tpl b/views/providers.tpl index bea3abec8..c9d2bb8d9 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -645,7 +645,7 @@ <div class="middle aligned row"> <div class="right aligned four wide column"> - <label>Titlovi (require anti-captcha)</label> + <label>Titlovi</label> </div> <div class="one wide column"> <div id="titlovi" class="ui toggle checkbox provider"> From 6616ecd592179e5e176ee5398b2c42acae1fb617 Mon Sep 17 00:00:00 2001 From: Halali <moravlvk@gmail.com> Date: Sat, 5 Oct 2019 23:15:57 +0200 Subject: [PATCH 7/9] Add login and password for Titlovi API --- bazarr/config.py | 8 ++++++-- bazarr/get_providers.py | 5 ++++- bazarr/main.py | 4 ++++ views/providers.tpl | 21 ++++++++++++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index d5b606e8e..7f96bd7a2 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -66,7 +66,7 @@ defaults = { 'full_update_day': '6', 'full_update_hour': '4', 'only_monitored': 'False', -}, + }, 'radarr': { 'ip': '127.0.0.1', 'port': '7878', @@ -77,7 +77,7 @@ defaults = { 'full_update_day': '6', 'full_update_hour': '5', 'only_monitored': 'False', -}, + }, 'proxy': { 'type': 'None', 'url': '', @@ -131,6 +131,10 @@ defaults = { }, 'analytics': { 'enabled': 'True' + }, + 'titlovi': { + 'username': '', + 'password': '' } } diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index 258a1c1db..420f8a557 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -126,7 +126,10 @@ def get_providers_auth(): 'betaseries': {'token': settings.betaseries.token}, 'titulky': {'username': settings.titulky.username, 'password': settings.titulky.password, - } + }, + 'titlovi': {'username': settings.titlovi.username, + 'password': settings.titlovi.password, + }, } return providers_auth diff --git a/bazarr/main.py b/bazarr/main.py index 5f1e7efb1..afd3f7781 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -391,6 +391,8 @@ def save_wizard(): settings.napisy24.password = request.forms.get('settings_napisy24_password') settings.subscene.username = request.forms.get('settings_subscene_username') settings.subscene.password = request.forms.get('settings_subscene_password') + settings.titlovi.username = request.forms.get('settings_titlovi_username') + settings.titlovi.password = request.forms.get('settings_titlovi_password') settings.betaseries.token = request.forms.get('settings_betaseries_token') settings_subliminal_languages = request.forms.getall('settings_subliminal_languages') @@ -1819,6 +1821,8 @@ def save_settings(): settings.napisy24.password = request.forms.get('settings_napisy24_password') settings.subscene.username = request.forms.get('settings_subscene_username') settings.subscene.password = request.forms.get('settings_subscene_password') + settings.titlovi.username = request.forms.get('settings_titlovi_username') + settings.titlovi.password = request.forms.get('settings_titlovi_password') settings.betaseries.token = request.forms.get('settings_betaseries_token') settings_subliminal_languages = request.forms.getall('settings_subliminal_languages') diff --git a/views/providers.tpl b/views/providers.tpl index c9d2bb8d9..89d929217 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -655,7 +655,26 @@ </div> </div> <div id="titlovi_option" class="ui grid container"> - + <div class="middle aligned row"> + <div class="right aligned six wide column"> + <label>Username</label> + </div> + <div class="six wide column"> + <div class="ui fluid input"> + <input name="settings_titlovi_username" type="text" value="{{settings.titlovi.username if settings.titlovi.username != None else ''}}"> + </div> + </div> + </div> + <div class="middle aligned row"> + <div class="right aligned six wide column"> + <label>Password</label> + </div> + <div class="six wide column"> + <div class="ui fluid input"> + <input name="settings_titlovi_password" type="password" value="{{settings.titlovi.password if settings.titlovi.password != None else ''}}"> + </div> + </div> + </div> </div> <div class="middle aligned row"> From 63d503756687e9834024a96f549b01060749dfb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sat, 5 Oct 2019 22:39:58 -0400 Subject: [PATCH 8/9] Added episodefile and moviefile id to database for future use. --- bazarr/database.py | 13 +++++++++++++ bazarr/get_episodes.py | 9 ++++++--- bazarr/get_movies.py | 12 ++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/bazarr/database.py b/bazarr/database.py index 6d2353b7b..5b1cce45a 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -4,6 +4,7 @@ import atexit from get_args import args from peewee import * from playhouse.sqliteq import SqliteQueueDatabase +from playhouse.migrate import * from helper import path_replace, path_replace_movie, path_replace_reverse, path_replace_reverse_movie @@ -14,6 +15,8 @@ database = SqliteQueueDatabase( queue_max_size=256, # Max. # of pending writes that can accumulate. results_timeout=30.0) # Max. time to wait for query to be executed. +migrator = SqliteMigrator(database) + @database.func('path_substitution') def path_substitution(path): @@ -78,6 +81,11 @@ class TableEpisodes(BaseModel): subtitles = TextField(null=True) title = TextField(null=True) video_codec = TextField(null=True) + episode_file_id = IntegerField(null=True) + + migrate( + migrator.add_column('table_episodes', 'episode_file_id', episode_file_id), + ) class Meta: table_name = 'table_episodes' @@ -109,6 +117,11 @@ class TableMovies(BaseModel): tmdb_id = TextField(column_name='tmdbId', primary_key=True, null=False) video_codec = TextField(null=True) year = TextField(null=True) + movie_file_id = IntegerField(null=True) + + migrate( + migrator.add_column('table_movies', 'movie_file_id', movie_file_id), + ) class Meta: table_name = 'table_movies' diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index 09a1220b8..cc6e5369a 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -116,7 +116,8 @@ def sync_episodes(): 'format': format, 'resolution': resolution, 'video_codec': videoCodec, - 'audio_codec': audioCodec}) + 'audio_codec': audioCodec, + 'episode_file_id': episode['episodeFile']['id']}) else: episodes_to_add.append({'sonarr_series_id': episode['seriesId'], 'sonarr_episode_id': episode['id'], @@ -129,7 +130,8 @@ def sync_episodes(): 'format': format, 'resolution': resolution, 'video_codec': videoCodec, - 'audio_codec': audioCodec}) + 'audio_codec': audioCodec, + 'episode_file_id': episode['episodeFile']['id']}) # Update existing episodes in DB episode_in_db_list = [] @@ -145,7 +147,8 @@ def sync_episodes(): TableEpisodes.format, TableEpisodes.resolution, TableEpisodes.video_codec, - TableEpisodes.audio_codec + TableEpisodes.audio_codec, + TableEpisodes.episode_file_id ).dicts() for item in episodes_in_db: diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index 6a754ae32..5f545580b 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -156,7 +156,8 @@ def update_movies(): 'video_codec': unicode(videoCodec), 'audio_codec': unicode(audioCodec), 'overview': unicode(overview), - 'imdb_id': unicode(imdbId)}) + 'imdb_id': unicode(imdbId), + 'movie_file_id': movie['movieFile']['id']}) else: if movie_default_enabled is True: movies_to_add.append({'radarr_id': movie["id"], @@ -180,7 +181,8 @@ def update_movies(): 'video_codec': videoCodec, 'audio_codec': audioCodec, 'imdb_id': imdbId, - 'forced': movie_default_forced}) + 'forced': movie_default_forced, + 'movie_file_id': movie['movieFile']['id']}) else: movies_to_add.append({'radarr_id': movie["id"], 'title': movie["title"], @@ -199,7 +201,8 @@ def update_movies(): 'resolution': resolution, 'video_codec': videoCodec, 'audio_codec': audioCodec, - 'imdb_id': imdbId}) + 'imdb_id': imdbId, + 'movie_file_id': movie['movieFile']['id']}) else: logging.error( 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator + @@ -225,7 +228,8 @@ def update_movies(): TableMovies.resolution, TableMovies.video_codec, TableMovies.audio_codec, - TableMovies.imdb_id + TableMovies.imdb_id, + TableMovies.movie_file_id ).dicts() for item in movies_in_db: From d65601d9cbc3f60276882583ea253520e51f7324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sat, 5 Oct 2019 22:48:04 -0400 Subject: [PATCH 9/9] Fix for #605. --- libs/subliminal_patch/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index 9e96754dd..77bcd1397 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -866,6 +866,8 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non logger.debug(u"Saving %r to %r", subtitle, subtitle_path) content = subtitle.get_modified_content(format=format, debug=debug_mods) if content: + if os.path.exists(subtitle_path): + os.remove(subtitle_path) with open(subtitle_path, 'w') as f: f.write(content) subtitle.storage_path = subtitle_path