From 6c4c124ae434a812f5291858297edb20cb3b7adf Mon Sep 17 00:00:00 2001 From: panni Date: Thu, 4 Apr 2019 17:01:37 +0200 Subject: [PATCH] core: update to subliminal_patch:head; fix subscene; add alternative titles support to subscene and opensubtitles --- libs/subliminal_patch/core.py | 10 +++ libs/subliminal_patch/http.py | 14 +++- .../providers/opensubtitles.py | 29 ++++++-- libs/subliminal_patch/providers/subscene.py | 71 +++++++++++++------ libs/subliminal_patch/subtitle.py | 8 ++- 5 files changed, 97 insertions(+), 35 deletions(-) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index 5dda9fb3c..df38b4e09 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -518,10 +518,20 @@ def scan_video(path, dont_use_actual_file=False, hints=None, providers=None, ski hints["expected_title"] = [hints["title"]] guessed_result = guessit(guess_from, options=hints) + logger.debug('GuessIt found: %s', json.dumps(guessed_result, cls=GuessitEncoder, indent=4, ensure_ascii=False)) video = Video.fromguess(path, guessed_result) video.hints = hints + # get possibly alternative title from the filename itself + alt_guess = guessit(filename, options=hints) + if "title" in alt_guess and alt_guess["title"] != guessed_result["title"]: + if video_type == "episode": + video.alternative_series.append(alt_guess["title"]) + else: + video.alternative_titles.append(alt_guess["title"]) + logger.debug("Adding alternative title: %s", alt_guess["title"]) + if dont_use_actual_file: return video diff --git a/libs/subliminal_patch/http.py b/libs/subliminal_patch/http.py index d6fddb358..d859cd31d 100644 --- a/libs/subliminal_patch/http.py +++ b/libs/subliminal_patch/http.py @@ -12,6 +12,7 @@ from requests import Session, exceptions from urllib3.util import connection from retry.api import retry_call from exceptions import APIThrottled +from cfscrape import CloudflareScraper from subzero.lib.io import get_viable_encoding @@ -30,12 +31,19 @@ custom_resolver = dns.resolver.Resolver(configure=False) custom_resolver.nameservers = ['8.8.8.8', '1.1.1.1'] -class CertifiSession(Session): +class CertifiSession(CloudflareScraper): timeout = 10 def __init__(self): super(CertifiSession, self).__init__() self.verify = pem_file + self.headers.update({ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache', + 'DNT': '1' + }) def request(self, *args, **kwargs): if kwargs.get('timeout') is None: @@ -47,7 +55,7 @@ class RetryingSession(CertifiSession): proxied_functions = ("get", "post") def __init__(self): - super(CertifiSession, self).__init__() + super(RetryingSession, self).__init__() self.verify = pem_file proxy = os.environ.get('SZ_HTTP_PROXY') @@ -62,7 +70,7 @@ class RetryingSession(CertifiSession): # fixme: may be a little loud logger.debug("Using proxy %s for: %s", self.proxies["http"], args[0]) - return retry_call(getattr(super(CertifiSession, self), method), fargs=args, fkwargs=kwargs, tries=3, delay=5, + return retry_call(getattr(super(RetryingSession, self), method), fargs=args, fkwargs=kwargs, tries=3, delay=5, exceptions=(exceptions.ConnectionError, exceptions.ProxyError, exceptions.SSLError, diff --git a/libs/subliminal_patch/providers/opensubtitles.py b/libs/subliminal_patch/providers/opensubtitles.py index 032b89058..4ce3aacea 100644 --- a/libs/subliminal_patch/providers/opensubtitles.py +++ b/libs/subliminal_patch/providers/opensubtitles.py @@ -11,8 +11,8 @@ from babelfish import language_converters from dogpile.cache.api import NO_VALUE from subliminal.exceptions import ConfigurationError, ServiceUnavailable from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSubtitlesProvider,\ - OpenSubtitlesSubtitle as _OpenSubtitlesSubtitle, Episode, ServerProxy, Unauthorized, NoSession, \ - DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError + OpenSubtitlesSubtitle as _OpenSubtitlesSubtitle, Episode, Movie, ServerProxy, Unauthorized, NoSession, \ + DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError, sanitize from mixins import ProviderRetryMixin from subliminal.subtitle import fix_line_ending from subliminal_patch.http import SubZeroRequestsTransport @@ -45,6 +45,19 @@ class OpenSubtitlesSubtitle(_OpenSubtitlesSubtitle): def get_matches(self, video, hearing_impaired=False): matches = super(OpenSubtitlesSubtitle, self).get_matches(video) + # episode + if isinstance(video, Episode) and self.movie_kind == 'episode': + # series + if video.series and (sanitize(self.series_name) in ( + sanitize(name) for name in [video.series] + video.alternative_series)): + matches.add('series') + # movie + elif isinstance(video, Movie) and self.movie_kind == 'movie': + # title + if video.title and (sanitize(self.movie_name) in ( + sanitize(name) for name in [video.title] + video.alternative_titles)): + matches.add('title') + sub_fps = None try: sub_fps = float(self.fps) @@ -205,19 +218,19 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): season = episode = None if isinstance(video, Episode): - query = video.series + query = [video.series] + video.alternative_series season = video.season episode = episode = min(video.episode) if isinstance(video.episode, list) else video.episode if video.is_special: season = None episode = None - query = u"%s %s" % (video.series, video.title) + query = [u"%s %s" % (series, video.title) for series in [video.series] + video.alternative_series] logger.info("%s: Searching for special: %r", self.__class__, query) # elif ('opensubtitles' not in video.hashes or not video.size) and not video.imdb_id: # query = video.name.split(os.sep)[-1] else: - query = video.title + query = [video.title] + video.alternative_titles return self.query(languages, hash=video.hashes.get('opensubtitles'), size=video.size, imdb_id=video.imdb_id, query=query, season=season, episode=episode, tag=video.original_name, @@ -238,9 +251,11 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider): else: criteria.append({'imdbid': imdb_id[2:]}) if query and season and episode: - criteria.append({'query': query.replace('\'', ''), 'season': season, 'episode': episode}) + for q in query: + criteria.append({'query': q.replace('\'', ''), 'season': season, 'episode': episode}) elif query: - criteria.append({'query': query.replace('\'', '')}) + for q in query: + criteria.append({'query': q.replace('\'', '')}) if not criteria: raise ValueError('Not enough information') diff --git a/libs/subliminal_patch/providers/subscene.py b/libs/subliminal_patch/providers/subscene.py index 38a97c579..7a17e1365 100644 --- a/libs/subliminal_patch/providers/subscene.py +++ b/libs/subliminal_patch/providers/subscene.py @@ -5,6 +5,7 @@ import logging import os import time import inflect +import cfscrape from random import randint from zipfile import ZipFile @@ -12,7 +13,9 @@ from zipfile import ZipFile from babelfish import language_converters from guessit import guessit from requests import Session +from dogpile.cache.api import NO_VALUE from subliminal import Episode, ProviderError +from subliminal.cache import region from subliminal.utils import sanitize_release_group from subliminal_patch.providers import Provider from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin @@ -125,6 +128,7 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin): self.session = Session() from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] + self.session.headers['Referer'] = "https://subscene.com" def terminate(self): logger.info("Closing session") @@ -197,44 +201,65 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin): vfn = get_video_filename(video) subtitles = [] logger.debug(u"Searching for: %s", vfn) + + cf_data = region.get("cf_data") + if cf_data is not NO_VALUE: + cf_cookies, user_agent = cf_data + logger.debug("Trying to use old cf cookies") + self.session.cookies.update(cf_cookies) + self.session.headers['User-Agent'] = user_agent + film = search(vfn, session=self.session) + + try: + cf_data = self.session.get_live_tokens("subscene.com") + except: + pass + else: + logger.debug("Storing cf cookies") + region.set("cf_data", cf_data) + if film and film.subtitles: logger.debug('Release results found: %s', len(film.subtitles)) subtitles = self.parse_results(video, film) else: logger.debug('No release results found') + time.sleep(self.search_throttle) + # re-search for episodes without explicit release name if isinstance(video, Episode): #term = u"%s S%02iE%02i" % (video.series, video.season, video.episode) - term = u"%s - %s Season" % (video.series, p.number_to_words("%sth" % video.season).capitalize()) - time.sleep(self.search_throttle) - logger.debug('Searching for alternative results: %s', term) - film = search(term, session=self.session, release=False) - if film and film.subtitles: - logger.debug('Alternative results found: %s', len(film.subtitles)) - subtitles += self.parse_results(video, film) - else: - logger.debug('No alternative results found') - - # packs - if video.season_fully_aired: - term = u"%s S%02i" % (video.series, video.season) - logger.debug('Searching for packs: %s', term) + for series in [video.series] + video.alternative_series: + term = u"%s - %s Season" % (series, p.number_to_words("%sth" % video.season).capitalize()) time.sleep(self.search_throttle) - film = search(term, session=self.session) + logger.debug('Searching for alternative results: %s', term) + film = search(term, session=self.session, release=False) if film and film.subtitles: - logger.debug('Pack results found: %s', len(film.subtitles)) + logger.debug('Alternative results found: %s', len(film.subtitles)) subtitles += self.parse_results(video, film) else: - logger.debug('No pack results found') - else: - logger.debug("Not searching for packs, because the season hasn't fully aired") + logger.debug('No alternative results found') + + # packs + if video.season_fully_aired: + term = u"%s S%02i" % (series, video.season) + logger.debug('Searching for packs: %s', term) + time.sleep(self.search_throttle) + film = search(term, session=self.session) + if film and film.subtitles: + logger.debug('Pack results found: %s', len(film.subtitles)) + subtitles += self.parse_results(video, film) + else: + logger.debug('No pack results found') + else: + logger.debug("Not searching for packs, because the season hasn't fully aired") else: - logger.debug('Searching for movie results: %s', video.title) - film = search(video.title, year=video.year, session=self.session, limit_to=None, release=False) - if film and film.subtitles: - subtitles += self.parse_results(video, film) + for title in [video.title] + video.alternative_titles: + logger.debug('Searching for movie results: %s', title) + film = search(title, year=video.year, session=self.session, limit_to=None, release=False) + if film and film.subtitles: + subtitles += self.parse_results(video, film) logger.info("%s subtitles found" % len(subtitles)) return subtitles diff --git a/libs/subliminal_patch/subtitle.py b/libs/subliminal_patch/subtitle.py index 9a165fe4b..69a3c1e5b 100644 --- a/libs/subliminal_patch/subtitle.py +++ b/libs/subliminal_patch/subtitle.py @@ -38,6 +38,8 @@ class Subtitle(Subtitle_): plex_media_fps = None skip_wrong_fps = False wrong_fps = False + wrong_series = False + wrong_season_ep = False is_pack = False asked_for_release_group = None asked_for_episode = None @@ -356,7 +358,8 @@ def guess_matches(video, guess, partial=False): matches = set() if isinstance(video, Episode): # series - if video.series and 'title' in guess and sanitize(guess['title']) == sanitize(video.series): + if video.series and 'title' in guess and sanitize(guess['title']) in ( + sanitize(name) for name in [video.series] + video.alternative_series): matches.add('series') # title if video.title and 'episode_title' in guess and sanitize(guess['episode_title']) == sanitize(video.title): @@ -384,7 +387,8 @@ def guess_matches(video, guess, partial=False): if video.year and 'year' in guess and guess['year'] == video.year: matches.add('year') # title - if video.title and 'title' in guess and sanitize(guess['title']) == sanitize(video.title): + if video.title and 'title' in guess and sanitize(guess['title']) in ( + sanitize(name) for name in [video.title] + video.alternative_titles): matches.add('title') # release_group