Merge branch 'development' into hermes

# Conflicts:
#	bazarr/main.py
#	views/providers.tpl
#	views/settings_general.tpl

README.md

@@ -67,7 +67,6 @@ If you need something that is not already part of Bazarr, feel free to create a
 * SubZ
 * Supersubtitles
 * Titlovi
-* Titrari.ro
 * TVSubtitles
 * XSubs
 * Zimuku

bazarr/get_providers.py

@@ -39,7 +39,7 @@ PROVIDER_THROTTLE_MAP = {
 }
 
 PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", "hosszupuska",
-                        "supersubtitles", "titlovi", "titrari", "argenteam", "assrt", "subscene"]
+                        "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
 
 throttle_count = {}
 

bazarr/get_subtitle.py

@@ -19,7 +19,8 @@ from subliminal_patch.core import SZAsyncProviderPool, download_best_subtitles,
     list_all_subtitles, get_subtitle_path
 from subliminal_patch.score import compute_score
 from subliminal.refiners.tvdb import series_re
-from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2
+from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2, \
+    alpha2_from_language, alpha3_from_language
 from config import settings
 from helper import path_replace, path_replace_movie, path_replace_reverse, \
     path_replace_reverse_movie, pp_replace, get_target_folder, force_unicode
@@ -97,7 +98,7 @@ def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_sco
     return min_score, max_score, set(scores)
 
 
-def download_subtitle(path, language, hi, forced, providers, providers_auth, sceneName, title, media_type,
+def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title, media_type,
                       forced_minimum_score=None, is_upgrade=False):
     # fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
     #  one query
@@ -207,6 +208,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
                             downloaded_language_code3 = subtitle.language.alpha3
                         downloaded_language = language_from_alpha3(downloaded_language_code3)
                         downloaded_language_code2 = alpha2_from_alpha3(downloaded_language_code3)
+                        audio_language_code2 = alpha2_from_language(audio_language)
+                        audio_language_code3 = alpha3_from_language(audio_language)
                         downloaded_path = subtitle.storage_path
                         is_forced_string = " forced" if subtitle.language.forced else ""
                         logging.debug('BAZARR Subtitles file saved to disk: ' + downloaded_path)
@@ -220,8 +223,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
                         
                         if use_postprocessing is True:
                             command = pp_replace(postprocessing_cmd, path, downloaded_path, downloaded_language,
-                                                 downloaded_language_code2, downloaded_language_code3,
-                                                 subtitle.language.forced)
+                                                 downloaded_language_code2, downloaded_language_code3, audio_language,
+                                                 audio_language_code2, audio_language_code3, subtitle.language.forced)
                             postprocessing(command, path)
                         
                         # fixme: support multiple languages at once
@@ -354,7 +357,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
     return final_subtitles
 
 
-def manual_download_subtitle(path, language, hi, forced, subtitle, provider, providers_auth, sceneName, title,
+def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName, title,
                              media_type):
     logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path)
     
@@ -411,6 +414,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
                             downloaded_language_code3 = subtitle.language.alpha3
                         downloaded_language = language_from_alpha3(downloaded_language_code3)
                         downloaded_language_code2 = alpha2_from_alpha3(downloaded_language_code3)
+                        audio_language_code2 = alpha2_from_language(audio_language)
+                        audio_language_code3 = alpha3_from_language(audio_language)
                         downloaded_path = saved_subtitle.storage_path
                         logging.debug('BAZARR Subtitles file saved to disk: ' + downloaded_path)
                         is_forced_string = " forced" if subtitle.language.forced else ""
@@ -419,8 +424,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
                         
                         if use_postprocessing is True:
                             command = pp_replace(postprocessing_cmd, path, downloaded_path, downloaded_language,
-                                                 downloaded_language_code2, downloaded_language_code3,
-                                                 subtitle.language.forced)
+                                                 downloaded_language_code2, downloaded_language_code3, audio_language,
+                                                 audio_language_code2, audio_language_code3, subtitle.language.forced)
                             postprocessing(command, path)
                         
                         if media_type == 'series':
@@ -500,7 +505,7 @@ def series_download_subtitles(no):
         logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no))
         return
 
-    series_details = database.execute("SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?",
+    series_details = database.execute("SELECT hearing_impaired, audio_language, title, forced FROM table_shows WHERE sonarrSeriesId=?",
                                       (no,), only_one=True)
     if not series_details:
         logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no))
@@ -519,6 +524,7 @@ def series_download_subtitles(no):
                                         length=count_episodes_details)
                     result = download_subtitle(path_replace(episode['path']),
                                                str(alpha3_from_alpha2(language.split(':')[0])),
+                                               series_details['audio_language'],
                                                series_details['hearing_impaired'],
                                                "True" if len(language.split(':')) > 1 else "False",
                                                providers_list,
@@ -555,7 +561,7 @@ def episode_download_subtitles(no):
     episodes_details = database.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, "
                                         "table_episodes.sonarrEpisodeId, table_episodes.scene_name, "
                                         "table_shows.hearing_impaired, table_shows.title, table_shows.sonarrSeriesId, "
-                                        "table_shows.forced FROM table_episodes LEFT JOIN table_shows on "
+                                        "table_shows.forced, table_shows.audio_language FROM table_episodes LEFT JOIN table_shows on "
                                         "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId "
                                         "WHERE sonarrEpisodeId=?" + episodes_details_clause, (no,))
 
@@ -575,6 +581,7 @@ def episode_download_subtitles(no):
                                             path_replace(episode['path']), queue='get_subtitle')
                     result = download_subtitle(path_replace(episode['path']),
                                                str(alpha3_from_alpha2(language.split(':')[0])),
+                                               episode['audio_language'],
                                                episode['hearing_impaired'],
                                                "True" if len(language.split(':')) > 1 else "False",
                                                providers_list,
@@ -604,7 +611,7 @@ def movies_download_subtitles(no):
     else:
         movie_details_clause = ''
 
-    movie = database.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title, forced "
+    movie = database.execute("SELECT path, missing_subtitles, audio_language, radarrId, sceneName, hearing_impaired, title, forced "
                              "FROM table_movies WHERE radarrId=?" + movie_details_clause, (no,), only_one=True)
 
     if not movie:
@@ -626,6 +633,7 @@ def movies_download_subtitles(no):
                                     length=count_movie)
                 result = download_subtitle(path_replace_movie(movie['path']),
                                            str(alpha3_from_alpha2(language.split(':')[0])),
+                                           movie['audio_language'],
                                            movie['hearing_impaired'],
                                            "True" if len(language.split(':')) > 1 else "False",
                                            providers_list,
@@ -656,7 +664,7 @@ def movies_download_subtitles(no):
 def wanted_download_subtitles(path, l, count_episodes):
     episodes_details = database.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, "
                                         "table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, "
-                                        "table_shows.hearing_impaired, table_episodes.scene_name,"
+                                        "table_shows.hearing_impaired, table_shows.audio_language, table_episodes.scene_name,"
                                         "table_episodes.failedAttempts, table_shows.title, table_shows.forced "
                                         "FROM table_episodes LEFT JOIN table_shows on "
                                         "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId "
@@ -689,6 +697,7 @@ def wanted_download_subtitles(path, l, count_episodes):
                                             length=count_episodes)
                         result = download_subtitle(path_replace(episode['path']),
                                                    str(alpha3_from_alpha2(language.split(':')[0])),
+                                                   episode['audio_language'],
                                                    episode['hearing_impaired'],
                                                    "True" if len(language.split(':')) > 1 else "False",
                                                    providers_list,
@@ -712,7 +721,7 @@ def wanted_download_subtitles(path, l, count_episodes):
 
 
 def wanted_download_subtitles_movie(path, l, count_movies):
-    movies_details = database.execute("SELECT path, missing_subtitles, radarrId, hearing_impaired, sceneName, "
+    movies_details = database.execute("SELECT path, missing_subtitles, radarrId, hearing_impaired, audio_language, sceneName, "
                                       "failedAttempts, title, forced FROM table_movies WHERE path = ? "
                                       "AND missing_subtitles != '[]'", (path_replace_reverse_movie(path),))
     
@@ -742,6 +751,7 @@ def wanted_download_subtitles_movie(path, l, count_movies):
                                             length=count_movies)
                         result = download_subtitle(path_replace_movie(movie['path']),
                                                    str(alpha3_from_alpha2(language.split(':')[0])),
+                                                   movie['audio_language'],
                                                    movie['hearing_impaired'],
                                                    "True" if len(language.split(':')) > 1 else "False",
                                                    providers_list,
@@ -944,7 +954,7 @@ def upgrade_subtitles():
 
     if settings.general.getboolean('use_sonarr'):
         upgradable_episodes = database.execute("SELECT table_history.video_path, table_history.language, "
-                                               "table_history.score, table_shows.hearing_impaired, "
+                                               "table_history.score, table_shows.hearing_impaired, table_shows.audio_language, "
                                                "table_episodes.scene_name, table_episodes.title,"
                                                "table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId,"
                                                "MAX(table_history.timestamp) as timestamp, "
@@ -978,7 +988,7 @@ def upgrade_subtitles():
 
     if settings.general.getboolean('use_radarr'):
         upgradable_movies = database.execute("SELECT table_history_movie.video_path, table_history_movie.language, "
-                                             "table_history_movie.score, table_movies.hearing_impaired, "
+                                             "table_history_movie.score, table_movies.hearing_impaired, table_movies.audio_language, "
                                              "table_movies.sceneName, table_movies.title, table_movies.radarrId, "
                                              "MAX(table_history_movie.timestamp) as timestamp, table_movies.languages, "
                                              "table_movies.forced FROM table_history_movie INNER JOIN "
@@ -1041,6 +1051,7 @@ def upgrade_subtitles():
                     
                     result = download_subtitle(path_replace(episode['video_path']),
                                                str(alpha3_from_alpha2(language)),
+                                               episode['audio_language'],
                                                episode['hearing_impaired'],
                                                is_forced,
                                                providers_list,
@@ -1092,6 +1103,7 @@ def upgrade_subtitles():
                     
                     result = download_subtitle(path_replace_movie(movie['video_path']),
                                                str(alpha3_from_alpha2(language)),
+                                               movie['audio_language'],
                                                movie['hearing_impaired'],
                                                is_forced,
                                                providers_list,

bazarr/helper.py

@@ -94,7 +94,7 @@ def path_replace_reverse_movie(path):
     return path
 
 
-def pp_replace(pp_command, episode, subtitles, language, language_code2, language_code3, forced):
+def pp_replace(pp_command, episode, subtitles, language, language_code2, language_code3, episode_language, episode_language_code2, episode_language_code3, forced):
     is_forced = ":forced" if forced else ""
     is_forced_string = " forced" if forced else ""
     pp_command = pp_command.replace('{{directory}}', os.path.dirname(episode))
@@ -104,6 +104,9 @@ def pp_replace(pp_command, episode, subtitles, language, language_code2, languag
     pp_command = pp_command.replace('{{subtitles_language}}', language + is_forced_string)
     pp_command = pp_command.replace('{{subtitles_language_code2}}', language_code2 + is_forced)
     pp_command = pp_command.replace('{{subtitles_language_code3}}', language_code3 + is_forced)
+    pp_command = pp_command.replace('{{episode_language}}', episode_language)
+    pp_command = pp_command.replace('{{episode_language_code2}}', episode_language_code2)
+    pp_command = pp_command.replace('{{episode_language_code3}}', episode_language_code3)
     return pp_command
 
 

bazarr/main.py

@@ -819,16 +819,15 @@ def movie_history(no):
 
 # Mute DeprecationWarning
 warnings.simplefilter("ignore", DeprecationWarning)
+# Mute Insecure HTTPS requests made to Sonarr and Radarr
+warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+# Mute Python3 BrokenPipeError
 warnings.simplefilter("ignore", BrokenPipeError)
-if args.dev:
-    server = app.run(
-        host=str(settings.general.ip), port=(int(args.port) if args.port else int(settings.general.port)))
-else:
-    server = CherryPyWSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app)
+
+server = CherryPyWSGIServer(host=str(settings.general.ip), port=int(args.port) if args.port else int(settings.general.port))), app)
 try:
     logging.info('BAZARR is started and waiting for request on http://' + str(settings.general.ip) + ':' + (str(
         args.port) if args.port else str(settings.general.port)) + str(base_url))
-    if not args.dev:
-        server.start()
+    server.start()
 except KeyboardInterrupt:
     doShutdown()

libs/pysubs2/exceptions.py

@@ -12,3 +12,6 @@ class UnknownFormatIdentifierError(Pysubs2Error):
 
 class FormatAutodetectionError(Pysubs2Error):
     """Subtitle format is ambiguous or unknown."""
+
+class ContentNotUsable(Pysubs2Error):
+    """Current content not usable for specified format"""

libs/pysubs2/ssastyle.py

@@ -41,6 +41,7 @@ class SSAStyle(object):
         self.italic = False #: Italic
         self.underline = False #: Underline (ASS only)
         self.strikeout = False #: Strikeout (ASS only)
+        self.drawing = False #: Drawing (ASS only, see http://docs.aegisub.org/3.1/ASS_Tags/#drawing-tags
         self.scalex = 100.0 #: Horizontal scaling (ASS only)
         self.scaley = 100.0 #: Vertical scaling (ASS only)
         self.spacing = 0.0 #: Letter spacing (ASS only)

libs/pysubs2/subrip.py

@@ -5,6 +5,7 @@ from .formatbase import FormatBase
 from .ssaevent import SSAEvent
 from .ssastyle import SSAStyle
 from .substation import parse_tags
+from .exceptions import ContentNotUsable
 from .time import ms_to_times, make_time, TIMESTAMP, timestamp_to_ms
 
 #: Largest timestamp allowed in SubRip, ie. 99:59:59,999.
@@ -81,6 +82,7 @@ class SubripFormat(FormatBase):
                 if sty.italic: fragment = "<i>%s</i>" % fragment
                 if sty.underline: fragment = "<u>%s</u>" % fragment
                 if sty.strikeout: fragment = "<s>%s</s>" % fragment
+                if sty.drawing: raise ContentNotUsable
                 body.append(fragment)
 
             return re.sub("\n+", "\n", "".join(body).strip())
@@ -90,7 +92,10 @@ class SubripFormat(FormatBase):
         for i, line in enumerate(visible_lines, 1):
             start = ms_to_timestamp(line.start)
             end = ms_to_timestamp(line.end)
-            text = prepare_text(line.text, subs.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            try:
+                text = prepare_text(line.text, subs.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            except ContentNotUsable:
+                continue
 
             print("%d" % i, file=fp) # Python 2.7 compat
             print(start, "-->", end, file=fp)

libs/pysubs2/substation.py

@@ -110,7 +110,7 @@ def parse_tags(text, style=SSAStyle.DEFAULT_STYLE, styles={}):
     
     def apply_overrides(all_overrides):
         s = style.copy()
-        for tag in re.findall(r"\\[ibus][10]|\\r[a-zA-Z_0-9 ]*", all_overrides):
+        for tag in re.findall(r"\\[ibusp][0-9]|\\r[a-zA-Z_0-9 ]*", all_overrides):
             if tag == r"\r":
                 s = style.copy() # reset to original line style
             elif tag.startswith(r"\r"):
@@ -122,6 +122,13 @@ def parse_tags(text, style=SSAStyle.DEFAULT_STYLE, styles={}):
                 elif "b" in tag: s.bold = "1" in tag
                 elif "u" in tag: s.underline = "1" in tag
                 elif "s" in tag: s.strikeout = "1" in tag
+                elif "p" in tag:
+                    try:
+                        scale = int(tag[2:])
+                    except (ValueError, IndexError):
+                        continue
+
+                    s.drawing = scale > 0
         return s
     
     overrides = SSAEvent.OVERRIDE_SEQUENCE.findall(text)

libs/subliminal_patch/providers/bsplayer.py

@@ -2,7 +2,6 @@
 from __future__ import absolute_import
 import logging
 import io
-import os
 
 from requests import Session
 from guessit import guessit
@@ -10,6 +9,7 @@ from subliminal_patch.providers import Provider
 from subliminal_patch.subtitle import Subtitle
 from subliminal.utils import sanitize_release_group
 from subliminal.subtitle import guess_matches
+from subliminal.video import Episode, Movie
 from subzero.language import Language
 
 import gzip
@@ -44,6 +44,52 @@ class BSPlayerSubtitle(Subtitle):
     def get_matches(self, video):
         matches = set()
         matches |= guess_matches(video, guessit(self.filename))
+
+        subtitle_filename = self.filename
+
+        # episode
+        if isinstance(video, Episode):
+            # already matched in search query
+            matches.update(['title', 'series', 'season', 'episode', 'year'])
+
+        # movie
+        elif isinstance(video, Movie):
+            # already matched in search query
+            matches.update(['title', 'year'])
+
+        # release_group
+        if video.release_group and video.release_group.lower() in subtitle_filename:
+            matches.add('release_group')
+
+        # resolution
+        if video.resolution and video.resolution.lower() in subtitle_filename:
+            matches.add('resolution')
+
+        # format
+        formats = []
+        if video.format:
+            formats = [video.format.lower()]
+            if formats[0] == "web-dl":
+                formats.append("webdl")
+                formats.append("webrip")
+                formats.append("web ")
+            for frmt in formats:
+                if frmt.lower() in subtitle_filename:
+                    matches.add('format')
+                    break
+
+        # video_codec
+        if video.video_codec:
+            video_codecs = [video.video_codec.lower()]
+            if video_codecs[0] == "h264":
+                formats.append("x264")
+            elif video_codecs[0] == "h265":
+                formats.append("x265")
+            for vc in formats:
+                if vc.lower() in subtitle_filename:
+                    matches.add('video_codec')
+                    break
+
         matches.add('hash')
 
         return matches

libs/subliminal_patch/providers/legendasdivx.py

@@ -15,6 +15,7 @@ from subliminal_patch.subtitle import Subtitle
 from subliminal.video import Episode, Movie
 from subliminal.subtitle import SUBTITLE_EXTENSIONS, fix_line_ending,guess_matches
 from subzero.language import Language
+from subliminal_patch.score import get_scores
 
 logger = logging.getLogger(__name__)
 
@@ -104,7 +105,8 @@ class LegendasdivxSubtitle(Subtitle):
                     matches.update(['video_codec'])
                     break
 
-        matches |= guess_matches(video, guessit(self.description))
+        # running guessit on a huge description may break guessit
+        # matches |= guess_matches(video, guessit(self.description))
         return matches
 
 
@@ -274,7 +276,7 @@ class LegendasdivxProvider(Provider):
 
             archive = self._get_archive(res.content)
             # extract the subtitle
-            subtitle_content = self._get_subtitle_from_archive(archive)
+            subtitle_content = self._get_subtitle_from_archive(archive, subtitle)
             subtitle.content = fix_line_ending(subtitle_content)
             subtitle.normalize()
 
@@ -297,11 +299,13 @@ class LegendasdivxProvider(Provider):
 
         return archive
 
-    def _get_subtitle_from_archive(self, archive):
+    def _get_subtitle_from_archive(self, archive, subtitle):
         # some files have a non subtitle with .txt extension
         _tmp = list(SUBTITLE_EXTENSIONS)
         _tmp.remove('.txt')
         _subtitle_extensions = tuple(_tmp)
+        _max_score = 0
+        _scores = get_scores (subtitle.video)
 
         for name in archive.namelist():
             # discard hidden files
@@ -312,7 +316,26 @@ class LegendasdivxProvider(Provider):
             if not name.lower().endswith(_subtitle_extensions):
                 continue
 
-            logger.debug("returning from archive: %s" % name)
-            return archive.read(name)
+            _guess = guessit (name)
+            if isinstance(subtitle.video, Episode):
+                logger.debug ("guessing %s" % name)
+                logger.debug("subtitle S{}E{} video S{}E{}".format(_guess['season'],_guess['episode'],subtitle.video.season,subtitle.video.episode))
+
+                if subtitle.video.episode != _guess['episode'] or subtitle.video.season != _guess['season']:
+                    logger.debug('subtitle does not match video, skipping')
+                    continue
+
+            matches = set()
+            matches |= guess_matches (subtitle.video, _guess)
+            logger.debug('srt matches: %s' % matches)
+            _score = sum ((_scores.get (match, 0) for match in matches))
+            if _score > _max_score:
+                _max_name = name
+                _max_score = _score
+                logger.debug("new max: {} {}".format(name, _score))
+
+        if _max_score > 0:
+            logger.debug("returning from archive: {} scored {}".format(_max_name, _max_score))
+            return archive.read(_max_name)
 
         raise ParseResponseError('Can not find the subtitle in the compressed file')

libs/subliminal_patch/subtitle.py

@@ -279,6 +279,12 @@ class Subtitle(Subtitle_):
 
     @classmethod
     def pysubs2_to_unicode(cls, sub, format="srt"):
+        """
+        this is a modified version of pysubs2.SubripFormat.to_file with special handling for drawing tags in ASS
+        :param sub:
+        :param format:
+        :return:
+        """
         def ms_to_timestamp(ms, mssep=","):
             """Convert ms to 'HH:MM:SS,mmm'"""
             # XXX throw on overflow/underflow?
@@ -293,6 +299,9 @@ class Subtitle(Subtitle_):
                 fragment = fragment.replace(r"\h", u" ")
                 fragment = fragment.replace(r"\n", u"\n")
                 fragment = fragment.replace(r"\N", u"\n")
+                if sty.drawing:
+                    raise pysubs2.ContentNotUsable
+
                 if format == "srt":
                     if sty.italic:
                         fragment = u"<i>%s</i>" % fragment
@@ -324,7 +333,10 @@ class Subtitle(Subtitle_):
         for i, line in enumerate(visible_lines, 1):
             start = ms_to_timestamp(line.start, mssep=mssep)
             end = ms_to_timestamp(line.end, mssep=mssep)
-            text = prepare_text(line.text, sub.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            try:
+                text = prepare_text(line.text, sub.styles.get(line.style, SSAStyle.DEFAULT_STYLE))
+            except pysubs2.ContentNotUsable:
+                continue
 
             out.append(u"%d\n" % i)
             out.append(u"%s --> %s\n" % (start, end))

libs/subzero/language.py

@@ -30,7 +30,6 @@ repl_map = {
     "rum": "ro",
     "slo": "sk",
     "tib": "bo",
-    "ron": "ro",
 }
 
 ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)))) + list(repl_map.values())