From 26b48b175999620ea572d05416eecf32d078d509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 18 Dec 2019 06:35:54 -0500 Subject: [PATCH 01/14] Fix for #707. --- libs/pyprobe/pyprobe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/pyprobe/pyprobe.py b/libs/pyprobe/pyprobe.py index 59a6e5001..d26bb5b27 100644 --- a/libs/pyprobe/pyprobe.py +++ b/libs/pyprobe/pyprobe.py @@ -179,7 +179,7 @@ class VideoFileParser: if PY3: command = [parser] + commandArgs + [inputFile] completedProcess = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True ) if completedProcess.returncode: raise IOError( From 3b360db1c0e5fdfb25cd867bdd98d24b8be741cc Mon Sep 17 00:00:00 2001 From: Mexx62 Date: Sat, 21 Dec 2019 22:56:45 +0100 Subject: [PATCH 02/14] Betaseries: Use release group to choose which subtitle to download in archive --- libs/subliminal_patch/providers/betaseries.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/libs/subliminal_patch/providers/betaseries.py b/libs/subliminal_patch/providers/betaseries.py index b44de5329..bbbf90760 100644 --- a/libs/subliminal_patch/providers/betaseries.py +++ b/libs/subliminal_patch/providers/betaseries.py @@ -75,6 +75,7 @@ class BetaSeriesProvider(Provider): def query(self, languages, video): # query the server result = None + self.video = video matches = set() if video.tvdb_id: params = {'key': self.token, @@ -137,8 +138,13 @@ class BetaSeriesProvider(Provider): r.raise_for_status() archive = _get_archive(r.content) - subtitle_content = _get_subtitle_from_archive( - archive) if archive else r.content + if archive: + subtitle_names = _get_subtitle_names_from_archive(archive) + subtitle_to_download = _choose_subtitle_with_release_group(subtitle_names, self.video.release_group) + logger.debug('Subtitle to download: ' + subtitle_to_download) + subtitle_content = archive.read(subtitle_to_download) + else: + subtitle_content = r.content if subtitle_content: subtitle.content = fix_line_ending(subtitle_content) @@ -160,7 +166,8 @@ def _get_archive(content): return archive -def _get_subtitle_from_archive(archive): +def _get_subtitle_names_from_archive(archive): + subtitlesToConsider = [] for name in archive.namelist(): # discard hidden files if os.path.split(name)[-1].startswith('.'): @@ -170,9 +177,13 @@ def _get_subtitle_from_archive(archive): if not name.lower().endswith(SUBTITLE_EXTENSIONS): continue - return archive.read(name) + subtitlesToConsider.append(name) - return None + if len(subtitlesToConsider)>0: + logger.debug('Subtitles in archive: ' + ' '.join(subtitlesToConsider)) + return subtitlesToConsider + else: + return None def _translateLanguageCodeToLanguage(languageCode): @@ -180,3 +191,11 @@ def _translateLanguageCodeToLanguage(languageCode): return Language.fromalpha2('en') elif languageCode.lower() == 'vf': return Language.fromalpha2('fr') + + +def _choose_subtitle_with_release_group(subtitle_names, release_group): + if release_group: + for subtitle in subtitle_names: + if release_group in subtitle: + return subtitle + return subtitle_names[0] From 133787cf231f1017e777fba0a19c8787f2720c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 23 Dec 2019 12:26:49 -0500 Subject: [PATCH 03/14] Fix for #709. --- libs/scandir.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/scandir.py b/libs/scandir.py index 403f52d7b..2177e7487 100644 --- a/libs/scandir.py +++ b/libs/scandir.py @@ -462,6 +462,15 @@ elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.plat @property def path(self): if self._path is None: + # Make sure both attributes are string + try: + self._scandir_path = self._scandir_path.decode(encoding="utf-8") + except (UnicodeDecodeError, AttributeError): + pass + try: + self.name = self.name.decode(encoding="utf-8") + except (UnicodeDecodeError, AttributeError): + pass self._path = join(self._scandir_path, self.name) return self._path From d4fdee1e9f81d0a3d0e07a0e6a9004953fb62cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 23 Dec 2019 14:03:31 -0500 Subject: [PATCH 04/14] Fix for #711. --- bazarr/config.py | 6 +++++- libs/simpleconfigparser/__init__.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index deea48731..ca61871e0 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -5,6 +5,7 @@ import os from simpleconfigparser import simpleconfigparser from get_args import args +from six import PY3 defaults = { 'general': { @@ -139,7 +140,10 @@ defaults = { } } -settings = simpleconfigparser(defaults=defaults) +if PY3: + settings = simpleconfigparser(defaults=defaults, interpolation=None) +else: + settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) base_url = settings.general.base_url diff --git a/libs/simpleconfigparser/__init__.py b/libs/simpleconfigparser/__init__.py index 9cce999b9..c84ccbacc 100644 --- a/libs/simpleconfigparser/__init__.py +++ b/libs/simpleconfigparser/__init__.py @@ -26,7 +26,7 @@ THE SOFTWARE. try: from configparser import ConfigParser as configparser, NoOptionError, NoSectionError except ImportError: - from ConfigParser import SafeConfigParser as configparser, NoOptionError, NoSectionError + from ConfigParser import RawConfigParser as configparser, NoOptionError, NoSectionError class simpleconfigparser(configparser): @@ -126,6 +126,6 @@ class simpleconfigparser(configparser): def get(self, section, option, raw=False, vars=None, fallback=None): try: # Strip out quotes from the edges - return configparser.get(self, section, option, raw=raw, vars=vars).strip('"\'') + return configparser.get(self, section, option).strip('"\'') except NoOptionError: return None From 5dc8facae7dfe5b50d8f35c92750f4c07e5d6e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 23 Dec 2019 16:42:06 -0500 Subject: [PATCH 05/14] Another fix for #709. --- libs/scandir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/scandir.py b/libs/scandir.py index 2177e7487..3d0bdf140 100644 --- a/libs/scandir.py +++ b/libs/scandir.py @@ -464,11 +464,11 @@ elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.plat if self._path is None: # Make sure both attributes are string try: - self._scandir_path = self._scandir_path.decode(encoding="utf-8") + self._scandir_path = self._scandir_path.decode('unicode_escape') except (UnicodeDecodeError, AttributeError): pass try: - self.name = self.name.decode(encoding="utf-8") + self.name = self.name.decode('unicode_escape') except (UnicodeDecodeError, AttributeError): pass self._path = join(self._scandir_path, self.name) From 64fa210688628e21c4c5fbc3b127d25cbe283b4c Mon Sep 17 00:00:00 2001 From: panni Date: Tue, 24 Dec 2019 00:25:02 +0100 Subject: [PATCH 06/14] core: update to subliminal_patch:head; core: try and fix #703 --- libs/subliminal_patch/core.py | 5 +++-- libs/subzero/language.py | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index fcd7ce3be..d4685c289 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -31,7 +31,7 @@ from subliminal.core import guessit, ProviderPool, io, is_windows_special_path, ThreadPoolExecutor, check_video from subliminal_patch.exceptions import TooManyRequests, APIThrottled -from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE +from subzero.language import Language, ENDSWITH_LANGUAGECODE_RE, FULL_LANGUAGE_LIST from scandir import scandir, scandir_generic as _scandir_generic import six @@ -610,7 +610,8 @@ def _search_external_subtitles(path, languages=None, only_one=False, scandir_gen forced = "forced" in adv_tag # remove possible language code for matching - p_root_bare = ENDSWITH_LANGUAGECODE_RE.sub("", p_root) + p_root_bare = ENDSWITH_LANGUAGECODE_RE.sub( + lambda m: "" if str(m.group(1)).lower() in FULL_LANGUAGE_LIST else m.group(0), p_root) p_root_lower = p_root_bare.lower() diff --git a/libs/subzero/language.py b/libs/subzero/language.py index d212a02f4..38cfb8113 100644 --- a/libs/subzero/language.py +++ b/libs/subzero/language.py @@ -4,7 +4,7 @@ import types import re from babelfish.exceptions import LanguageError -from babelfish import Language as Language_, basestr +from babelfish import Language as Language_, basestr, LANGUAGE_MATRIX from six.moves import zip repl_map = { @@ -33,6 +33,12 @@ repl_map = { } +ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)) + repl_map.values())) +ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)) + + filter(lambda x: len(x) == 3, repl_map.keys()))) +FULL_LANGUAGE_LIST = ALPHA2_LIST + ALPHA3b_LIST + + def language_from_stream(l): if not l: raise LanguageError() From a612335a44531a487862e4aa5e42bcab6a5cf1f8 Mon Sep 17 00:00:00 2001 From: panni Date: Tue, 24 Dec 2019 00:46:34 +0100 Subject: [PATCH 07/14] core: update to subliminal_patch:head; core: try and fix #660 --- libs/subliminal_patch/subtitle.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libs/subliminal_patch/subtitle.py b/libs/subliminal_patch/subtitle.py index 1b3ce002a..8116697bf 100644 --- a/libs/subliminal_patch/subtitle.py +++ b/libs/subliminal_patch/subtitle.py @@ -20,6 +20,15 @@ from subliminal import Subtitle as Subtitle_ from subliminal.subtitle import Episode, Movie, sanitize_release_group, get_equivalent_release_groups from subliminal_patch.utils import sanitize from ftfy import fix_text +from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE + +BOMS = ( + (BOM_UTF8, "UTF-8"), + (BOM_UTF32_BE, "UTF-32-BE"), + (BOM_UTF32_LE, "UTF-32-LE"), + (BOM_UTF16_BE, "UTF-16-BE"), + (BOM_UTF16_LE, "UTF-16-LE"), +) logger = logging.getLogger(__name__) @@ -106,6 +115,9 @@ class Subtitle(Subtitle_): # normalize line endings self.content = self.content.replace(b"\r\n", b"\n").replace(b'\r', b'\n') + def _check_bom(self, data): + return [encoding for bom, encoding in BOMS if data.startswith(bom)] + def guess_encoding(self): """Guess encoding using the language, falling back on chardet. @@ -120,6 +132,11 @@ class Subtitle(Subtitle_): encodings = ['utf-8'] + # check UTF BOMs + bom_encodings = self._check_bom(self.content) + if bom_encodings: + encodings = list(set(enc.lower() for enc in bom_encodings + encodings)) + # add language-specific encodings # http://scratchpad.wikia.com/wiki/Character_Encoding_Recommendation_for_Languages From d0ccd08aef5b30cf6870af323f81ece9170d76e1 Mon Sep 17 00:00:00 2001 From: panni Date: Tue, 24 Dec 2019 02:33:22 +0100 Subject: [PATCH 08/14] core: fix values casting --- libs/subzero/language.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/subzero/language.py b/libs/subzero/language.py index 38cfb8113..34c6c882a 100644 --- a/libs/subzero/language.py +++ b/libs/subzero/language.py @@ -33,9 +33,9 @@ repl_map = { } -ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)) + repl_map.values())) +ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)) + list(repl_map.values()))) ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)) + - filter(lambda x: len(x) == 3, repl_map.keys()))) + filter(lambda x: len(x) == 3, list(repl_map.keys())))) FULL_LANGUAGE_LIST = ALPHA2_LIST + ALPHA3b_LIST From e4cc062dd377c5e8ee9dc2e7f528cfe37ae2c2ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 23 Dec 2019 21:01:18 -0500 Subject: [PATCH 09/14] Another fix for #703. --- libs/subzero/language.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/subzero/language.py b/libs/subzero/language.py index 34c6c882a..12d41d26e 100644 --- a/libs/subzero/language.py +++ b/libs/subzero/language.py @@ -32,10 +32,9 @@ repl_map = { "tib": "bo", } - -ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)) + list(repl_map.values()))) -ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)) + - filter(lambda x: len(x) == 3, list(repl_map.keys())))) +ALPHA2_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha2, LANGUAGE_MATRIX)))) + list(repl_map.values()) +ALPHA3b_LIST = list(set(filter(lambda x: x, map(lambda x: x.alpha3, LANGUAGE_MATRIX)))) + \ + list(set(filter(lambda x: len(x) == 3, list(repl_map.keys())))) FULL_LANGUAGE_LIST = ALPHA2_LIST + ALPHA3b_LIST From ca59bca40c7965b94298efe185b1e1cb35a45aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 24 Dec 2019 09:00:42 -0500 Subject: [PATCH 10/14] Added more logging of error for #705. --- bazarr/list_subtitles.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index da206c769..5615d4fe6 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -190,6 +190,9 @@ def list_missing_subtitles(no=None, epno=None): "FROM table_episodes LEFT JOIN table_shows " "on table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId" + episodes_subtitles_clause) + if isinstance(episodes_subtitles, six.string_types): + logging.debug("BAZARR list missing subtitles query to DB returned this instead of rows: " + episodes_subtitles) + return missing_subtitles_global = [] use_embedded_subs = settings.general.getboolean('use_embedded_subs') @@ -244,6 +247,9 @@ def list_missing_subtitles_movies(no=None): movies_subtitles = database.execute("SELECT radarrId, subtitles, languages, forced FROM table_movies" + movies_subtitles_clause) + if isinstance(movies_subtitles, six.string_types): + logging.error("BAZARR list missing subtitles query to DB returned this instead of rows: " + movies_subtitles) + return missing_subtitles_global = [] use_embedded_subs = settings.general.getboolean('use_embedded_subs') From cca888fa940424bb7c2e0fc335a165a8dcf7aa9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 24 Dec 2019 09:02:08 -0500 Subject: [PATCH 11/14] Added more logging of error for #705. --- bazarr/list_subtitles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index 5615d4fe6..abf47920b 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -191,7 +191,7 @@ def list_missing_subtitles(no=None, epno=None): "on table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId" + episodes_subtitles_clause) if isinstance(episodes_subtitles, six.string_types): - logging.debug("BAZARR list missing subtitles query to DB returned this instead of rows: " + episodes_subtitles) + logging.error("BAZARR list missing subtitles query to DB returned this instead of rows: " + episodes_subtitles) return missing_subtitles_global = [] From 67ad65ed76a1e0be632696232f5f97dd6999e944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 25 Dec 2019 09:06:05 -0500 Subject: [PATCH 12/14] Fix for #716. --- libs/pyprobe/pyprobe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/pyprobe/pyprobe.py b/libs/pyprobe/pyprobe.py index d26bb5b27..b280ce551 100644 --- a/libs/pyprobe/pyprobe.py +++ b/libs/pyprobe/pyprobe.py @@ -179,7 +179,7 @@ class VideoFileParser: if PY3: command = [parser] + commandArgs + [inputFile] completedProcess = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, encoding="utf-8" ) if completedProcess.returncode: raise IOError( From 49e2d9482d5e4897fb246a1c566c40c38acc8439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 25 Dec 2019 14:02:14 -0500 Subject: [PATCH 13/14] Updated Python 2.7.x end of support date and deprecation warning. --- views/menu.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/views/menu.tpl b/views/menu.tpl index b13e44326..923f8fbc7 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -237,15 +237,15 @@
Python deprecation warning
- Bazarr is now compatible with Python 3.6 and newer. You should upgrade Python as we'll drop support for Python 2.7.x by the end of 2019. + Bazarr is now compatible with Python 3.6 and newer. You should upgrade Python as we'll drop support for Python 2.7.x by the end of January 2020.
% if os.name == 'posix': -
If you are running under Docker, don't worry, we'll take care of this for you. Just pull the new image.
+
If you are running under Docker, don't worry, we'll take care of this for you. Just pull the new image that should be available within a couple of days.
% end % if os.name == 'nt': -
If you have installed using the Windows Installer, we'll provide a new installer soon.
+
If you have installed using the Windows Installer, we'll provide a new installer soon that will upgrade your current installation (make sure to not change installation directory).
% end -
If you are running from source, it's up to you to install Python 3 and use it to run Bazarr.
+
If you are running from source, it's up to you to install Python 3 (don't forget requirements.txt) and use it to run Bazarr.
From 376d57e4e7abb1bd34e885bc13d9b77200637030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 26 Dec 2019 18:22:05 -0500 Subject: [PATCH 14/14] Updated Python 2.7.x end of support date and deprecation warning. --- views/menu.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/views/menu.tpl b/views/menu.tpl index 923f8fbc7..60be0aecb 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -243,7 +243,7 @@
If you are running under Docker, don't worry, we'll take care of this for you. Just pull the new image that should be available within a couple of days.
% end % if os.name == 'nt': -
If you have installed using the Windows Installer, we'll provide a new installer soon that will upgrade your current installation (make sure to not change installation directory).
+
If you have installed using the Windows Installer, just download the new installer that will upgrade your current installation (make sure to not change installation directory).
% end
If you are running from source, it's up to you to install Python 3 (don't forget requirements.txt) and use it to run Bazarr.