From 0427c83cdade5622e176ba81d16f6a7eaad2309b Mon Sep 17 00:00:00 2001 From: Vitiko Date: Wed, 4 May 2022 22:56:35 -0400 Subject: [PATCH 1/4] no log: update fese library --- libs/fese/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/fese/__init__.py b/libs/fese/__init__.py index 0c30ad013..fbe3f44b7 100755 --- a/libs/fese/__init__.py +++ b/libs/fese/__init__.py @@ -14,7 +14,7 @@ from babelfish import Language from babelfish.exceptions import LanguageError import pysubs2 -__version__ = "0.1.3" +__version__ = "0.1.4" logger = logging.getLogger(__name__) @@ -127,6 +127,11 @@ class FFprobeSubtitleStream: self.time_base = stream.get("time_base") self.tags = stream.get("tags", {}) self.start_time = float(stream.get("start_time", 0)) + # TODO: separate tags + self.number_of_frames = int(self.tags.get("NUMBER_OF_FRAMES", 0)) + self.number_of_frames_eng = int( + self.tags.get("NUMBER_OF_FRAMES-eng", self.number_of_frames) + ) self.duration, self.duration_ts = 0, 0 From e186bd1a0f754738873eec995271522dc1df2263 Mon Sep 17 00:00:00 2001 From: Vitiko Date: Wed, 4 May 2022 23:04:32 -0400 Subject: [PATCH 2/4] no log: update embedded subtitles provider release info representation --- libs/subliminal_patch/providers/embeddedsubtitles.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libs/subliminal_patch/providers/embeddedsubtitles.py b/libs/subliminal_patch/providers/embeddedsubtitles.py index aca0bb31b..324438a1c 100644 --- a/libs/subliminal_patch/providers/embeddedsubtitles.py +++ b/libs/subliminal_patch/providers/embeddedsubtitles.py @@ -40,7 +40,7 @@ class EmbeddedSubtitle(Subtitle): self.container: FFprobeVideoContainer = container self.forced = stream.disposition.forced self.page_link = self.container.path - self.release_info = os.path.basename(self.page_link) + self.release_info = _get_pretty_release_name(stream, container) self.media_type = media_type self._matches: set = matches @@ -272,3 +272,8 @@ def _is_fuse_rclone_mount(path: str): # https://forum.rclone.org/t/fuse-inode-number-aufs/215/5 # https://pkg.go.dev/bazil.org/fuse/fs?utm_source=godoc#GenerateDynamicInode return len(str(os.stat(path).st_ino)) > 18 + + +def _get_pretty_release_name(stream, container): + bname = os.path.basename(container.path) + return f"{os.path.splitext(bname)[0]}.{stream.suffix}" From 134613711ac1ede510cfc023072f0712a175b858 Mon Sep 17 00:00:00 2001 From: Vitiko Date: Thu, 5 May 2022 00:02:34 -0400 Subject: [PATCH 3/4] Embedded Subtitles provider: improve detection of potentially incomplete subtitles --- .../providers/embeddedsubtitles.py | 35 ++++++++++++++++++ .../test_embeddedsubtitles.py | 36 ++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/libs/subliminal_patch/providers/embeddedsubtitles.py b/libs/subliminal_patch/providers/embeddedsubtitles.py index 324438a1c..63e7c8fde 100644 --- a/libs/subliminal_patch/providers/embeddedsubtitles.py +++ b/libs/subliminal_patch/providers/embeddedsubtitles.py @@ -117,6 +117,8 @@ class EmbeddedSubtitlesProvider(Provider): self._blacklist.add(path) streams = [] + streams = _discard_possible_incomplete_subtitles(list(streams)) + if not streams: logger.debug("No subtitles found for container: %s", video) @@ -260,6 +262,39 @@ def _check_hi_fallback(streams, languages): logger.debug("HI fallback not needed: %s", compatible_streams) +def _discard_possible_incomplete_subtitles(streams): + """Check number_of_frames attributes from subtitle streams in order to find + supposedly incomplete subtitles""" + try: + max_frames = max(stream.number_of_frames for stream in streams) + except ValueError: + return [] + + # Blatantly assume there's nothing to discard as some ffprobe streams don't + # have number_of_frames tags + if not max_frames: + return streams + + logger.debug("Checking possible incomplete subtitles (max frames: %d)", max_frames) + + valid_streams = [] + + for stream in streams: + # 500 < 1200 + if stream.number_of_frames < max_frames // 2: + logger.debug( + "Possible bad subtitle found: %s (%s frames - %s frames)", + stream, + stream.number_of_frames, + max_frames, + ) + continue + + valid_streams.append(stream) + + return valid_streams + + def _is_fuse_rclone_mount(path: str): # Experimental! diff --git a/tests/subliminal_patch/test_embeddedsubtitles.py b/tests/subliminal_patch/test_embeddedsubtitles.py index 942c5a56a..0bda12072 100644 --- a/tests/subliminal_patch/test_embeddedsubtitles.py +++ b/tests/subliminal_patch/test_embeddedsubtitles.py @@ -9,10 +9,11 @@ import subliminal_patch from subliminal_patch.core import Episode from subliminal_patch.core import Movie from subliminal_patch.exceptions import MustGetBlacklisted -from subliminal_patch.providers.embeddedsubtitles import \ - _MemoizedFFprobeVideoContainer -from subliminal_patch.providers.embeddedsubtitles import \ - EmbeddedSubtitlesProvider +from subliminal_patch.providers.embeddedsubtitles import _MemoizedFFprobeVideoContainer +from subliminal_patch.providers.embeddedsubtitles import EmbeddedSubtitlesProvider +from subliminal_patch.providers.embeddedsubtitles import ( + _discard_possible_incomplete_subtitles, +) from subzero.language import Language @@ -249,3 +250,30 @@ def test_memoized(video_single_language, mocker): ] is not None ) + + +@pytest.mark.parametrize( + "number_of_frames,expected_len", + [((34, 811), 1), ((0, 0), 2), ((811, 34), 1), ((900, 1000), 2), ((0, 900), 1)], +) +def test_discard_possible_incomplete_subtitles(number_of_frames, expected_len): + subtitle_1 = FFprobeSubtitleStream( + { + "index": 1, + "codec_name": "subrip", + "codec_long_name": "SubRip subtitle", + "disposition": {}, + "tags": {"language": "eng", "NUMBER_OF_FRAMES": number_of_frames[0]}, + } + ) + subtitle_2 = FFprobeSubtitleStream( + { + "index": 2, + "codec_name": "subrip", + "codec_long_name": "SubRip subtitle", + "disposition": {}, + "tags": {"language": "eng", "NUMBER_OF_FRAMES": number_of_frames[1]}, + } + ) + new_list = _discard_possible_incomplete_subtitles([subtitle_1, subtitle_2]) + assert len(new_list) == expected_len From a783515ad4fc5f54a27c97a2955679c6555a5839 Mon Sep 17 00:00:00 2001 From: Vitiko Date: Thu, 5 May 2022 14:16:30 -0400 Subject: [PATCH 4/4] Avoid NoneType on providers pool updates --- libs/subliminal_patch/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index 01778375d..c5f11f75c 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -105,10 +105,12 @@ class SZProviderPool(ProviderPool): # Check if the pool was initialized enough hours ago self._check_lifetime() + providers = set(providers or []) + # Check if any new provider has been added - updated = set(providers) != self.providers or ban_list != self.ban_list - removed_providers = list(sorted(self.providers - set(providers))) - new_providers = list(sorted(set(providers) - self.providers)) + updated = providers != self.providers or ban_list != self.ban_list + removed_providers = list(sorted(self.providers - providers)) + new_providers = list(sorted(providers - self.providers)) # Terminate and delete removed providers from instance for removed in removed_providers: @@ -128,8 +130,6 @@ class SZProviderPool(ProviderPool): self.providers.difference_update(removed_providers) self.providers.update(list(providers)) - self.blacklist = blacklist - # Restart providers with new configs for key, val in provider_configs.items(): # Don't restart providers that are not enabled @@ -154,6 +154,9 @@ class SZProviderPool(ProviderPool): self.provider_configs = provider_configs + self.blacklist = blacklist or [] + self.ban_list = ban_list or {'must_contain': [], 'must_not_contain': []} + return updated def _check_lifetime(self):