From 63b326aa2f12df482f9537a0fec2f7755a152bfc Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sat, 11 Dec 2021 07:44:53 -0500 Subject: [PATCH] Implemented words/regex ban list for subtitles --- bazarr/api/episodes/episodes_subtitles.py | 5 +-- bazarr/api/movies/movies_subtitles.py | 5 +-- bazarr/api/providers/providers_episodes.py | 5 +-- bazarr/api/providers/providers_movies.py | 5 +-- bazarr/api/system/settings.py | 8 +++-- bazarr/database.py | 36 +++++++++++++++++++--- bazarr/get_subtitle.py | 18 +++++++++-- frontend/src/@types/api.d.ts | 2 ++ frontend/src/Settings/Languages/modal.tsx | 25 +++++++++++++++ frontend/src/Settings/Languages/table.tsx | 34 ++++++++++++++++++++ libs/subliminal_patch/core.py | 17 ++++++++-- 11 files changed, 141 insertions(+), 19 deletions(-) diff --git a/bazarr/api/episodes/episodes_subtitles.py b/bazarr/api/episodes/episodes_subtitles.py index 86f8b0024..70f59857c 100644 --- a/bazarr/api/episodes/episodes_subtitles.py +++ b/bazarr/api/episodes/episodes_subtitles.py @@ -6,7 +6,7 @@ from flask import request from flask_restful import Resource from subliminal_patch.core import SUBTITLE_EXTENSIONS -from database import TableEpisodes, get_audio_profile_languages +from database import TableEpisodes, get_audio_profile_languages, get_profile_id from ..utils import authenticate from helper import path_mappings from get_providers import get_providers, get_providers_auth @@ -55,7 +55,8 @@ class EpisodesSubtitles(Resource): try: result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list, - providers_auth, sceneName, title, 'series') + providers_auth, sceneName, title, 'series', + profile_id=get_profile_id(episode_id=sonarrEpisodeId)) if result is not None: message = result[0] path = result[1] diff --git a/bazarr/api/movies/movies_subtitles.py b/bazarr/api/movies/movies_subtitles.py index cc2a07e75..c4f1a1e2a 100644 --- a/bazarr/api/movies/movies_subtitles.py +++ b/bazarr/api/movies/movies_subtitles.py @@ -6,7 +6,7 @@ from flask import request from flask_restful import Resource from subliminal_patch.core import SUBTITLE_EXTENSIONS -from database import TableMovies, get_audio_profile_languages +from database import TableMovies, get_audio_profile_languages, get_profile_id from ..utils import authenticate from helper import path_mappings from get_providers import get_providers, get_providers_auth @@ -57,7 +57,8 @@ class MoviesSubtitles(Resource): try: result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list, - providers_auth, sceneName, title, 'movie') + providers_auth, sceneName, title, 'movie', + profile_id=get_profile_id(movie_id=radarrId)) if result is not None: message = result[0] path = result[1] diff --git a/bazarr/api/providers/providers_episodes.py b/bazarr/api/providers/providers_episodes.py index feedc0134..283cc160d 100644 --- a/bazarr/api/providers/providers_episodes.py +++ b/bazarr/api/providers/providers_episodes.py @@ -3,7 +3,7 @@ from flask import request, jsonify from flask_restful import Resource -from database import TableEpisodes, TableShows, get_audio_profile_languages +from database import TableEpisodes, TableShows, get_audio_profile_languages, get_profile_id from helper import path_mappings from get_providers import get_providers, get_providers_auth from get_subtitle import manual_search, manual_download_subtitle @@ -76,7 +76,8 @@ class ProviderEpisodes(Resource): try: result = manual_download_subtitle(episodePath, language, audio_language, hi, forced, subtitle, - selected_provider, providers_auth, sceneName, title, 'series') + selected_provider, providers_auth, sceneName, title, 'series', + profile_id=get_profile_id(episode_id=sonarrEpisodeId)) if result is not None: message = result[0] path = result[1] diff --git a/bazarr/api/providers/providers_movies.py b/bazarr/api/providers/providers_movies.py index c189ab2c4..d5c31c1c4 100644 --- a/bazarr/api/providers/providers_movies.py +++ b/bazarr/api/providers/providers_movies.py @@ -3,7 +3,7 @@ from flask import request, jsonify from flask_restful import Resource -from database import TableMovies, get_audio_profile_languages +from database import TableMovies, get_audio_profile_languages, get_profile_id from helper import path_mappings from get_providers import get_providers, get_providers_auth from get_subtitle import manual_search, manual_download_subtitle @@ -77,7 +77,8 @@ class ProviderMovies(Resource): try: result = manual_download_subtitle(moviePath, language, audio_language, hi, forced, subtitle, - selected_provider, providers_auth, sceneName, title, 'movie') + selected_provider, providers_auth, sceneName, title, 'movie', + profile_id=get_profile_id(movie_id=radarrId)) if result is not None: message = result[0] path = result[1] diff --git a/bazarr/api/system/settings.py b/bazarr/api/system/settings.py index c8d9574fc..50f5d7473 100644 --- a/bazarr/api/system/settings.py +++ b/bazarr/api/system/settings.py @@ -56,7 +56,9 @@ class SystemSettings(Resource): TableLanguagesProfiles.update({ TableLanguagesProfiles.name: item['name'], TableLanguagesProfiles.cutoff: item['cutoff'] if item['cutoff'] != 'null' else None, - TableLanguagesProfiles.items: json.dumps(item['items']) + TableLanguagesProfiles.items: json.dumps(item['items']), + TableLanguagesProfiles.mustContain: item['mustContain'], + TableLanguagesProfiles.mustNotContain: item['mustNotContain'], })\ .where(TableLanguagesProfiles.profileId == item['profileId'])\ .execute() @@ -67,7 +69,9 @@ class SystemSettings(Resource): TableLanguagesProfiles.profileId: item['profileId'], TableLanguagesProfiles.name: item['name'], TableLanguagesProfiles.cutoff: item['cutoff'] if item['cutoff'] != 'null' else None, - TableLanguagesProfiles.items: json.dumps(item['items']) + TableLanguagesProfiles.items: json.dumps(item['items']), + TableLanguagesProfiles.mustContain: item['must_contain'], + TableLanguagesProfiles.mustNotContain: item['must_not_contain'], }).execute() for profileId in existing: # Unassign this profileId from series and movies diff --git a/bazarr/database.py b/bazarr/database.py index 85f420110..b80c1ee7d 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -136,6 +136,8 @@ class TableLanguagesProfiles(BaseModel): items = TextField() name = TextField() profileId = AutoField() + mustContain = TextField(null=True) + mustNotContain = TextField(null=True) class Meta: table_name = 'table_languages_profiles' @@ -329,7 +331,9 @@ def migrate_db(): migrator.add_column('table_history_movie', 'provider', TextField(null=True)), migrator.add_column('table_history_movie', 'score', TextField(null=True)), migrator.add_column('table_history_movie', 'subs_id', TextField(null=True)), - migrator.add_column('table_history_movie', 'subtitles_path', TextField(null=True)) + migrator.add_column('table_history_movie', 'subtitles_path', TextField(null=True)), + migrator.add_column('table_languages_profiles', 'mustContain', TextField(null=True)), + migrator.add_column('table_languages_profiles', 'mustNotContain', TextField(null=True)), ) @@ -394,10 +398,16 @@ def update_profile_id_list(): profile_id_list = TableLanguagesProfiles.select(TableLanguagesProfiles.profileId, TableLanguagesProfiles.name, TableLanguagesProfiles.cutoff, - TableLanguagesProfiles.items).dicts() + TableLanguagesProfiles.items, + TableLanguagesProfiles.mustContain, + TableLanguagesProfiles.mustNotContain).dicts() profile_id_list = list(profile_id_list) for profile in profile_id_list: profile['items'] = json.loads(profile['items']) + profile['mustContain'] = ast.literal_eval(profile['mustContain']) if profile['mustContain'] else \ + profile['mustContain'] + profile['mustNotContain'] = ast.literal_eval(profile['mustNotContain']) if profile['mustNotContain'] else \ + profile['mustNotContain'] def get_profiles_list(profile_id=None): @@ -422,7 +432,7 @@ def get_desired_languages(profile_id): if profile_id and profile_id != 'null': for profile in profile_id_list: - profileId, name, cutoff, items = profile.values() + profileId, name, cutoff, items, mustContain, mustNotContain = profile.values() if profileId == int(profile_id): languages = [x['language'] for x in items] break @@ -438,7 +448,7 @@ def get_profile_id_name(profile_id): if profile_id and profile_id != 'null': for profile in profile_id_list: - profileId, name, cutoff, items = profile.values() + profileId, name, cutoff, items, mustContain, mustNotContain = profile.values() if profileId == int(profile_id): name_from_id = name break @@ -455,7 +465,7 @@ def get_profile_cutoff(profile_id): if profile_id and profile_id != 'null': cutoff_language = [] for profile in profile_id_list: - profileId, name, cutoff, items = profile.values() + profileId, name, cutoff, items, mustContain, mustNotContain = profile.values() if cutoff: if profileId == int(profile_id): for item in items: @@ -498,6 +508,22 @@ def get_audio_profile_languages(series_id=None, episode_id=None, movie_id=None): return audio_languages +def get_profile_id(series_id=None, episode_id=None, movie_id=None): + if series_id: + profileId = TableShows.get(TableShows.sonarrSeriesId == series_id).profileId + elif episode_id: + profileId = TableShows.select(TableShows.profileId)\ + .join(TableEpisodes, on=(TableShows.sonarrSeriesId == TableEpisodes.sonarrSeriesId))\ + .where(TableEpisodes.sonarrEpisodeId == episode_id)\ + .get().profileId + elif movie_id: + profileId = TableMovies.get(TableMovies.radarrId == movie_id).profileId + else: + return None + + return profileId + + def convert_list_to_clause(arr: list): if isinstance(arr, list): return f"({','.join(str(x) for x in arr)})" diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 8e5d35db9..ff8c0cb9a 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -84,7 +84,7 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"): def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title, - media_type, forced_minimum_score=None, is_upgrade=False): + media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None): # fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in # one query @@ -158,6 +158,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro compute_score=compute_score, throttle_time=None, # fixme blacklist=get_blacklist(media_type=media_type), + ban_list=get_ban_list(profile_id), throttle_callback=provider_throttle, score_obj=handler, pre_download_hook=None, # fixme @@ -361,6 +362,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, providers=providers, provider_configs=providers_auth, blacklist=get_blacklist(media_type=media_type), + ban_list=get_ban_list(profileId), throttle_callback=provider_throttle, language_hook=None) # fixme @@ -375,6 +377,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, providers=['subscene'], provider_configs=providers_auth, blacklist=get_blacklist(media_type=media_type), + ban_list=get_ban_list(profileId), throttle_callback=provider_throttle, language_hook=None) # fixme providers_auth['subscene']['only_foreign'] = False @@ -466,7 +469,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title, def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName, - title, media_type): + title, media_type, profile_id): logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path) if settings.general.getboolean('utf8_encode'): @@ -498,6 +501,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl provider_configs=providers_auth, pool_class=provider_pool(), blacklist=get_blacklist(media_type=media_type), + ban_list=get_ban_list(profile_id), throttle_callback=provider_throttle) logging.debug('BAZARR Subtitles file downloaded for this file:' + path) else: @@ -1706,6 +1710,7 @@ def _get_lang_obj(alpha3): return sub.subzero_language() + def _get_scores(media_type, min_movie=None, min_ep=None): series = "series" == media_type handler = series_score if series else movie_score @@ -1713,3 +1718,12 @@ def _get_scores(media_type, min_movie=None, min_ep=None): min_ep = min_ep or (240 * 100 / handler.max_score) min_score_ = int(min_ep if series else min_movie) return handler.get_scores(min_score_) + + +def get_ban_list(profile_id): + if profile_id: + profile = get_profiles_list(profile_id) + if profile: + return {'must_contain': profile['mustContain'] or [], + 'must_not_contain': profile['mustNotContain'] or []} + return None diff --git a/frontend/src/@types/api.d.ts b/frontend/src/@types/api.d.ts index 043753879..2d820460a 100644 --- a/frontend/src/@types/api.d.ts +++ b/frontend/src/@types/api.d.ts @@ -33,6 +33,8 @@ declare namespace Language { profileId: number; cutoff: number | null; items: ProfileItem[]; + mustContain: string[]; + mustNotContain: string[]; } } diff --git a/frontend/src/Settings/Languages/modal.tsx b/frontend/src/Settings/Languages/modal.tsx index 20d1943a9..7371267dc 100644 --- a/frontend/src/Settings/Languages/modal.tsx +++ b/frontend/src/Settings/Languages/modal.tsx @@ -13,6 +13,7 @@ import { ActionButton, BaseModal, BaseModalProps, + Chips, LanguageSelector, Selector, SimpleTable, @@ -31,6 +32,8 @@ function createDefaultProfile(): Language.Profile { name: "", items: [], cutoff: null, + mustContain: [], + mustNotContain: [], }; } @@ -260,6 +263,28 @@ const LanguagesProfileModal: FunctionComponent = ( > Ignore others if existing + + updateProfile("mustContain", mc)} + > + + Subtitles release info must include one of those words or they will be + excluded from search results (regex supported). + + + + { + updateProfile("mustNotContain", mnc); + }} + > + + Subtitles release info including one of those words (case insensitive) + will be excluded from search results (regex supported). + + ); }; diff --git a/frontend/src/Settings/Languages/table.tsx b/frontend/src/Settings/Languages/table.tsx index 4547e3198..09c75d138 100644 --- a/frontend/src/Settings/Languages/table.tsx +++ b/frontend/src/Settings/Languages/table.tsx @@ -94,6 +94,40 @@ const Table: FunctionComponent = () => { }); }, }, + { + Header: "Must contain", + accessor: "mustContain", + Cell: (row) => { + const items = row.value; + if (!items) { + return false; + } + return items.map((v) => { + return ( + + {v} + + ); + }); + }, + }, + { + Header: "Must not contain", + accessor: "mustNotContain", + Cell: (row) => { + const items = row.value; + if (!items) { + return false; + } + return items.map((v) => { + return ( + + {v} + + ); + }); + }, + }, { accessor: "profileId", Cell: ({ row, update }) => { diff --git a/libs/subliminal_patch/core.py b/libs/subliminal_patch/core.py index b6629526a..aa82db1f9 100644 --- a/libs/subliminal_patch/core.py +++ b/libs/subliminal_patch/core.py @@ -66,7 +66,7 @@ def remove_crap_from_fn(fn): class SZProviderPool(ProviderPool): - def __init__(self, providers=None, provider_configs=None, blacklist=None, throttle_callback=None, + def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None, pre_download_hook=None, post_download_hook=None, language_hook=None): #: Name of providers to use self.providers = providers @@ -82,6 +82,9 @@ class SZProviderPool(ProviderPool): self.blacklist = blacklist or [] + #: Should be a dict of 2 lists of strings + self.ban_list = ban_list or {'must_contain': [], 'must_not_contain': []} + self.throttle_callback = throttle_callback self.pre_download_hook = pre_download_hook @@ -184,6 +187,16 @@ class SZProviderPool(ProviderPool): if (str(provider), str(s.id)) in self.blacklist: logger.info("Skipping blacklisted subtitle: %s", s) continue + if hasattr(s, 'release_info'): + if s.release_info is not None: + if any([x for x in self.ban_list["must_not_contain"] + if re.search(x, s.release_info, flags=re.IGNORECASE) is not None]): + logger.info("Skipping subtitle because release name contains prohibited string: %s", s) + continue + if any([x for x in self.ban_list["must_contain"] + if re.search(x, s.release_info, flags=re.IGNORECASE) is None]): + logger.info("Skipping subtitle because release name does not contains required string: %s", s) + continue if s.id in seen: continue s.plex_media_fps = float(video.fps) if video.fps else None @@ -506,7 +519,7 @@ class SZAsyncProviderPool(SZProviderPool): return provider, provider_subtitles - def list_subtitles(self, video, languages, blacklist=None): + def list_subtitles(self, video, languages, blacklist=None, ban_list=None): if is_windows_special_path: return super(SZAsyncProviderPool, self).list_subtitles(video, languages)