From ae35529de8cab8214cced92dddb039a1484e93da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Fri, 11 Oct 2019 00:00:23 -0400 Subject: [PATCH] Continuing development. --- bazarr/get_episodes.py | 4 +-- bazarr/get_movies.py | 2 +- bazarr/get_subtitle.py | 14 ++++---- bazarr/helper.py | 39 +++++++++++++------- bazarr/list_subtitles.py | 77 +++++++++++++++++++--------------------- bazarr/main.py | 16 ++++----- 6 files changed, 81 insertions(+), 71 deletions(-) diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index 9851ecc5c..2525007dd 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -5,7 +5,7 @@ import requests import logging import re from queueconfig import notifications -from database import TableShows, TableEpisodes, wal_cleaning +from database import database, TableShows, TableEpisodes, wal_cleaning from get_args import args from config import settings, url_sonarr @@ -188,7 +188,7 @@ def sync_episodes(): for i, altered_episode in enumerate(altered_episodes, 1): notifications.write(msg='Indexing episodes embedded subtitles...', queue='get_episodes', item=i, length=len(altered_episodes)) - store_subtitles(path_replace(altered_episode[1])) + store_subtitles(altered_episode[1], path_replace(altered_episode[1])) list_missing_subtitles(altered_episode[2]) logging.debug('BAZARR All episodes synced from Sonarr into database.') diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index 38a9cca13..b35bb6148 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -270,7 +270,7 @@ def update_movies(): for i, altered_movie in enumerate(altered_movies, 1): notifications.write(msg='Indexing movies embedded subtitles...', queue='get_movies', item=i, length=len(altered_movies)) - store_subtitles_movie(path_replace_movie(altered_movie[1])) + store_subtitles_movie(altered_movie[1], path_replace_movie(altered_movie[1])) list_missing_subtitles_movies(altered_movie[2]) logging.debug('BAZARR All movies synced from Radarr into database.') diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 3b1ecd27e..4fa67fcb4 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -607,7 +607,7 @@ def series_download_subtitles(no): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles(path_replace(episode.path)) + store_subtitles(episode.path, path_replace(episode.path)) history_log(1, no, episode.sonarr_episode_id, message, path, language_code, provider, score) send_notifications(no, episode.sonarr_episode_id, message) else: @@ -671,7 +671,7 @@ def episode_download_subtitles(no): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles(path_replace(episode.path)) + store_subtitles(episode.path, path_replace(episode.path)) history_log(1, episode.sonarr_series_id, episode.sonarr_episode_id, message, path, language_code, provider, score) send_notifications(episode.sonarr_series_id, episode.sonarr_episode_id, message) list_missing_subtitles(episode.sonarr_series_id) @@ -720,7 +720,7 @@ def movies_download_subtitles(no): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles_movie(path_replace_movie(movie.path)) + store_subtitles_movie(movie.path, path_replace_movie(movie.path)) history_log_movie(1, no, message, path, language_code, provider, score) send_notifications_movie(no, message) else: @@ -798,7 +798,7 @@ def wanted_download_subtitles(path, l, count_episodes): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles(path_replace(episode.path)) + store_subtitles(episode.path, path_replace(episode.path)) list_missing_subtitles(episode.sonarr_series_id.sonarr_series_id) history_log(1, episode.sonarr_series_id.sonarr_series_id, episode.sonarr_episode_id, message, path, language_code, provider, score) send_notifications(episode.sonarr_series_id.sonarr_series_id, episode.sonarr_episode_id, message) @@ -867,7 +867,7 @@ def wanted_download_subtitles_movie(path, l, count_movies): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles_movie(path_replace_movie(movie.path)) + store_subtitles_movie(movie.path, path_replace_movie(movie.path)) list_missing_subtitles_movies(movie.radarr_id) history_log_movie(1, movie.radarr_id, message, path, language_code, provider, score) send_notifications_movie(movie.radarr_id, message) @@ -1222,7 +1222,7 @@ def upgrade_subtitles(): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles(path_replace(episode['video_path'])) + store_subtitles(episode['video_path'], path_replace(episode['video_path'])) history_log(3, episode['sonarr_series_id'], episode['sonarr_episode_id'], message, path, language_code, provider, score) send_notifications(episode['sonarr_series_id'], episode['sonarr_episode_id'], message) @@ -1271,6 +1271,6 @@ def upgrade_subtitles(): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] - store_subtitles_movie(path_replace_movie(movie['video_path'])) + store_subtitles_movie(movie['video_path'], path_replace_movie(movie['video_path'])) history_log_movie(3, movie['radarr_id'], message, path, language_code, provider, score) send_notifications_movie(movie['radarr_id'], message) diff --git a/bazarr/helper.py b/bazarr/helper.py index e307e8f9a..e4ba127ad 100644 --- a/bazarr/helper.py +++ b/bazarr/helper.py @@ -53,21 +53,34 @@ def sonarr_path_mapping_reverse_regex(): def radarr_path_mapping_regex(): global path_mapping_movie global radarr_regex - - path_mapping_movie = dict(ast.literal_eval(settings.general.path_mappings_movie)) - radarr_regex = re.compile("|".join(map(re.escape, path_mapping_movie.keys()))) + global radarr_use_path_mapping + + path_mapping_movie = ast.literal_eval(settings.general.path_mappings_movie) + path_mapping_movie = sorted(path_mapping_movie, key=operator.itemgetter(0), reverse=True) + path_mapping_movie = OrderedDict((mapping[0], mapping[1]) for mapping in path_mapping_movie if mapping[0] != '') + if any(item for sublist in path_mapping_movie for item in sublist): + radarr_use_path_mapping = True + radarr_regex = re.compile("|".join(path_mapping_movie.keys())) + else: + radarr_use_path_mapping = False def radarr_path_mapping_reverse_regex(): global radarr_platform global path_mapping_reverse_movie global radarr_reverse_regex + global radarr_use_path_mapping - radarr_platform = get_radarr_platform() + radarr_platform = get_sonarr_platform() - path_mapping_reverse_movie_temp = ast.literal_eval(settings.general.path_mappings_movie) - path_mapping_reverse_movie = dict([sublist[::-1] for sublist in path_mapping_reverse_movie_temp]) - radarr_reverse_regex = re.compile("|".join(map(re.escape, path_mapping_reverse_movie.keys()))) + path_mapping_reverse_movie = ast.literal_eval(settings.general.path_mappings) + path_mapping_reverse_movie = sorted(path_mapping_reverse_movie, key=operator.itemgetter(0), reverse=True) + path_mapping_reverse_movie = OrderedDict((mapping[1], mapping[0]) for mapping in path_mapping_reverse_movie if mapping[0] != '') + if any(item for sublist in path_mapping_reverse_movie for item in sublist): + radarr_use_path_mapping = True + radarr_reverse_regex = re.compile("|".join(map(re.escape, path_mapping_reverse_movie.keys()))) + else: + radarr_use_path_mapping = False def path_replace(path): @@ -98,7 +111,7 @@ def path_replace_reverse(path): def path_replace_movie(path): - if path is None: + if path is None or radarr_use_path_mapping is False: return None reverted_path = radarr_regex.sub(lambda match: path_mapping_movie[match.group(0)], path, count=1) @@ -109,17 +122,17 @@ def path_replace_movie(path): def path_replace_reverse_movie(path): - if path is None: - return None + if path is None or radarr_use_path_mapping is False: + return path - reverted_path_movie_temp = radarr_reverse_regex.sub(lambda match: path_mapping_reverse_movie[match.group(0)], path, count=1) + reverted_path_temp = radarr_reverse_regex.sub(lambda match: path_mapping_reverse_movie[match.group(0)], path, count=1) if radarr_platform == 'posix': from posixpath import normpath - reverted_path = reverted_path_movie_temp.replace('\\', '/') + reverted_path = reverted_path_temp.replace('\\', '/') elif radarr_platform == 'nt': from ntpath import normpath - reverted_path = reverted_path_movie_temp.replace('/', '\\') + reverted_path = reverted_path_temp.replace('/', '\\') return normpath(reverted_path) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index e75d85096..542ec600b 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -31,14 +31,14 @@ import six gc.enable() -def store_subtitles(file): - logging.debug('BAZARR started subtitles indexing for this file: ' + file) +def store_subtitles(original_path, reversed_path): + logging.debug('BAZARR started subtitles indexing for this file: ' + reversed_path) actual_subtitles = [] - if os.path.exists(file): + if os.path.exists(reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(file) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages: try: if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle": @@ -56,39 +56,39 @@ def store_subtitles(file): pass except Exception as e: logging.exception( - "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(file)[1], file)) + "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(reversed_path)[1], reversed_path)) pass - + brazilian_portuguese = [".pt-br", ".pob", "pb"] brazilian_portuguese_forced = [".pt-br.forced", ".pob.forced", "pb.forced"] try: dest_folder = get_subtitle_destination_folder() subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] - subtitles = search_external_subtitles(file, languages=get_language_set(), + subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), only_one=settings.general.getboolean('single_language')) except Exception as e: logging.exception("BAZARR unable to index external subtitles.") pass else: for subtitle, language in six.iteritems(subtitles): - subtitle_path = get_external_subtitles_path(file, subtitle) + subtitle_path = get_external_subtitles_path(reversed_path, subtitle) if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)): logging.debug("BAZARR external subtitles detected: " + "pb") actual_subtitles.append( - [str("pb"), path_replace_reverse(subtitle_path)]) + [str("pb"), original_path]) elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)): logging.debug("BAZARR external subtitles detected: " + "pb:forced") actual_subtitles.append( - [str("pb:forced"), path_replace_reverse(subtitle_path)]) - + [str("pb:forced"), original_path]) + elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append( - [str(language), path_replace_reverse(subtitle_path)]) + [str(language), original_path]) else: if os.path.splitext(subtitle)[1] != ".sub": logging.debug("BAZARR falling back to file content analysis to detect language.") - with open(os.path.join(os.path.dirname(file), subtitle), 'r') as f: + with open(os.path.join(os.path.dirname(reversed_path), subtitle), 'r') as f: text = list(islice(f, 100)) text = ' '.join(text) encoding = UnicodeDammit(text) @@ -98,7 +98,7 @@ def store_subtitles(file): except Exception as e: logging.exception( 'BAZARR Error trying to detect language for this subtitles file: ' + - os.path.join(os.path.dirname(file), subtitle) + + os.path.join(os.path.dirname(reversed_path), subtitle) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') else: if len(detected_language) > 0: @@ -106,14 +106,14 @@ def store_subtitles(file): "BAZARR external subtitles detected and analysis guessed this language: " + str( detected_language)) actual_subtitles.append([str(detected_language), path_replace_reverse( - os.path.join(os.path.dirname(file), subtitle))]) + os.path.join(os.path.dirname(reversed_path), subtitle))]) update_count = TableEpisodes.update( { TableEpisodes.subtitles: str(actual_subtitles) } ).where( - TableEpisodes.path == path_replace_reverse(file) + TableEpisodes.path == original_path ).execute() if update_count > 0: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) @@ -122,19 +122,19 @@ def store_subtitles(file): else: logging.debug("BAZARR this file doesn't seems to exist or isn't accessible.") - logging.debug('BAZARR ended subtitles indexing for this file: ' + file) + logging.debug('BAZARR ended subtitles indexing for this file: ' + reversed_path) return actual_subtitles -def store_subtitles_movie(file): - logging.debug('BAZARR started subtitles indexing for this file: ' + file) +def store_subtitles_movie(original_path, reversed_path): + logging.debug('BAZARR started subtitles indexing for this file: ' + reversed_path) actual_subtitles = [] - if os.path.exists(file): + if os.path.exists(reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(file) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path) for subtitle_language, subtitle_forced, subtitle_codec in subtitle_languages: try: if settings.general.getboolean("ignore_pgs_subs") and subtitle_codec == "hdmv_pgs_subtitle": @@ -152,7 +152,7 @@ def store_subtitles_movie(file): pass except Exception as e: logging.exception( - "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(file)[1], file)) + "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(reversed_path)[1], reversed_path)) pass dest_folder = get_subtitle_destination_folder() or '' @@ -160,32 +160,29 @@ def store_subtitles_movie(file): brazilian_portuguese = [".pt-br", ".pob", "pb"] brazilian_portuguese_forced = [".pt-br.forced", ".pob.forced", "pb.forced"] try: - subtitles = search_external_subtitles(file, languages=get_language_set(), + subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), only_one=settings.general.getboolean('single_language')) except Exception as e: logging.exception("BAZARR unable to index external subtitles.") pass else: for subtitle, language in six.iteritems(subtitles): - if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)) is True: + if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)): logging.debug("BAZARR external subtitles detected: " + "pb") actual_subtitles.append( - [str("pb"), - path_replace_reverse_movie(os.path.join(os.path.dirname(file), dest_folder, subtitle))]) - elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)) is True: + [str("pb"), original_path]) + elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)): logging.debug("BAZARR external subtitles detected: " + "pb:forced") actual_subtitles.append( - [str("pb:forced"), - path_replace_reverse_movie(os.path.join(os.path.dirname(file), dest_folder, subtitle))]) + [str("pb:forced"), original_path]) elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append( - [str(language), - path_replace_reverse_movie(os.path.join(os.path.dirname(file), dest_folder, subtitle))]) + [str(language), original_path]) else: if os.path.splitext(subtitle)[1] != ".sub": logging.debug("BAZARR falling back to file content analysis to detect language.") - with open(os.path.join(os.path.dirname(file), dest_folder, subtitle), 'r') as f: + with open(os.path.join(os.path.dirname(reversed_path), dest_folder, subtitle), 'r') as f: text = list(islice(f, 100)) text = ' '.join(text) encoding = UnicodeDammit(text) @@ -195,7 +192,7 @@ def store_subtitles_movie(file): except Exception as e: logging.exception( 'BAZARR Error trying to detect language for this subtitles file: ' + - os.path.join(os.path.dirname(file), subtitle) + + os.path.join(os.path.dirname(reversed_path), subtitle) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') else: if len(detected_language) > 0: @@ -203,14 +200,14 @@ def store_subtitles_movie(file): "BAZARR external subtitles detected and analysis guessed this language: " + str( detected_language)) actual_subtitles.append([str(detected_language), path_replace_reverse_movie( - os.path.join(os.path.dirname(file), dest_folder, subtitle))]) + os.path.join(os.path.dirname(reversed_path), dest_folder, subtitle))]) update_count = TableMovies.update( { TableMovies.subtitles: str(actual_subtitles) } ).where( - TableMovies.path == path_replace_reverse_movie(file) + TableMovies.path == original_path ).execute() if update_count > 0: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) @@ -219,7 +216,7 @@ def store_subtitles_movie(file): else: logging.debug("BAZARR this file doesn't seems to exist or isn't accessible.") - logging.debug('BAZARR ended subtitles indexing for this file: ' + file) + logging.debug('BAZARR ended subtitles indexing for this file: ' + reversed_path) return actual_subtitles @@ -363,7 +360,7 @@ def series_full_scan_subtitles(): for i, episode in enumerate(episodes, 1): notifications.write(msg='Updating all episodes subtitles from disk...', queue='list_subtitles_series', item=i, length=count_episodes) - store_subtitles(path_replace(episode.path)) + store_subtitles(episode.path, path_replace(episode.path)) gc.collect() @@ -377,7 +374,7 @@ def movies_full_scan_subtitles(): for i, movie in enumerate(movies, 1): notifications.write(msg='Updating all movies subtitles from disk...', queue='list_subtitles_movies', item=i, length=count_movies) - store_subtitles_movie(path_replace_movie(movie.path)) + store_subtitles_movie(movie.path, path_replace_movie(movie.path)) gc.collect() @@ -390,7 +387,7 @@ def series_scan_subtitles(no): ) for episode in episodes: - store_subtitles(path_replace(episode.path)) + store_subtitles(episode.path, path_replace(episode.path)) list_missing_subtitles(no) @@ -403,7 +400,7 @@ def movies_scan_subtitles(no): ) for movie in movies: - store_subtitles_movie(path_replace_movie(movie.path)) + store_subtitles_movie(movie.path, path_replace_movie(movie.path)) list_missing_subtitles_movies(no) diff --git a/bazarr/main.py b/bazarr/main.py index 94cdb67c3..dd78a67e6 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -2060,7 +2060,7 @@ def remove_subtitles(): history_log(0, sonarrSeriesId, sonarrEpisodeId, result) except OSError as e: logging.exception('BAZARR cannot delete subtitles file: ' + subtitlesPath) - store_subtitles(six.text_type(episodePath)) + store_subtitles(episodePath, six.text_type(episodePath)) list_missing_subtitles(sonarrSeriesId) @@ -2079,7 +2079,7 @@ def remove_subtitles_movie(): history_log_movie(0, radarrId, result) except OSError as e: logging.exception('BAZARR cannot delete subtitles file: ' + subtitlesPath) - store_subtitles_movie(six.text_type(moviePath)) + store_subtitles_movie(moviePath, six.text_type(moviePath)) list_missing_subtitles_movies(radarrId) @@ -2113,7 +2113,7 @@ def get_subtitle(): score = result[4] history_log(1, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) - store_subtitles(six.text_type(episodePath)) + store_subtitles(episodePath, six.text_type(episodePath)) list_missing_subtitles(sonarrSeriesId) redirect(ref) except OSError: @@ -2171,7 +2171,7 @@ def manual_get_subtitle(): score = result[4] history_log(2, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) - store_subtitles(six.text_type(episodePath)) + store_subtitles(episodePath, six.text_type(episodePath)) list_missing_subtitles(sonarrSeriesId) redirect(ref) except OSError: @@ -2215,7 +2215,7 @@ def perform_manual_upload_subtitle(): score = 360 history_log(4, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) - store_subtitles(six.text_type(episodePath)) + store_subtitles(episodePath, six.text_type(episodePath)) list_missing_subtitles(sonarrSeriesId) redirect(ref) @@ -2252,7 +2252,7 @@ def get_subtitle_movie(): score = result[4] history_log_movie(1, radarrId, message, path, language_code, provider, score) send_notifications_movie(radarrId, message) - store_subtitles_movie(six.text_type(moviePath)) + store_subtitles_movie(moviePath, six.text_type(moviePath)) list_missing_subtitles_movies(radarrId) redirect(ref) except OSError: @@ -2308,7 +2308,7 @@ def manual_get_subtitle_movie(): score = result[4] history_log_movie(2, radarrId, message, path, language_code, provider, score) send_notifications_movie(radarrId, message) - store_subtitles_movie(six.text_type(moviePath)) + store_subtitles_movie(moviePath, six.text_type(moviePath)) list_missing_subtitles_movies(radarrId) redirect(ref) except OSError: @@ -2351,7 +2351,7 @@ def perform_manual_upload_subtitle_movie(): score = 120 history_log_movie(4, radarrId, message, path, language_code, provider, score) send_notifications_movie(radarrId, message) - store_subtitles_movie(six.text_type(moviePath)) + store_subtitles_movie(moviePath, six.text_type(moviePath)) list_missing_subtitles_movies(radarrId) redirect(ref)