From 8e5dbcd8f9ffcaf47209290fa22811b6b4fa5696 Mon Sep 17 00:00:00 2001 From: Abdulmohsen Date: Thu, 29 Apr 2021 02:37:46 +0300 Subject: [PATCH] Added caching of video files analyzing to prevent useless disk IO --- bazarr/database.py | 2 + bazarr/embedded_subs_reader.py | 26 ++++++---- bazarr/get_episodes.py | 65 +++++++++++++------------ bazarr/get_movies.py | 86 ++++++++++++++++------------------ bazarr/list_subtitles.py | 4 +- bazarr/utils.py | 48 ++++++++++++++++++- 6 files changed, 144 insertions(+), 87 deletions(-) diff --git a/bazarr/database.py b/bazarr/database.py index c98e37eab..f99b5ca2e 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -108,6 +108,7 @@ def db_upgrade(): ['table_episodes', 'audio_codec', 'text'], ['table_episodes', 'episode_file_id', 'integer'], ['table_episodes', 'audio_language', 'text'], + ['table_episodes', 'file_ffprobe', 'text'], ['table_movies', 'sortTitle', 'text'], ['table_movies', 'year', 'text'], ['table_movies', 'alternativeTitles', 'text'], @@ -120,6 +121,7 @@ def db_upgrade(): ['table_movies', 'movie_file_id', 'integer'], ['table_movies', 'tags', 'text', '[]'], ['table_movies', 'profileId', 'integer'], + ['table_movies', 'file_ffprobe', 'text'], ['table_history', 'video_path', 'text'], ['table_history', 'language', 'text'], ['table_history', 'provider', 'text'], diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index fda402d4a..2193b2bed 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -10,21 +10,28 @@ from knowit import api class EmbeddedSubsReader: def __init__(self): self.ffprobe = None - - def list_languages(self, file): - from utils import get_binary + self.cache = None + self.data = None + + def list_languages(self, file, original_path): + from utils import get_binary, cache_get_ffprobe, cache_save_ffprobe + self.cache = cache_get_ffprobe(original_path) + if self.cache['ffprobe'] is not None: + logging.debug('returning cached results for: %s', original_path) + return self.cache['ffprobe'] + self.ffprobe = get_binary("ffprobe") subtitles_list = [] if self.ffprobe: api.initialize({'provider': 'ffmpeg', 'ffmpeg': self.ffprobe}) - data = api.know(file) + self.data = api.know(file) traditional_chinese = ["cht", "tc", "traditional", "zht", "hant", "big5", u"繁", u"雙語"] brazilian_portuguese = ["pt-br", "pob", "pb", "brazilian", "brasil", "brazil"] - if 'subtitle' in data: - for detected_language in data['subtitle']: + if 'subtitle' in self.data: + for detected_language in self.data['subtitle']: if 'language' in detected_language: language = detected_language['language'].alpha3 if language == 'zho' and 'name' in detected_language: @@ -44,11 +51,11 @@ class EmbeddedSubsReader: if os.path.splitext(file)[1] == '.mkv': with open(file, 'rb') as f: try: - mkv = enzyme.MKV(f) + self.data = enzyme.MKV(f) except MalformedMKVError: logging.error('BAZARR cannot analyze this MKV with our built-in MKV parser, you should install ffmpeg: ' + file) else: - for subtitle_track in mkv.subtitle_tracks: + for subtitle_track in self.data.subtitle_tracks: hearing_impaired = False if subtitle_track.name: if 'sdh' in subtitle_track.name.lower(): @@ -56,6 +63,9 @@ class EmbeddedSubsReader: subtitles_list.append([subtitle_track.language, subtitle_track.forced, hearing_impaired, subtitle_track.codec_id]) + if subtitles_list is not None: + cache_save_ffprobe(original_path, self.cache['id'], self.cache['type'], subtitles_list) + return subtitles_list diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index 0840715a1..c7efc2d47 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -24,7 +24,7 @@ def sync_episodes(): apikey_sonarr = settings.sonarr.apikey # Get current episodes id in DB - current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId FROM table_episodes") + current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId, file_ffprobe FROM table_episodes") current_episodes_db_list = [x['sonarrEpisodeId'] for x in current_episodes_db] @@ -32,7 +32,14 @@ def sync_episodes(): episodes_to_update = [] episodes_to_add = [] altered_episodes = [] - + path_episodes = {} + + for item in current_episodes_db: + path_episodes[str(item['sonarrEpisodeId'])] = { + 'path': item['path'], + 'file_ffprobe': item['file_ffprobe'] + } + # Get sonarrId for each series from database seriesIdList = database.execute("SELECT sonarrSeriesId, title FROM table_shows") @@ -102,37 +109,33 @@ def sync_episodes(): # Add episodes in sonarr to current episode list current_episodes_sonarr.append(episode['id']) - + + info = { + 'sonarrSeriesId': episode['seriesId'], + 'sonarrEpisodeId': episode['id'], + 'title': episode['title'], + 'path': episode['episodeFile']['path'], + 'season': episode['seasonNumber'], + 'episode': episode['episodeNumber'], + 'scene_name': sceneName, + 'monitored': str(bool(episode['monitored'])), + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'episode_file_id': episode['episodeFile']['id'], + 'audio_language': str(audio_language), + 'file_ffprobe': None, + } + if episode['id'] in current_episodes_db_list: - episodes_to_update.append({'sonarrSeriesId': episode['seriesId'], - 'sonarrEpisodeId': episode['id'], - 'title': episode['title'], - 'path': episode['episodeFile']['path'], - 'season': episode['seasonNumber'], - 'episode': episode['episodeNumber'], - 'scene_name': sceneName, - 'monitored': str(bool(episode['monitored'])), - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'episode_file_id': episode['episodeFile']['id'], - 'audio_language': str(audio_language)}) + if info['path'] == path_episodes[str(info['sonarrEpisodeId'])]['path'] and path_episodes[str(info['sonarrEpisodeId'])]['file_ffprobe'] is not None: + logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['sonarrEpisodeId'])) + info['file_ffprobe'] = path_episodes[str(episode['id'])]['file_ffprobe'] + + episodes_to_update.append(info) else: - episodes_to_add.append({'sonarrSeriesId': episode['seriesId'], - 'sonarrEpisodeId': episode['id'], - 'title': episode['title'], - 'path': episode['episodeFile']['path'], - 'season': episode['seasonNumber'], - 'episode': episode['episodeNumber'], - 'scene_name': sceneName, - 'monitored': str(bool(episode['monitored'])), - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'episode_file_id': episode['episodeFile']['id'], - 'audio_language': str(audio_language)}) + episodes_to_add.append(info) # Remove old episodes from DB removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index 9ab7fd7d0..e9b329118 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -63,14 +63,21 @@ def update_movies(): return else: # Get current movies in DB - current_movies_db = database.execute("SELECT tmdbId, path, radarrId FROM table_movies") - + current_movies_db = database.execute("SELECT tmdbId, path, radarrId, file_ffprobe FROM table_movies") + current_movies_db_list = [x['tmdbId'] for x in current_movies_db] current_movies_radarr = [] movies_to_update = [] movies_to_add = [] altered_movies = [] + path_movies = {} + + for item in current_movies_db: + path_movies[str(item['tmdbId'])] = { + 'path': item['path'], + 'file_ffprobe': item['file_ffprobe'] + } moviesIdListLength = len(r.json()) for i, movie in enumerate(r.json(), 1): @@ -167,51 +174,40 @@ def update_movies(): # Add movies in radarr to current movies list current_movies_radarr.append(str(movie['tmdbId'])) - + + info = { + 'radarrId': int(movie['id']), + 'title': movie['title'], + 'path': movie['path'] + separator + movie['movieFile']['relativePath'], + 'tmdbId': str(movie['tmdbId']), + 'poster': poster, + 'fanart': fanart, + 'audio_language': str(audio_language), + 'sceneName': sceneName, + 'monitored': str(bool(movie['monitored'])), + 'year': str(movie['year']), + 'sortTitle': movie['sortTitle'], + 'alternativeTitles': alternativeTitles, + 'format': format, + 'resolution': resolution, + 'video_codec': videoCodec, + 'audio_codec': audioCodec, + 'overview': overview, + 'imdbId': imdbId, + 'movie_file_id': int(movie['movieFile']['id']), + 'tags': str(tags), + 'file_ffprobe': None, + } + if str(movie['tmdbId']) in current_movies_db_list: - movies_to_update.append({'radarrId': int(movie["id"]), - 'title': movie["title"], - 'path': movie["path"] + separator + movie['movieFile']['relativePath'], - 'tmdbId': str(movie["tmdbId"]), - 'poster': poster, - 'fanart': fanart, - 'audio_language': str(audio_language), - 'sceneName': sceneName, - 'monitored': str(bool(movie['monitored'])), - 'year': str(movie['year']), - 'sortTitle': movie['sortTitle'], - 'alternativeTitles': alternativeTitles, - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'overview': overview, - 'imdbId': imdbId, - 'movie_file_id': int(movie['movieFile']['id']), - 'tags': str(tags)}) + if info['path'] == path_movies[str(info['tmdbId'])]['path'] and path_movies[str(info['tmdbId'])]['file_ffprobe'] is not None: + logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['tmdbId'])) + info['file_ffprobe'] = path_movies[str(info['tmdbId'])]['file_ffprobe'] + + movies_to_update.append(info) else: - movies_to_add.append({'radarrId': int(movie["id"]), - 'title': movie["title"], - 'path': movie["path"] + separator + movie['movieFile']['relativePath'], - 'tmdbId': str(movie["tmdbId"]), - 'subtitles': '[]', - 'overview': overview, - 'poster': poster, - 'fanart': fanart, - 'audio_language': str(audio_language), - 'sceneName': sceneName, - 'monitored': str(bool(movie['monitored'])), - 'sortTitle': movie['sortTitle'], - 'year': str(movie['year']), - 'alternativeTitles': alternativeTitles, - 'format': format, - 'resolution': resolution, - 'video_codec': videoCodec, - 'audio_codec': audioCodec, - 'imdbId': imdbId, - 'movie_file_id': int(movie['movieFile']['id']), - 'tags': str(tags), - 'profileId': movie_default_profile}) + info['profileId'] = movie_default_profile + movies_to_add.append(info) else: logging.error( 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator + diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index ff0793173..9e9f8d32d 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -31,7 +31,7 @@ def store_subtitles(original_path, reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(reversed_path) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path) for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages: try: if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \ @@ -145,7 +145,7 @@ def store_subtitles_movie(original_path, reversed_path): if settings.general.getboolean('use_embedded_subs'): logging.debug("BAZARR is trying to index embedded subtitles.") try: - subtitle_languages = embedded_subs_reader.list_languages(reversed_path) + subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path) for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages: try: if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \ diff --git a/bazarr/utils.py b/bazarr/utils.py index ede3cc2d1..8518cf5c2 100644 --- a/bazarr/utils.py +++ b/bazarr/utils.py @@ -399,7 +399,53 @@ def translate_subtitles_file(video_path, source_srt_file, to_lang, forced, hi): return dest_srt_file + def check_credentials(user, pw): username = settings.auth.username password = settings.auth.password - return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username \ No newline at end of file + return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username + + +def cache_get_ffprobe(file): + record = {} + + item = database.execute("SELECT sonarrEpisodeId, file_ffprobe FROM table_episodes WHERE path = ?", + (file,), only_one=True) + + if item is not None: + record['id'] = item['sonarrEpisodeId'] + record['type'] = 'episode' + + if item['file_ffprobe']: + record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe']) + else: + record['ffprobe'] = None + + return record + + item = database.execute("SELECT tmdbId, file_ffprobe FROM table_movies WHERE path = ?", + (file,), only_one=True) + if item is not None: + record['id'] = item['tmdbId'] + record['type'] = 'movie' + + if item['file_ffprobe']: + record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe']) + else: + record['ffprobe'] = None + + return record + + return {'type': None, 'id': None, 'ffprobe': None} + + +def cache_save_ffprobe(file, record_id, record_type, ffprobe): + logging.debug('Saving ffprobe records [%s, %s, %s] data: %s', record_type, record_id, file, ffprobe) + + if record_type == 'movie': + database.execute("UPDATE table_movies SET file_ffprobe = ? WHERE path = ? AND tmdbId = ?", + (json.JSONEncoder().encode(ffprobe), file, record_id)) + + if record_type == 'episode': + database.execute("UPDATE table_episodes SET file_ffprobe = ? WHERE path = ? AND sonarrEpisodeId = ?", + (json.JSONEncoder().encode(ffprobe), file, record_id))