Added caching of video files analyzing to prevent useless disk IO

bazarr/database.py

@@ -108,6 +108,7 @@ def db_upgrade():
         ['table_episodes', 'audio_codec', 'text'],
         ['table_episodes', 'episode_file_id', 'integer'],
         ['table_episodes', 'audio_language', 'text'],
+        ['table_episodes', 'file_ffprobe', 'text'],
         ['table_movies', 'sortTitle', 'text'],
         ['table_movies', 'year', 'text'],
         ['table_movies', 'alternativeTitles', 'text'],
@@ -120,6 +121,7 @@ def db_upgrade():
         ['table_movies', 'movie_file_id', 'integer'],
         ['table_movies', 'tags', 'text', '[]'],
         ['table_movies', 'profileId', 'integer'],
+        ['table_movies', 'file_ffprobe', 'text'],
         ['table_history', 'video_path', 'text'],
         ['table_history', 'language', 'text'],
         ['table_history', 'provider', 'text'],

bazarr/embedded_subs_reader.py

@@ -10,21 +10,28 @@ from knowit import api
 class EmbeddedSubsReader:
     def __init__(self):
         self.ffprobe = None
-    
-    def list_languages(self, file):
-        from utils import get_binary
+        self.cache = None
+        self.data = None
+
+    def list_languages(self, file, original_path):
+        from utils import get_binary, cache_get_ffprobe, cache_save_ffprobe
+        self.cache = cache_get_ffprobe(original_path)
+        if self.cache['ffprobe'] is not None:
+            logging.debug('returning cached results for: %s', original_path)
+            return self.cache['ffprobe']
+
         self.ffprobe = get_binary("ffprobe")
 
         subtitles_list = []
         if self.ffprobe:
             api.initialize({'provider': 'ffmpeg', 'ffmpeg': self.ffprobe})
-            data = api.know(file)
+            self.data = api.know(file)
 
             traditional_chinese = ["cht", "tc", "traditional", "zht", "hant", "big5", u"繁", u"雙語"]
             brazilian_portuguese = ["pt-br", "pob", "pb", "brazilian", "brasil", "brazil"]
 
-            if 'subtitle' in data:
-                for detected_language in data['subtitle']:
+            if 'subtitle' in self.data:
+                for detected_language in self.data['subtitle']:
                     if 'language' in detected_language:
                         language = detected_language['language'].alpha3
                         if language == 'zho' and 'name' in detected_language:
@@ -44,11 +51,11 @@ class EmbeddedSubsReader:
             if os.path.splitext(file)[1] == '.mkv':
                 with open(file, 'rb') as f:
                     try:
-                        mkv = enzyme.MKV(f)
+                        self.data = enzyme.MKV(f)
                     except MalformedMKVError:
                         logging.error('BAZARR cannot analyze this MKV with our built-in MKV parser, you should install ffmpeg: ' + file)
                     else:
-                        for subtitle_track in mkv.subtitle_tracks:
+                        for subtitle_track in self.data.subtitle_tracks:
                             hearing_impaired = False
                             if subtitle_track.name:
                                 if 'sdh' in subtitle_track.name.lower():
@@ -56,6 +63,9 @@ class EmbeddedSubsReader:
                             subtitles_list.append([subtitle_track.language, subtitle_track.forced, hearing_impaired,
                                                    subtitle_track.codec_id])
 
+        if subtitles_list is not None:
+            cache_save_ffprobe(original_path, self.cache['id'], self.cache['type'], subtitles_list)
+
         return subtitles_list
 
 

bazarr/get_episodes.py

@@ -24,7 +24,7 @@ def sync_episodes():
     apikey_sonarr = settings.sonarr.apikey
     
     # Get current episodes id in DB
-    current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId FROM table_episodes")
+    current_episodes_db = database.execute("SELECT sonarrEpisodeId, path, sonarrSeriesId, file_ffprobe FROM table_episodes")
 
     current_episodes_db_list = [x['sonarrEpisodeId'] for x in current_episodes_db]
 
@@ -32,7 +32,14 @@ def sync_episodes():
     episodes_to_update = []
     episodes_to_add = []
     altered_episodes = []
-    
+    path_episodes = {}
+
+    for item in current_episodes_db:
+        path_episodes[str(item['sonarrEpisodeId'])] = {
+            'path': item['path'],
+            'file_ffprobe': item['file_ffprobe']
+        }
+
     # Get sonarrId for each series from database
     seriesIdList = database.execute("SELECT sonarrSeriesId, title FROM table_shows")
     
@@ -102,37 +109,33 @@ def sync_episodes():
 
                                 # Add episodes in sonarr to current episode list
                                 current_episodes_sonarr.append(episode['id'])
-                                
+
+                                info = {
+                                    'sonarrSeriesId': episode['seriesId'],
+                                    'sonarrEpisodeId': episode['id'],
+                                    'title': episode['title'],
+                                    'path': episode['episodeFile']['path'],
+                                    'season': episode['seasonNumber'],
+                                    'episode': episode['episodeNumber'],
+                                    'scene_name': sceneName,
+                                    'monitored': str(bool(episode['monitored'])),
+                                    'format': format,
+                                    'resolution': resolution,
+                                    'video_codec': videoCodec,
+                                    'audio_codec': audioCodec,
+                                    'episode_file_id': episode['episodeFile']['id'],
+                                    'audio_language': str(audio_language),
+                                    'file_ffprobe': None,
+                                }
+
                                 if episode['id'] in current_episodes_db_list:
-                                    episodes_to_update.append({'sonarrSeriesId': episode['seriesId'],
-                                                               'sonarrEpisodeId': episode['id'],
-                                                               'title': episode['title'],
-                                                               'path': episode['episodeFile']['path'],
-                                                               'season': episode['seasonNumber'],
-                                                               'episode': episode['episodeNumber'],
-                                                               'scene_name': sceneName,
-                                                               'monitored': str(bool(episode['monitored'])),
-                                                               'format': format,
-                                                               'resolution': resolution,
-                                                               'video_codec': videoCodec,
-                                                               'audio_codec': audioCodec,
-                                                               'episode_file_id': episode['episodeFile']['id'],
-                                                               'audio_language': str(audio_language)})
+                                    if info['path'] == path_episodes[str(info['sonarrEpisodeId'])]['path'] and path_episodes[str(info['sonarrEpisodeId'])]['file_ffprobe'] is not None:
+                                        logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['sonarrEpisodeId']))
+                                        info['file_ffprobe'] = path_episodes[str(episode['id'])]['file_ffprobe']
+
+                                    episodes_to_update.append(info)
                                 else:
-                                    episodes_to_add.append({'sonarrSeriesId': episode['seriesId'],
-                                                            'sonarrEpisodeId': episode['id'],
-                                                            'title': episode['title'],
-                                                            'path': episode['episodeFile']['path'],
-                                                            'season': episode['seasonNumber'],
-                                                            'episode': episode['episodeNumber'],
-                                                            'scene_name': sceneName,
-                                                            'monitored': str(bool(episode['monitored'])),
-                                                            'format': format,
-                                                            'resolution': resolution,
-                                                            'video_codec': videoCodec,
-                                                            'audio_codec': audioCodec,
-                                                            'episode_file_id': episode['episodeFile']['id'],
-                                                            'audio_language': str(audio_language)})
+                                    episodes_to_add.append(info)
 
     # Remove old episodes from DB
     removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr))

bazarr/get_movies.py

@@ -63,14 +63,21 @@ def update_movies():
             return
         else:
             # Get current movies in DB
-            current_movies_db = database.execute("SELECT tmdbId, path, radarrId FROM table_movies")
-            
+            current_movies_db = database.execute("SELECT tmdbId, path, radarrId, file_ffprobe FROM table_movies")
+
             current_movies_db_list = [x['tmdbId'] for x in current_movies_db]
 
             current_movies_radarr = []
             movies_to_update = []
             movies_to_add = []
             altered_movies = []
+            path_movies = {}
+
+            for item in current_movies_db:
+                path_movies[str(item['tmdbId'])] = {
+                    'path': item['path'],
+                    'file_ffprobe': item['file_ffprobe']
+                }
 
             moviesIdListLength = len(r.json())
             for i, movie in enumerate(r.json(), 1):
@@ -167,51 +174,40 @@ def update_movies():
 
                             # Add movies in radarr to current movies list
                             current_movies_radarr.append(str(movie['tmdbId']))
-                            
+
+                            info = {
+                                'radarrId': int(movie['id']),
+                                'title': movie['title'],
+                                'path': movie['path'] + separator + movie['movieFile']['relativePath'],
+                                'tmdbId': str(movie['tmdbId']),
+                                'poster': poster,
+                                'fanart': fanart,
+                                'audio_language': str(audio_language),
+                                'sceneName': sceneName,
+                                'monitored': str(bool(movie['monitored'])),
+                                'year': str(movie['year']),
+                                'sortTitle': movie['sortTitle'],
+                                'alternativeTitles': alternativeTitles,
+                                'format': format,
+                                'resolution': resolution,
+                                'video_codec': videoCodec,
+                                'audio_codec': audioCodec,
+                                'overview': overview,
+                                'imdbId': imdbId,
+                                'movie_file_id': int(movie['movieFile']['id']),
+                                'tags': str(tags),
+                                'file_ffprobe': None,
+                            }
+
                             if str(movie['tmdbId']) in current_movies_db_list:
-                                movies_to_update.append({'radarrId': int(movie["id"]),
-                                                         'title': movie["title"],
-                                                         'path': movie["path"] + separator + movie['movieFile']['relativePath'],
-                                                         'tmdbId': str(movie["tmdbId"]),
-                                                         'poster': poster,
-                                                         'fanart': fanart,
-                                                         'audio_language': str(audio_language),
-                                                         'sceneName': sceneName,
-                                                         'monitored': str(bool(movie['monitored'])),
-                                                         'year': str(movie['year']),
-                                                         'sortTitle': movie['sortTitle'],
-                                                         'alternativeTitles': alternativeTitles,
-                                                         'format': format,
-                                                         'resolution': resolution,
-                                                         'video_codec': videoCodec,
-                                                         'audio_codec': audioCodec,
-                                                         'overview': overview,
-                                                         'imdbId': imdbId,
-                                                         'movie_file_id': int(movie['movieFile']['id']),
-                                                         'tags': str(tags)})
+                                if info['path'] == path_movies[str(info['tmdbId'])]['path'] and path_movies[str(info['tmdbId'])]['file_ffprobe'] is not None:
+                                    logging.debug('path is the same. Preserving ffprobe old results for: %s.', str(info['tmdbId']))
+                                    info['file_ffprobe'] = path_movies[str(info['tmdbId'])]['file_ffprobe']
+
+                                movies_to_update.append(info)
                             else:
-                                movies_to_add.append({'radarrId': int(movie["id"]),
-                                                      'title': movie["title"],
-                                                      'path': movie["path"] + separator + movie['movieFile']['relativePath'],
-                                                      'tmdbId': str(movie["tmdbId"]),
-                                                      'subtitles': '[]',
-                                                      'overview': overview,
-                                                      'poster': poster,
-                                                      'fanart': fanart,
-                                                      'audio_language': str(audio_language),
-                                                      'sceneName': sceneName,
-                                                      'monitored': str(bool(movie['monitored'])),
-                                                      'sortTitle': movie['sortTitle'],
-                                                      'year': str(movie['year']),
-                                                      'alternativeTitles': alternativeTitles,
-                                                      'format': format,
-                                                      'resolution': resolution,
-                                                      'video_codec': videoCodec,
-                                                      'audio_codec': audioCodec,
-                                                      'imdbId': imdbId,
-                                                      'movie_file_id': int(movie['movieFile']['id']),
-                                                      'tags': str(tags),
-                                                      'profileId': movie_default_profile})
+                                info['profileId'] = movie_default_profile
+                                movies_to_add.append(info)
                         else:
                             logging.error(
                                 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator +

bazarr/list_subtitles.py

@@ -31,7 +31,7 @@ def store_subtitles(original_path, reversed_path):
         if settings.general.getboolean('use_embedded_subs'):
             logging.debug("BAZARR is trying to index embedded subtitles.")
             try:
-                subtitle_languages = embedded_subs_reader.list_languages(reversed_path)
+                subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path)
                 for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages:
                     try:
                         if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \
@@ -145,7 +145,7 @@ def store_subtitles_movie(original_path, reversed_path):
         if settings.general.getboolean('use_embedded_subs'):
             logging.debug("BAZARR is trying to index embedded subtitles.")
             try:
-                subtitle_languages = embedded_subs_reader.list_languages(reversed_path)
+                subtitle_languages = embedded_subs_reader.list_languages(reversed_path, original_path)
                 for subtitle_language, subtitle_forced, subtitle_hi, subtitle_codec in subtitle_languages:
                     try:
                         if (settings.general.getboolean("ignore_pgs_subs") and subtitle_codec.lower() == "pgs") or \

bazarr/utils.py

@@ -399,7 +399,53 @@ def translate_subtitles_file(video_path, source_srt_file, to_lang, forced, hi):
 
     return dest_srt_file
 
+
 def check_credentials(user, pw):
     username = settings.auth.username
     password = settings.auth.password
-    return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username
+    return hashlib.md5(pw.encode('utf-8')).hexdigest() == password and user == username
+
+
+def cache_get_ffprobe(file):
+    record = {}
+
+    item = database.execute("SELECT sonarrEpisodeId, file_ffprobe FROM table_episodes WHERE path = ?",
+                            (file,), only_one=True)
+
+    if item is not None:
+        record['id'] = item['sonarrEpisodeId']
+        record['type'] = 'episode'
+
+        if item['file_ffprobe']:
+            record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe'])
+        else:
+            record['ffprobe'] = None
+
+        return record
+
+    item = database.execute("SELECT tmdbId, file_ffprobe FROM table_movies WHERE path = ?",
+                            (file,), only_one=True)
+    if item is not None:
+        record['id'] = item['tmdbId']
+        record['type'] = 'movie'
+
+        if item['file_ffprobe']:
+            record['ffprobe'] = json.JSONDecoder().decode(item['file_ffprobe'])
+        else:
+            record['ffprobe'] = None
+
+        return record
+
+    return {'type': None, 'id': None, 'ffprobe': None}
+
+
+def cache_save_ffprobe(file, record_id, record_type, ffprobe):
+    logging.debug('Saving ffprobe records [%s, %s, %s] data: %s', record_type, record_id, file, ffprobe)
+
+    if record_type == 'movie':
+        database.execute("UPDATE table_movies SET file_ffprobe = ? WHERE path = ? AND tmdbId = ?",
+                         (json.JSONEncoder().encode(ffprobe), file, record_id))
+
+    if record_type == 'episode':
+        database.execute("UPDATE table_episodes SET file_ffprobe = ? WHERE path = ? AND sonarrEpisodeId = ?",
+                         (json.JSONEncoder().encode(ffprobe), file, record_id))