Added validation for incomplete indexing of subtitles or calculation of missing subtitles before searching.

bazarr/api/badges/badges.py

@@ -36,7 +36,7 @@ class Badges(Resource):
     def get(self):
         """Get badges count to update the UI"""
         episodes_conditions = [(TableEpisodes.missing_subtitles.is_not(None)),
-                               (TableEpisodes.missing_subtitles != '[]')]
+                               (TableEpisodes.missing_subtitles.is_not('[]'))]
         episodes_conditions += get_exclusion_clause('series')
         missing_episodes = database.execute(
             select(TableEpisodes.missing_subtitles)
@@ -49,7 +49,7 @@ class Badges(Resource):
             missing_episodes_count += len(ast.literal_eval(episode.missing_subtitles))
 
         movies_conditions = [(TableMovies.missing_subtitles.is_not(None)),
-                             (TableMovies.missing_subtitles != '[]')]
+                             (TableMovies.missing_subtitles.is_not('[]'))]
         movies_conditions += get_exclusion_clause('movie')
         missing_movies = database.execute(
             select(TableMovies.missing_subtitles)

bazarr/api/episodes/wanted.py

@@ -48,7 +48,8 @@ class EpisodesWanted(Resource):
         args = self.get_request_parser.parse_args()
         episodeid = args.get('episodeid[]')
 
-        wanted_conditions = [(TableEpisodes.missing_subtitles != '[]')]
+        wanted_conditions = [(TableEpisodes.missing_subtitles.is_not(None)),
+                               (TableEpisodes.missing_subtitles.is_not('[]'))]
         if len(episodeid) > 0:
             wanted_conditions.append((TableEpisodes.sonarrEpisodeId in episodeid))
             start = 0

bazarr/api/movies/wanted.py

@@ -45,7 +45,7 @@ class MoviesWanted(Resource):
         args = self.get_request_parser.parse_args()
         radarrid = args.get("radarrid[]")
 
-        wanted_conditions = [(TableMovies.missing_subtitles != '[]')]
+        wanted_conditions = [(TableMovies.missing_subtitles.is_not('[]'))]
         if len(radarrid) > 0:
             wanted_conditions.append((TableMovies.radarrId.in_(radarrid)))
             start = 0

bazarr/api/providers/providers_episodes.py

@@ -11,7 +11,7 @@ from subtitles.manual import manual_search, manual_download_subtitle
 from sonarr.history import history_log
 from app.config import settings
 from app.notifier import send_notifications
-from subtitles.indexer.series import store_subtitles
+from subtitles.indexer.series import store_subtitles, list_missing_subtitles
 from subtitles.processing import ProcessSubtitlesResult
 
 from ..utils import authenticate
@@ -50,18 +50,27 @@ class ProviderEpisodes(Resource):
         """Search manually for an episode subtitles"""
         args = self.get_request_parser.parse_args()
         sonarrEpisodeId = args.get('episodeid')
-        episodeInfo = database.execute(
-            select(TableEpisodes.path,
+        stmt = select(TableEpisodes.path,
                       TableEpisodes.sceneName,
                       TableShows.title,
-                   TableShows.profileId)
-            .select_from(TableEpisodes)
-            .join(TableShows)
-            .where(TableEpisodes.sonarrEpisodeId == sonarrEpisodeId)) \
-            .first()
+                      TableShows.profileId,
+                      TableEpisodes.subtitles,
+                      TableEpisodes.missing_subtitles) \
+            .select_from(TableEpisodes) \
+            .join(TableShows) \
+            .where(TableEpisodes.sonarrEpisodeId == sonarrEpisodeId)
+        episodeInfo = database.execute(stmt).first()
 
         if not episodeInfo:
             return 'Episode not found', 404
+        elif episodeInfo.subtitles is None:
+            # subtitles indexing for this episode is incomplete, we'll do it again
+            store_subtitles(episodeInfo.path, path_mappings.path_replace(episodeInfo.path))
+            episodeInfo = database.execute(stmt).first()
+        elif episodeInfo.missing_subtitles is None:
+            # missing subtitles calculation for this episode is incomplete, we'll do it again
+            list_missing_subtitles(epno=sonarrEpisodeId)
+            episodeInfo = database.execute(stmt).first()
 
         title = episodeInfo.title
         episodePath = path_mappings.path_replace(episodeInfo.path)

bazarr/api/providers/providers_movies.py

@@ -11,7 +11,7 @@ from subtitles.manual import manual_search, manual_download_subtitle
 from radarr.history import history_log_movie
 from app.config import settings
 from app.notifier import send_notifications_movie
-from subtitles.indexer.movies import store_subtitles_movie
+from subtitles.indexer.movies import store_subtitles_movie, list_missing_subtitles_movies
 from subtitles.processing import ProcessSubtitlesResult
 
 from ..utils import authenticate
@@ -51,16 +51,25 @@ class ProviderMovies(Resource):
         """Search manually for a movie subtitles"""
         args = self.get_request_parser.parse_args()
         radarrId = args.get('radarrid')
-        movieInfo = database.execute(
-            select(TableMovies.title,
+        stmt = select(TableMovies.title,
                       TableMovies.path,
                       TableMovies.sceneName,
-                   TableMovies.profileId)
-            .where(TableMovies.radarrId == radarrId)) \
-            .first()
+                      TableMovies.profileId,
+                      TableMovies.subtitles,
+                      TableMovies.missing_subtitles) \
+            .where(TableMovies.radarrId == radarrId)
+        movieInfo = database.execute(stmt).first()
 
         if not movieInfo:
             return 'Movie not found', 404
+        elif movieInfo.subtitles is None:
+            # subtitles indexing for this movie is incomplete, we'll do it again
+            store_subtitles_movie(movieInfo.path, path_mappings.path_replace_movie(movieInfo.path))
+            movieInfo = database.execute(stmt).first()
+        elif movieInfo.missing_subtitles is None:
+            # missing subtitles calculation for this movie is incomplete, we'll do it again
+            list_missing_subtitles_movies(no=radarrId)
+            movieInfo = database.execute(stmt).first()
 
         title = movieInfo.title
         moviePath = path_mappings.path_replace_movie(movieInfo.path)

bazarr/api/series/series.py

@@ -73,7 +73,7 @@ class Series(Resource):
             .group_by(TableShows.sonarrSeriesId)\
             .subquery()
 
-        episodes_missing_conditions = [(TableEpisodes.missing_subtitles != '[]')]
+        episodes_missing_conditions = [(TableEpisodes.missing_subtitles.is_not('[]'))]
         episodes_missing_conditions += get_exclusion_clause('series')
 
         episodeMissingCount = select(TableShows.sonarrSeriesId,

bazarr/subtitles/mass_download/movies.py

@@ -9,7 +9,7 @@ import os
 from functools import reduce
 
 from utilities.path_mappings import path_mappings
-from subtitles.indexer.movies import store_subtitles_movie
+from subtitles.indexer.movies import store_subtitles_movie, list_missing_subtitles_movies
 from radarr.history import history_log_movie
 from app.notifier import send_notifications_movie
 from app.get_providers import get_providers
@@ -20,10 +20,9 @@ from ..download import generate_subtitles
 
 
 def movies_download_subtitles(no):
-    conditions = [(TableMovies.radarrId == no)]
+    conditions = [(TableMovies.radarrId.is_(no))]
     conditions += get_exclusion_clause('movie')
-    movie = database.execute(
-        select(TableMovies.path,
+    stmt = select(TableMovies.path,
                   TableMovies.missing_subtitles,
                   TableMovies.audio_language,
                   TableMovies.radarrId,
@@ -31,12 +30,22 @@ def movies_download_subtitles(no):
                   TableMovies.title,
                   TableMovies.tags,
                   TableMovies.monitored,
-               TableMovies.profileId)
-        .where(reduce(operator.and_, conditions))) \
-        .first()
+                  TableMovies.profileId,
+                  TableMovies.subtitles) \
+        .where(reduce(operator.and_, conditions))
+    movie = database.execute(stmt).first()
+
     if not movie:
-        logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no))
+        logging.debug(f"BAZARR no movie with that radarrId can be found in database: {no}")
         return
+    elif movie.subtitles is None:
+        # subtitles indexing for this movie is incomplete, we'll do it again
+        store_subtitles_movie(movie.path, path_mappings.path_replace_movie(movie.path))
+        movie = database.execute(stmt).first()
+    elif movie.missing_subtitles is None:
+        # missing subtitles calculation for this movie is incomplete, we'll do it again
+        list_missing_subtitles_movies(no=no)
+        movie = database.execute(stmt).first()
 
     moviePath = path_mappings.path_replace_movie(movie.path)
 

bazarr/subtitles/mass_download/series.py

@@ -9,7 +9,7 @@ import os
 from functools import reduce
 
 from utilities.path_mappings import path_mappings
-from subtitles.indexer.series import store_subtitles
+from subtitles.indexer.series import store_subtitles, list_missing_subtitles
 from sonarr.history import history_log
 from app.notifier import send_notifications
 from app.get_providers import get_providers
@@ -29,22 +29,15 @@ def series_download_subtitles(no):
         raise OSError
 
     conditions = [(TableEpisodes.sonarrSeriesId == no),
-                  (TableEpisodes.missing_subtitles != '[]')]
+                  (TableEpisodes.missing_subtitles.is_not('[]'))]
     conditions += get_exclusion_clause('series')
     episodes_details = database.execute(
-        select(TableEpisodes.path,
-               TableEpisodes.missing_subtitles,
-               TableEpisodes.monitored,
-               TableEpisodes.sonarrEpisodeId,
-               TableEpisodes.sceneName,
-               TableShows.tags,
-               TableShows.seriesType,
-               TableEpisodes.audio_language,
+        select(TableEpisodes.sonarrEpisodeId,
                TableShows.title,
                TableEpisodes.season,
                TableEpisodes.episode,
                TableEpisodes.title.label('episodeTitle'),
-               TableShows.profileId)
+               TableEpisodes.missing_subtitles)
         .select_from(TableEpisodes)
         .join(TableShows)
         .where(reduce(operator.and_, conditions))) \
@@ -66,36 +59,7 @@ def series_download_subtitles(no):
                           value=i,
                           count=count_episodes_details)
 
-            audio_language_list = get_audio_profile_languages(episode.audio_language)
-            if len(audio_language_list) > 0:
-                audio_language = audio_language_list[0]['name']
-            else:
-                audio_language = 'None'
-
-            languages = []
-            for language in ast.literal_eval(episode.missing_subtitles):
-                if language is not None:
-                    hi_ = "True" if language.endswith(':hi') else "False"
-                    forced_ = "True" if language.endswith(':forced') else "False"
-                    languages.append((language.split(":")[0], hi_, forced_))
-
-            if not languages:
-                continue
-
-            for result in generate_subtitles(path_mappings.path_replace(episode.path),
-                                             languages,
-                                             audio_language,
-                                             str(episode.sceneName),
-                                             episode.title,
-                                             'series',
-                                             episode.profileId,
-                                             check_if_still_required=True):
-                if result:
-                    if isinstance(result, tuple) and len(result):
-                        result = result[0]
-                    store_subtitles(episode.path, path_mappings.path_replace(episode.path))
-                    history_log(1, no, episode.sonarrEpisodeId, result)
-                    send_notifications(no, episode.sonarrEpisodeId, result.message)
+            episode_download_subtitles(no=episode.sonarrEpisodeId, send_progress=False, providers_list=providers_list)
         else:
             logging.info("BAZARR All providers are throttled")
             break
@@ -103,11 +67,10 @@ def series_download_subtitles(no):
     hide_progress(id=f'series_search_progress_{no}')
 
 
-def episode_download_subtitles(no, send_progress=False):
+def episode_download_subtitles(no, send_progress=False, providers_list=None):
     conditions = [(TableEpisodes.sonarrEpisodeId == no)]
     conditions += get_exclusion_clause('series')
-    episodes_details = database.execute(
-        select(TableEpisodes.path,
+    stmt = select(TableEpisodes.path,
                   TableEpisodes.missing_subtitles,
                   TableEpisodes.monitored,
                   TableEpisodes.sonarrEpisodeId,
@@ -120,16 +83,26 @@ def episode_download_subtitles(no, send_progress=False):
                   TableEpisodes.title.label('episodeTitle'),
                   TableEpisodes.season,
                   TableEpisodes.episode,
-               TableShows.profileId)
-        .select_from(TableEpisodes)
-        .join(TableShows)
-        .where(reduce(operator.and_, conditions))) \
-        .all()
-    if not episodes_details:
+                  TableShows.profileId,
+                  TableEpisodes.subtitles) \
+        .select_from(TableEpisodes) \
+        .join(TableShows) \
+        .where(reduce(operator.and_, conditions))
+    episode = database.execute(stmt).first()
+
+    if not episode:
         logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no))
         return
-
-    for episode in episodes_details:
+    elif episode.subtitles is None:
+        # subtitles indexing for this episode is incomplete, we'll do it again
+        store_subtitles(episode.path, path_mappings.path_replace_movie(episode.path))
+        episode = database.execute(stmt).first()
+    elif episode.missing_subtitles is None:
+        # missing subtitles calculation for this episode is incomplete, we'll do it again
+        list_missing_subtitles(epno=no)
+        episode = database.execute(stmt).first()
+
+    if not providers_list:
         providers_list = get_providers()
 
     if providers_list:
@@ -154,7 +127,7 @@ def episode_download_subtitles(no, send_progress=False):
                 languages.append((language.split(":")[0], hi_, forced_))
 
         if not languages:
-                continue
+            return
 
         for result in generate_subtitles(path_mappings.path_replace(episode.path),
                                          languages,
@@ -175,4 +148,3 @@ def episode_download_subtitles(no, send_progress=False):
             hide_progress(id=f'episode_search_progress_{no}')
     else:
         logging.info("BAZARR All providers are throttled")
-            break

bazarr/subtitles/wanted/movies.py

@@ -8,7 +8,7 @@ import operator
 from functools import reduce
 
 from utilities.path_mappings import path_mappings
-from subtitles.indexer.movies import store_subtitles_movie
+from subtitles.indexer.movies import store_subtitles_movie, list_missing_subtitles_movies
 from radarr.history import history_log_movie
 from app.notifier import send_notifications_movie
 from app.get_providers import get_providers
@@ -63,30 +63,40 @@ def _wanted_movie(movie):
 
 
 def wanted_download_subtitles_movie(radarr_id):
-    movies_details = database.execute(
-        select(TableMovies.path,
+    stmt = select(TableMovies.path,
                   TableMovies.missing_subtitles,
                   TableMovies.radarrId,
                   TableMovies.audio_language,
                   TableMovies.sceneName,
                   TableMovies.failedAttempts,
                   TableMovies.title,
-               TableMovies.profileId)
-        .where(TableMovies.radarrId == radarr_id)) \
-        .all()
+                  TableMovies.profileId,
+                  TableMovies.subtitles) \
+        .where(TableMovies.radarrId == radarr_id)
+    movie = database.execute(stmt).first()
+
+    if not movie:
+        logging.debug(f"BAZARR no movie with that radarrId can be found in database: {radarr_id}")
+        return
+    elif movie.subtitles is None:
+        # subtitles indexing for this movie is incomplete, we'll do it again
+        store_subtitles_movie(movie.path, path_mappings.path_replace_movie(movie.path))
+        movie = database.execute(stmt).first()
+    elif movie.missing_subtitles is None:
+        # missing subtitles calculation for this movie is incomplete, we'll do it again
+        list_missing_subtitles_movies(no=radarr_id)
+        movie = database.execute(stmt).first()
 
-    for movie in movies_details:
     providers_list = get_providers()
 
     if providers_list:
         _wanted_movie(movie)
     else:
         logging.info("BAZARR All providers are throttled")
-            break
 
 
 def wanted_search_missing_subtitles_movies():
-    conditions = [(TableMovies.missing_subtitles != '[]')]
+    conditions = [(TableMovies.missing_subtitles.is_not('[]'))]
     conditions += get_exclusion_clause('movie')
     movies = database.execute(
         select(TableMovies.radarrId,

bazarr/subtitles/wanted/series.py

@@ -8,6 +8,7 @@ import operator
 from functools import reduce
 
 from utilities.path_mappings import path_mappings
+from subtitles.indexer.series import store_subtitles, list_missing_subtitles
 from subtitles.indexer.series import store_subtitles
 from sonarr.history import history_log
 from app.notifier import send_notifications
@@ -64,8 +65,7 @@ def _wanted_episode(episode):
 
 
 def wanted_download_subtitles(sonarr_episode_id):
-    episodes_details = database.execute(
-        select(TableEpisodes.path,
+    stmt = select(TableEpisodes.path,
                   TableEpisodes.missing_subtitles,
                   TableEpisodes.sonarrEpisodeId,
                   TableEpisodes.sonarrSeriesId,
@@ -73,24 +73,35 @@ def wanted_download_subtitles(sonarr_episode_id):
                   TableEpisodes.sceneName,
                   TableEpisodes.failedAttempts,
                   TableShows.title,
-               TableShows.profileId)
-        .select_from(TableEpisodes)
-        .join(TableShows)
-        .where((TableEpisodes.sonarrEpisodeId == sonarr_episode_id))) \
-        .all()
+                  TableShows.profileId,
+                  TableEpisodes.subtitles) \
+        .select_from(TableEpisodes) \
+        .join(TableShows) \
+        .where((TableEpisodes.sonarrEpisodeId == sonarr_episode_id))
+    episode_details = database.execute(stmt).first()
+
+    if not episode_details:
+        logging.debug(f"BAZARR no episode with that sonarrId can be found in database: {sonarr_episode_id}")
+        return
+    elif episode_details.subtitles is None:
+        # subtitles indexing for this episode is incomplete, we'll do it again
+        store_subtitles(episode_details.path, path_mappings.path_replace(episode_details.path))
+        episode_details = database.execute(stmt).first()
+    elif episode_details.missing_subtitles is None:
+        # missing subtitles calculation for this episode is incomplete, we'll do it again
+        list_missing_subtitles(epno=sonarr_episode_id)
+        episode_details = database.execute(stmt).first()
 
-    for episode in episodes_details:
     providers_list = get_providers()
 
     if providers_list:
-            _wanted_episode(episode)
+        _wanted_episode(episode_details)
     else:
         logging.info("BAZARR All providers are throttled")
-            break
 
 
 def wanted_search_missing_subtitles_series():
-    conditions = [(TableEpisodes.missing_subtitles != '[]')]
+    conditions = [(TableEpisodes.missing_subtitles.is_not('[]'))]
     conditions += get_exclusion_clause('series')
     episodes = database.execute(
         select(TableEpisodes.sonarrSeriesId,