From 3b96b6b6eae842473fbe9b34f2e6c2e8e5ae3245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sun, 19 Jul 2020 16:02:38 -0400 Subject: [PATCH] Added subtitles blacklisting. --- bazarr/api.py | 411 +++++++++++++++++++++++++++------ bazarr/database.py | 10 +- bazarr/embedded_subs_reader.py | 7 +- bazarr/get_subtitle.py | 48 ++-- bazarr/main.py | 12 + bazarr/subsyncer.py | 4 +- bazarr/utils.py | 97 +++++++- views/_main.html | 11 + views/blacklistmovies.html | 107 +++++++++ views/blacklistseries.html | 111 +++++++++ views/episodes.html | 63 ++++- views/historymovies.html | 37 ++- views/historyseries.html | 38 ++- views/movie.html | 70 +++++- 14 files changed, 920 insertions(+), 106 deletions(-) create mode 100644 views/blacklistmovies.html create mode 100644 views/blacklistseries.html diff --git a/bazarr/api.py b/bazarr/api.py index 7581182ea..97add7a3f 100644 --- a/bazarr/api.py +++ b/bazarr/api.py @@ -24,11 +24,13 @@ from get_languages import language_from_alpha3, language_from_alpha2, alpha2_fro alpha3_from_language, alpha3_from_alpha2 from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ manual_search, manual_download_subtitle, manual_upload_subtitle, wanted_search_missing_subtitles_series, \ - wanted_search_missing_subtitles_movies + wanted_search_missing_subtitles_movies, episode_download_subtitles, movies_download_subtitles from notifier import send_notifications, send_notifications_movie from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies -from utils import history_log, history_log_movie, get_sonarr_version, get_radarr_version +from utils import history_log, history_log_movie, blacklist_log, blacklist_delete, blacklist_delete_all, \ + blacklist_log_movie, blacklist_delete_movie, blacklist_delete_all_movie, get_sonarr_version, get_radarr_version, \ + delete_subtitles from get_providers import get_providers, get_providers_auth, list_throttled_providers, reset_throttled_providers from event_handler import event_stream from scheduler import scheduler @@ -471,22 +473,22 @@ class EpisodesSubtitlesDelete(Resource): def delete(self): episodePath = request.form.get('episodePath') language = request.form.get('language') + forced = request.form.get('forced') subtitlesPath = request.form.get('subtitlesPath') sonarrSeriesId = request.form.get('sonarrSeriesId') sonarrEpisodeId = request.form.get('sonarrEpisodeId') - try: - os.remove(path_mappings.path_replace(subtitlesPath)) - result = language_from_alpha3(language) + " subtitles deleted from disk." - history_log(0, sonarrSeriesId, sonarrEpisodeId, result, language=alpha2_from_alpha3(language), - video_path=path_mappings.path_replace_reverse(episodePath)) - store_subtitles(path_mappings.path_replace_reverse(episodePath), episodePath) - return result, 202 - except OSError as e: - logging.exception('BAZARR cannot delete subtitles file: ' + subtitlesPath) - - store_subtitles(path_mappings.path_replace_reverse(episodePath), episodePath) - return '', 204 + result = delete_subtitles(media_type='series', + language=language, + forced=forced, + media_path=episodePath, + subtitles_path=subtitlesPath, + sonarr_series_id=sonarrSeriesId, + sonarr_episode_id=sonarrEpisodeId) + if result: + return '', 202 + else: + return '', 204 class EpisodesSubtitlesDownload(Resource): @@ -518,7 +520,8 @@ class EpisodesSubtitlesDownload(Resource): provider = result[3] score = result[4] subs_id = result[6] - history_log(1, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score, subs_id) + subs_path = result[7] + history_log(1, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score, subs_id, subs_path) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) store_subtitles(path, episodePath) else: @@ -586,7 +589,8 @@ class EpisodesSubtitlesManualDownload(Resource): provider = result[3] score = result[4] subs_id = result[6] - history_log(2, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score, subs_id) + subs_path = result[7] + history_log(2, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score, subs_id, subs_path) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) store_subtitles(path, episodePath) return result, 201 @@ -629,10 +633,11 @@ class EpisodesSubtitlesUpload(Resource): if result is not None: message = result[0] path = result[1] + subs_path = result[2] language_code = language + ":forced" if forced else language provider = "manual" score = 360 - history_log(4, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score) + history_log(4, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score, subtitles_path=subs_path) send_notifications(sonarrSeriesId, sonarrEpisodeId, message) store_subtitles(path, episodePath) @@ -664,7 +669,8 @@ class EpisodesHistory(Resource): def get(self): episodeid = request.args.get('episodeid') - episode_history = database.execute("SELECT action, timestamp, language, provider, score FROM table_history " + episode_history = database.execute("SELECT action, timestamp, language, provider, score, sonarrSeriesId, " + "sonarrEpisodeId, subs_id, video_path, subtitles_path FROM table_history " "WHERE sonarrEpisodeId=? ORDER BY timestamp DESC", (episodeid,)) for item in episode_history: item['timestamp'] = "
" + \ pretty.date(datetime.datetime.fromtimestamp(item['timestamp'])) + "
" if item['language']: - item['language'] = language_from_alpha2(item['language']) - else: - item['language'] = "undefined" + language = item['language'].split(':') + item['language'] = {"name": language_from_alpha2(language[0]), + "code2": language[0], + "code3": alpha3_from_alpha2(language[0]), + "forced": True if len(language) > 1 else False} if item['score']: item['score'] = str(round((int(item['score']) * 100 / 360), 2)) + "%" + if item['video_path']: + # Provide mapped path + mapped_path = path_mappings.path_replace(item['video_path']) + item.update({"mapped_path": mapped_path}) + + # Confirm if path exist + item.update({"exist": os.path.isfile(mapped_path)}) + else: + item.update({"mapped_path": None}) + item.update({"exist": False}) + + if item['subtitles_path']: + # Provide mapped subtitles path + mapped_subtitles_path = path_mappings.path_replace_movie(item['subtitles_path']) + item.update({"mapped_subtitles_path": mapped_subtitles_path}) + else: + item.update({"mapped_subtitles_path": None}) + + # Check if subtitles is blacklisted + if item['action'] not in [0, 4, 5]: + blacklist_db = database.execute( + "SELECT provider, subs_id FROM table_blacklist WHERE provider=? AND " + "subs_id=?", (item['provider'], item['subs_id'])) + else: + blacklist_db = [] + + if len(blacklist_db): + item.update({"blacklisted": True}) + else: + item.update({"blacklisted": False}) + return jsonify(data=episode_history) @@ -875,21 +914,20 @@ class MovieSubtitlesDelete(Resource): def delete(self): moviePath = request.form.get('moviePath') language = request.form.get('language') + forced = request.form.get('forced') subtitlesPath = request.form.get('subtitlesPath') radarrId = request.form.get('radarrId') - try: - os.remove(path_mappings.path_replace_movie(subtitlesPath)) - result = language_from_alpha3(language) + " subtitles deleted from disk." - history_log_movie(0, radarrId, result, language=alpha2_from_alpha3(language), - video_path=path_mappings.path_replace_reverse_movie(moviePath)) - store_subtitles_movie(path_mappings.path_replace_reverse_movie(moviePath), moviePath) - return result, 202 - except OSError as e: - logging.exception('BAZARR cannot delete subtitles file: ' + subtitlesPath) - - store_subtitles_movie(path_mappings.path_replace_reverse_movie(moviePath), moviePath) - return '', 204 + result = delete_subtitles(media_type='movie', + language=language, + forced=forced, + media_path=moviePath, + subtitles_path=subtitlesPath, + radarr_id=radarrId) + if result: + return '', 202 + else: + return '', 204 class MovieSubtitlesDownload(Resource): @@ -920,7 +958,8 @@ class MovieSubtitlesDownload(Resource): provider = result[3] score = result[4] subs_id = result[6] - history_log_movie(1, radarrId, message, path, language_code, provider, score, subs_id) + subs_path = result[7] + history_log_movie(1, radarrId, message, path, language_code, provider, score, subs_id, subs_path) send_notifications_movie(radarrId, message) store_subtitles_movie(path, moviePath) else: @@ -987,7 +1026,8 @@ class MovieSubtitlesManualDownload(Resource): provider = result[3] score = result[4] subs_id = result[6] - history_log_movie(2, radarrId, message, path, language_code, provider, score, subs_id) + subs_path = result[7] + history_log_movie(2, radarrId, message, path, language_code, provider, score, subs_id, subs_path) send_notifications_movie(radarrId, message) store_subtitles_movie(path, moviePath) return result, 201 @@ -1029,10 +1069,11 @@ class MovieSubtitlesUpload(Resource): if result is not None: message = result[0] path = result[1] + subs_path = result[2] language_code = language + ":forced" if forced else language provider = "manual" score = 120 - history_log_movie(4, radarrId, message, path, language_code, provider, score) + history_log_movie(4, radarrId, message, path, language_code, provider, score, subtitles_path=subs_path) send_notifications_movie(radarrId, message) store_subtitles_movie(path, moviePath) @@ -1064,21 +1105,54 @@ class MovieHistory(Resource): def get(self): radarrid = request.args.get('radarrid') - movie_history = database.execute("SELECT action, timestamp, language, provider, score " - "FROM table_history_movie WHERE radarrId=? ORDER BY timestamp DESC", - (radarrid,)) + movie_history = database.execute("SELECT action, timestamp, language, provider, score, radarrId, subs_id, " + "video_path, subtitles_path FROM table_history_movie WHERE radarrId=? ORDER " + "BY timestamp DESC", (radarrid,)) for item in movie_history: item['timestamp'] = "
" + \ pretty.date(datetime.datetime.fromtimestamp(item['timestamp'])) + "
" if item['language']: - item['language'] = language_from_alpha2(item['language']) - else: - item['language'] = "undefined" + language = item['language'].split(':') + item['language'] = {"name": language_from_alpha2(language[0]), + "code2": language[0], + "code3": alpha3_from_alpha2(language[0]), + "forced": True if len(language) > 1 else False} if item['score']: item['score'] = str(round((int(item['score']) * 100 / 120), 2)) + "%" + if item['video_path']: + # Provide mapped path + mapped_path = path_mappings.path_replace(item['video_path']) + item.update({"mapped_path": mapped_path}) + + # Confirm if path exist + item.update({"exist": os.path.isfile(mapped_path)}) + else: + item.update({"mapped_path": None}) + item.update({"exist": False}) + + if item['subtitles_path']: + # Provide mapped subtitles path + mapped_subtitles_path = path_mappings.path_replace_movie(item['subtitles_path']) + item.update({"mapped_subtitles_path": mapped_subtitles_path}) + else: + item.update({"mapped_subtitles_path": None}) + + # Check if subtitles is blacklisted + if item['action'] not in [0, 4, 5]: + blacklist_db = database.execute( + "SELECT provider, subs_id FROM table_blacklist_movie WHERE provider=? AND " + "subs_id=?", (item['provider'], item['subs_id'])) + else: + blacklist_db = [] + + if len(blacklist_db): + item.update({"blacklisted": True}) + else: + item.update({"blacklisted": False}) + return jsonify(data=movie_history) @@ -1130,12 +1204,11 @@ class HistorySeries(Resource): upgradable_episodes = database.execute( "SELECT video_path, MAX(timestamp) as timestamp, score, table_shows.tags, table_episodes.monitored, " - "table_shows.seriesType FROM table_history INNER JOIN table_episodes on table_episodes.sonarrEpisodeId " - "= table_history.sonarrEpisodeId INNER JOIN table_shows on table_shows.sonarrSeriesId = " - "table_episodes.sonarrSeriesId WHERE action IN (" + - ','.join(map(str, query_actions)) + ") AND timestamp > ? AND " - "score is not null GROUP BY table_history.video_path, table_history.language", - (minimum_timestamp,)) + "table_shows.seriesType FROM table_history INNER JOIN table_episodes on " + "table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId INNER JOIN table_shows on " + "table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE action IN (" + + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND score is not null GROUP BY " + "table_history.video_path, table_history.language", (minimum_timestamp,)) upgradable_episodes = filter_exclusions(upgradable_episodes, 'series') for upgradable_episode in upgradable_episodes: @@ -1151,15 +1224,17 @@ class HistorySeries(Resource): row_count = database.execute("SELECT COUNT(*) as count FROM table_history LEFT JOIN table_episodes " "on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId WHERE " "table_episodes.title is not NULL", only_one=True)['count'] - data = database.execute("SELECT table_history.action, table_shows.title as seriesTitle, table_episodes.monitored, " + data = database.execute("SELECT table_shows.title as seriesTitle, table_episodes.monitored, " "table_episodes.season || 'x' || table_episodes.episode as episode_number, " - "table_episodes.title as episodeTitle, table_history.timestamp, " + "table_episodes.title as episodeTitle, table_history.timestamp, table_history.subs_id, " "table_history.description, table_history.sonarrSeriesId, table_episodes.path, " - "table_history.language, table_history.score, table_shows.tags FROM table_history " - "LEFT JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId " - "LEFT JOIN table_episodes on table_episodes.sonarrEpisodeId = " - "table_history.sonarrEpisodeId WHERE table_episodes.title is not NULL ORDER BY " - "timestamp DESC LIMIT ? OFFSET ?", (length, start)) + "table_history.language, table_history.score, table_shows.tags, table_history.action, " + "table_history.subtitles_path, table_history.sonarrEpisodeId, table_history.provider " + "FROM table_history LEFT JOIN table_shows on table_shows.sonarrSeriesId = " + "table_history.sonarrSeriesId LEFT JOIN table_episodes on " + "table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId WHERE " + "table_episodes.title is not NULL ORDER BY timestamp DESC LIMIT ? OFFSET ?", + (length, start)) for item in data: # Mark episode as upgradable or not @@ -1180,12 +1255,35 @@ class HistorySeries(Resource): if item['timestamp']: item['timestamp'] = pretty.date(int(item['timestamp'])) - # Provide mapped path - mapped_path = path_mappings.path_replace(item['path']) - item.update({"mapped_path": mapped_path}) + if item['path']: + # Provide mapped path + mapped_path = path_mappings.path_replace(item['path']) + item.update({"mapped_path": mapped_path}) - # Confirm if path exist - item.update({"exist": os.path.isfile(mapped_path)}) + # Confirm if path exist + item.update({"exist": os.path.isfile(mapped_path)}) + else: + item.update({"mapped_path": None}) + item.update({"exist": False}) + + if item['subtitles_path']: + # Provide mapped subtitles path + mapped_subtitles_path = path_mappings.path_replace_movie(item['subtitles_path']) + item.update({"mapped_subtitles_path": mapped_subtitles_path}) + else: + item.update({"mapped_subtitles_path": None}) + + # Check if subtitles is blacklisted + if item['action'] not in [0, 4, 5]: + blacklist_db = database.execute("SELECT provider, subs_id FROM table_blacklist WHERE provider=? AND " + "subs_id=?", (item['provider'], item['subs_id'])) + else: + blacklist_db = [] + + if len(blacklist_db): + item.update({"blacklisted": True}) + else: + item.update({"blacklisted": False}) return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=data) @@ -1230,11 +1328,13 @@ class HistoryMovies(Resource): "table_movies.radarrId = table_history_movie.radarrId WHERE table_movies.title " "is not NULL", only_one=True)['count'] data = database.execute("SELECT table_history_movie.action, table_movies.title, table_history_movie.timestamp, " - "table_history_movie.description, table_history_movie.radarrId, table_movies.monitored, " - "table_history_movie.video_path, table_history_movie.language, table_movies.tags, " - "table_history_movie.score FROM table_history_movie LEFT JOIN table_movies on " - "table_movies.radarrId = table_history_movie.radarrId WHERE table_movies.title " - "is not NULL ORDER BY timestamp DESC LIMIT ? OFFSET ?", (length, start)) + "table_history_movie.description, table_history_movie.radarrId, table_movies.monitored," + " table_history_movie.video_path, table_history_movie.language, table_movies.tags, " + "table_history_movie.score, table_history_movie.subs_id, table_history_movie.provider, " + "table_history_movie.subtitles_path, table_history_movie.subtitles_path FROM " + "table_history_movie LEFT JOIN table_movies on table_movies.radarrId = " + "table_history_movie.radarrId WHERE table_movies.title is not NULL ORDER BY timestamp " + "DESC LIMIT ? OFFSET ?", (length, start)) for item in data: # Mark movies as upgradable or not @@ -1266,6 +1366,25 @@ class HistoryMovies(Resource): item.update({"mapped_path": None}) item.update({"exist": False}) + if item['subtitles_path']: + # Provide mapped subtitles path + mapped_subtitles_path = path_mappings.path_replace_movie(item['subtitles_path']) + item.update({"mapped_subtitles_path": mapped_subtitles_path}) + else: + item.update({"mapped_subtitles_path": None}) + + # Check if subtitles is blacklisted + if item['action'] not in [0, 4, 5]: + blacklist_db = database.execute("SELECT provider, subs_id FROM table_blacklist_movie WHERE provider=? " + "AND subs_id=?", (item['provider'], item['subs_id'])) + else: + blacklist_db = [] + + if len(blacklist_db): + item.update({"blacklisted": True}) + else: + item.update({"blacklisted": False}) + return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=data) @@ -1419,6 +1538,159 @@ class SearchWantedMovies(Resource): return '', 200 +class BlacklistSeries(Resource): + @authenticate + def get(self): + start = request.args.get('start') or 0 + length = request.args.get('length') or -1 + draw = request.args.get('draw') + + row_count = database.execute("SELECT COUNT(*) as count FROM table_blacklist", only_one=True)['count'] + data = database.execute("SELECT table_shows.title as seriesTitle, table_episodes.season || 'x' || " + "table_episodes.episode as episode_number, table_episodes.title as episodeTitle, " + "table_episodes.sonarrSeriesId, table_blacklist.provider, table_blacklist.subs_id, " + "table_blacklist.language, table_blacklist.timestamp FROM table_blacklist INNER JOIN " + "table_episodes on table_episodes.sonarrEpisodeId = table_blacklist.sonarr_episode_id " + "INNER JOIN table_shows on table_shows.sonarrSeriesId = " + "table_blacklist.sonarr_series_id ORDER BY table_blacklist.timestamp DESC LIMIT ? " + "OFFSET ?", (length, start)) + + for item in data: + # Make timestamp pretty + item.update({'timestamp': pretty.date(datetime.datetime.fromtimestamp(item['timestamp']))}) + + # Convert language code2 to name + if item['language']: + language = item['language'].split(':') + item['language'] = {"name": language_from_alpha2(language[0]), + "code2": language[0], + "code3": alpha3_from_alpha2(language[0]), + "forced": True if len(language) > 1 else False} + + return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=data) + + +class BlacklistEpisodeSubtitlesAdd(Resource): + @authenticate + def post(self): + sonarr_series_id = int(request.form.get('sonarr_series_id')) + sonarr_episode_id = int(request.form.get('sonarr_episode_id')) + provider = request.form.get('provider') + subs_id = request.form.get('subs_id') + language = request.form.get('language') + forced = request.form.get('forced') + language_str = language + ':forced' if forced == 'true' else language + media_path = request.form.get('video_path') + subtitles_path = request.form.get('subtitles_path') + + blacklist_log(sonarr_series_id=sonarr_series_id, + sonarr_episode_id=sonarr_episode_id, + provider=provider, + subs_id=subs_id, + language=language_str) + delete_subtitles(media_type='series', + language=alpha3_from_alpha2(language), + forced=forced, + media_path=path_mappings.path_replace(media_path), + subtitles_path=path_mappings.path_replace(subtitles_path), + sonarr_series_id=sonarr_series_id, + sonarr_episode_id=sonarr_episode_id) + episode_download_subtitles(sonarr_episode_id) + event_stream(type='episodeHistory') + return '', 200 + + +class BlacklistEpisodeSubtitlesRemove(Resource): + @authenticate + def delete(self): + provider = request.form.get('provider') + subs_id = request.form.get('subs_id') + + blacklist_delete(provider=provider, subs_id=subs_id) + return '', 200 + + +class BlacklistEpisodeSubtitlesRemoveAll(Resource): + @authenticate + def delete(self): + blacklist_delete_all() + return '', 200 + + +class BlacklistMovies(Resource): + @authenticate + def get(self): + start = request.args.get('start') or 0 + length = request.args.get('length') or -1 + draw = request.args.get('draw') + + row_count = database.execute("SELECT COUNT(*) as count FROM table_blacklist_movie", only_one=True)['count'] + data = database.execute("SELECT table_movies.title, table_movies.radarrId, table_blacklist_movie.provider, " + "table_blacklist_movie.subs_id, table_blacklist_movie.language, " + "table_blacklist_movie.timestamp FROM table_blacklist_movie INNER JOIN " + "table_movies on table_movies.radarrId = table_blacklist_movie.radarr_id " + "ORDER BY table_blacklist_movie.timestamp DESC LIMIT ? " + "OFFSET ?", (length, start)) + + for item in data: + # Make timestamp pretty + item.update({'timestamp': pretty.date(datetime.datetime.fromtimestamp(item['timestamp']))}) + + # Convert language code2 to name + if item['language']: + language = item['language'].split(':') + item['language'] = {"name": language_from_alpha2(language[0]), + "code2": language[0], + "code3": alpha3_from_alpha2(language[0]), + "forced": True if len(language) > 1 else False} + + return jsonify(draw=draw, recordsTotal=row_count, recordsFiltered=row_count, data=data) + + +class BlacklistMovieSubtitlesAdd(Resource): + @authenticate + def post(self): + radarr_id = int(request.form.get('radarr_id')) + provider = request.form.get('provider') + subs_id = request.form.get('subs_id') + language = request.form.get('language') + forced = request.form.get('forced') + language_str = language + ':forced' if forced == 'true' else language + media_path = request.form.get('video_path') + subtitles_path = request.form.get('subtitles_path') + + blacklist_log_movie(radarr_id=radarr_id, + provider=provider, + subs_id=subs_id, + language=language_str) + delete_subtitles(media_type='movie', + language=alpha3_from_alpha2(language), + forced=forced, + media_path=path_mappings.path_replace_movie(media_path), + subtitles_path=path_mappings.path_replace_movie(subtitles_path), + radarr_id=radarr_id) + movies_download_subtitles(radarr_id) + event_stream(type='movieHistory') + return '', 200 + + +class BlacklistMovieSubtitlesRemove(Resource): + @authenticate + def delete(self): + provider = request.form.get('provider') + subs_id = request.form.get('subs_id') + + blacklist_delete_movie(provider=provider, subs_id=subs_id) + return '', 200 + + +class BlacklistMovieSubtitlesRemoveAll(Resource): + @authenticate + def delete(self): + blacklist_delete_all_movie() + return '', 200 + + class SyncSubtitles(Resource): @authenticate def post(self): @@ -1529,6 +1801,15 @@ api.add_resource(WantedMovies, '/wanted_movies') api.add_resource(SearchWantedSeries, '/search_wanted_series') api.add_resource(SearchWantedMovies, '/search_wanted_movies') +api.add_resource(BlacklistSeries, '/blacklist_series') +api.add_resource(BlacklistEpisodeSubtitlesAdd, '/blacklist_episode_subtitles_add') +api.add_resource(BlacklistEpisodeSubtitlesRemove, '/blacklist_episode_subtitles_remove') +api.add_resource(BlacklistEpisodeSubtitlesRemoveAll, '/blacklist_episode_subtitles_remove_all') +api.add_resource(BlacklistMovies, '/blacklist_movies') +api.add_resource(BlacklistMovieSubtitlesAdd, '/blacklist_movie_subtitles_add') +api.add_resource(BlacklistMovieSubtitlesRemove, '/blacklist_movie_subtitles_remove') +api.add_resource(BlacklistMovieSubtitlesRemoveAll, '/blacklist_movie_subtitles_remove_all') + api.add_resource(SyncSubtitles, '/sync_subtitles') api.add_resource(BrowseBazarrFS, '/browse_bazarr_filesystem') diff --git a/bazarr/database.py b/bazarr/database.py index 16833e460..15e9c2d60 100644 --- a/bazarr/database.py +++ b/bazarr/database.py @@ -115,11 +115,13 @@ def db_upgrade(): ['table_history', 'provider', 'text'], ['table_history', 'score', 'text'], ['table_history', 'subs_id', 'text'], + ['table_history', 'subtitles_path', 'text'], ['table_history_movie', 'video_path', 'text'], ['table_history_movie', 'language', 'text'], ['table_history_movie', 'provider', 'text'], ['table_history_movie', 'score', 'text'], - ['table_history_movie', 'subs_id', 'text'] + ['table_history_movie', 'subs_id', 'text'], + ['table_history_movie', 'subtitles_path', 'text'] ] for column in columnToAdd: @@ -139,6 +141,12 @@ def db_upgrade(): database.execute("UPDATE table_movies SET hearing_impaired = 'False' WHERE hearing_impaired is null") database.execute("UPDATE table_movies SET forced = 'False' WHERE forced is null") + # Create blacklist tables + database.execute("CREATE TABLE IF NOT EXISTS table_blacklist (sonarr_series_id integer, sonarr_episode_id integer, " + "timestamp integer, provider text, subs_id text, language text)") + database.execute("CREATE TABLE IF NOT EXISTS table_blacklist_movie (radarr_id integer, timestamp integer, " + "provider text, subs_id text, language text)") + def filter_exclusions(dicts_list, type): if type == 'series': diff --git a/bazarr/embedded_subs_reader.py b/bazarr/embedded_subs_reader.py index 3857e5fa0..b4467fe74 100644 --- a/bazarr/embedded_subs_reader.py +++ b/bazarr/embedded_subs_reader.py @@ -4,14 +4,15 @@ import logging import os from knowit import api -from utils import get_binary - class EmbeddedSubsReader: def __init__(self): - self.ffprobe = get_binary("ffprobe") + self.ffprobe = None def list_languages(self, file): + from utils import get_binary + self.ffprobe = get_binary("ffprobe") + subtitles_list = [] if self.ffprobe: diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index eeae8a2c1..a34816369 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -23,7 +23,7 @@ from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_ from config import settings from helper import path_mappings, pp_replace, get_target_folder, force_unicode from list_subtitles import store_subtitles, list_missing_subtitles, store_subtitles_movie, list_missing_subtitles_movies -from utils import history_log, history_log_movie, get_binary +from utils import history_log, history_log_movie, get_binary, get_blacklist from notifier import send_notifications, send_notifications_movie from get_providers import get_providers, get_providers_auth, provider_throttle, provider_pool from knowit import api @@ -167,7 +167,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro pool_class=provider_pool(), compute_score=compute_score, throttle_time=None, # fixme - blacklist=None, # fixme + blacklist=get_blacklist(media_type=media_type), throttle_callback=provider_throttle, pre_download_hook=None, # fixme post_download_hook=None, # fixme @@ -263,13 +263,16 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro # fixme: support multiple languages at once if media_type == 'series': reversed_path = path_mappings.path_replace_reverse(path) + reversed_subtitles_path = path_mappings.path_replace_reverse(downloaded_path) + else: reversed_path = path_mappings.path_replace_reverse_movie(path) + reversed_subtitles_path = path_mappings.path_replace_reverse_movie(downloaded_path) track_event(category=downloaded_provider, action=action, label=downloaded_language) return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \ - subtitle.language.forced, subtitle.id + subtitle.language.forced, subtitle.id, reversed_subtitles_path if not saved_any: logging.debug('BAZARR No Subtitles were found for this file: ' + path) @@ -332,6 +335,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa subtitles = list_all_subtitles([video], language_set, providers=providers, provider_configs=providers_auth, + blacklist=get_blacklist(media_type=media_type), throttle_callback=provider_throttle, language_hook=None) # fixme else: @@ -420,8 +424,12 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl min_score, max_score, scores = get_scores(video, media_type) try: if provider: - download_subtitles([subtitle], providers={provider}, provider_configs=providers_auth, - pool_class=provider_pool(), throttle_callback=provider_throttle) + download_subtitles([subtitle], + providers={provider}, + provider_configs=providers_auth, + pool_class=provider_pool(), + blacklist=get_blacklist(media_type=media_type), + throttle_callback=provider_throttle) logging.debug('BAZARR Subtitles file downloaded for this file:' + path) else: logging.info("BAZARR All providers are throttled") @@ -507,14 +515,16 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl if media_type == 'series': reversed_path = path_mappings.path_replace_reverse(path) + reversed_subtitles_path = path_mappings.path_replace_reverse(downloaded_path) else: reversed_path = path_mappings.path_replace_reverse_movie(path) + reversed_subtitles_path = path_mappings.path_replace_reverse_movie(downloaded_path) track_event(category=downloaded_provider, action="manually_downloaded", label=downloaded_language) return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \ - subtitle.language.forced, subtitle.id + subtitle.language.forced, subtitle.id, reversed_subtitles_path else: logging.error( "BAZARR Tried to manually download a Subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str( @@ -625,10 +635,12 @@ def manual_upload_subtitle(path, language, forced, title, scene_name, media_type if media_type == 'series': reversed_path = path_mappings.path_replace_reverse(path) + reversed_subtitles_path = path_mappings.path_replace_reverse(subtitle_path) else: reversed_path = path_mappings.path_replace_reverse_movie(path) + reversed_subtitles_path = path_mappings.path_replace_reverse_movie(subtitle_path) - return message, reversed_path + return message, reversed_path, reversed_subtitles_path def series_download_subtitles(no): @@ -676,9 +688,10 @@ def series_download_subtitles(no): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score, - subs_id) + subs_id, subs_path) send_notifications(no, episode['sonarrEpisodeId'], message) else: logging.info("BAZARR All providers are throttled") @@ -722,9 +735,10 @@ def episode_download_subtitles(no): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, - language_code, provider, score, subs_id) + language_code, provider, score, subs_id, subs_path) send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) else: logging.info("BAZARR All providers are throttled") @@ -771,8 +785,9 @@ def movies_download_subtitles(no): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) - history_log_movie(1, no, message, path, language_code, provider, score, subs_id) + history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path) send_notifications_movie(no, message) else: logging.info("BAZARR All providers are throttled") @@ -828,9 +843,11 @@ def wanted_download_subtitles(path, l, count_episodes): language_code = result[2] + ":forced" if forced else result[2] provider = result[3] score = result[4] + subs_id = result[6] + subs_path = result[7] store_subtitles(episode['path'], path_mappings.path_replace(episode['path'])) history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, - language_code, provider, score) + language_code, provider, score, subs_id, subs_path) send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) else: logging.debug( @@ -884,9 +901,10 @@ def wanted_download_subtitles_movie(path, l, count_movies): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path'])) history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score, - subs_id) + subs_id, subs_path) send_notifications_movie(movie['radarrId'], message) else: logging.info( @@ -1167,9 +1185,10 @@ def upgrade_subtitles(): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles(episode['video_path'], path_mappings.path_replace(episode['video_path'])) history_log(3, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path, - language_code, provider, score, subs_id) + language_code, provider, score, subs_id, subs_path) send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message) if settings.general.getboolean('use_radarr'): @@ -1217,9 +1236,10 @@ def upgrade_subtitles(): provider = result[3] score = result[4] subs_id = result[6] + subs_path = result[7] store_subtitles_movie(movie['video_path'], path_mappings.path_replace_movie(movie['video_path'])) - history_log_movie(3, movie['radarrId'], message, path, language_code, provider, score, subs_id) + history_log_movie(3, movie['radarrId'], message, path, language_code, provider, score, subs_id, subs_path) send_notifications_movie(movie['radarrId'], message) diff --git a/bazarr/main.py b/bazarr/main.py index 1a4048d3e..b748101ad 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -274,6 +274,18 @@ def historystats(): data_languages=sorted(data_languages_list, key=lambda i: i['name'])) +@app.route('/blacklist/series/') +@login_required +def blacklistseries(): + return render_template('blacklistseries.html') + + +@app.route('/blacklist/movies/') +@login_required +def blacklistmovies(): + return render_template('blacklistmovies.html') + + @app.route('/wanted/series/') @login_required def wantedseries(): diff --git a/bazarr/subsyncer.py b/bazarr/subsyncer.py index fafe64c87..8932b6204 100644 --- a/bazarr/subsyncer.py +++ b/bazarr/subsyncer.py @@ -105,11 +105,11 @@ class SubSyncer: if media_type == 'series': history_log(action=5, sonarr_series_id=sonarr_series_id, sonarr_episode_id=sonarr_episode_id, description=message, video_path=path_mappings.path_replace_reverse(self.reference), - language=alpha2_from_alpha3(srt_lang)) + language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) else: history_log_movie(action=5, radarr_id=radarr_id, description=message, video_path=path_mappings.path_replace_reverse_movie(self.reference), - language=alpha2_from_alpha3(srt_lang)) + language=alpha2_from_alpha3(srt_lang), subtitles_path=srt_path) else: logging.error('BAZARR unable to sync subtitles: ' + self.srtin) diff --git a/bazarr/utils.py b/bazarr/utils.py index 109e76ba8..bfc16dec5 100644 --- a/bazarr/utils.py +++ b/bazarr/utils.py @@ -11,6 +11,9 @@ from get_args import args from config import settings, url_sonarr, url_radarr from database import database from event_handler import event_stream +from get_languages import alpha2_from_alpha3, language_from_alpha3 +from helper import path_mappings +from list_subtitles import store_subtitles, store_subtitles_movie from subliminal import region as subliminal_cache_region import datetime @@ -22,22 +25,55 @@ class BinaryNotFound(Exception): def history_log(action, sonarr_series_id, sonarr_episode_id, description, video_path=None, language=None, provider=None, - score=None, subs_id=None): + score=None, subs_id=None, subtitles_path=None): database.execute("INSERT INTO table_history (action, sonarrSeriesId, sonarrEpisodeId, timestamp, description," - "video_path, language, provider, score, subs_id) VALUES (?,?,?,?,?,?,?,?,?,?)", + "video_path, language, provider, score, subs_id, subtitles_path) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (action, sonarr_series_id, sonarr_episode_id, time.time(), description, video_path, language, - provider, score, subs_id)) + provider, score, subs_id, subtitles_path)) event_stream(type='episodeHistory') +def blacklist_log(sonarr_series_id, sonarr_episode_id, provider, subs_id, language): + database.execute("INSERT INTO table_blacklist (sonarr_series_id, sonarr_episode_id, timestamp, provider, " + "subs_id, language) VALUES (?,?,?,?,?,?)", + (sonarr_series_id, sonarr_episode_id, time.time(), provider, subs_id, language)) + event_stream(type='episodeBlacklist') + + +def blacklist_delete(provider, subs_id): + database.execute("DELETE FROM table_blacklist WHERE provider=? AND subs_id=?", (provider, subs_id)) + event_stream(type='episodeBlacklist') + + +def blacklist_delete_all(): + database.execute("DELETE FROM table_blacklist") + event_stream(type='episodeBlacklist') + + def history_log_movie(action, radarr_id, description, video_path=None, language=None, provider=None, score=None, - subs_id=None): + subs_id=None, subtitles_path=None): database.execute("INSERT INTO table_history_movie (action, radarrId, timestamp, description, video_path, language, " - "provider, score, subs_id) VALUES (?,?,?,?,?,?,?,?,?)", - (action, radarr_id, time.time(), description, video_path, language, provider, score, subs_id)) + "provider, score, subs_id, subtitles_path) VALUES (?,?,?,?,?,?,?,?,?,?)", + (action, radarr_id, time.time(), description, video_path, language, provider, score, subs_id, subtitles_path)) event_stream(type='movieHistory') +def blacklist_log_movie(radarr_id, provider, subs_id, language): + database.execute("INSERT INTO table_blacklist_movie (radarr_id, timestamp, provider, subs_id, language) " + "VALUES (?,?,?,?,?)", (radarr_id, time.time(), provider, subs_id, language)) + event_stream(type='movieBlacklist') + + +def blacklist_delete_movie(provider, subs_id): + database.execute("DELETE FROM table_blacklist_movie WHERE provider=? AND subs_id=?", (provider, subs_id)) + event_stream(type='movieBlacklist') + + +def blacklist_delete_all_movie(): + database.execute("DELETE FROM table_blacklist_movie") + event_stream(type='movieBlacklist') + + def get_binary(name): binaries_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'bin')) @@ -65,6 +101,19 @@ def get_binary(name): raise BinaryNotFound +def get_blacklist(media_type): + if media_type == 'series': + blacklist_db = database.execute("SELECT provider, subs_id FROM table_blacklist") + else: + blacklist_db = database.execute("SELECT provider, subs_id FROM table_blacklist_movie") + + blacklist_list = [] + for item in blacklist_db: + blacklist_list.append((item['provider'], item['subs_id'])) + + return blacklist_list + + def cache_maintenance(): main_cache_validity = 14 # days pack_cache_validity = 4 # days @@ -139,3 +188,39 @@ def get_radarr_platform(): except Exception: logging.debug('BAZARR cannot get Radarr platform') return radarr_platform + + +def delete_subtitles(media_type, language, forced, media_path, subtitles_path, sonarr_series_id=None, + sonarr_episode_id=None, radarr_id=None): + if not subtitles_path.endswith('.srt'): + logging.error('BAZARR can only delete .srt files.') + return False + language_log = alpha2_from_alpha3(language) + ':forced' if forced in [True, 'true'] else alpha2_from_alpha3(language) + language_string = language_from_alpha3(language) + ' forced' if forced in [True, 'true'] else language_from_alpha3(language) + result = language_string + " subtitles deleted from disk." + + if media_type == 'series': + try: + os.remove(path_mappings.path_replace(subtitles_path)) + except OSError: + logging.exception('BAZARR cannot delete subtitles file: ' + subtitles_path) + store_subtitles(path_mappings.path_replace_reverse(media_path), media_path) + return False + else: + history_log(0, sonarr_series_id, sonarr_episode_id, result, language=language_log, + video_path=path_mappings.path_replace_reverse(media_path), + subtitles_path=path_mappings.path_replace_reverse(subtitles_path)) + store_subtitles(path_mappings.path_replace_reverse(media_path), media_path) + else: + try: + os.remove(path_mappings.path_replace_movie(subtitles_path)) + except OSError: + logging.exception('BAZARR cannot delete subtitles file: ' + subtitles_path) + store_subtitles_movie(path_mappings.path_replace_reverse_movie(media_path), media_path) + return False + else: + history_log_movie(0, radarr_id, result, language=language_log, + video_path=path_mappings.path_replace_reverse_movie(media_path), + subtitles_path=path_mappings.path_replace_reverse_movie(subtitles_path)) + store_subtitles_movie(path_mappings.path_replace_reverse_movie(media_path), media_path) + return True diff --git a/views/_main.html b/views/_main.html index 67cf1fb69..cf6a4d625 100644 --- a/views/_main.html +++ b/views/_main.html @@ -232,6 +232,17 @@ +
  • Blacklist + +
    • +
  • Wanted