From 53f3338ef7940313580a18d44ed903c5e4669fc7 Mon Sep 17 00:00:00 2001 From: halali Date: Mon, 27 Aug 2018 02:53:02 +0200 Subject: [PATCH] Adaptive searching (#130) * Move settings from Database to Config file * Delete unused config and forgot upload init * Fix * Update get_subtitles and fix provider auth * Revert "Update get_subtitles and fix provider auth" This reverts commit c904bf6b5a61b291d4e5764a885578e66ae7d5d3. * Update get_sebtitles * more fixes * Update requirements * Initial Commit * Finished --- bazarr.py | 9 +++-- create_db.sql | 2 ++ get_episodes.py | 1 + get_movies.py | 4 +++ get_settings.py | 11 ++++-- get_subtitle.py | 95 +++++++++++++++++++++++++++++++++++++++++-------- init.py | 2 +- update_db.py | 6 ++++ 8 files changed, 110 insertions(+), 20 deletions(-) diff --git a/bazarr.py b/bazarr.py index f05bb3bb2..945074172 100644 --- a/bazarr.py +++ b/bazarr.py @@ -755,6 +755,11 @@ def save_settings(): settings_general_only_monitored = 'False' else: settings_general_only_monitored = 'True' + settings_general_adaptive_searching = request.forms.get('settings_general_adaptive_searching') + if settings_general_adaptive_searching is None: + settings_general_adaptive_searching = 'False' + else: + settings_general_adaptive_searching = 'True' settings_general_minimum_score = request.forms.get('settings_general_minimum_score') settings_general_minimum_score_movies = request.forms.get('settings_general_minimum_score_movies') settings_general_use_postprocessing = request.forms.get('settings_general_use_postprocessing') @@ -805,8 +810,8 @@ def save_settings(): cfg.set('general', 'minimum_score_movie', text_type(settings_general_minimum_score_movies)) cfg.set('general', 'use_embedded_subs', text_type(settings_general_embedded)) cfg.set('general', 'only_monitored', text_type(settings_general_only_monitored)) - # cfg.set('general', 'configured', text_type(configured)) - # cfg.set('general', 'updated', text_type(updated)) + cfg.set('general', 'adaptive_searching', text_type(settings_general_adaptive_searching)) + if after != before: configured() diff --git a/create_db.sql b/create_db.sql index 96561f9fd..93c815ee4 100644 --- a/create_db.sql +++ b/create_db.sql @@ -46,6 +46,7 @@ CREATE TABLE "table_episodes" ( `missing_subtitles` TEXT, `scene_name` TEXT, `monitored` TEXT + 'failedAttempts' 'text' ); CREATE TABLE "table_movies" ( `tmdbId` TEXT NOT NULL UNIQUE, @@ -62,6 +63,7 @@ CREATE TABLE "table_movies" ( `audio_language` "text", `sceneName` TEXT, `monitored` TEXT, PRIMARY KEY(`tmdbId`) + 'failedAttempts' 'text' ); CREATE TABLE "table_history_movie" ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, diff --git a/get_episodes.py b/get_episodes.py index e3b16e115..18fea0c87 100644 --- a/get_episodes.py +++ b/get_episodes.py @@ -21,6 +21,7 @@ def update_all_movies(): logging.info('All missing movie subtitles updated in database.') def sync_episodes(): + logging.debug('Starting episode sync from Sonarr.') from get_settings import get_sonarr_settings url_sonarr = get_sonarr_settings()[6] apikey_sonarr = get_sonarr_settings()[4] diff --git a/get_movies.py b/get_movies.py index dbf8b5de1..c16b2e536 100644 --- a/get_movies.py +++ b/get_movies.py @@ -9,6 +9,7 @@ from get_settings import get_general_settings, path_replace_movie from list_subtitles import store_subtitles_movie, list_missing_subtitles_movies def update_movies(): + logging.debug('Starting movie sync from Radarr.') from get_settings import get_radarr_settings url_radarr = get_radarr_settings()[6] # url_radarr_short = get_radarr_settings()[7] @@ -105,7 +106,10 @@ def update_movies(): # Close database connection db.close() + logging.debug('All movies synced from Radarr into database.') + list_missing_subtitles_movies() + logging.debug('All movie missing subtitles updated in database.') def get_profile_list(): from get_settings import get_radarr_settings diff --git a/get_settings.py b/get_settings.py index 6eae7569d..e00bf5661 100644 --- a/get_settings.py +++ b/get_settings.py @@ -141,6 +141,11 @@ def get_general_settings(): only_monitored = cfg.getboolean('general', 'only_monitored') else: only_monitored = False + + if cfg.has_option('general', 'adaptive_searching'): + adaptive_searching = cfg.getboolean('general', 'adaptive_searching') + else: + adaptive_searching = False else: ip = '0.0.0.0' @@ -168,8 +173,9 @@ def get_general_settings(): minimum_score_movie = '100' use_embedded_subs = False only_monitored = False + adaptive_searching = False - return [ip, port, base_url, path_mappings, log_level, branch, auto_update, single_language, minimum_score, use_scenename, use_postprocessing, postprocessing_cmd, use_sonarr, use_radarr, path_mappings_movie, serie_default_enabled, serie_default_language, serie_default_hi, movie_default_enabled,movie_default_language, movie_default_hi, page_size, minimum_score_movie, use_embedded_subs, only_monitored] + return [ip, port, base_url, path_mappings, log_level, branch, auto_update, single_language, minimum_score, use_scenename, use_postprocessing, postprocessing_cmd, use_sonarr, use_radarr, path_mappings_movie, serie_default_enabled, serie_default_language, serie_default_hi, movie_default_enabled,movie_default_language, movie_default_hi, page_size, minimum_score_movie, use_embedded_subs, only_monitored, adaptive_searching] def get_auth_settings(): @@ -413,4 +419,5 @@ movie_default_hi = result[20] page_size = result[21] minimum_score_movie = result[22] use_embedded_subs = result[23] -only_monitored = result[24] \ No newline at end of file +only_monitored = result[24] +adaptive_searching = result[25] \ No newline at end of file diff --git a/get_subtitle.py b/get_subtitle.py index eee90e4c4..f9f19bc8c 100644 --- a/get_subtitle.py +++ b/get_subtitle.py @@ -5,6 +5,8 @@ import sqlite3 import ast import logging import subprocess +import time +from datetime import datetime, timedelta from babelfish import Language from subliminal import region, scan_video, Video, download_best_subtitles, compute_score, save_subtitles from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2 @@ -137,7 +139,7 @@ def series_download_subtitles(no): else: providers_list = None providers_auth = None - + for episode in episodes_details: for language in ast.literal_eval(episode[1]): if language is not None: @@ -186,7 +188,7 @@ def movies_download_subtitles(no): def wanted_download_subtitles(path): conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) c_db = conn_db.cursor() - episodes_details = c_db.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, table_shows.hearing_impaired, table_episodes.scene_name FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.path = ? AND missing_subtitles != '[]'", (path_replace_reverse(path),)).fetchall() + episodes_details = c_db.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, table_shows.hearing_impaired, table_episodes.scene_name, table_episodes.failedAttempts FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.path = ? AND missing_subtitles != '[]'", (path_replace_reverse(path),)).fetchall() enabled_providers = c_db.execute("SELECT * FROM table_settings_providers WHERE enabled = 1").fetchall() c_db.close() @@ -208,19 +210,41 @@ def wanted_download_subtitles(path): providers_auth = None for episode in episodes_details: + attempt = episode[6] + if type(attempt) == unicode: + attempt = ast.literal_eval(attempt) for language in ast.literal_eval(episode[1]): - message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), episode[4], providers_list, providers_auth, episode[5], 'series') - if message is not None: - store_subtitles(path_replace(episode[0])) - list_missing_subtitles(episode[3]) - history_log(1, episode[3], episode[2], message) - send_notifications(episode[3], episode[2], message) + if attempt is None: + attempt = [] + attempt.append([language, time.time()]) + else: + att = zip(*attempt)[0] + if language not in att: + attempt.append([language, time.time()]) + + conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) + c_db = conn_db.cursor() + c_db.execute('UPDATE table_episodes SET failedAttempts = ? WHERE sonarrEpisodeId = ?', (unicode(attempt), episode[2])) + conn_db.commit() + c_db.close() + + for i in range(len(attempt)): + if attempt[i][0] == language: + if search_active(attempt[i][1]) is True: + message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), episode[4], providers_list, providers_auth, episode[5], 'series') + if message is not None: + store_subtitles(path_replace(episode[0])) + list_missing_subtitles(episode[3]) + history_log(1, episode[3], episode[2], message) + send_notifications(episode[3], episode[2], message) + else: + logging.debug('Search is not active for episode ' + episode[0] + ' Language: ' + attempt[i][0]) def wanted_download_subtitles_movie(path): conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) c_db = conn_db.cursor() - movies_details = c_db.execute("SELECT path, missing_subtitles, radarrId, radarrId, hearing_impaired, sceneName FROM table_movies WHERE path = ? AND missing_subtitles != '[]'", (path_replace_reverse_movie(path),)).fetchall() + movies_details = c_db.execute("SELECT path, missing_subtitles, radarrId, radarrId, hearing_impaired, sceneName, failedAttempts FROM table_movies WHERE path = ? AND missing_subtitles != '[]'", (path_replace_reverse_movie(path),)).fetchall() enabled_providers = c_db.execute("SELECT * FROM table_settings_providers WHERE enabled = 1").fetchall() c_db.close() @@ -242,13 +266,35 @@ def wanted_download_subtitles_movie(path): providers_auth = None for movie in movies_details: + attempt = movie[6] + if type(attempt) == unicode: + attempt = ast.literal_eval(attempt) for language in ast.literal_eval(movie[1]): - message = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], providers_list, providers_auth, movie[5], 'movies') - if message is not None: - store_subtitles_movie(path_replace_movie(movie[0])) - list_missing_subtitles_movies(movie[3]) - history_log_movie(1, movie[3], message) - send_notifications_movie(movie[3], message) + if attempt is None: + attempt = [] + attempt.append([language, time.time()]) + else: + att = zip(*attempt)[0] + if language not in att: + attempt.append([language, time.time()]) + + conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) + c_db = conn_db.cursor() + c_db.execute('UPDATE table_movies SET failedAttempts = ? WHERE radarrId = ?', (unicode(attempt), movie[2])) + conn_db.commit() + c_db.close() + + for i in range(len(attempt)): + if attempt[i][0] == language: + if search_active(attempt[i][1]) is True: + message = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], providers_list, providers_auth, movie[5], 'movies') + if message is not None: + store_subtitles_movie(path_replace_movie(movie[0])) + list_missing_subtitles_movies(movie[3]) + history_log_movie(1, movie[3], message) + send_notifications_movie(movie[3], message) + else: + logging.info('Search is not active for movie ' + movie[0] + ' Language: ' + attempt[i][0]) def wanted_search_missing_subtitles(): @@ -281,3 +327,22 @@ def wanted_search_missing_subtitles(): wanted_download_subtitles_movie(movie[0]) logging.info('Finished searching for missing subtitles. Check histories for more information.') + + +def search_active(timestamp): + if get_general_settings()[25] is True: + search_deadline = timedelta(weeks=3) + search_delta = timedelta(weeks=1) + aa = datetime.fromtimestamp(float(timestamp)) + attempt_datetime = datetime.strptime(str(aa).split(".")[0], '%Y-%m-%d %H:%M:%S') + attempt_search_deadline = attempt_datetime + search_deadline + today = datetime.today() + attempt_age_in_days = (today.date() - attempt_search_deadline.date()).days + if today.date() <= attempt_search_deadline.date(): + return True + elif attempt_age_in_days % search_delta.days == 0: + return True + else: + return False + else: + return True diff --git a/init.py b/init.py index f57d07010..12249563b 100644 --- a/init.py +++ b/init.py @@ -124,7 +124,6 @@ try: logging.info('Config file succesfully migrated from database') except sqlite3.OperationalError: -# if not os.path.exists(config_file): if os.path.exists(config_file) is False: cfg = ConfigParser() @@ -158,6 +157,7 @@ except sqlite3.OperationalError: cfg.set(section, 'minimum_score_movie', "100") cfg.set(section, 'use_embedded_subs', "False") cfg.set(section, 'only_monitored', "False") + cfg.set(section, 'adaptive_searching', "False") section = 'auth' diff --git a/update_db.py b/update_db.py index 667fecb30..9c77a56f9 100644 --- a/update_db.py +++ b/update_db.py @@ -69,6 +69,12 @@ if os.path.exists(os.path.join(config_dir, 'db/bazarr.db')) == True: except: pass + try: + c.execute('alter table table_movies add column "failedAttempts" "text"') + c.execute('alter table table_episodes add column "failedAttempts" "text"') + except: + pass + # Commit change to db db.commit()