Merge remote-tracking branch 'origin/development' into updater

# Conflicts:
#	bazarr/config.py
#	bazarr/scheduler.py

bazarr/config.py

@@ -26,19 +26,24 @@ defaults = {
         'serie_default_language': '[]',
         'serie_default_hi': 'False',
         'movie_default_enabled': 'False',
-        'movie_default_language': [],
+        'movie_default_language': '[]',
         'movie_default_hi': 'False',
         'page_size': '25',
         'minimum_score_movie': '70',
         'use_embedded_subs': 'True',
         'adaptive_searching': 'False',
         'enabled_providers': '',
-        'throtteled_providers': '',
+        'throtteled_providers': '{}',
         'multithreading': 'True',
+        'chmod_enabled': 'False',
         'chmod': '0640',
         'subfolder': 'current',
         'subfolder_custom': '',
-        'update_restart': 'True'
+        'update_restart': 'True',
+        'subfolder_custom': '',
+        'upgrade_subs': 'True',
+        'days_to_upgrade_subs': '7',
+        'upgrade_manual': 'True'
     },
     'auth': {
         'type': 'None',

bazarr/get_episodes.py

@@ -81,7 +81,10 @@ def sync_episodes():
                                     format, resolution = episode['episodeFile']['quality']['quality']['name'].split('-')
                                 except:
                                     format = episode['episodeFile']['quality']['quality']['name']
-                                    resolution = str(episode['episodeFile']['quality']['quality']['resolution']) + 'p'
+                                    try:
+                                        resolution = str(episode['episodeFile']['quality']['quality']['resolution']) + 'p'
+                                    except:
+                                        resolution = None
 
                                 if 'mediaInfo' in episode['episodeFile']:
                                     if 'videoCodec' in episode['episodeFile']['mediaInfo']:

bazarr/get_movies.py

@@ -84,7 +84,10 @@ def update_movies():
                                 format, resolution = movie['movieFile']['quality']['quality']['name'].split('-')
                             except:
                                 format = movie['movieFile']['quality']['quality']['name']
+                            try:
                                 resolution = movie['movieFile']['quality']['quality']['resolution'].lstrip('r').lower()
+                            except:
+                                resolution = None
 
                             if 'mediaInfo' in movie['movieFile']:
                                 videoFormat = videoCodecID = videoProfile = videoCodecLibrary = None

bazarr/get_providers.py

@@ -3,6 +3,7 @@ import os
 import datetime
 import logging
 import subliminal_patch
+import pretty
 
 from get_args import args
 from config import settings
@@ -139,3 +140,33 @@ def provider_throttle(name, exception):
     
     logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
                  throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
+
+def update_throttled_provider():
+    changed = False
+    if settings.general.enabled_providers:
+        for provider in settings.general.enabled_providers.lower().split(','):
+            reason, until, throttle_desc = tp.get(provider, (None, None, None))
+
+            if reason:
+                now = datetime.datetime.now()
+                if now < until:
+                    pass
+                else:
+                    logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
+                    del tp[provider]
+                    settings.general.throtteled_providers = str(tp)
+                    changed = True
+
+        if changed:
+            with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
+                settings.write(handle)
+
+
+def list_throttled_providers():
+    update_throttled_provider()
+    throttled_providers = []
+    if settings.general.enabled_providers:
+        for provider in settings.general.enabled_providers.lower().split(','):
+            reason, until, throttle_desc = tp.get(provider, (None, None, None))
+            throttled_providers.append([provider, reason, pretty.date(until)])
+    return throttled_providers

bazarr/get_subtitle.py

@@ -102,7 +102,7 @@ def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_sco
     return min_score, max_score, set(scores)
 
 
-def download_subtitle(path, language, hi, providers, providers_auth, sceneName, title, media_type):
+def download_subtitle(path, language, hi, providers, providers_auth, sceneName, title, media_type, forced_minimum_score=None, is_upgrade=False):
     # fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
     #  one query
 
@@ -112,7 +112,7 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
     else:
         hi = False
     language_set = set()
-    
+
     if not isinstance(language, types.ListType):
         language = [language]
     
@@ -145,6 +145,8 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
                                                   min_score_series_perc=int(minimum_score))
 
         if providers:
+            if forced_minimum_score:
+                min_score = int(forced_minimum_score) + 1
             downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
                                                            providers=providers,
                                                            provider_configs=providers_auth,
@@ -168,7 +170,7 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
                 
                 try:
                     fld = get_target_folder(path)
-                    chmod = int(settings.general.chmod, 8) if not sys.platform.startswith('win') else None
+                    chmod = int(settings.general.chmod, 8) if not sys.platform.startswith('win') and settings.general.getboolean('chmod_enabled') else None
                     saved_subtitles = save_subtitles(video.original_path, subtitles, single=single,
                                                      tags=None,  # fixme
                                                      directory=fld,
@@ -183,16 +185,23 @@ def download_subtitle(path, language, hi, providers, providers_auth, sceneName,
                     saved_any = True
                     for subtitle in saved_subtitles:
                         downloaded_provider = subtitle.provider_name
-                        downloaded_language = language_from_alpha3(subtitle.language.alpha3)
+                        if subtitle.language == 'pt-BR':
-                        downloaded_language_code2 = alpha2_from_alpha3(subtitle.language.alpha3)
+                            downloaded_language_code3 = 'pob'
-                        downloaded_language_code3 = subtitle.language.alpha3
+                        else:
+                            downloaded_language_code3 = subtitle.language.alpha3
+                        downloaded_language = language_from_alpha3(downloaded_language_code3)
+                        downloaded_language_code2 = alpha2_from_alpha3(downloaded_language_code3)
                         downloaded_path = subtitle.storage_path
                         logging.debug('BAZARR Subtitles file saved to disk: ' + downloaded_path)
+                        if is_upgrade:
+                            action = "upgraded"
+                        else:
+                            action = "downloaded"
                         if video.used_scene_name:
-                            message = downloaded_language + " subtitles downloaded from " + downloaded_provider + " with a score of " + unicode(
+                            message = downloaded_language + " subtitles " + action + " from " + downloaded_provider + " with a score of " + unicode(
                                 round(subtitle.score * 100 / max_score, 2)) + "% using this scene name: " + sceneName
                         else:
-                            message = downloaded_language + " subtitles downloaded from " + downloaded_provider + " with a score of " + unicode(
+                            message = downloaded_language + " subtitles " + action + " from " + downloaded_provider + " with a score of " + unicode(
                                 round(subtitle.score * 100 / max_score, 2)) + "% using filename guessing."
                         
                         if use_postprocessing is True:
@@ -352,7 +361,7 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
             try:
                 score = round(subtitle.score / max_score * 100, 2)
                 fld = get_target_folder(path)
-                chmod = int(settings.general.chmod, 8) if not sys.platform.startswith('win') else None
+                chmod = int(settings.general.chmod, 8) if not sys.platform.startswith('win') and settings.general.getboolean('chmod_enabled') else None
                 saved_subtitles = save_subtitles(video.original_path, [subtitle], single=single,
                                                  tags=None,  # fixme
                                                  directory=fld,
@@ -367,9 +376,12 @@ def manual_download_subtitle(path, language, hi, subtitle, provider, providers_a
                 if saved_subtitles:
                     for saved_subtitle in saved_subtitles:
                         downloaded_provider = saved_subtitle.provider_name
-                        downloaded_language = language_from_alpha3(saved_subtitle.language.alpha3)
+                        if saved_subtitle.language == 'pt-BR':
-                        downloaded_language_code2 = alpha2_from_alpha3(saved_subtitle.language.alpha3)
+                            downloaded_language_code3 = 'pob'
-                        downloaded_language_code3 = saved_subtitle.language.alpha3
+                        else:
+                            downloaded_language_code3 = subtitle.language.alpha3
+                        downloaded_language = language_from_alpha3(downloaded_language_code3)
+                        downloaded_language_code2 = alpha2_from_alpha3(downloaded_language_code3)
                         downloaded_path = saved_subtitle.storage_path
                         logging.debug('BAZARR Subtitles file saved to disk: ' + downloaded_path)
                         message = downloaded_language + " subtitles downloaded from " + downloaded_provider + " with a score of " + unicode(
@@ -707,3 +719,88 @@ def refine_from_db(path, video):
                 if data[6]: video.audio_codec = data[6]
 
     return video
+
+
+def upgrade_subtitles():
+    days_to_upgrade_subs = settings.general.days_to_upgrade_subs
+    minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) -
+                         datetime(1970, 1, 1)).total_seconds()
+
+    if settings.general.getboolean('upgrade_manual'):
+        query_actions = [1, 2, 3]
+    else:
+        query_actions = [1, 3]
+
+    db = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30)
+    c = db.cursor()
+    episodes_list = c.execute("""SELECT table_history.video_path, table_history.language, table_history.score,
+                                        table_shows.hearing_impaired, table_episodes.scene_name, table_episodes.title,
+                                        table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId,
+                                        MAX(table_history.timestamp), table_shows.languages
+                                   FROM table_history
+                             INNER JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId
+                             INNER JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId
+                                  WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND timestamp > ? AND 
+                                        score is not null
+                               GROUP BY table_history.video_path, table_history.language""",
+                              (minimum_timestamp,)).fetchall()
+    movies_list = c.execute("""SELECT table_history_movie.video_path, table_history_movie.language,
+                                      table_history_movie.score, table_movies.hearing_impaired, table_movies.sceneName,
+                                      table_movies.title, table_movies.radarrId, MAX(table_history_movie.timestamp), 
+                                      table_movies.languages
+                                 FROM table_history_movie
+                           INNER JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId
+                                WHERE action  IN (""" + ','.join(map(str, query_actions)) + """) AND timestamp > ? AND 
+                                      score is not null
+                             GROUP BY table_history_movie.video_path, table_history_movie.language""",
+                            (minimum_timestamp,)).fetchall()
+    db.close()
+
+    episodes_to_upgrade = []
+    for episode in episodes_list:
+        if os.path.exists(path_replace(episode[0])) and int(episode[2]) < 360:
+            episodes_to_upgrade.append(episode)
+
+    movies_to_upgrade = []
+    for movie in movies_list:
+        if os.path.exists(path_replace_movie(movie[0])) and int(movie[2]) < 120:
+            movies_to_upgrade.append(movie)
+
+    providers_list = get_providers()
+    providers_auth = get_providers_auth()
+
+    for episode in episodes_to_upgrade:
+        if episode[1] in ast.literal_eval(str(episode[9])):
+            notifications.write(
+                msg='Searching to upgrade ' + str(language_from_alpha2(episode[1])) + ' subtitles for this episode: ' +
+                    path_replace(episode[0]), queue='get_subtitle')
+            result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(episode[1])),
+                                       episode[3], providers_list, providers_auth, str(episode[4]),
+                                       episode[5], 'series', forced_minimum_score=int(episode[2]), is_upgrade=True)
+            if result is not None:
+                message = result[0]
+                path = result[1]
+                language_code = result[2]
+                provider = result[3]
+                score = result[4]
+                store_subtitles(path_replace(episode[0]))
+                history_log(3, episode[6], episode[7], message, path, language_code, provider, score)
+                send_notifications(episode[6], episode[7], message)
+
+    for movie in movies_to_upgrade:
+        if movie[1] in ast.literal_eval(str(movie[8])):
+            notifications.write(
+                msg='Searching to upgrade ' + str(language_from_alpha2(movie[1])) + ' subtitles for this movie: ' +
+                    path_replace_movie(movie[0]), queue='get_subtitle')
+            result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(movie[1])),
+                                       movie[3], providers_list, providers_auth, str(movie[4]),
+                                       movie[5], 'movie', forced_minimum_score=int(movie[2]), is_upgrade=True)
+            if result is not None:
+                message = result[0]
+                path = result[1]
+                language_code = result[2]
+                provider = result[3]
+                score = result[4]
+                store_subtitles_movie(path_replace_movie(movie[0]))
+                history_log_movie(3, movie[6], message, path, language_code, provider, score)
+                send_notifications_movie(movie[6], message)

bazarr/init.py

@@ -127,10 +127,14 @@ try:
     settings.general.enabled_providers = u'' if not providers_list else ','.join(providers_list)
     with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
         settings.write(handle)
-
 except:
     pass
 
+if settings.general.throtteled_providers == '' or None:
+    settings.general.throtteled_providers = '{}'
+    with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
+        settings.write(handle)
+
 if not os.path.exists(os.path.normpath(os.path.join(args.config_dir, 'config', 'users.json'))):
     cork = Cork(os.path.normpath(os.path.join(args.config_dir, 'config')), initialize=True)
     

bazarr/main.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-bazarr_version = '0.7.2.1'
+bazarr_version = '0.7.3'
 
 import gc
 import sys
@@ -53,14 +53,14 @@ from cork import Cork
 from bottle import route, run, template, static_file, request, redirect, response, HTTPError, app, hook, abort
 from datetime import datetime, timedelta
 from get_languages import load_language_in_db, language_from_alpha3
-from get_providers import get_providers, get_providers_auth
+from get_providers import get_providers, get_providers_auth, list_throttled_providers
 from get_series import *
 from get_episodes import *
 
 from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \
     list_missing_subtitles, list_missing_subtitles_movies
 from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \
-    wanted_download_subtitles, wanted_search_missing_subtitles, manual_search, manual_download_subtitle
+    wanted_download_subtitles, wanted_search_missing_subtitles, manual_search, manual_download_subtitle, upgrade_subtitles
 from utils import history_log, history_log_movie
 from scheduler import *
 from notifier import send_notifications, send_notifications_movie
@@ -276,7 +276,18 @@ def save_wizard():
         settings_general_embedded = 'True'
     settings_subfolder = request.forms.get('settings_subfolder')
     settings_subfolder_custom = request.forms.get('settings_subfolder_custom')
-    
+    settings_upgrade_subs = request.forms.get('settings_upgrade_subs')
+    if settings_upgrade_subs is None:
+        settings_upgrade_subs = 'False'
+    else:
+        settings_upgrade_subs = 'True'
+    settings_days_to_upgrade_subs = request.forms.get('settings_days_to_upgrade_subs')
+    settings_upgrade_manual = request.forms.get('settings_upgrade_manual')
+    if settings_upgrade_manual is None:
+        settings_upgrade_manual = 'False'
+    else:
+        settings_upgrade_manual = 'True'
+
     settings.general.ip = text_type(settings_general_ip)
     settings.general.port = text_type(settings_general_port)
     settings.general.base_url = text_type(settings_general_baseurl)
@@ -288,7 +299,10 @@ def save_wizard():
     settings.general.subfolder = text_type(settings_subfolder)
     settings.general.subfolder_custom = text_type(settings_subfolder_custom)
     settings.general.use_embedded_subs = text_type(settings_general_embedded)
-    
+    settings.general.upgrade_subs = text_type(settings_upgrade_subs)
+    settings.general.days_to_upgrade_subs = text_type(settings_days_to_upgrade_subs)
+    settings.general.upgrade_manual = text_type(settings_upgrade_manual)
+
     settings_sonarr_ip = request.forms.get('settings_sonarr_ip')
     settings_sonarr_port = request.forms.get('settings_sonarr_port')
     settings_sonarr_baseurl = request.forms.get('settings_sonarr_baseurl')
@@ -937,7 +951,7 @@ def historyseries():
     today = []
     thisweek = []
     thisyear = []
-    stats = c.execute("SELECT timestamp FROM table_history WHERE action LIKE '1'").fetchall()
+    stats = c.execute("SELECT timestamp FROM table_history WHERE action > 0").fetchall()
     total = len(stats)
     for stat in stats:
         if now - timedelta(hours=24) <= datetime.fromtimestamp(stat[0]) <= now:
@@ -949,14 +963,44 @@ def historyseries():
     stats = [len(today), len(thisweek), len(thisyear), total]
 
     c.execute(
-        "SELECT table_history.action, table_shows.title, table_episodes.season || 'x' || table_episodes.episode, table_episodes.title, table_history.timestamp, table_history.description, table_history.sonarrSeriesId FROM table_history LEFT JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId LEFT JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId ORDER BY id DESC LIMIT ? OFFSET ?",
+        "SELECT table_history.action, table_shows.title, table_episodes.season || 'x' || table_episodes.episode, table_episodes.title, table_history.timestamp, table_history.description, table_history.sonarrSeriesId, table_episodes.path, table_shows.languages, table_history.language, table_history.score FROM table_history LEFT JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId LEFT JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId ORDER BY id DESC LIMIT ? OFFSET ?",
         (page_size, offset,))
     data = c.fetchall()
+
+    upgradable_episodes = []
+    if settings.general.getboolean('upgrade_subs'):
+        days_to_upgrade_subs = settings.general.days_to_upgrade_subs
+        minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) -
+                             datetime(1970, 1, 1)).total_seconds()
+
+        if settings.general.getboolean('upgrade_manual'):
+            query_actions = [1, 2, 3]
+        else:
+            query_actions = [1, 3]
+
+        upgradable_episodes = c.execute("""SELECT video_path, MAX(timestamp), score
+                                             FROM table_history
+                                            WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND 
+                                                  timestamp > ? AND score is not null
+                                         GROUP BY table_history.video_path, table_history.language""",
+                                        (minimum_timestamp,)).fetchall()
+        upgradable_episodes_not_perfect = []
+        for upgradable_episode in upgradable_episodes:
+            try:
+                int(upgradable_episode[2])
+            except ValueError:
+                pass
+            else:
+                if int(upgradable_episode[2]) < 360:
+                    upgradable_episodes_not_perfect.append(upgradable_episode)
+
     c.close()
+
     data = reversed(sorted(data, key=operator.itemgetter(4)))
+
     return template('historyseries', bazarr_version=bazarr_version, rows=data, row_count=row_count,
                     page=page, max_page=max_page, stats=stats, base_url=base_url, page_size=page_size,
-                    current_port=settings.general.port)
+                    current_port=settings.general.port, upgradable_episodes=upgradable_episodes_not_perfect)
 
 
 @route(base_url + 'historymovies')
@@ -980,7 +1024,7 @@ def historymovies():
     today = []
     thisweek = []
     thisyear = []
-    stats = c.execute("SELECT timestamp FROM table_history_movie WHERE action LIKE '1'").fetchall()
+    stats = c.execute("SELECT timestamp FROM table_history_movie WHERE action > 0").fetchall()
     total = len(stats)
     for stat in stats:
         if now - timedelta(hours=24) <= datetime.fromtimestamp(stat[0]) <= now:
@@ -992,14 +1036,42 @@ def historymovies():
     stats = [len(today), len(thisweek), len(thisyear), total]
 
     c.execute(
-        "SELECT table_history_movie.action, table_movies.title, table_history_movie.timestamp, table_history_movie.description, table_history_movie.radarrId FROM table_history_movie LEFT JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId ORDER BY id DESC LIMIT ? OFFSET ?",
+        "SELECT table_history_movie.action, table_movies.title, table_history_movie.timestamp, table_history_movie.description, table_history_movie.radarrId, table_history_movie.video_path, table_movies.languages, table_history_movie.language, table_history_movie.score FROM table_history_movie LEFT JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId ORDER BY id DESC LIMIT ? OFFSET ?",
         (page_size, offset,))
     data = c.fetchall()
+
+    upgradable_movies = []
+    if settings.general.getboolean('upgrade_subs'):
+        days_to_upgrade_subs = settings.general.days_to_upgrade_subs
+        minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) -
+                             datetime(1970, 1, 1)).total_seconds()
+
+        if settings.general.getboolean('upgrade_manual'):
+            query_actions = [1, 2, 3]
+        else:
+            query_actions = [1, 3]
+
+        upgradable_movies = c.execute("""SELECT video_path, MAX(timestamp), score
+                                           FROM table_history_movie
+                                          WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND 
+                                                timestamp > ? AND score is not null
+                                       GROUP BY video_path, language""",
+                                      (minimum_timestamp,)).fetchall()
+        upgradable_movies_not_perfect = []
+        for upgradable_movie in upgradable_movies:
+            try:
+                int(upgradable_movie[2])
+            except ValueError:
+                pass
+            else:
+                if int(upgradable_movie[2]) < 120:
+                    upgradable_movies_not_perfect.append(upgradable_movie)
+
     c.close()
     data = reversed(sorted(data, key=operator.itemgetter(2)))
     return template('historymovies', bazarr_version=bazarr_version, rows=data, row_count=row_count,
                     page=page, max_page=max_page, stats=stats, base_url=base_url, page_size=page_size,
-                    current_port=settings.general.port)
+                    current_port=settings.general.port, upgradable_movies=upgradable_movies_not_perfect)
 
 
 @route(base_url + 'wanted')
@@ -1123,6 +1195,11 @@ def save_settings():
         settings_general_debug = 'False'
     else:
         settings_general_debug = 'True'
+    settings_general_chmod_enabled = request.forms.get('settings_general_chmod_enabled')
+    if settings_general_chmod_enabled is None:
+        settings_general_chmod_enabled = 'False'
+    else:
+        settings_general_chmod_enabled = 'True'
     settings_general_chmod = request.forms.get('settings_general_chmod')
     settings_general_sourcepath = request.forms.getall('settings_general_sourcepath')
     settings_general_destpath = request.forms.getall('settings_general_destpath')
@@ -1190,7 +1267,18 @@ def save_settings():
     settings_page_size = request.forms.get('settings_page_size')
     settings_subfolder = request.forms.get('settings_subfolder')
     settings_subfolder_custom = request.forms.get('settings_subfolder_custom')
-    
+    settings_upgrade_subs = request.forms.get('settings_upgrade_subs')
+    if settings_upgrade_subs is None:
+        settings_upgrade_subs = 'False'
+    else:
+        settings_upgrade_subs = 'True'
+    settings_days_to_upgrade_subs = request.forms.get('settings_days_to_upgrade_subs')
+    settings_upgrade_manual = request.forms.get('settings_upgrade_manual')
+    if settings_upgrade_manual is None:
+        settings_upgrade_manual = 'False'
+    else:
+        settings_upgrade_manual = 'True'
+
     before = (unicode(settings.general.ip), int(settings.general.port), unicode(settings.general.base_url),
               unicode(settings.general.path_mappings), unicode(settings.general.getboolean('use_sonarr')),
               unicode(settings.general.getboolean('use_radarr')), unicode(settings.general.path_mappings_movie))
@@ -1203,6 +1291,7 @@ def save_settings():
     settings.general.base_url = text_type(settings_general_baseurl)
     settings.general.path_mappings = text_type(settings_general_pathmapping)
     settings.general.debug = text_type(settings_general_debug)
+    settings.general.chmod_enabled = text_type(settings_general_chmod_enabled)
     settings.general.chmod = text_type(settings_general_chmod)
     settings.general.branch = text_type(settings_general_branch)
     settings.general.auto_update = text_type(settings_general_automatic)
@@ -1218,6 +1307,9 @@ def save_settings():
     settings.general.page_size = text_type(settings_page_size)
     settings.general.subfolder = text_type(settings_subfolder)
     settings.general.subfolder_custom = text_type(settings_subfolder_custom)
+    settings.general.upgrade_subs = text_type(settings_upgrade_subs)
+    settings.general.days_to_upgrade_subs = text_type(settings_days_to_upgrade_subs)
+    settings.general.upgrade_manual = text_type(settings_upgrade_manual)
     settings.general.minimum_score_movie = text_type(settings_general_minimum_score_movies)
     settings.general.use_embedded_subs = text_type(settings_general_embedded)
     settings.general.adaptive_searching = text_type(settings_general_adaptive_searching)
@@ -1558,6 +1650,8 @@ def system():
         elif job.trigger.__str__().startswith('cron'):
             task_list.append([job.name, get_time_from_cron(job.trigger.fields), next_run, job.id])
 
+    throttled_providers = list_throttled_providers()
+
     i = 0
     with open(os.path.join(args.config_dir, 'log', 'bazarr.log')) as f:
         for i, l in enumerate(f, 1):
@@ -1566,8 +1660,12 @@ def system():
         page_size = int(settings.general.page_size)
         max_page = int(math.ceil(row_count / (page_size + 0.0)))
 
-    with open(os.path.join(args.config_dir, 'config', 'releases.txt'), 'r') as f:
+    try:
-        releases = ast.literal_eval(f.read())
+        with open(os.path.join(args.config_dir, 'config', 'releases.txt'), 'r') as f:
+            releases = ast.literal_eval(f.read())
+    except Exception as e:
+        releases = []
+        logging.exception('BAZARR cannot parse releases caching file: ' + os.path.join(args.config_dir, 'config', 'releases.txt'))
 
     use_sonarr = settings.general.getboolean('use_sonarr')
     apikey_sonarr = settings.sonarr.apikey
@@ -1594,7 +1692,7 @@ def system():
                     operating_system=platform.platform(), python_version=platform.python_version(),
                     config_dir=args.config_dir, bazarr_dir=os.path.normcase(os.getcwd()),
                     base_url=base_url, task_list=task_list, row_count=row_count, max_page=max_page, page_size=page_size,
-                    releases=releases, current_port=settings.general.port)
+                    releases=releases, current_port=settings.general.port, throttled_providers=throttled_providers)
 
 
 @route(base_url + 'logs/<page:int>')
@@ -1745,7 +1843,7 @@ def manual_get_subtitle():
             language_code = result[2]
             provider = result[3]
             score = result[4]
-            history_log(1, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score)
+            history_log(2, sonarrSeriesId, sonarrEpisodeId, message, path, language_code, provider, score)
             send_notifications(sonarrSeriesId, sonarrEpisodeId, message)
             store_subtitles(unicode(episodePath))
             list_missing_subtitles(sonarrSeriesId)
@@ -1834,7 +1932,7 @@ def manual_get_subtitle_movie():
             language_code = result[2]
             provider = result[3]
             score = result[4]
-            history_log_movie(1, radarrId, message, path, language_code, provider, score)
+            history_log_movie(2, radarrId, message, path, language_code, provider, score)
             send_notifications_movie(radarrId, message)
             store_subtitles_movie(unicode(moviePath))
             list_missing_subtitles_movies(radarrId)

bazarr/scheduler.py

@@ -4,7 +4,7 @@ from get_episodes import sync_episodes, update_all_episodes, update_all_movies
 from get_movies import update_movies
 from get_series import update_series
 from config import settings
-from get_subtitle import wanted_search_missing_subtitles
+from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles
 from get_args import args
 
 
@@ -110,6 +110,10 @@ if settings.general.getboolean('use_sonarr') or settings.general.getboolean('use
     scheduler.add_job(wanted_search_missing_subtitles, IntervalTrigger(hours=3), max_instances=1, coalesce=True,
                       misfire_grace_time=15, id='wanted_search_missing_subtitles', name='Search for wanted subtitles')
 
+if settings.general.getboolean('upgrade_subs'):
+    scheduler.add_job(upgrade_subtitles, IntervalTrigger(hours=12), max_instances=1, coalesce=True,
+                      misfire_grace_time=15, id='upgrade_subtitles', name='Upgrade previously downloaded subtitles')
+
 schedule_update_job()
 sonarr_full_update()
 radarr_full_update()

views/episodes.tpl

@@ -467,6 +467,9 @@
         hi = $(this).attr("data-hi");
 		sonarrSeriesId = $(this).attr("data-sonarrSeriesId");
 		sonarrEpisodeId = $(this).attr("data-sonarrEpisodeId");
+		var languages = Array.from({{!subs_languages_list}});
+		var is_pb = languages.includes('pb');
+		var is_pt = languages.includes('pt');
 
 		const values = {
 			episodePath: episodePath,
@@ -505,7 +508,15 @@
         			return data +'%';
     				}
 				},
-				{ data: 'language' },
+				{ data: null,
+				render: function ( data, type, row ) {
+		    		if ( data.language === "pt" && is_pb === true && is_pt === false) {
+		    			return 'pb'
+					} else {
+		    			return data.language
+					}
+					}
+				},
 				{ data: 'hearing_impaired' },
 				{ data: null,
 				render: function ( data, type, row ) {

views/historymovies.tpl

@@ -52,6 +52,7 @@
 					</tr>
 				</thead>
 				<tbody>
+				%import ast
 				%import time
 				%import pretty
 				%for row in rows:
@@ -65,6 +66,14 @@
 							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been downloaded." data-inverted="" data-position="top left">
 								<i class="ui download icon"></i>
 							</div>
+						%elif row[0] == 2:
+							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been manually downloaded." data-inverted="" data-position="top left">
+								<i class="ui user icon"></i>
+							</div>
+						%elif row[0] == 3:
+							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been upgraded." data-inverted="" data-position="top left">
+								<i class="ui recycle icon"></i>
+							</div>
 						%end
 						</td>
 						<td>
@@ -75,7 +84,20 @@
 								{{pretty.date(int(row[2]))}}
 							</div>
 						</td>
-						<td>{{row[3]}}</td>
+						<td>
+							% upgradable_criteria = (row[5], row[2], row[8])
+							% if upgradable_criteria in upgradable_movies:
+							% if row[7] in ast.literal_eval(str(row[6])):
+							<div class="ui inverted basic compact icon" data-tooltip="This subtitles is eligible to an upgrade." data-inverted="" data-position="top left">
+								<i class="ui green recycle icon upgrade"></i>{{row[3]}}
+							</div>
+							% else:
+							{{row[3]}}
+							% end
+							% else:
+							{{row[3]}}
+							%end
+						</td>
 					</tr>
 				%end
 				</tbody>
@@ -174,7 +196,7 @@
 	    sessionStorage.clear();
 	}
 
-	$('a, i').on('click', function(){
+	$('a').on('click', function(){
 		sessionStorage.scrolly=$(window).scrollTop();
 
 		$('#loader').addClass('active');

views/historyseries.tpl

@@ -54,6 +54,7 @@
 					</tr>
 				</thead>
 				<tbody>
+				%import ast
 				%import time
 				%import pretty
 				%for row in rows:
@@ -67,6 +68,14 @@
 							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been downloaded." data-inverted="" data-position="top left">
 								<i class="ui download icon"></i>
 							</div>
+						%elif row[0] == 2:
+							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been manually downloaded." data-inverted="" data-position="top left">
+								<i class="ui user icon"></i>
+							</div>
+						%elif row[0] == 3:
+							<div class="ui inverted basic compact icon" data-tooltip="Subtitles file have been upgraded." data-inverted="" data-position="top left">
+								<i class="ui recycle icon"></i>
+							</div>
 						%end
 						</td>
 						<td>
@@ -90,7 +99,20 @@
 								{{pretty.date(int(row[4]))}}
 							</div>
 						</td>
-						<td>{{row[5]}}</td>
+						<td>
+							% upgradable_criteria = (row[7], row[4], row[10])
+							% if upgradable_criteria in upgradable_episodes:
+							% if row[9] in ast.literal_eval(str(row[8])):
+							<div class="ui inverted basic compact icon" data-tooltip="This subtitles is eligible to an upgrade." data-inverted="" data-position="top left">
+								<i class="ui green recycle icon upgrade"></i>{{row[5]}}
+							</div>
+							% else:
+							{{row[5]}}
+							% end
+							% else:
+							{{row[5]}}
+							%end
+						</td>
 					</tr>
 				%end
 				</tbody>
@@ -189,7 +211,7 @@
 	    sessionStorage.clear();
 	}
 
-	$('a, i').on('click', function(){
+	$('a').on('click', function(){
 		sessionStorage.scrolly=$(window).scrollTop();
 
 		$('#loader').addClass('active');

views/menu.tpl

@@ -25,7 +25,11 @@
     </head>
     <body>
 		% from get_args import args
+		% from get_providers import update_throttled_provider
+		% update_throttled_provider()
 
+		% import ast
+		% import datetime
 		% import os
 		% import sqlite3
         % from config import settings
@@ -46,7 +50,8 @@
     	% c = conn.cursor()
 		% wanted_series = c.execute("SELECT COUNT(*) FROM table_episodes WHERE missing_subtitles != '[]'" + monitored_only_query_string_sonarr).fetchone()
 		% wanted_movies = c.execute("SELECT COUNT(*) FROM table_movies WHERE missing_subtitles != '[]'" + monitored_only_query_string_radarr).fetchone()
-
+		% from get_providers import list_throttled_providers
+		% throttled_providers_count = len(eval(str(settings.general.throtteled_providers)))
 		<div id="divmenu" class="ui container">
 			<div class="ui grid">
 				<div class="middle aligned row">
@@ -96,7 +101,13 @@
 												Settings
 											</a>
 											<a class="item" href="{{base_url}}system">
-												<i class="laptop icon"></i>
+												<i class="laptop icon">
+													% if throttled_providers_count:
+													<div class="floating ui tiny yellow label" style="left:90% !important;top:0.5em !important;">
+														{{throttled_providers_count}}
+													</div>
+													% end
+												</i>
 												System
 											</a>
 											<a id="donate" class="item" href="https://beerpay.io/morpheus65535/bazarr">

views/movie.tpl

@@ -416,6 +416,9 @@
 		language = $(this).attr("data-language");
 		hi = $(this).attr("data-hi");
 		radarrId = $(this).attr("data-radarrId");
+		var languages = Array.from({{!subs_languages_list}});
+		var is_pb = languages.includes('pb');
+		var is_pt = languages.includes('pt');
 
 		const values = {
 			moviePath: moviePath,
@@ -453,7 +456,15 @@
         			return data +'%';
     				}
 				},
-				{ data: 'language' },
+				{ data: null,
+				render: function ( data, type, row ) {
+		    		if ( data.language === "pt" && is_pb === true && is_pt === false) {
+		    			return 'pb'
+					} else {
+		    			return data.language
+					}
+					}
+				},
 				{ data: 'hearing_impaired' },
 				{ data: null,
 				render: function ( data, type, row ) {

views/settings.tpl

@@ -152,6 +152,17 @@
                                 </div>
                             </div>
                         </div>
+                        <div id="chmod_enabled" class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Enable chmod</label>
+                            </div>
+                            <div class="five wide column">
+                                <div id="settings_chmod_enabled" class="ui toggle checkbox" data-chmod={{settings.general.getboolean('chmod_enabled')}}>
+                                    <input name="settings_general_chmod_enabled" type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+                        </div>
                         <div id="chmod" class="middle aligned row">
                             <div class="right aligned four wide column">
                                 <label>Set subtitle file permissions to</label>
@@ -599,7 +610,7 @@
                             </div>
                         </div>
 
-                        <div class="middle aligned row">
+                        <div class="middle aligned row postprocessing">
                             <div class="right aligned four wide column">
                                 <label>Post-processing command</label>
                             </div>
@@ -610,7 +621,7 @@
                             </div>
                         </div>
 
-                        <div class="middle aligned row">
+                        <div class="middle aligned row postprocessing">
                             <div class="right aligned four wide column">
                                 <label>Variables you can use in your command (include the double curly brace):</label>
                             </div>
@@ -1103,6 +1114,7 @@
                         </div>
 
                         <div class="middle aligned row subfolder">
+                            <div class="two wide column"></div>
                             <div class="right aligned four wide column">
                                 <label>Custom Subtitle folder</label>
                             </div>
@@ -1123,6 +1135,62 @@
                             </div>
                         </div>
 
+                        <div class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Upgrade previously downloaded subtitles</label>
+                            </div>
+                            <div class="one wide column">
+                                <div id="settings_upgrade_subs" class="ui toggle checkbox" data-upgrade={{settings.general.getboolean('upgrade_subs')}}>
+                                    <input name="settings_upgrade_subs" type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+
+                            <div class="collapsed center aligned column">
+                                <div class="ui basic icon"
+                                     data-tooltip='Schedule a task that run every 12 hours to upgrade subtitles previously downloaded by Bazarr.'
+                                     data-inverted="">
+                                    <i class="help circle large icon"></i>
+                                </div>
+                            </div>
+                        </div>
+
+                        <div class="middle aligned row upgrade_subs">
+                            <div class="two wide column"></div>
+                            <div class="right aligned four wide column">
+                                <label>Number of days to go back in history to upgrade subtitles (up to 30)</label>
+                            </div>
+                            <div class="five wide column">
+                                <div class='field'>
+                                    <div class="ui fluid input">
+                                        <input id="settings_days_to_upgrade_subs" name="settings_days_to_upgrade_subs"
+                                               type="text" value="{{ settings.general.days_to_upgrade_subs }}">
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+
+                        <div class="middle aligned row upgrade_subs">
+                            <div class="two wide column"></div>
+                            <div class="right aligned four wide column">
+                                <label>Upgrade manually downloaded subtitles</label>
+                            </div>
+                            <div class="one wide column">
+                                <div id="settings_upgrade_manual" class="ui toggle checkbox" data-upgrade-manual={{settings.general.getboolean('upgrade_manual')}}>
+                                    <input name="settings_upgrade_manual" type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+
+                            <div class="collapsed center aligned column">
+                                <div class="ui basic icon"
+                                     data-tooltip='Enable or disable upgrade of manually searched and downloaded subtitles.'
+                                     data-inverted="">
+                                    <i class="help circle large icon"></i>
+                                </div>
+                            </div>
+                        </div>
+
                         <div class="middle aligned row">
                             <div class="right aligned four wide column">
                                 <label>Use embedded subtitles</label>
@@ -2001,6 +2069,7 @@
     % import sys
     % if sys.platform.startswith('win'):
     $("#chmod").hide();
+    $("#chmod_enabled").hide();
     % end
 
     $('.menu .item')
@@ -2045,6 +2114,12 @@
                 $("#settings_debug").checkbox('uncheck');
             }
 
+    if ($('#settings_chmod_enabled').data("chmod") === "True") {
+                $("#settings_chmod_enabled").checkbox('check');
+            } else {
+                $("#settings_chmod_enabled").checkbox('uncheck');
+            }
+
     if ($('#settings_single_language').data("single-language") === "True") {
                 $("#settings_single_language").checkbox('check');
             } else {
@@ -2057,6 +2132,18 @@
                 $("#settings_scenename").checkbox('uncheck');
             }
 
+    if ($('#settings_upgrade_subs').data("upgrade") === "True") {
+                $("#settings_upgrade_subs").checkbox('check');
+            } else {
+                $("#settings_upgrade_subs").checkbox('uncheck');
+            }
+
+    if ($('#settings_upgrade_manual').data("upgrade-manual") === "True") {
+                $("#settings_upgrade_manual").checkbox('check');
+            } else {
+                $("#settings_upgrade_manual").checkbox('uncheck');
+            }
+
     if ($('#settings_embedded').data("embedded") === "True") {
                 $("#settings_embedded").checkbox('check');
             } else {
@@ -2180,6 +2267,51 @@
         }
     });
 
+    if ($('#settings_use_postprocessing').data("postprocessing") === "True") {
+                $('.postprocessing').show();
+            } else {
+                $('.postprocessing').hide();
+            }
+
+    $('#settings_use_postprocessing').checkbox({
+        onChecked: function() {
+            $('.postprocessing').show();
+        },
+        onUnchecked: function() {
+            $('.postprocessing').hide();
+        }
+    });
+
+    if ($('#settings_upgrade_subs').data("upgrade") === "True") {
+                $('.upgrade_subs').show();
+            } else {
+                $('.upgrade_subs').hide();
+            }
+
+    $('#settings_upgrade_subs').checkbox({
+        onChecked: function() {
+            $('.upgrade_subs').show();
+        },
+        onUnchecked: function() {
+            $('.upgrade_subs').hide();
+        }
+    });
+
+    if ($('#settings_chmod_enabled').data("chmod") === "True") {
+                $('#chmod').show();
+            } else {
+                $('#chmod').hide();
+            }
+
+    $('#settings_chmod_enabled').checkbox({
+        onChecked: function() {
+            $('#chmod').show();
+        },
+        onUnchecked: function() {
+            $('#chmod').hide();
+        }
+    });
+
     if ($('#settings_auth_type').val() === "None") {
         $('.auth_option').hide();
     }
@@ -2400,7 +2532,7 @@
                         }
                     ]
                 },
-                % if not sys.platform.startswith('win'):
+                % if not sys.platform.startswith('win') and settings.general.getboolean('chmod_enabled'):
                 settings_general_chmod: {
                     rules: [
                         {
@@ -2514,6 +2646,16 @@
                             type : 'empty'
                         }
                     ]
+                },
+                settings_days_to_upgrade_subs : {
+                    rules : [
+                        {
+                            type : 'integer[1..30]'
+                        },
+                        {
+                            type : 'empty'
+                        }
+                    ]
                 }
             },
             inline : true,

views/system.tpl

@@ -54,9 +54,18 @@
                     <div id="logout" class="ui icon button" data-tooltip="Logout" data-inverted=""><i class="sign-out icon"></i></div>
                 % end
 			</div>
-			<div class="ui top attached tabular menu">
+			% import datetime
+            % throttled_providers_count = len(eval(str(settings.general.throtteled_providers)))
+            <div class="ui top attached tabular menu">
 				<a class="tabs item active" data-tab="tasks">Tasks</a>
 				<a class="tabs item" data-tab="logs">Logs</a>
+                <a class="tabs item" data-tab="providers">Providers
+                    % if throttled_providers_count:
+                    <div class="ui tiny yellow label">
+                        {{throttled_providers_count}}
+                    </div>
+                    % end
+                </a>
                 <a class="tabs item" data-tab="status">Status</a>
 				<a class="tabs item" data-tab="releases">Releases</a>
 			</div>
@@ -120,6 +129,28 @@
                     %end
 				</div>
 			</div>
+            <div class="ui bottom attached tab segment" data-tab="providers">
+				<div class="content">
+					<table class="ui very basic table">
+						<thead>
+							<tr>
+								<th>Name</th>
+								<th>Status</th>
+								<th>Next retry</th>
+							</tr>
+						</thead>
+						<tbody>
+						%for provider in throttled_providers:
+							<tr>
+								<td>{{provider[0]}}</td>
+								<td>{{provider[1] if provider[1] is not None else "Good"}}</td>
+								<td>{{provider[2] if provider[2] is not "now" else "-"}}</td>
+							</tr>
+						%end
+						</tbody>
+					</table>
+				</div>
+			</div>
             <div class="ui bottom attached tab segment" data-tab="status">
 				<div class="ui dividing header">About</div>
                 <div class="twelve wide column">