Optimize provider throttling.

bazarr/get_providers.py

@@ -4,6 +4,7 @@ import datetime
 import logging
 import subliminal_patch
 import pretty
+import time
 
 from get_args import args
 from config import settings
@@ -33,6 +34,8 @@ PROVIDER_THROTTLE_MAP = {
 PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska",
                         "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
 
+throttle_count = {}
+
 if not settings.general.throtteled_providers:
     tp = {}
 else:
@@ -137,8 +140,8 @@ def provider_throttle(name, exception):
     throttle_delta, throttle_description = throttle_data
     throttle_until = datetime.datetime.now() + throttle_delta
     
+    if throttled_count(name):
         tp[name] = (cls_name, throttle_until, throttle_description)
-    
         settings.general.throtteled_providers = str(tp)
         with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
             settings.write(handle)
@@ -146,6 +149,37 @@ def provider_throttle(name, exception):
         logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
                      throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
 
+
+def throttled_count(name):
+    global throttle_count
+    if name in throttle_count.keys():
+        if 'count' in throttle_count[name].keys():
+            for key, value in throttle_count[name].items():
+                if key == 'count':
+                    value += 1
+                    throttle_count[name]['count'] = value
+        else:
+            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
+    
+    else:
+        throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
+    
+    if throttle_count[name]['count'] < 5:
+        if throttle_count[name]['time'] > datetime.datetime.now():
+            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
+                         throttle_count[name]['count'])
+            time.sleep(5)
+            return False
+        else:
+            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
+            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
+                         throttle_count[name]['count'])
+            time.sleep(5)
+            return False
+    else:
+        return True
+
+
 def update_throttled_provider():
     changed = False
     if settings.general.enabled_providers:

bazarr/get_subtitle.py

@@ -5,7 +5,6 @@ import sys
 import sqlite3
 import ast
 import logging
-import operator
 import subprocess
 import time
 import cPickle as pickle
@@ -13,7 +12,6 @@ import codecs
 import types
 import re
 import subliminal
-import subliminal_patch
 from datetime import datetime, timedelta
 from subzero.language import Language
 from subzero.video import parse_video
@@ -58,7 +56,7 @@ def get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=No
         path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1])
         used_scene_name = True
         hash_from = original_path
-    if providers:
+    
     try:
         video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name,
                             hash_from=hash_from)
@@ -75,9 +73,6 @@ def get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=No
     
     except:
         logging.exception("BAZARR Error trying to get video information for this file: " + path)
-    else:
-        logging.info("BAZARR All providers are throttled")
-        return None
 
 
 def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_score_series_perc=240 * 100 / 360.0,
@@ -125,9 +120,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         language = [language]
     
     if forced == "True":
-        providers_auth['podnapisi']['only_foreign'] = True
+        providers_auth['podnapisi']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
-        providers_auth['subscene']['only_foreign'] = True
+        providers_auth['subscene']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
-        providers_auth['opensubtitles']['only_foreign'] = True
+        providers_auth['opensubtitles']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
     else:
         providers_auth['podnapisi']['only_foreign'] = False
         providers_auth['subscene']['only_foreign'] = False
@@ -161,8 +156,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         post_download_hook=None,
         language_hook=None
     """
-    
+    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
-    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
+                      media_type=media_type)
     if video:
         min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
                                                   min_score_series_perc=int(minimum_score))
@@ -184,6 +179,7 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         else:
             downloaded_subtitles = None
             logging.info("BAZARR All providers are throttled")
+            return None
         
         saved_any = False
         if downloaded_subtitles:
@@ -317,8 +313,12 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
     minimum_score_movie = settings.general.minimum_score_movie
     use_postprocessing = settings.general.getboolean('use_postprocessing')
     postprocessing_cmd = settings.general.postprocessing_cmd
-    
+    if providers:
-    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
+        video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
+                          media_type=media_type)
+    else:
+        logging.info("BAZARR All providers are throttled")
+        return None
     if video:
         min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
                                                   min_score_series_perc=int(minimum_score))
@@ -333,6 +333,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
             else:
                 subtitles = []
                 logging.info("BAZARR All providers are throttled")
+                return None
         except Exception as e:
             logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
         else:
@@ -389,7 +390,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
     use_postprocessing = settings.general.getboolean('use_postprocessing')
     postprocessing_cmd = settings.general.postprocessing_cmd
     single = settings.general.getboolean('single_language')
-    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider}, media_type=media_type)
+    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider},
+                      media_type=media_type)
     if video:
         min_score, max_score, scores = get_scores(video, media_type)
         try:
@@ -510,13 +512,15 @@ def series_download_subtitles(no):
     count_episodes_details = len(episodes_details)
     
     for i, episode in enumerate(episodes_details, 1):
+        if providers_list:
             for language in ast.literal_eval(episode[1]):
                 if language is not None:
                     notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i,
                                         length=count_episodes_details)
                     result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
                                                series_details[0], series_details[2], providers_list,
-                                           providers_auth, str(episode[3]), series_details[1], 'series')
+                                               providers_auth, str(episode[3]), series_details[1],
+                                               'series')
                     if result is not None:
                         message = result[0]
                         path = result[1]
@@ -527,6 +531,10 @@ def series_download_subtitles(no):
                         store_subtitles(path_replace(episode[0]))
                         history_log(1, no, episode[2], message, path, language_code, provider, score)
                         send_notifications(no, episode[2], message)
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
+            break
     list_missing_subtitles(no)
     
     if count_episodes_details:
@@ -551,6 +559,7 @@ def episode_download_subtitles(no):
     providers_auth = get_providers_auth()
     
     for episode in episodes_details:
+        if providers_list:
             for language in ast.literal_eval(episode[1]):
                 if language is not None:
                     notifications.write(msg='Searching for ' + str(
@@ -569,8 +578,11 @@ def episode_download_subtitles(no):
                         store_subtitles(path_replace(episode[0]))
                         history_log(1, episode[6], episode[2], message, path, language_code, provider, score)
                         send_notifications(episode[6], episode[2], message)
-        
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
             list_missing_subtitles(episode[6])
+            break
 
 
 def movies_download_subtitles(no):
@@ -587,8 +599,10 @@ def movies_download_subtitles(no):
     count_movie = len(ast.literal_eval(movie[1]))
     
     for i, language in enumerate(ast.literal_eval(movie[1]), 1):
+        if providers_list:
             if language is not None:
-            notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i, length=count_movie)
+                notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i,
+                                    length=count_movie)
                 result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4],
                                            movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie')
                 if result is not None:
@@ -601,6 +615,10 @@ def movies_download_subtitles(no):
                     store_subtitles_movie(path_replace_movie(movie[0]))
                     history_log_movie(1, no, message, path, language_code, provider, score)
                     send_notifications_movie(no, message)
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
+            break
     list_missing_subtitles_movies(no)
     
     if count_movie:
@@ -744,24 +762,28 @@ def wanted_search_missing_subtitles():
     movies = c.fetchall()
     
     c.close()
-    providers = get_providers()
     if settings.general.getboolean('use_sonarr'):
-        if providers:
+        
         count_episodes = len(episodes)
         for i, episode in enumerate(episodes, 1):
+            providers = get_providers()
+            if providers:
                 wanted_download_subtitles(episode[0], i, count_episodes)
             else:
                 notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
                 logging.info("BAZARR All providers are throttled")
+                return
     
     if settings.general.getboolean('use_radarr'):
-        if providers:
         count_movies = len(movies)
         for i, movie in enumerate(movies, 1):
+            providers = get_providers()
+            if providers:
                 wanted_download_subtitles_movie(movie[0], i, count_movies)
             else:
                 notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
                 logging.info("BAZARR All providers are throttled")
+                return
     
     logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.')
     
@@ -920,6 +942,11 @@ def upgrade_subtitles():
     
     if settings.general.getboolean('use_sonarr'):
         for i, episode in enumerate(episodes_to_upgrade, 1):
+            providers = get_providers()
+            if not providers:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
             if episode[9] != "None":
                 desired_languages = ast.literal_eval(str(episode[9]))
                 if episode[10] == "True":
@@ -957,6 +984,11 @@ def upgrade_subtitles():
     
     if settings.general.getboolean('use_radarr'):
         for i, movie in enumerate(movies_to_upgrade, 1):
+            providers = get_providers()
+            if not providers:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
             if movie[8] != "None":
                 desired_languages = ast.literal_eval(str(movie[8]))
                 if movie[9] == "True":