Merge remote-tracking branch 'origin/development' into development

bazarr/get_providers.py

@@ -4,6 +4,7 @@ import datetime
 import logging
 import subliminal_patch
 import pretty
+import time
 
 from get_args import args
 from config import settings
@@ -33,6 +34,8 @@ PROVIDER_THROTTLE_MAP = {
 PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendastv", "napiprojekt", "shooter", "hosszupuska",
                         "supersubtitles", "titlovi", "argenteam", "assrt", "subscene"]
 
+throttle_count = {}
+
 if not settings.general.throtteled_providers:
     tp = {}
 else:
@@ -44,7 +47,7 @@ def provider_pool():
         return subliminal_patch.core.SZAsyncProviderPool
     return subliminal_patch.core.SZProviderPool
 
-    
+
 def get_providers():
     changed = False
     providers_list = []
@@ -52,23 +55,23 @@ def get_providers():
         for provider in settings.general.enabled_providers.lower().split(','):
             reason, until, throttle_desc = tp.get(provider, (None, None, None))
             providers_list.append(provider)
-    
+            
             if reason:
                 now = datetime.datetime.now()
                 if now < until:
                     logging.debug("Not using %s until %s, because of: %s", provider,
-                                 until.strftime("%y/%m/%d %H:%M"), reason)
+                                  until.strftime("%y/%m/%d %H:%M"), reason)
                     providers_list.remove(provider)
                 else:
                     logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
                     del tp[provider]
                     settings.general.throtteled_providers = str(tp)
                     changed = True
-
+        
         if changed:
             with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
                 settings.write(handle)
-
+        
         # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles
         #     for provider in providers_list:
         #         if provider in PROVIDERS_FORCED_OFF:
@@ -109,7 +112,7 @@ def get_providers_auth():
                        },
         'xsubs': {'username': settings.xsubs.username,
                   'password': settings.xsubs.password,
-                       },
+                  },
         'assrt': {'token': settings.assrt.token, },
         'napisy24': {'username': settings.napisy24.username,
                      'password': settings.napisy24.password,
@@ -137,21 +140,52 @@ def provider_throttle(name, exception):
     throttle_delta, throttle_description = throttle_data
     throttle_until = datetime.datetime.now() + throttle_delta
     
-    tp[name] = (cls_name, throttle_until, throttle_description)
+    if throttled_count(name):
+        tp[name] = (cls_name, throttle_until, throttle_description)
+        settings.general.throtteled_providers = str(tp)
+        with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
+            settings.write(handle)
+        
+        logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
+                     throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
+
+
+def throttled_count(name):
+    global throttle_count
+    if name in throttle_count.keys():
+        if 'count' in throttle_count[name].keys():
+            for key, value in throttle_count[name].items():
+                if key == 'count':
+                    value += 1
+                    throttle_count[name]['count'] = value
+        else:
+            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
     
-    settings.general.throtteled_providers = str(tp)
-    with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
-        settings.write(handle)
+    else:
+        throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
     
-    logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name, throttle_description,
-                 throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.message)
+    if throttle_count[name]['count'] < 5:
+        if throttle_count[name]['time'] > datetime.datetime.now():
+            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
+                         throttle_count[name]['count'])
+            time.sleep(5)
+            return False
+        else:
+            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
+            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
+                         throttle_count[name]['count'])
+            time.sleep(5)
+            return False
+    else:
+        return True
+
 
 def update_throttled_provider():
     changed = False
     if settings.general.enabled_providers:
         for provider in settings.general.enabled_providers.lower().split(','):
             reason, until, throttle_desc = tp.get(provider, (None, None, None))
-
+            
             if reason:
                 now = datetime.datetime.now()
                 if now < until:
@@ -161,7 +195,7 @@ def update_throttled_provider():
                     del tp[provider]
                     settings.general.throtteled_providers = str(tp)
                     changed = True
-
+        
         if changed:
             with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
                 settings.write(handle)

bazarr/get_subtitle.py

@@ -5,7 +5,6 @@ import sys
 import sqlite3
 import ast
 import logging
-import operator
 import subprocess
 import time
 import cPickle as pickle
@@ -13,7 +12,6 @@ import codecs
 import types
 import re
 import subliminal
-import subliminal_patch
 from datetime import datetime, timedelta
 from subzero.language import Language
 from subzero.video import parse_video
@@ -58,26 +56,23 @@ def get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=No
         path = os.path.join(os.path.dirname(path), sceneName + os.path.splitext(path)[1])
         used_scene_name = True
         hash_from = original_path
-    if providers:
-        try:
-            video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name,
-                                hash_from=hash_from)
-            video.used_scene_name = used_scene_name
-            video.original_name = original_name
-            video.original_path = original_path
-            refine_from_db(original_path, video)
-
-            if use_mediainfo:
-                refine_from_mediainfo(original_path, video)
-
-            logging.debug('BAZARR is using those video object properties: %s', vars(video))
-            return video
+    
+    try:
+        video = parse_video(path, hints=hints, providers=providers, dry_run=used_scene_name,
+                            hash_from=hash_from)
+        video.used_scene_name = used_scene_name
+        video.original_name = original_name
+        video.original_path = original_path
+        refine_from_db(original_path, video)
         
-        except:
-            logging.exception("BAZARR Error trying to get video information for this file: " + path)
-    else:
-        logging.info("BAZARR All providers are throttled")
-        return None
+        if use_mediainfo:
+            refine_from_mediainfo(original_path, video)
+        
+        logging.debug('BAZARR is using those video object properties: %s', vars(video))
+        return video
+    
+    except:
+        logging.exception("BAZARR Error trying to get video information for this file: " + path)
 
 
 def get_scores(video, media_type, min_score_movie_perc=60 * 100 / 120.0, min_score_series_perc=240 * 100 / 360.0,
@@ -125,9 +120,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         language = [language]
     
     if forced == "True":
-        providers_auth['podnapisi']['only_foreign'] = True
-        providers_auth['subscene']['only_foreign'] = True
-        providers_auth['opensubtitles']['only_foreign'] = True
+        providers_auth['podnapisi']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
+        providers_auth['subscene']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
+        providers_auth['opensubtitles']['only_foreign'] = True  ## fixme: This is also in get_providers_auth()
     else:
         providers_auth['podnapisi']['only_foreign'] = False
         providers_auth['subscene']['only_foreign'] = False
@@ -161,8 +156,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         post_download_hook=None,
         language_hook=None
     """
-    
-    video = get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
+    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
+                      media_type=media_type)
     if video:
         min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
                                                   min_score_series_perc=int(minimum_score))
@@ -184,6 +179,7 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         else:
             downloaded_subtitles = None
             logging.info("BAZARR All providers are throttled")
+            return None
         
         saved_any = False
         if downloaded_subtitles:
@@ -273,9 +269,9 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
         if not saved_any:
             logging.debug('BAZARR No subtitles were found for this file: ' + path)
             return None
-
+    
     subliminal.region.backend.sync()
-
+    
     logging.debug('BAZARR Ended searching subtitles for file: ' + path)
 
 
@@ -317,8 +313,12 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
     minimum_score_movie = settings.general.minimum_score_movie
     use_postprocessing = settings.general.getboolean('use_postprocessing')
     postprocessing_cmd = settings.general.postprocessing_cmd
-    
-    video = get_video(path, title, sceneName, use_scenename, use_mediainfo, providers=providers, media_type=media_type)
+    if providers:
+        video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers=providers,
+                          media_type=media_type)
+    else:
+        logging.info("BAZARR All providers are throttled")
+        return None
     if video:
         min_score, max_score, scores = get_scores(video, media_type, min_score_movie_perc=int(minimum_score_movie),
                                                   min_score_series_perc=int(minimum_score))
@@ -333,6 +333,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
             else:
                 subtitles = []
                 logging.info("BAZARR All providers are throttled")
+                return None
         except Exception as e:
             logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
         else:
@@ -368,9 +369,9 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
             final_subtitles = sorted(subtitles_list, key=lambda x: x['score'], reverse=True)
             logging.debug('BAZARR ' + str(len(final_subtitles)) + " subtitles have been found for this file: " + path)
             logging.debug('BAZARR Ended searching subtitles for this file: ' + path)
-
+    
     subliminal.region.backend.sync()
-
+    
     return final_subtitles
 
 
@@ -389,7 +390,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
     use_postprocessing = settings.general.getboolean('use_postprocessing')
     postprocessing_cmd = settings.general.postprocessing_cmd
     single = settings.general.getboolean('single_language')
-    video = get_video(path, title, sceneName, use_scenename, use_mediainfo, providers={provider}, media_type=media_type)
+    video = get_video(force_unicode(path), title, sceneName, use_scenename, use_mediainfo, providers={provider},
+                      media_type=media_type)
     if video:
         min_score, max_score, scores = get_scores(video, media_type)
         try:
@@ -483,9 +485,9 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro
                         "BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(
                             subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
                     return None
-
+    
     subliminal.region.backend.sync()
-
+    
     logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path)
 
 
@@ -510,23 +512,29 @@ def series_download_subtitles(no):
     count_episodes_details = len(episodes_details)
     
     for i, episode in enumerate(episodes_details, 1):
-        for language in ast.literal_eval(episode[1]):
-            if language is not None:
-                notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i,
-                                    length=count_episodes_details)
-                result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
-                                           series_details[0], series_details[2], providers_list,
-                                           providers_auth, str(episode[3]), series_details[1], 'series')
-                if result is not None:
-                    message = result[0]
-                    path = result[1]
-                    forced = result[5]
-                    language_code = result[2] + ":forced" if forced else result[2]
-                    provider = result[3]
-                    score = result[4]
-                    store_subtitles(path_replace(episode[0]))
-                    history_log(1, no, episode[2], message, path, language_code, provider, score)
-                    send_notifications(no, episode[2], message)
+        if providers_list:
+            for language in ast.literal_eval(episode[1]):
+                if language is not None:
+                    notifications.write(msg='Searching for series subtitles...', queue='get_subtitle', item=i,
+                                        length=count_episodes_details)
+                    result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
+                                               series_details[0], series_details[2], providers_list,
+                                               providers_auth, str(episode[3]), series_details[1],
+                                               'series')
+                    if result is not None:
+                        message = result[0]
+                        path = result[1]
+                        forced = result[5]
+                        language_code = result[2] + ":forced" if forced else result[2]
+                        provider = result[3]
+                        score = result[4]
+                        store_subtitles(path_replace(episode[0]))
+                        history_log(1, no, episode[2], message, path, language_code, provider, score)
+                        send_notifications(no, episode[2], message)
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
+            break
     list_missing_subtitles(no)
     
     if count_episodes_details:
@@ -551,26 +559,30 @@ def episode_download_subtitles(no):
     providers_auth = get_providers_auth()
     
     for episode in episodes_details:
-        for language in ast.literal_eval(episode[1]):
-            if language is not None:
-                notifications.write(msg='Searching for ' + str(
-                    language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]),
-                                    queue='get_subtitle')
-                result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
-                                           episode[4], episode[7], providers_list, providers_auth, str(episode[3]),
-                                           episode[5], 'series')
-                if result is not None:
-                    message = result[0]
-                    path = result[1]
-                    forced = result[5]
-                    language_code = result[2] + ":forced" if forced else result[2]
-                    provider = result[3]
-                    score = result[4]
-                    store_subtitles(path_replace(episode[0]))
-                    history_log(1, episode[6], episode[2], message, path, language_code, provider, score)
-                    send_notifications(episode[6], episode[2], message)
-        
-        list_missing_subtitles(episode[6])
+        if providers_list:
+            for language in ast.literal_eval(episode[1]):
+                if language is not None:
+                    notifications.write(msg='Searching for ' + str(
+                        language_from_alpha2(language)) + ' subtitles for this episode: ' + path_replace(episode[0]),
+                                        queue='get_subtitle')
+                    result = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)),
+                                               episode[4], episode[7], providers_list, providers_auth, str(episode[3]),
+                                               episode[5], 'series')
+                    if result is not None:
+                        message = result[0]
+                        path = result[1]
+                        forced = result[5]
+                        language_code = result[2] + ":forced" if forced else result[2]
+                        provider = result[3]
+                        score = result[4]
+                        store_subtitles(path_replace(episode[0]))
+                        history_log(1, episode[6], episode[2], message, path, language_code, provider, score)
+                        send_notifications(episode[6], episode[2], message)
+                        list_missing_subtitles(episode[6])
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
+            break
 
 
 def movies_download_subtitles(no):
@@ -587,20 +599,26 @@ def movies_download_subtitles(no):
     count_movie = len(ast.literal_eval(movie[1]))
     
     for i, language in enumerate(ast.literal_eval(movie[1]), 1):
-        if language is not None:
-            notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i, length=count_movie)
-            result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4],
-                                       movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie')
-            if result is not None:
-                message = result[0]
-                path = result[1]
-                forced = result[5]
-                language_code = result[2] + ":forced" if forced else result[2]
-                provider = result[3]
-                score = result[4]
-                store_subtitles_movie(path_replace_movie(movie[0]))
-                history_log_movie(1, no, message, path, language_code, provider, score)
-                send_notifications_movie(no, message)
+        if providers_list:
+            if language is not None:
+                notifications.write(msg='Searching for movies subtitles', queue='get_subtitle', item=i,
+                                    length=count_movie)
+                result = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4],
+                                           movie[6], providers_list, providers_auth, str(movie[3]), movie[5], 'movie')
+                if result is not None:
+                    message = result[0]
+                    path = result[1]
+                    forced = result[5]
+                    language_code = result[2] + ":forced" if forced else result[2]
+                    provider = result[3]
+                    score = result[4]
+                    store_subtitles_movie(path_replace_movie(movie[0]))
+                    history_log_movie(1, no, message, path, language_code, provider, score)
+                    send_notifications_movie(no, message)
+        else:
+            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+            logging.info("BAZARR All providers are throttled")
+            break
     list_missing_subtitles_movies(no)
     
     if count_movie:
@@ -744,24 +762,28 @@ def wanted_search_missing_subtitles():
     movies = c.fetchall()
     
     c.close()
-    providers = get_providers()
     if settings.general.getboolean('use_sonarr'):
-        if providers:
-            count_episodes = len(episodes)
-            for i, episode in enumerate(episodes, 1):
+        
+        count_episodes = len(episodes)
+        for i, episode in enumerate(episodes, 1):
+            providers = get_providers()
+            if providers:
                 wanted_download_subtitles(episode[0], i, count_episodes)
-        else:
-            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
-            logging.info("BAZARR All providers are throttled")
+            else:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
     
     if settings.general.getboolean('use_radarr'):
-        if providers:
-            count_movies = len(movies)
-            for i, movie in enumerate(movies, 1):
+        count_movies = len(movies)
+        for i, movie in enumerate(movies, 1):
+            providers = get_providers()
+            if providers:
                 wanted_download_subtitles_movie(movie[0], i, count_movies)
-        else:
-            notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
-            logging.info("BAZARR All providers are throttled")
+            else:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
     
     logging.info('BAZARR Finished searching for missing subtitles. Check histories for more information.')
     
@@ -841,23 +863,23 @@ def refine_from_db(path, video):
 def refine_from_mediainfo(path, video):
     if video.fps:
         return
-
+    
     exe = get_binary('mediainfo')
     if not exe:
         logging.debug('BAZARR MediaInfo library not found!')
         return
     else:
         logging.debug('BAZARR MediaInfo library used is %s', exe)
-
+    
     media_info = MediaInfo.parse(path, library_file=exe)
-
+    
     video_track = next((t for t in media_info.tracks if t.track_type == 'Video'), None)
     if not video_track:
         logging.debug('BAZARR MediaInfo was unable to find video tracks in the file!')
         return
-
+    
     logging.debug('MediaInfo found: %s', video_track.to_data())
-
+    
     if not video.fps:
         if video_track.frame_rate:
             video.fps = float(video_track.frame_rate)
@@ -920,6 +942,11 @@ def upgrade_subtitles():
     
     if settings.general.getboolean('use_sonarr'):
         for i, episode in enumerate(episodes_to_upgrade, 1):
+            providers = get_providers()
+            if not providers:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
             if episode[9] != "None":
                 desired_languages = ast.literal_eval(str(episode[9]))
                 if episode[10] == "True":
@@ -957,6 +984,11 @@ def upgrade_subtitles():
     
     if settings.general.getboolean('use_radarr'):
         for i, movie in enumerate(movies_to_upgrade, 1):
+            providers = get_providers()
+            if not providers:
+                notifications.write(msg='BAZARR All providers are throttled', queue='get_subtitle', duration='long')
+                logging.info("BAZARR All providers are throttled")
+                return
             if movie[8] != "None":
                 desired_languages = ast.literal_eval(str(movie[8]))
                 if movie[9] == "True":

libs/subliminal_patch/http.py

@@ -354,7 +354,6 @@ def patch_create_connection():
                         return _orig_create_connection((ip, port), *args, **kwargs)
                     except dns.exception.DNSException:
                         logger.warning("DNS: Couldn't resolve %s with DNS: %s", host, custom_resolver.nameservers)
-                        raise
 
         logger.debug("DNS: Falling back to default DNS or IP on %s", host)
         return _orig_create_connection((host, port), *args, **kwargs)

libs/subliminal_patch/providers/titlovi.py

@@ -226,7 +226,7 @@ class TitloviProvider(Provider, ProviderSubtitleArchiveMixin):
                         page_link = self.server_url + sub.a.attrs['href']
                         # subtitle language
                         _lang = sub.select_one('.lang')
-                        match = lang_re.search(_lang.attrs.get('src', _lang.attrs.get('cfsrc', '')))
+                        match = lang_re.search(_lang.attrs.get('src', _lang.attrs.get('data-cfsrc', '')))
                         if match:
                             try:
                                 # decode language

views/providers.tpl

@@ -134,7 +134,7 @@
                         <div id="betaseries_option" class="ui grid container">
                             <div class="middle aligned row">
                                 <div class="right aligned six wide column">
-                                    <label>Token</label>
+                                    <label>Token/Secret Key</label>
                                 </div>
                                 <div class="six wide column">
                                     <div class="ui fluid input">
@@ -808,4 +808,4 @@
                             $('#'+$(this).parent().attr('id')+'_option').hide();
                         }
                     });
-                </script>
+                </script>