bazarr/bazarr/get_providers.py

# coding=utf-8
import os
import datetime
import logging
import subliminal_patch
import pretty
import time

from get_args import args
from config import settings
from subliminal_patch.exceptions import TooManyRequests, APIThrottled, ParseResponseError
from subliminal.exceptions import DownloadLimitExceeded, ServiceUnavailable
from subliminal import region as subliminal_cache_region

VALID_THROTTLE_EXCEPTIONS = (TooManyRequests, DownloadLimitExceeded, ServiceUnavailable, APIThrottled,
                             ParseResponseError)
VALID_COUNT_EXCEPTIONS = ('TooManyRequests', 'ServiceUnavailable', 'APIThrottled')

PROVIDER_THROTTLE_MAP = {
    "default": {
        TooManyRequests: (datetime.timedelta(hours=1), "1 hour"),
        DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
        ServiceUnavailable: (datetime.timedelta(minutes=20), "20 minutes"),
        APIThrottled: (datetime.timedelta(minutes=10), "10 minutes"),
        ParseResponseError: (datetime.timedelta(hours=6), "6 hours"),
    },
    "opensubtitles": {
        TooManyRequests: (datetime.timedelta(hours=3), "3 hours"),
        DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"),
        APIThrottled: (datetime.timedelta(seconds=15), "15 seconds"),
    },
    "addic7ed": {
        DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"),
        TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"),
    },
    "titulky": {
        DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours")
    }
}

PROVIDERS_FORCED_OFF = ["addic7ed", "tvsubtitles", "legendasdivx", "legendastv", "napiprojekt", "shooter", "hosszupuska",
                        "supersubtitles", "titlovi", "titrari", "argenteam", "assrt", "subscene"]

throttle_count = {}

if not settings.general.throtteled_providers:
    tp = {}
else:
    tp = eval(str(settings.general.throtteled_providers))


def provider_pool():
    if settings.general.getboolean('multithreading'):
        return subliminal_patch.core.SZAsyncProviderPool
    return subliminal_patch.core.SZProviderPool


def get_providers():
    changed = False
    providers_list = []
    if settings.general.enabled_providers:
        for provider in settings.general.enabled_providers.lower().split(','):
            reason, until, throttle_desc = tp.get(provider, (None, None, None))
            providers_list.append(provider)
            
            if reason:
                now = datetime.datetime.now()
                if now < until:
                    logging.debug("Not using %s until %s, because of: %s", provider,
                                  until.strftime("%y/%m/%d %H:%M"), reason)
                    providers_list.remove(provider)
                else:
                    logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
                    del tp[provider]
                    settings.general.throtteled_providers = str(tp)
                    changed = True
        
        if changed:
            with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
                settings.write(handle)
        
        # if forced only is enabled: # fixme: Prepared for forced only implementation to remove providers with don't support forced only subtitles
        #     for provider in providers_list:
        #         if provider in PROVIDERS_FORCED_OFF:
        #             providers_list.remove(provider)
    
    if not providers_list:
        providers_list = None
    
    return providers_list


def get_providers_auth():
    providers_auth = {
        'addic7ed': {'username': settings.addic7ed.username,
                     'password': settings.addic7ed.password,
                     'use_random_agents': settings.addic7ed.getboolean('random_agents'),
                     },
        'opensubtitles': {'username': settings.opensubtitles.username,
                          'password': settings.opensubtitles.password,
                          'use_tag_search': settings.opensubtitles.getboolean('use_tag_search'),
                          'only_foreign': False,  # fixme
                          'also_foreign': False,  # fixme
                          'is_vip': settings.opensubtitles.getboolean('vip'),
                          'use_ssl': settings.opensubtitles.getboolean('ssl'),
                          'timeout': int(settings.opensubtitles.timeout) or 15,
                          'skip_wrong_fps': settings.opensubtitles.getboolean('skip_wrong_fps'),
                          },
        'podnapisi': {
            'only_foreign': False,  # fixme
            'also_foreign': False,  # fixme
        },
        'subscene': {'username': settings.subscene.username,
                     'password': settings.subscene.password,
                     'only_foreign': False,  # fixme
                     },
        'legendasdivx': {'username': settings.legendasdivx.username,
                       'password': settings.legendasdivx.password,
                       },
        'legendastv': {'username': settings.legendastv.username,
                       'password': settings.legendastv.password,
                       },
        'xsubs': {'username': settings.xsubs.username,
                  'password': settings.xsubs.password,
                  },
        'assrt': {'token': settings.assrt.token, },
        'napisy24': {'username': settings.napisy24.username,
                     'password': settings.napisy24.password,
                     },
        'betaseries': {'token': settings.betaseries.token},
        'titulky': {'username': settings.titulky.username,
                    'password': settings.titulky.password,
                    },
        'titlovi': {'username': settings.titlovi.username,
                  'password': settings.titlovi.password,
                  },
    }
    
    return providers_auth


def provider_throttle(name, exception):
    cls = getattr(exception, "__class__")
    cls_name = getattr(cls, "__name__")
    if cls not in VALID_THROTTLE_EXCEPTIONS:
        for valid_cls in VALID_THROTTLE_EXCEPTIONS:
            if isinstance(cls, valid_cls):
                cls = valid_cls
    
    throttle_data = PROVIDER_THROTTLE_MAP.get(name, PROVIDER_THROTTLE_MAP["default"]).get(cls, None) or \
                    PROVIDER_THROTTLE_MAP["default"].get(cls, None)
    
    if throttle_data:
        throttle_delta, throttle_description = throttle_data
    else:
        throttle_delta, throttle_description = datetime.timedelta(minutes=10), "10 minutes"
    
    throttle_until = datetime.datetime.now() + throttle_delta
    
    if cls_name not in VALID_COUNT_EXCEPTIONS or throttled_count(name):
        if cls_name == 'ValueError' and exception.args[0].startswith('unsupported pickle protocol'):
            for fn in subliminal_cache_region.backend.all_filenames:
                try:
                    os.remove(fn)
                except (IOError, OSError):
                    logging.debug("Couldn't remove cache file: %s", os.path.basename(fn))
        else:
            tp[name] = (cls_name, throttle_until, throttle_description)
            settings.general.throtteled_providers = str(tp)
            with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
                settings.write(handle)

            logging.info("Throttling %s for %s, until %s, because of: %s. Exception info: %r", name,
                         throttle_description, throttle_until.strftime("%y/%m/%d %H:%M"), cls_name, exception.args[0]
                         if exception.args else None)


def throttled_count(name):
    global throttle_count
    if name in list(throttle_count.keys()):
        if 'count' in list(throttle_count[name].keys()):
            for key, value in throttle_count[name].items():
                if key == 'count':
                    value += 1
                    throttle_count[name]['count'] = value
        else:
            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
    
    else:
        throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
    
    if throttle_count[name]['count'] < 5:
        if throttle_count[name]['time'] > datetime.datetime.now():
            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
                         throttle_count[name]['count'])
            time.sleep(5)
            return False
        else:
            throttle_count[name] = {"count": 1, "time": (datetime.datetime.now() + datetime.timedelta(seconds=120))}
            logging.info("Provider %s throttle count %s of 5, waiting 5sec and trying again", name,
                         throttle_count[name]['count'])
            time.sleep(5)
            return False
    else:
        return True


def update_throttled_provider():
    changed = False
    if settings.general.enabled_providers:
        for provider in list(tp):
            if provider not in settings.general.enabled_providers:
                del tp[provider]
                settings.general.throtteled_providers = str(tp)
                changed = True

            reason, until, throttle_desc = tp.get(provider, (None, None, None))

            if reason:
                now = datetime.datetime.now()
                if now < until:
                    pass
                else:
                    logging.info("Using %s again after %s, (disabled because: %s)", provider, throttle_desc, reason)
                    del tp[provider]
                    settings.general.throtteled_providers = str(tp)
                    changed = True
        
        if changed:
            with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle:
                settings.write(handle)


def list_throttled_providers():
    update_throttled_provider()
    throttled_providers = []
    if settings.general.enabled_providers:
        for provider in settings.general.enabled_providers.lower().split(','):
            reason, until, throttle_desc = tp.get(provider, (None, None, None))
            throttled_providers.append([provider, reason, pretty.date(until)])
    return throttled_providers