Added vip and daily download limit to addic7ed and try to slow down a little to prevent being ip blocked by the provider.

bazarr/config.py

@@ -132,7 +132,8 @@ defaults = {
     },
     'addic7ed': {
         'username': '',
-        'password': ''
+        'password': '',
+        'vip': 'False'
     },
     'podnapisi': {
         'verify_ssl': 'True'

bazarr/get_providers.py

@@ -127,6 +127,7 @@ def get_providers_auth():
         'addic7ed'        : {
             'username': settings.addic7ed.username,
             'password': settings.addic7ed.password,
+            'is_vip': settings.addic7ed.getboolean('vip'),
         },
         'opensubtitles'   : {
             'username'      : settings.opensubtitles.username,

frontend/src/Settings/Providers/list.ts

@@ -21,6 +21,10 @@ export const ProviderList: Readonly<ProviderInfo[]> = [
     defaultKey: {
       username: "",
       password: "",
+      vip: false,
+    },
+    keyNameOverride: {
+      vip: "VIP",
     },
   },
   { key: "argenteam", description: "LATAM Spanish Subtitles Provider" },

libs/subliminal_patch/providers/addic7ed.py

@@ -23,6 +23,8 @@ from subzero.language import Language
 
 logger = logging.getLogger(__name__)
 
+show_cells_re = re.compile(b'<td class="(?:version|vr)">.*?</td>', re.DOTALL)
+
 #: Series header parsing regex
 series_year_re = re.compile(r'^(?P<series>[ \w\'.:(),*&!?-]+?)(?: \((?P<year>\d{4})\))?$')
 
@@ -68,6 +70,7 @@ class Addic7edProvider(_Addic7edProvider):
     languages.update(set(Language.rebuild(l, hi=True) for l in languages))
 
     video_types = (Episode, Movie)
+    vip = False
     USE_ADDICTED_RANDOM_AGENTS = False
     hearing_impaired_verifiable = True
     subtitle_class = Addic7edSubtitle
@@ -76,9 +79,10 @@ class Addic7edProvider(_Addic7edProvider):
     sanitize_characters = {'-', ':', '(', ')', '.', '/'}
     last_show_ids_fetch_key = "addic7ed_last_id_fetch"
 
-    def __init__(self, username=None, password=None, use_random_agents=False):
+    def __init__(self, username=None, password=None, use_random_agents=False, is_vip=False):
         super(Addic7edProvider, self).__init__(username=username, password=password)
         self.USE_ADDICTED_RANDOM_AGENTS = use_random_agents
+        self.vip = is_vip
 
         if not all((username, password)):
             raise ConfigurationError('Username and password must be specified')
@@ -95,7 +99,7 @@ class Addic7edProvider(_Addic7edProvider):
         # login
         if self.username and self.password:
             def check_verification(cache_region):
-                rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=60,
+                rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10,
                                       headers={"Referer": self.server_url})
                 if rr.status_code == 302:
                     logger.info('Addic7ed: Login expired')
@@ -115,7 +119,7 @@ class Addic7edProvider(_Addic7edProvider):
             tries = 0
             while tries <= 3:
                 tries += 1
-                r = self.session.get(self.server_url + 'login.php', timeout=60, headers={"Referer": self.server_url})
+                r = self.session.get(self.server_url + 'login.php', timeout=10, headers={"Referer": self.server_url})
                 if "g-recaptcha" in r.text or "grecaptcha" in r.text:
                     logger.info('Addic7ed: Solving captcha. This might take a couple of minutes, but should only '
                                 'happen once every so often')
@@ -139,11 +143,13 @@ class Addic7edProvider(_Addic7edProvider):
                         if tries >= 3:
                             raise Exception("Addic7ed: Couldn't solve captcha!")
                         logger.info("Addic7ed: Couldn't solve captcha! Retrying")
+                        time.sleep(4)
                         continue
 
                     data[g] = result
 
-                r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=60,
+                time.sleep(1)
+                r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10,
                                       headers={"Referer": self.server_url + "login.php"})
 
                 if "relax, slow down" in r.text:
@@ -157,6 +163,7 @@ class Addic7edProvider(_Addic7edProvider):
                         logger.error("Addic7ed: Something went wrong when logging in")
                         raise AuthenticationError(self.username)
                     logger.info("Addic7ed: Something went wrong when logging in; retrying")
+                    time.sleep(4)
                     continue
                 break
 
@@ -165,6 +172,8 @@ class Addic7edProvider(_Addic7edProvider):
             logger.debug('Addic7ed: Logged in')
             self.logged_in = True
 
+            time.sleep(2)
+
     def terminate(self):
         self.session.close()
 
@@ -238,7 +247,7 @@ class Addic7edProvider(_Addic7edProvider):
         # get the movie id
         logger.info('Getting movie id')
 
-        r = self.session.get(self.server_url + 'search.php?search=' + quote_plus(movie), timeout=60)
+        r = self.session.get(self.server_url + 'search.php?search=' + quote_plus(movie), timeout=10)
         r.raise_for_status()
 
         soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser'])
@@ -285,10 +294,18 @@ class Addic7edProvider(_Addic7edProvider):
         logger.info('Getting show ids')
         region.set(self.last_show_ids_fetch_key, datetime.datetime.now())
 
-        r = self.session.get(self.server_url + 'shows.php', timeout=60)
+        r = self.session.get(self.server_url + 'shows.php', timeout=10)
         r.raise_for_status()
 
-        soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser'])
+        # LXML parser seems to fail when parsing Addic7ed.com HTML markup.
+        # Last known version to work properly is 3.6.4 (next version, 3.7.0, fails)
+        # Assuming the site's markup is bad, and stripping it down to only contain what's needed.
+        show_cells = re.findall(show_cells_re, r.content)
+        if show_cells:
+            soup = ParserBeautifulSoup(''.join(show_cells).decode('utf-8', 'ignore'), ['lxml', 'html.parser'])
+        else:
+            # If RegEx fails, fall back to original r.content and use 'html.parser'
+            soup = ParserBeautifulSoup(r.content, ['html.parser'])
 
         # populate the show ids
         show_ids = {}
@@ -345,7 +362,7 @@ class Addic7edProvider(_Addic7edProvider):
                 headers = {
                     "referer": self.server_url + "srch.php"
                 }
-            r = self.session.get(self.server_url + endpoint, params=params, timeout=60, headers=headers)
+            r = self.session.get(self.server_url + endpoint, params=params, timeout=10, headers=headers)
             r.raise_for_status()
 
             if r.text and "Sorry, your search" not in r.text:
@@ -386,7 +403,7 @@ class Addic7edProvider(_Addic7edProvider):
         logger.info('Getting the page of show id %d, season %d', show_id, season)
         r = self.session.get(self.server_url + 'ajax_loadShow.php',
                              params={'show': show_id, 'season': season},
-                             timeout=60,
+                             timeout=10,
                              headers={
                                  "referer": "%sshow/%s" % (self.server_url, show_id),
                                  "X-Requested-With": "XMLHttpRequest"
@@ -446,7 +463,7 @@ class Addic7edProvider(_Addic7edProvider):
         # get the page of the movie
         logger.info('Getting the page of movie id %d', movie_id)
         r = self.session.get(self.server_url + 'movie/' + movie_id,
-                             timeout=60,
+                             timeout=10,
                              headers={
                                  "referer": self.server_url,
                                  "X-Requested-With": "XMLHttpRequest"
@@ -550,9 +567,30 @@ class Addic7edProvider(_Addic7edProvider):
         return []
 
     def download_subtitle(self, subtitle):
+        last_dls = region.get("addic7ed_dls")
+        now = datetime.datetime.now()
+        one_day = datetime.timedelta(hours=24)
+
+        def raise_limit():
+            logger.info("Addic7ed: Downloads per day exceeded (%s)", cap)
+            raise DownloadLimitPerDayExceeded
+
+        if type(last_dls) is not list:
+            last_dls = []
+        else:
+            # filter all non-expired DLs
+            last_dls = list(filter(lambda t: t + one_day > now, last_dls))
+            region.set("addic7ed_dls", last_dls)
+
+        cap = self.vip and 80 or 40
+        amount = len(last_dls)
+
+        if amount >= cap:
+            raise_limit()
+
         # download the subtitle
         r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link},
-                             timeout=60)
+                             timeout=10)
         r.raise_for_status()
 
         if r.status_code == 304:
@@ -569,3 +607,9 @@ class Addic7edProvider(_Addic7edProvider):
             raise DownloadLimitExceeded
 
         subtitle.content = fix_line_ending(r.content)
+        last_dls.append(datetime.datetime.now())
+        region.set("addic7ed_dls", last_dls)
+        logger.info("Addic7ed: Used %s/%s downloads", amount + 1, cap)
+
+        if amount + 1 >= cap:
+            raise_limit()