Fixed HI subtitles identification when downloading and improved some constants. #2386

bazarr/app/signalr_client.py

@@ -12,7 +12,7 @@ from signalrcore.hub_connection_builder import HubConnectionBuilder
 from collections import deque
 from time import sleep
 
-from constants import headers
+from constants import HEADERS
 from app.event_handler import event_stream
 from sonarr.sync.episodes import sync_episodes, sync_one_episode
 from sonarr.sync.series import update_series, update_one_series
@@ -39,7 +39,7 @@ class SonarrSignalrClientLegacy:
         self.session = Session()
         self.session.timeout = 60
         self.session.verify = False
-        self.session.headers = headers
+        self.session.headers = HEADERS
         self.connection = None
         self.connected = False
 
@@ -162,7 +162,7 @@ class SonarrSignalrClient:
             .with_url(f"{url_sonarr()}/signalr/messages?access_token={self.apikey_sonarr}",
                       options={
                           "verify_ssl": False,
-                          "headers": headers
+                          "headers": HEADERS
                       }) \
             .with_automatic_reconnect({
                 "type": "raw",
@@ -229,7 +229,7 @@ class RadarrSignalrClient:
             .with_url(f"{url_radarr()}/signalr/messages?access_token={self.apikey_radarr}",
                       options={
                           "verify_ssl": False,
-                          "headers": headers
+                          "headers": HEADERS
                       }) \
             .with_automatic_reconnect({
                 "type": "raw",

bazarr/app/ui.py

@@ -9,7 +9,7 @@ from flask import (request, abort, render_template, Response, session, send_file
 from functools import wraps
 from urllib.parse import unquote
 
-from constants import headers
+from constants import HEADERS
 from literals import FILE_LOG
 from sonarr.info import url_api_sonarr
 from radarr.info import url_api_radarr
@@ -118,7 +118,7 @@ def series_images(url):
     baseUrl = settings.sonarr.base_url
     url_image = f'{url_api_sonarr()}{url.lstrip(baseUrl)}?apikey={apikey}'.replace('poster-250', 'poster-500')
     try:
-        req = requests.get(url_image, stream=True, timeout=15, verify=False, headers=headers)
+        req = requests.get(url_image, stream=True, timeout=15, verify=False, headers=HEADERS)
     except Exception:
         return '', 404
     else:
@@ -132,7 +132,7 @@ def movies_images(url):
     baseUrl = settings.radarr.base_url
     url_image = f'{url_api_radarr()}{url.lstrip(baseUrl)}?apikey={apikey}'
     try:
-        req = requests.get(url_image, stream=True, timeout=15, verify=False, headers=headers)
+        req = requests.get(url_image, stream=True, timeout=15, verify=False, headers=HEADERS)
     except Exception:
         return '', 404
     else:
@@ -173,7 +173,7 @@ def proxy(protocol, url):
     url = f'{protocol}://{unquote(url)}'
     params = request.args
     try:
-        result = requests.get(url, params, allow_redirects=False, verify=False, timeout=5, headers=headers)
+        result = requests.get(url, params, allow_redirects=False, verify=False, timeout=5, headers=HEADERS)
     except Exception as e:
         return dict(status=False, error=repr(e))
     else:

bazarr/constants.py

@@ -4,10 +4,10 @@ import os
 import re
 
 # set Bazarr user-agent used to make requests
-headers = {"User-Agent": os.environ["SZ_USER_AGENT"]}
-
-# hearing-impaired detection regex
-hi_regex = re.compile(r'[*¶♫♪].{3,}[*¶♫♪]|[\[\(\{].{3,}[\]\)\}](?<!{\\an\d})')
+HEADERS = {"User-Agent": os.environ["SZ_USER_AGENT"]}
 
 # minimum file size for Bazarr to consider it a video
 MINIMUM_VIDEO_SIZE = 20480
+
+# maximum size for a subtitles file
+MAXIMUM_SUBTITLE_SIZE = 1 * 1024 * 1024

bazarr/radarr/filesystem.py

@@ -5,7 +5,7 @@ import logging
 
 from app.config import settings
 from radarr.info import url_api_radarr
-from constants import headers
+from constants import HEADERS
 
 
 def browse_radarr_filesystem(path='#'):
@@ -16,7 +16,7 @@ def browse_radarr_filesystem(path='#'):
                                  f"includeFiles=false&apikey={settings.radarr.apikey}")
     try:
         r = requests.get(url_radarr_api_filesystem, timeout=int(settings.radarr.http_timeout), verify=False,
-                         headers=headers)
+                         headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get series from Radarr. Http error.")

bazarr/radarr/info.py

@@ -8,7 +8,7 @@ from requests.exceptions import JSONDecodeError
 from dogpile.cache import make_region
 
 from app.config import settings, empty_values
-from constants import headers
+from constants import HEADERS
 
 region = make_region().configure('dogpile.cache.memory')
 
@@ -30,7 +30,7 @@ class GetRadarrInfo:
             try:
                 rv = f"{url_radarr()}/api/system/status?apikey={settings.radarr.apikey}"
                 radarr_json = requests.get(rv, timeout=int(settings.radarr.http_timeout), verify=False,
-                                           headers=headers).json()
+                                           headers=HEADERS).json()
                 if 'version' in radarr_json:
                     radarr_version = radarr_json['version']
                 else:
@@ -39,7 +39,7 @@ class GetRadarrInfo:
                 try:
                     rv = f"{url_radarr()}/api/v3/system/status?apikey={settings.radarr.apikey}"
                     radarr_version = requests.get(rv, timeout=int(settings.radarr.http_timeout), verify=False,
-                                                  headers=headers).json()['version']
+                                                  headers=HEADERS).json()['version']
                 except JSONDecodeError:
                     logging.debug('BAZARR cannot get Radarr version')
                     radarr_version = 'unknown'

bazarr/radarr/notify.py

@@ -5,7 +5,7 @@ import requests
 
 from app.config import settings
 from radarr.info import url_api_radarr
-from constants import headers
+from constants import HEADERS
 
 
 def notify_radarr(radarr_id):
@@ -15,6 +15,6 @@ def notify_radarr(radarr_id):
             'name': 'RescanMovie',
             'movieId': int(radarr_id)
         }
-        requests.post(url, json=data, timeout=int(settings.radarr.http_timeout), verify=False, headers=headers)
+        requests.post(url, json=data, timeout=int(settings.radarr.http_timeout), verify=False, headers=HEADERS)
     except Exception:
         logging.exception('BAZARR cannot notify Radarr')

bazarr/radarr/rootfolder.py

@@ -8,7 +8,7 @@ from app.config import settings
 from utilities.path_mappings import path_mappings
 from app.database import TableMoviesRootfolder, TableMovies, database, delete, update, insert, select
 from radarr.info import url_api_radarr
-from constants import headers
+from constants import HEADERS
 
 
 def get_radarr_rootfolder():
@@ -19,7 +19,7 @@ def get_radarr_rootfolder():
     url_radarr_api_rootfolder = f"{url_api_radarr()}rootfolder?apikey={apikey_radarr}"
 
     try:
-        rootfolder = requests.get(url_radarr_api_rootfolder, timeout=int(settings.radarr.http_timeout), verify=False, headers=headers)
+        rootfolder = requests.get(url_radarr_api_rootfolder, timeout=int(settings.radarr.http_timeout), verify=False, headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get rootfolder from Radarr. Connection Error.")
         return []

bazarr/radarr/sync/utils.py

@@ -5,7 +5,7 @@ import logging
 
 from app.config import settings
 from radarr.info import get_radarr_info, url_api_radarr
-from constants import headers
+from constants import HEADERS
 
 
 def get_profile_list():
@@ -16,7 +16,7 @@ def get_profile_list():
                              f"apikey={apikey_radarr}")
 
     try:
-        profiles_json = requests.get(url_radarr_api_movies, timeout=int(settings.radarr.http_timeout), verify=False, headers=headers)
+        profiles_json = requests.get(url_radarr_api_movies, timeout=int(settings.radarr.http_timeout), verify=False, headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get profiles from Radarr. Connection Error.")
     except requests.exceptions.Timeout:
@@ -45,7 +45,7 @@ def get_tags():
     url_radarr_api_series = f"{url_api_radarr()}tag?apikey={apikey_radarr}"
 
     try:
-        tagsDict = requests.get(url_radarr_api_series, timeout=int(settings.radarr.http_timeout), verify=False, headers=headers)
+        tagsDict = requests.get(url_radarr_api_series, timeout=int(settings.radarr.http_timeout), verify=False, headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get tags from Radarr. Connection Error.")
         return []
@@ -69,7 +69,7 @@ def get_movies_from_radarr_api(apikey_radarr, radarr_id=None):
     url_radarr_api_movies = f'{url_api_radarr()}movie{f"/{radarr_id}" if radarr_id else ""}?apikey={apikey_radarr}'
 
     try:
-        r = requests.get(url_radarr_api_movies, timeout=int(settings.radarr.http_timeout), verify=False, headers=headers)
+        r = requests.get(url_radarr_api_movies, timeout=int(settings.radarr.http_timeout), verify=False, headers=HEADERS)
         if r.status_code == 404:
             return
         r.raise_for_status()
@@ -100,7 +100,7 @@ def get_history_from_radarr_api(apikey_radarr, movie_id):
 
     try:
         r = requests.get(url_radarr_api_history, timeout=int(settings.sonarr.http_timeout), verify=False,
-                         headers=headers)
+                         headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get history from Radarr. Http error.")

bazarr/sonarr/filesystem.py

@@ -5,7 +5,7 @@ import logging
 
 from app.config import settings
 from sonarr.info import url_api_sonarr
-from constants import headers
+from constants import HEADERS
 
 
 def browse_sonarr_filesystem(path='#'):
@@ -15,7 +15,7 @@ def browse_sonarr_filesystem(path='#'):
                                  f"includeFiles=false&apikey={settings.sonarr.apikey}")
     try:
         r = requests.get(url_sonarr_api_filesystem, timeout=int(settings.sonarr.http_timeout), verify=False,
-                         headers=headers)
+                         headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get series from Sonarr. Http error.")

bazarr/sonarr/info.py

@@ -8,7 +8,7 @@ from requests.exceptions import JSONDecodeError
 from dogpile.cache import make_region
 
 from app.config import settings, empty_values
-from constants import headers
+from constants import HEADERS
 
 region = make_region().configure('dogpile.cache.memory')
 
@@ -30,7 +30,7 @@ class GetSonarrInfo:
             try:
                 sv = f"{url_sonarr()}/api/system/status?apikey={settings.sonarr.apikey}"
                 sonarr_json = requests.get(sv, timeout=int(settings.sonarr.http_timeout), verify=False,
-                                           headers=headers).json()
+                                           headers=HEADERS).json()
                 if 'version' in sonarr_json:
                     sonarr_version = sonarr_json['version']
                 else:
@@ -39,7 +39,7 @@ class GetSonarrInfo:
                 try:
                     sv = f"{url_sonarr()}/api/v3/system/status?apikey={settings.sonarr.apikey}"
                     sonarr_version = requests.get(sv, timeout=int(settings.sonarr.http_timeout), verify=False,
-                                                  headers=headers).json()['version']
+                                                  headers=HEADERS).json()['version']
                 except JSONDecodeError:
                     logging.debug('BAZARR cannot get Sonarr version')
                     sonarr_version = 'unknown'

bazarr/sonarr/notify.py

@@ -5,7 +5,7 @@ import requests
 
 from app.config import settings
 from sonarr.info import url_api_sonarr
-from constants import headers
+from constants import HEADERS
 
 
 def notify_sonarr(sonarr_series_id):
@@ -15,6 +15,6 @@ def notify_sonarr(sonarr_series_id):
             'name': 'RescanSeries',
             'seriesId': int(sonarr_series_id)
         }
-        requests.post(url, json=data, timeout=int(settings.sonarr.http_timeout), verify=False, headers=headers)
+        requests.post(url, json=data, timeout=int(settings.sonarr.http_timeout), verify=False, headers=HEADERS)
     except Exception:
         logging.exception('BAZARR cannot notify Sonarr')

bazarr/sonarr/rootfolder.py

@@ -8,7 +8,7 @@ from app.config import settings
 from app.database import TableShowsRootfolder, TableShows, database, insert, update, delete, select
 from utilities.path_mappings import path_mappings
 from sonarr.info import url_api_sonarr
-from constants import headers
+from constants import HEADERS
 
 
 def get_sonarr_rootfolder():
@@ -19,7 +19,7 @@ def get_sonarr_rootfolder():
     url_sonarr_api_rootfolder = f"{url_api_sonarr()}rootfolder?apikey={apikey_sonarr}"
 
     try:
-        rootfolder = requests.get(url_sonarr_api_rootfolder, timeout=int(settings.sonarr.http_timeout), verify=False, headers=headers)
+        rootfolder = requests.get(url_sonarr_api_rootfolder, timeout=int(settings.sonarr.http_timeout), verify=False, headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get rootfolder from Sonarr. Connection Error.")
         return []

bazarr/sonarr/sync/utils.py

@@ -5,7 +5,7 @@ import logging
 
 from app.config import settings
 from sonarr.info import get_sonarr_info, url_api_sonarr
-from constants import headers
+from constants import HEADERS
 
 
 def get_profile_list():
@@ -23,7 +23,7 @@ def get_profile_list():
 
     try:
         profiles_json = requests.get(url_sonarr_api_series, timeout=int(settings.sonarr.http_timeout), verify=False,
-                                     headers=headers)
+                                     headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get profiles from Sonarr. Connection Error.")
         return None
@@ -53,7 +53,7 @@ def get_tags():
     url_sonarr_api_series = f"{url_api_sonarr()}tag?apikey={apikey_sonarr}"
 
     try:
-        tagsDict = requests.get(url_sonarr_api_series, timeout=int(settings.sonarr.http_timeout), verify=False, headers=headers)
+        tagsDict = requests.get(url_sonarr_api_series, timeout=int(settings.sonarr.http_timeout), verify=False, headers=HEADERS)
     except requests.exceptions.ConnectionError:
         logging.exception("BAZARR Error trying to get tags from Sonarr. Connection Error.")
         return []
@@ -71,7 +71,7 @@ def get_series_from_sonarr_api(apikey_sonarr, sonarr_series_id=None):
     url_sonarr_api_series = (f"{url_api_sonarr()}series/{sonarr_series_id if sonarr_series_id else ''}?"
                              f"apikey={apikey_sonarr}")
     try:
-        r = requests.get(url_sonarr_api_series, timeout=int(settings.sonarr.http_timeout), verify=False, headers=headers)
+        r = requests.get(url_sonarr_api_series, timeout=int(settings.sonarr.http_timeout), verify=False, headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError as e:
         if e.response.status_code:
@@ -110,7 +110,7 @@ def get_episodes_from_sonarr_api(apikey_sonarr, series_id=None, episode_id=None)
         return
 
     try:
-        r = requests.get(url_sonarr_api_episode, timeout=int(settings.sonarr.http_timeout), verify=False, headers=headers)
+        r = requests.get(url_sonarr_api_episode, timeout=int(settings.sonarr.http_timeout), verify=False, headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get episodes from Sonarr. Http error.")
@@ -144,7 +144,7 @@ def get_episodesFiles_from_sonarr_api(apikey_sonarr, series_id=None, episode_fil
 
     try:
         r = requests.get(url_sonarr_api_episodeFiles, timeout=int(settings.sonarr.http_timeout), verify=False,
-                         headers=headers)
+                         headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get episodeFiles from Sonarr. Http error.")
@@ -173,7 +173,7 @@ def get_history_from_sonarr_api(apikey_sonarr, episode_id):
 
     try:
         r = requests.get(url_sonarr_api_history, timeout=int(settings.sonarr.http_timeout), verify=False,
-                         headers=headers)
+                         headers=HEADERS)
         r.raise_for_status()
     except requests.exceptions.HTTPError:
         logging.exception("BAZARR Error trying to get history from Sonarr. Http error.")

bazarr/subtitles/indexer/utils.py

@@ -9,8 +9,8 @@ from subliminal_patch import core
 from subzero.language import Language
 from charset_normalizer import detect
 
+from constants import MAXIMUM_SUBTITLE_SIZE
 from app.config import settings
-from constants import hi_regex
 from utilities.path_mappings import path_mappings
 
 
@@ -68,7 +68,7 @@ def guess_external_subtitles(dest_folder, subtitles, media_type, previously_inde
                 forced = True if os.path.splitext(os.path.splitext(subtitle)[0])[1] == '.forced' else False
 
                 # to improve performance, skip detection of files larger that 1M
-                if os.path.getsize(subtitle_path) > 1 * 1024 * 1024:
+                if os.path.getsize(subtitle_path) > MAXIMUM_SUBTITLE_SIZE:
                     logging.debug(f"BAZARR subtitles file is too large to be text based. Skipping this file: "
                                   f"{subtitle_path}")
                     continue
@@ -119,7 +119,7 @@ def guess_external_subtitles(dest_folder, subtitles, media_type, previously_inde
             # check if file exist:
             if os.path.exists(subtitle_path) and os.path.splitext(subtitle_path)[1] in core.SUBTITLE_EXTENSIONS:
                 # to improve performance, skip detection of files larger that 1M
-                if os.path.getsize(subtitle_path) > 1 * 1024 * 1024:
+                if os.path.getsize(subtitle_path) > MAXIMUM_SUBTITLE_SIZE:
                     logging.debug(f"BAZARR subtitles file is too large to be text based. Skipping this file: "
                                   f"{subtitle_path}")
                     continue
@@ -136,6 +136,6 @@ def guess_external_subtitles(dest_folder, subtitles, media_type, previously_inde
                     continue
                 text = text.decode(encoding)
 
-                if bool(re.search(hi_regex, text)):
+                if bool(re.search(core.HI_REGEX, text)):
                     subtitles[subtitle] = Language.rebuild(subtitles[subtitle], forced=False, hi=True)
     return subtitles

custom_libs/subliminal_patch/core.py

@@ -49,6 +49,8 @@ SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '
 
 _POOL_LIFETIME = datetime.timedelta(hours=12)
 
+HI_REGEX = re.compile(r'[*¶♫♪].{3,}[*¶♫♪]|[\[\(\{].{3,}[\]\)\}](?<!{\\an\d})')
+
 
 def remove_crap_from_fn(fn):
     # in case of the second regex part, the legit release group name will be in group(2), if it's followed by [string]
@@ -1201,6 +1203,8 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non
             continue
 
         # create subtitle path
+        if bool(re.search(HI_REGEX, subtitle.text)):
+            subtitle.language.hi = True
         subtitle_path = get_subtitle_path(file_path, None if single else subtitle.language,
                                           forced_tag=subtitle.language.forced,
                                           hi_tag=False if must_remove_hi else subtitle.language.hi, tags=tags)