From 43d9d43224c84609a4fbd050c8d82e9500743a68 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Wed, 18 Dec 2024 16:34:29 -0500 Subject: [PATCH 01/12] Added languages profile creation and assignment to health check. --- bazarr/utilities/health.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/bazarr/utilities/health.py b/bazarr/utilities/health.py index c1d3a6a3d..84a313cf0 100644 --- a/bazarr/utilities/health.py +++ b/bazarr/utilities/health.py @@ -2,8 +2,11 @@ import json +from sqlalchemy import func + from app.config import settings -from app.database import TableShowsRootfolder, TableMoviesRootfolder, TableLanguagesProfiles, database, select +from app.database import (TableShowsRootfolder, TableMoviesRootfolder, TableLanguagesProfiles, database, select, + TableShows, TableMovies) from app.event_handler import event_stream from .path_mappings import path_mappings from sonarr.rootfolder import check_sonarr_rootfolder @@ -66,4 +69,19 @@ def get_health_issues(): else: languages_profile_ids.append(items['id']) + # check if there's at least one languages profile created + languages_profiles_count = database.execute(select(func.count(TableLanguagesProfiles.profileId))).scalar() + series_with_profile = database.execute(select(func.count(TableShows.sonarrSeriesId)) + .where(TableShows.profileId.is_not(None))).scalar() + movies_with_profile = database.execute(select(func.count(TableMovies.radarrId)) + .where(TableMovies.profileId.is_not(None))).scalar() + if languages_profiles_count == 0: + health_issues.append({'object': 'Missing languages profile', + 'issue': 'You must create at least one languages profile and assign it to your content.'}) + elif languages_profiles_count > 0 and ((settings.general.use_sonarr and series_with_profile == 0) or + (settings.general.use_radarr and movies_with_profile == 0)): + health_issues.append({'object': 'No assigned languages profile', + 'issue': 'Although you have created at least one languages profile, you must assign it ' + 'to your content.'}) + return health_issues From 9d62d84ef590c65e7e9392fe82f1f2022b90bede Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Fri, 20 Dec 2024 20:32:13 -0500 Subject: [PATCH 02/12] Fixed titrari provider to prevent abuse and throttle properly. #2709 --- bazarr/app/get_providers.py | 3 +++ custom_libs/subliminal_patch/providers/titrari.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/bazarr/app/get_providers.py b/bazarr/app/get_providers.py index f45118326..c4c9b5be4 100644 --- a/bazarr/app/get_providers.py +++ b/bazarr/app/get_providers.py @@ -100,6 +100,9 @@ def provider_throttle_map(): "titlovi": { TooManyRequests: (datetime.timedelta(minutes=5), "5 minutes"), }, + "titrari": { + TooManyRequests: (datetime.timedelta(minutes=10), "10 minutes"), + }, "titulky": { DownloadLimitExceeded: ( titulky_limit_reset_timedelta(), diff --git a/custom_libs/subliminal_patch/providers/titrari.py b/custom_libs/subliminal_patch/providers/titrari.py index 7caed684d..a9976df21 100644 --- a/custom_libs/subliminal_patch/providers/titrari.py +++ b/custom_libs/subliminal_patch/providers/titrari.py @@ -5,18 +5,18 @@ import os import io import logging import re -import rarfile -from random import randint from zipfile import ZipFile, is_zipfile from rarfile import RarFile, is_rarfile from guessit import guessit +from time import sleep + from subliminal_patch.providers import Provider from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin from subliminal_patch.subtitle import Subtitle, guess_matches from subliminal_patch.utils import sanitize, fix_inconsistent_naming as _fix_inconsistent_naming -from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST from subliminal.exceptions import ProviderError +from subliminal_patch.exceptions import TooManyRequests from subliminal.providers import ParserBeautifulSoup from subliminal.video import Episode, Movie from subliminal.subtitle import SUBTITLE_EXTENSIONS @@ -147,6 +147,10 @@ class TitrariProvider(Provider, ProviderSubtitleArchiveMixin): params = self.getQueryParams(imdb_id, title, language) search_response = self.session.get(self.api_url, params=params, timeout=15) + + if search_response.status_code == 404 and 'Too many requests' in search_response.content: + raise TooManyRequests(search_response.content) + search_response.raise_for_status() if not search_response.content: @@ -215,6 +219,8 @@ class TitrariProvider(Provider, ProviderSubtitleArchiveMixin): ordered_subs = self.order(subtitles) + sleep(5) # prevent being blocked for too many requests + return ordered_subs @staticmethod From 320935548c28e85bbd9b0b0a6a073f1439deba31 Mon Sep 17 00:00:00 2001 From: destpstrzy Date: Sat, 21 Dec 2024 02:54:54 +0100 Subject: [PATCH 03/12] Fixed download error with Napiprojekt provider --- .../subliminal_patch/providers/napiprojekt.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/custom_libs/subliminal_patch/providers/napiprojekt.py b/custom_libs/subliminal_patch/providers/napiprojekt.py index 8fabed885..b663348d8 100644 --- a/custom_libs/subliminal_patch/providers/napiprojekt.py +++ b/custom_libs/subliminal_patch/providers/napiprojekt.py @@ -47,8 +47,6 @@ class NapiProjektProvider(_NapiProjektProvider): self.only_real_names = only_real_names def query(self, language, hash): - if self.only_authors or self.only_real_names: - return None params = { 'v': 'dreambox', 'kolejka': 'false', @@ -74,11 +72,23 @@ class NapiProjektProvider(_NapiProjektProvider): return subtitle def list_subtitles(self, video, languages): - def flatten(l): - return [item for sublist in l for item in sublist] + def flatten(nested_list): + """Flatten a nested list.""" + return [item for sublist in nested_list for item in sublist] + + # Determine the source of subtitles based on conditions + hash_subtitles = [] + if not (self.only_authors or self.only_real_names): + hash_subtitles = [ + subtitle + for language in languages + if (subtitle := self.query(language, video.hashes.get('napiprojekt'))) is not None + ] + + # Scrape additional subtitles + scraped_subtitles = flatten([self._scrape(video, language) for language in languages]) - return [s for s in [self.query(l, video.hashes['napiprojekt']) for l in languages] if s is not None] + \ - flatten([self._scrape(video, l) for l in languages]) + return hash_subtitles + scraped_subtitles def download_subtitle(self, subtitle): if subtitle.content is not None: @@ -125,7 +135,7 @@ class NapiProjektProvider(_NapiProjektProvider): author = "" if self.only_authors: - if author.lower() in ["brak", "automat", "si", "chatgpt", "ai", "robot"]: + if author.lower() in ["brak", "automat", "si", "chatgpt", "ai", "robot", "maszynowe", "tłumaczenie maszynowe"]: continue if self.only_real_names: From d5a290c7a2bf4c0d7d923bb967d1825de2bbaba3 Mon Sep 17 00:00:00 2001 From: Alessandro Vitali <98644809+alvitali@users.noreply.github.com> Date: Mon, 23 Dec 2024 20:42:59 +0100 Subject: [PATCH 04/12] Reduced throttle from 24 hours to 6 before retrying Opensubtitles.com API on DownloadLimitExceeded --- bazarr/app/get_providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/app/get_providers.py b/bazarr/app/get_providers.py index c4c9b5be4..a4316a9d5 100644 --- a/bazarr/app/get_providers.py +++ b/bazarr/app/get_providers.py @@ -90,7 +90,7 @@ def provider_throttle_map(): }, "opensubtitlescom": { TooManyRequests: (datetime.timedelta(minutes=1), "1 minute"), - DownloadLimitExceeded: (datetime.timedelta(hours=24), "24 hours"), + DownloadLimitExceeded: (datetime.timedelta(hours=6), "6 hours"), }, "addic7ed": { DownloadLimitExceeded: (datetime.timedelta(hours=3), "3 hours"), From b59b5a6fd7e555326d53b272f98aeaef1252b38d Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 24 Dec 2024 09:13:41 -0500 Subject: [PATCH 05/12] no log: minor UI fixes --- frontend/src/pages/Episodes/index.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/Episodes/index.tsx b/frontend/src/pages/Episodes/index.tsx index 2a518ae4c..017a8a15e 100644 --- a/frontend/src/pages/Episodes/index.tsx +++ b/frontend/src/pages/Episodes/index.tsx @@ -167,6 +167,7 @@ const SeriesEpisodesView: FunctionComponent = () => { series.profileId === null || !available } + loading={hasTask} > Search @@ -195,7 +196,8 @@ const SeriesEpisodesView: FunctionComponent = () => { series === undefined || series.episodeFileCount === 0 || series.profileId === null || - !available + !available || + hasTask } icon={faCloudUploadAlt} onClick={() => openDropzone.current?.()} From 96618d58434dd96370fba05a5bd3bfa71edaf85f Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 24 Dec 2024 09:53:46 -0500 Subject: [PATCH 06/12] no log: minor UI fixes --- frontend/src/pages/Movies/Details/index.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/pages/Movies/Details/index.tsx b/frontend/src/pages/Movies/Details/index.tsx index 709f03905..9bd56c660 100644 --- a/frontend/src/pages/Movies/Details/index.tsx +++ b/frontend/src/pages/Movies/Details/index.tsx @@ -141,6 +141,7 @@ const MovieDetailView: FunctionComponent = () => { { if (movie) { task.create(movie.title, TaskGroup.SearchSubtitle, action, { From 2247c55bfa9fd38bf3fbe80330a5c37f4988ee6a Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Tue, 24 Dec 2024 09:57:17 -0500 Subject: [PATCH 07/12] Added opensubtitles specific throttling for server under maintenance (http 506) --- custom_libs/subliminal_patch/providers/opensubtitles.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/custom_libs/subliminal_patch/providers/opensubtitles.py b/custom_libs/subliminal_patch/providers/opensubtitles.py index 678ec882e..84141757a 100644 --- a/custom_libs/subliminal_patch/providers/opensubtitles.py +++ b/custom_libs/subliminal_patch/providers/opensubtitles.py @@ -3,7 +3,6 @@ from __future__ import absolute_import import base64 import logging import os -import traceback import re import zlib import time @@ -411,6 +410,8 @@ def checked(fn, raise_api_limit=False): except requests.RequestException as e: status_code = e.response.status_code + if status_code == 503 and "Server under maintenance" in e.response.text: + status_code = 506 else: status_code = int(response['status'][:3]) except: @@ -437,6 +438,8 @@ def checked(fn, raise_api_limit=False): raise APIThrottled if status_code == 503: raise ServiceUnavailable(str(status_code)) + if status_code == 506: + raise ServiceUnavailable("Server under maintenance") if status_code != 200: if response and "status" in response: raise OpenSubtitlesError(response['status']) From 8346ea9dc857d6f9bcd2ed3a452e0cbd1c38cfd9 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Wed, 25 Dec 2024 14:22:30 -0500 Subject: [PATCH 08/12] Improved Sonarr and Radarr syncing to prevent database integrity exception being raised by relying on proper primary keys instead of other values. --- bazarr/radarr/sync/movies.py | 16 ++++++++-------- bazarr/sonarr/sync/series.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bazarr/radarr/sync/movies.py b/bazarr/radarr/sync/movies.py index c4cb5ab96..7dc6b3a03 100644 --- a/bazarr/radarr/sync/movies.py +++ b/bazarr/radarr/sync/movies.py @@ -53,7 +53,7 @@ def update_movie(updated_movie, send_event): updated_movie['updated_at_timestamp'] = datetime.now() database.execute( update(TableMovies).values(updated_movie) - .where(TableMovies.tmdbId == updated_movie['tmdbId'])) + .where(TableMovies.radarrId == updated_movie['radarrId'])) except IntegrityError as e: logging.error(f"BAZARR cannot update movie {updated_movie['path']} because of {e}") else: @@ -66,7 +66,7 @@ def update_movie(updated_movie, send_event): def get_movie_monitored_status(movie_id): existing_movie_monitored = database.execute( select(TableMovies.monitored) - .where(TableMovies.tmdbId == str(movie_id)))\ + .where(TableMovies.radarrId == str(movie_id)))\ .first() if existing_movie_monitored is None: return True @@ -124,16 +124,16 @@ def update_movies(send_event=True): return else: # Get current movies in DB - current_movies_id_db = [x.tmdbId for x in + current_movies_id_db = [x.radarrId for x in database.execute( - select(TableMovies.tmdbId)) + select(TableMovies.radarrId)) .all()] current_movies_db_kv = [x.items() for x in [y._asdict()['TableMovies'].__dict__ for y in database.execute( select(TableMovies)) .all()]] - current_movies_radarr = [str(movie['tmdbId']) for movie in movies if movie['hasFile'] and + current_movies_radarr = [str(movie['id']) for movie in movies if movie['hasFile'] and 'movieFile' in movie and (movie['movieFile']['size'] > MINIMUM_VIDEO_SIZE or get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE)] @@ -143,7 +143,7 @@ def update_movies(send_event=True): movies_deleted = [] if len(movies_to_delete): try: - database.execute(delete(TableMovies).where(TableMovies.tmdbId.in_(movies_to_delete))) + database.execute(delete(TableMovies).where(TableMovies.radarrId.in_(movies_to_delete))) except IntegrityError as e: logging.error(f"BAZARR cannot delete movies because of {e}") else: @@ -172,7 +172,7 @@ def update_movies(send_event=True): if movie['hasFile'] is True: if 'movieFile' in movie: if sync_monitored: - if get_movie_monitored_status(movie['tmdbId']) != movie['monitored']: + if get_movie_monitored_status(movie['id']) != movie['monitored']: # monitored status is not the same as our DB trace(f"{i}: (Monitor Status Mismatch) {movie['title']}") elif not movie['monitored']: @@ -184,7 +184,7 @@ def update_movies(send_event=True): get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE): # Add/update movies from Radarr that have a movie file to current movies list trace(f"{i}: (Processing) {movie['title']}") - if str(movie['tmdbId']) in current_movies_id_db: + if str(movie['id']) in current_movies_id_db: parsed_movie = movieParser(movie, action='update', tags_dict=tagsDict, language_profiles=language_profiles, diff --git a/bazarr/sonarr/sync/series.py b/bazarr/sonarr/sync/series.py index 065fdaa21..f04a4d375 100644 --- a/bazarr/sonarr/sync/series.py +++ b/bazarr/sonarr/sync/series.py @@ -34,7 +34,7 @@ def get_language_profiles(): def get_series_monitored_table(): series_monitored = database.execute( - select(TableShows.tvdbId, TableShows.monitored))\ + select(TableShows.sonarrSeriesId, TableShows.monitored))\ .all() series_dict = dict((x, y) for x, y in series_monitored) return series_dict @@ -95,7 +95,7 @@ def update_series(send_event=True): if sync_monitored: try: - monitored_status_db = bool_map[series_monitored[show['tvdbId']]] + monitored_status_db = bool_map[series_monitored[show['id']]] except KeyError: monitored_status_db = None if monitored_status_db is None: From b71daad7fb6a8a0a9d47a29471c24989eabef88c Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Thu, 26 Dec 2024 09:46:22 -0500 Subject: [PATCH 09/12] Fixed Bazarr not starting when configured IP isn't available for binding. --- bazarr/app/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazarr/app/server.py b/bazarr/app/server.py index 85c2eb680..11cadbe1c 100644 --- a/bazarr/app/server.py +++ b/bazarr/app/server.py @@ -49,12 +49,12 @@ class Server: threads=100) self.connected = True except OSError as error: - if error.errno == 49: + if error.errno == errno.EADDRNOTAVAIL: logging.exception("BAZARR cannot bind to specified IP, trying with 0.0.0.0") self.address = '0.0.0.0' self.connected = False super(Server, self).__init__() - elif error.errno == 48: + elif error.errno == errno.EADDRINUSE: if self.port != '6767': logging.exception("BAZARR cannot bind to specified TCP port, trying with default (6767)") self.port = '6767' @@ -64,7 +64,7 @@ class Server: logging.exception("BAZARR cannot bind to default TCP port (6767) because it's already in use, " "exiting...") self.shutdown(EXIT_PORT_ALREADY_IN_USE_ERROR) - elif error.errno == 97: + elif error.errno == errno.ENOLINK: logging.exception("BAZARR cannot bind to IPv6 (*), trying with 0.0.0.0") self.address = '0.0.0.0' self.connected = False From 4809b403ca3b98f0f38f4e7ef62565b5a40e9aed Mon Sep 17 00:00:00 2001 From: Xewdy Date: Thu, 26 Dec 2024 13:49:49 -0600 Subject: [PATCH 10/12] Added check to opensubtitles.com for SDH subtitles being classified as forced --- .../subliminal_patch/providers/opensubtitlescom.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/custom_libs/subliminal_patch/providers/opensubtitlescom.py b/custom_libs/subliminal_patch/providers/opensubtitlescom.py index 0f0c2eaff..39171b6af 100644 --- a/custom_libs/subliminal_patch/providers/opensubtitlescom.py +++ b/custom_libs/subliminal_patch/providers/opensubtitlescom.py @@ -290,6 +290,10 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): if not title_id: logger.debug(f'No match found for {title}') + @staticmethod + def is_real_forced(attributes): + return attributes['foreign_parts_only'] and not attributes['hearing_impaired'] + def query(self, languages, video): self.video = video if self.use_hash: @@ -363,11 +367,11 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): # filter out forced subtitles or not depending on the required languages if all([lang.forced for lang in languages]): # only forced - result['data'] = [x for x in result['data'] if x['attributes']['foreign_parts_only']] + result['data'] = [x for x in result['data'] if self.is_real_forced(x['attributes'])] elif any([lang.forced for lang in languages]): # also forced pass else: # not forced - result['data'] = [x for x in result['data'] if not x['attributes']['foreign_parts_only']] + result['data'] = [x for x in result['data'] if not self.is_real_forced(x['attributes'])] logger.debug(f"Query returned {len(result['data'])} subtitles") @@ -407,7 +411,7 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider): if len(item['attributes']['files']): subtitle = OpenSubtitlesComSubtitle( language=Language.fromietf(from_opensubtitlescom(item['attributes']['language'])), - forced=item['attributes']['foreign_parts_only'], + forced=self.is_real_forced(item['attributes']), hearing_impaired=item['attributes']['hearing_impaired'], page_link=item['attributes']['url'], file_id=item['attributes']['files'][0]['file_id'], From 60febe3d12b75d534b64d56839ee79ee063958fd Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sat, 28 Dec 2024 14:01:25 -0500 Subject: [PATCH 11/12] Fixed sync issue with Radarr that would remove all movies from database on every sync. --- bazarr/radarr/sync/movies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazarr/radarr/sync/movies.py b/bazarr/radarr/sync/movies.py index 7dc6b3a03..72d18f511 100644 --- a/bazarr/radarr/sync/movies.py +++ b/bazarr/radarr/sync/movies.py @@ -133,7 +133,7 @@ def update_movies(send_event=True): select(TableMovies)) .all()]] - current_movies_radarr = [str(movie['id']) for movie in movies if movie['hasFile'] and + current_movies_radarr = [movie['id'] for movie in movies if movie['hasFile'] and 'movieFile' in movie and (movie['movieFile']['size'] > MINIMUM_VIDEO_SIZE or get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE)] @@ -184,7 +184,7 @@ def update_movies(send_event=True): get_movie_file_size_from_db(movie['movieFile']['path']) > MINIMUM_VIDEO_SIZE): # Add/update movies from Radarr that have a movie file to current movies list trace(f"{i}: (Processing) {movie['title']}") - if str(movie['id']) in current_movies_id_db: + if movie['id'] in current_movies_id_db: parsed_movie = movieParser(movie, action='update', tags_dict=tagsDict, language_profiles=language_profiles, From 0413dbaa4cf632df0f560f5df2cd67851c3eee02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Lofj=C3=A4rd?= Date: Mon, 30 Dec 2024 16:43:57 +0100 Subject: [PATCH 12/12] Improved languages profile health check to include default assigned profiles --- bazarr/utilities/health.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bazarr/utilities/health.py b/bazarr/utilities/health.py index 84a313cf0..ae32d3a0a 100644 --- a/bazarr/utilities/health.py +++ b/bazarr/utilities/health.py @@ -75,11 +75,13 @@ def get_health_issues(): .where(TableShows.profileId.is_not(None))).scalar() movies_with_profile = database.execute(select(func.count(TableMovies.radarrId)) .where(TableMovies.profileId.is_not(None))).scalar() + default_series_profile_empty = settings.general.serie_default_enabled and settings.general.serie_default_profile == '' + default_movies_profile_empty = settings.general.movie_default_enabled and settings.general.movie_default_profile == '' if languages_profiles_count == 0: health_issues.append({'object': 'Missing languages profile', 'issue': 'You must create at least one languages profile and assign it to your content.'}) - elif languages_profiles_count > 0 and ((settings.general.use_sonarr and series_with_profile == 0) or - (settings.general.use_radarr and movies_with_profile == 0)): + elif languages_profiles_count > 0 and ((settings.general.use_sonarr and series_with_profile == 0 and default_series_profile_empty) or + (settings.general.use_radarr and movies_with_profile == 0 and default_movies_profile_empty)): health_issues.append({'object': 'No assigned languages profile', 'issue': 'Although you have created at least one languages profile, you must assign it ' 'to your content.'})