From 552cae74838a3f67df59680c469ad0146bda6083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 29 Nov 2018 12:55:43 -0500 Subject: [PATCH 1/2] Fix to prevent manual search to return None instead of an empty json string. --- bazarr/get_subtitle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 9d310810a..f4bad4b0e 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -245,7 +245,7 @@ def manual_search(path, language, hi, providers, providers_auth, sceneName, medi subtitles_dict = sorted(subtitles_list, key=lambda x: x['score'], reverse=True) logging.debug('BAZARR ' + str(len(subtitles_dict)) + " subtitles have been found for this file: " + path) logging.debug('BAZARR Ended searching subtitles for this file: ' + path) - return(subtitles_dict) + return(subtitles_dict) def manual_download_subtitle(path, language, hi, subtitle, provider, providers_auth, sceneName, media_type): logging.debug('BAZARR Manually downloading subtitles for this file: ' + path) From b0e6136de16729cf69fe590c95fd9d6b541469a5 Mon Sep 17 00:00:00 2001 From: morpheus65535 <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 4 Dec 2018 20:30:03 -0500 Subject: [PATCH 2/2] Added more debug logging for the existing subtitles indexing process. --- bazarr/list_subtitles.py | 135 +++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 49 deletions(-) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index 96c91276f..5b6af3620 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -18,10 +18,11 @@ from get_languages import alpha2_from_alpha3 gc.enable() def store_subtitles(file): - # languages = [] + logging.debug('BAZARR started subtitles indexing for this file: ' + file) actual_subtitles = [] if os.path.exists(file): if os.path.splitext(file)[1] == '.mkv': + logging.debug("BAZARR is trying to index embedded subtitles.") try: with open(file, 'rb') as f: mkv = enzyme.MKV(f) @@ -29,54 +30,71 @@ def store_subtitles(file): for subtitle_track in mkv.subtitle_tracks: try: if alpha2_from_alpha3(subtitle_track.language) != None: - actual_subtitles.append([str(alpha2_from_alpha3(subtitle_track.language)),None]) + lang = str(alpha2_from_alpha3(subtitle_track.language)) + logging.debug("BAZARR embedded subtitles detected: " + lang) + actual_subtitles.append([lang,None]) except: + logging.debug("BAZARR unable to index this unrecognized language: " + subtitle_track.language) pass - except: + except Exception as e: + logging.exception("BAZARR error when trying to analyze this mkv file: " + file) pass + else: + logging.debug("BAZARR This file isn't an .mkv file.") brazilian_portuguese = [".pt-br", ".pob", "pb"] try: subtitles = core.search_external_subtitles(file) - except: + except Exception as e: + logging.exception("BAZARR unable to index external subtitles.") pass else: for subtitle, language in subtitles.iteritems(): if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)) is True: + logging.debug("BAZARR external subtitles detected: " + "pb") actual_subtitles.append([str("pb"), path_replace_reverse(os.path.join(os.path.dirname(file), subtitle))]) elif str(language) != 'und': + logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append([str(language), path_replace_reverse(os.path.join(os.path.dirname(file), subtitle))]) else: - with open(path_replace(os.path.join(os.path.dirname(file), subtitle)), 'r') as f: - text = list(islice(f, 100)) - text = ' '.join(text) - encoding = UnicodeDammit(text) - try: - text = text.decode(encoding.original_encoding) - detected_language = langdetect.detect(text) - except Exception as e: - logging.exception('BAZARR Error trying to detect character encoding for this subtitles file: ' + path_replace(os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') - else: - if len(detected_language) > 0: - actual_subtitles.append([str(detected_language), path_replace_reverse(os.path.join(os.path.dirname(file), subtitle))]) - - conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) - c_db = conn_db.cursor() - - c_db.execute("UPDATE table_episodes SET subtitles = ? WHERE path = ?", (str(actual_subtitles), path_replace_reverse(file))) - conn_db.commit() - - c_db.close() + if os.path.splitext(subtitle)[1] != ".sub": + logging.debug("BAZARR falling back to file content analysis to detect language.") + with open(path_replace(os.path.join(os.path.dirname(file), subtitle)), 'r') as f: + text = list(islice(f, 100)) + text = ' '.join(text) + encoding = UnicodeDammit(text) + try: + text = text.decode(encoding.original_encoding) + detected_language = langdetect.detect(text) + except Exception as e: + logging.exception('BAZARR Error trying to detect language for this subtitles file: ' + path_replace(os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') + else: + if len(detected_language) > 0: + logging.debug("BAZARR external subtitles detected and analysis guessed this language: " + str(detected_language)) + actual_subtitles.append([str(detected_language), path_replace_reverse(os.path.join(os.path.dirname(file), subtitle))]) + + conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) + c_db = conn_db.cursor() + logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) + c_db.execute("UPDATE table_episodes SET subtitles = ? WHERE path = ?", (str(actual_subtitles), path_replace_reverse(file))) + conn_db.commit() + + c_db.close() + else: + logging.debug("BAZARR this file doesn't seems to exist or isn't accessible.") + + logging.debug('BAZARR ended subtitles indexing for this file: ' + file) return actual_subtitles def store_subtitles_movie(file): - # languages = [] + logging.debug('BAZARR started subtitles indexing for this file: ' + file) actual_subtitles = [] if os.path.exists(file): if os.path.splitext(file)[1] == '.mkv': + logging.debug("BAZARR is trying to index embedded subtitles.") try: with open(file, 'rb') as f: mkv = enzyme.MKV(f) @@ -84,42 +102,61 @@ def store_subtitles_movie(file): for subtitle_track in mkv.subtitle_tracks: try: if alpha2_from_alpha3(subtitle_track.language) != None: - actual_subtitles.append([str(alpha2_from_alpha3(subtitle_track.language)), None]) + lang = str(alpha2_from_alpha3(subtitle_track.language)) + logging.debug("BAZARR embedded subtitles detected: " + lang) + actual_subtitles.append([lang, None]) except: + logging.debug("BAZARR unable to index this unrecognized language: " + subtitle_track.language) pass - except: + except Exception as e: + logging.exception("BAZARR error when trying to analyze this mkv file: " + file) pass + else: + logging.debug("BAZARR This file isn't an .mkv file.") - subtitles = core.search_external_subtitles(file) - brazilian_portuguese = [".pt-br", ".pob", "pb"] - for subtitle, language in subtitles.iteritems(): - if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)) is True: - actual_subtitles.append([str("pb"), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) - elif str(language) != 'und': - actual_subtitles.append([str(language), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) - else: - if os.path.splitext(subtitle)[1] != ".sub": - with open(path_replace_movie(os.path.join(os.path.dirname(file), subtitle)), 'r') as f: - text = list(islice(f, 100)) - text = ' '.join(text) - encoding = UnicodeDammit(text) - try: - text = text.decode(encoding.original_encoding) - detected_language = langdetect.detect(text) - except Exception as e: - logging.exception('BAZARR Error trying to detect character encoding for this subtitles file: ' + path_replace_movie(os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') - else: - if len(detected_language) > 0: - actual_subtitles.append([str(detected_language), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) + brazilian_portuguese = [".pt-br", ".pob", "pb"] + try: + subtitles = core.search_external_subtitles(file) + except Exception as e: + logging.exception("BAZARR unable to index external subtitles.") + pass + else: + for subtitle, language in subtitles.iteritems(): + if str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese)) is True: + logging.debug("BAZARR external subtitles detected: " + "pb") + actual_subtitles.append([str("pb"), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) + elif str(language) != 'und': + logging.debug("BAZARR external subtitles detected: " + str(language)) + actual_subtitles.append([str(language), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) + else: + if os.path.splitext(subtitle)[1] != ".sub": + logging.debug("BAZARR falling back to file content analysis to detect language.") + with open(path_replace_movie(os.path.join(os.path.dirname(file), subtitle)), 'r') as f: + text = list(islice(f, 100)) + text = ' '.join(text) + encoding = UnicodeDammit(text) + try: + text = text.decode(encoding.original_encoding) + detected_language = langdetect.detect(text) + except Exception as e: + logging.exception('BAZARR Error trying to detect language for this subtitles file: ' + path_replace(os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.') + else: + if len(detected_language) > 0: + logging.debug("BAZARR external subtitles detected and analysis guessed this language: " + str(detected_language)) + actual_subtitles.append([str(detected_language), path_replace_reverse_movie(os.path.join(os.path.dirname(file), subtitle))]) conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) c_db = conn_db.cursor() - + logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) c_db.execute("UPDATE table_movies SET subtitles = ? WHERE path = ?", (str(actual_subtitles), path_replace_reverse_movie(file))) conn_db.commit() c_db.close() + else: + logging.debug("BAZARR this file doesn't seems to exist or isn't accessible.") + + logging.debug('BAZARR ended subtitles indexing for this file: ' + file) return actual_subtitles