Merge branch 'development' into python3

# Conflicts:
#	bazarr/list_subtitles.py
#	bazarr/main.py

bazarr/check_update.py

@@ -7,7 +7,7 @@ import requests
 import tarfile
 
 from get_args import args
-from config import settings, bazarr_url
+from config import settings
 from queueconfig import notifications
 from database import database
 

bazarr/config.py

@@ -143,36 +143,66 @@ settings = simpleconfigparser(defaults=defaults)
 settings.read(os.path.join(args.config_dir, 'config', 'config.ini'))
 
 base_url = settings.general.base_url
-bazarr_url = 'http://localhost:' + (str(args.port) if args.port else settings.general.port) + base_url
-
-# sonarr url
-if settings.sonarr.getboolean('ssl'):
-    protocol_sonarr = "https"
-else:
-    protocol_sonarr = "http"
-
-if settings.sonarr.base_url == '':
-    settings.sonarr.base_url = "/"
-if not settings.sonarr.base_url.startswith("/"):
-    settings.sonarr.base_url = "/" + settings.sonarr.base_url
-if settings.sonarr.base_url.endswith("/"):
-    settings.sonarr.base_url = settings.sonarr.base_url[:-1]
-
-url_sonarr = protocol_sonarr + "://" + settings.sonarr.ip + ":" + settings.sonarr.port + settings.sonarr.base_url
-url_sonarr_short = protocol_sonarr + "://" + settings.sonarr.ip + ":" + settings.sonarr.port
-
-# radarr url
-if settings.radarr.getboolean('ssl'):
-    protocol_radarr = "https"
-else:
-    protocol_radarr = "http"
-
-if settings.radarr.base_url == '':
-    settings.radarr.base_url = "/"
-if not settings.radarr.base_url.startswith("/"):
-    settings.radarr.base_url = "/" + settings.radarr.base_url
-if settings.radarr.base_url.endswith("/"):
-    settings.radarr.base_url = settings.radarr.base_url[:-1]
-
-url_radarr = protocol_radarr + "://" + settings.radarr.ip + ":" + settings.radarr.port + settings.radarr.base_url
-url_radarr_short = protocol_radarr + "://" + settings.radarr.ip + ":" + settings.radarr.port
+
+
+def url_sonarr():
+    if settings.sonarr.getboolean('ssl'):
+        protocol_sonarr = "https"
+    else:
+        protocol_sonarr = "http"
+
+    if settings.sonarr.base_url == '':
+        settings.sonarr.base_url = "/"
+    if not settings.sonarr.base_url.startswith("/"):
+        settings.sonarr.base_url = "/" + settings.sonarr.base_url
+    if settings.sonarr.base_url.endswith("/"):
+        settings.sonarr.base_url = settings.sonarr.base_url[:-1]
+
+    return protocol_sonarr + "://" + settings.sonarr.ip + ":" + settings.sonarr.port + settings.sonarr.base_url
+
+
+def url_sonarr_short():
+    if settings.sonarr.getboolean('ssl'):
+        protocol_sonarr = "https"
+    else:
+        protocol_sonarr = "http"
+
+    if settings.sonarr.base_url == '':
+        settings.sonarr.base_url = "/"
+    if not settings.sonarr.base_url.startswith("/"):
+        settings.sonarr.base_url = "/" + settings.sonarr.base_url
+    if settings.sonarr.base_url.endswith("/"):
+        settings.sonarr.base_url = settings.sonarr.base_url[:-1]
+    return protocol_sonarr + "://" + settings.sonarr.ip + ":" + settings.sonarr.port
+
+
+def url_radarr():
+    if settings.radarr.getboolean('ssl'):
+        protocol_radarr = "https"
+    else:
+        protocol_radarr = "http"
+
+    if settings.radarr.base_url == '':
+        settings.radarr.base_url = "/"
+    if not settings.radarr.base_url.startswith("/"):
+        settings.radarr.base_url = "/" + settings.radarr.base_url
+    if settings.radarr.base_url.endswith("/"):
+        settings.radarr.base_url = settings.radarr.base_url[:-1]
+
+    return protocol_radarr + "://" + settings.radarr.ip + ":" + settings.radarr.port + settings.radarr.base_url
+
+
+def url_radarr_short():
+    if settings.radarr.getboolean('ssl'):
+        protocol_radarr = "https"
+    else:
+        protocol_radarr = "http"
+
+    if settings.radarr.base_url == '':
+        settings.radarr.base_url = "/"
+    if not settings.radarr.base_url.startswith("/"):
+        settings.radarr.base_url = "/" + settings.radarr.base_url
+    if settings.radarr.base_url.endswith("/"):
+        settings.radarr.base_url = settings.radarr.base_url[:-1]
+
+    return protocol_radarr + "://" + settings.radarr.ip + ":" + settings.radarr.port

bazarr/embedded_subs_reader.py

@@ -35,8 +35,9 @@ class EmbeddedSubsReader:
                         mkv = enzyme.MKV(f)
                     except MalformedMKVError:
                         logging.error('BAZARR cannot analyze this MKV with our built-in MKV parser, you should install ffmpeg: ' + file)
-                for subtitle_track in mkv.subtitle_tracks:
-                    subtitles_list.append([subtitle_track.language, subtitle_track.forced, subtitle_track.codec_id])
+                    else:
+                        for subtitle_track in mkv.subtitle_tracks:
+                            subtitles_list.append([subtitle_track.language, subtitle_track.forced, subtitle_track.codec_id])
 
         return subtitles_list
 

bazarr/get_episodes.py

@@ -41,7 +41,7 @@ def sync_episodes():
     for i, seriesId in enumerate(seriesIdList, 1):
         notifications.write(msg='Getting episodes data from Sonarr...', queue='get_episodes', item=i, length=seriesIdListLength)
         # Get episodes data for a series from Sonarr
-        url_sonarr_api_episode = url_sonarr + "/api/episode?seriesId=" + str(seriesId['sonarrSeriesId']) + "&apikey=" + apikey_sonarr
+        url_sonarr_api_episode = url_sonarr() + "/api/episode?seriesId=" + str(seriesId['sonarrSeriesId']) + "&apikey=" + apikey_sonarr
         try:
             r = requests.get(url_sonarr_api_episode, timeout=60, verify=False)
             r.raise_for_status()
@@ -155,7 +155,7 @@ def sync_episodes():
         result = database.execute(
             '''INSERT OR IGNORE INTO table_episodes(''' + query.keys_insert + ''') VALUES(''' + query.question_marks +
             ''')''', query.values)
-        if result:
+        if result > 0:
             altered_episodes.append([added_episode['sonarrEpisodeId'], added_episode['path']])
         else:
             logging.debug('BAZARR unable to insert this episode into the database:',

bazarr/get_movies.py

@@ -36,7 +36,7 @@ def update_movies():
         audio_profiles = get_profile_list()
         
         # Get movies data from radarr
-        url_radarr_api_movies = url_radarr + "/api/movie?apikey=" + apikey_radarr
+        url_radarr_api_movies = url_radarr() + "/api/movie?apikey=" + apikey_radarr
         try:
             r = requests.get(url_radarr_api_movies, timeout=60, verify=False)
             r.raise_for_status()
@@ -240,7 +240,7 @@ def update_movies():
                 result = database.execute(
                     '''INSERT OR IGNORE INTO table_movies(''' + query.keys_insert + ''') VALUES(''' +
                     query.question_marks + ''')''', query.values)
-                if result:
+                if result > 0:
                     altered_movies.append([added_movie['tmdbId'],
                                            added_movie['path'],
                                            added_movie['radarrId'],
@@ -276,7 +276,7 @@ def get_profile_list():
     profiles_list = []
     # Get profiles data from radarr
 
-    url_radarr_api_movies = url_radarr + "/api/profile?apikey=" + apikey_radarr
+    url_radarr_api_movies = url_radarr() + "/api/profile?apikey=" + apikey_radarr
     try:
         profiles_json = requests.get(url_radarr_api_movies, timeout=60, verify=False)
     except requests.exceptions.ConnectionError as errc:

bazarr/get_series.py

@@ -33,7 +33,7 @@ def update_series():
         audio_profiles = get_profile_list()
         
         # Get shows data from Sonarr
-        url_sonarr_api_series = url_sonarr + "/api/series?apikey=" + apikey_sonarr
+        url_sonarr_api_series = url_sonarr() + "/api/series?apikey=" + apikey_sonarr
         try:
             r = requests.get(url_sonarr_api_series, timeout=60, verify=False)
             r.raise_for_status()
@@ -168,9 +168,9 @@ def get_profile_list():
     # Get profiles data from Sonarr
 
     if sonarr_version.startswith('2'):
-        url_sonarr_api_series = url_sonarr + "/api/profile?apikey=" + apikey_sonarr
+        url_sonarr_api_series = url_sonarr() + "/api/profile?apikey=" + apikey_sonarr
     elif sonarr_version.startswith('3'):
-        url_sonarr_api_series = url_sonarr + "/api/v3/languageprofile?apikey=" + apikey_sonarr
+        url_sonarr_api_series = url_sonarr() + "/api/v3/languageprofile?apikey=" + apikey_sonarr
 
     try:
         profiles_json = requests.get(url_sonarr_api_series, timeout=60, verify=False)

bazarr/get_subtitle.py

@@ -182,7 +182,7 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
                                                            post_download_hook=None,  # fixme
                                                            language_hook=None)  # fixme
             for provider in providers:
-                track_event(category=provider, action='search', label=language[0])
+                track_event(category=provider, action='search', label=language_from_alpha3(language[0]))
         else:
             downloaded_subtitles = None
             logging.info("BAZARR All providers are throttled")
@@ -340,7 +340,7 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
                                                throttle_callback=provider_throttle,
                                                language_hook=None)  # fixme
                 for provider in providers:
-                    track_event(category=provider, action='search', label=language[0])
+                    track_event(category=provider, action='search', label=language_from_alpha3(lang))
             else:
                 subtitles = []
                 logging.info("BAZARR All providers are throttled")

bazarr/init.py

@@ -2,7 +2,6 @@
 
 from __future__ import absolute_import
 import os
-import logging
 import time
 import rarfile
 
@@ -10,6 +9,7 @@ from cork import Cork
 from backports import configparser2
 from config import settings
 from get_args import args
+from logger import configure_logging
 
 from dogpile.cache.region import register_backend as register_cache_backend
 import subliminal
@@ -34,24 +34,21 @@ if not os.path.exists(args.config_dir):
     # Create config_dir directory tree
     try:
         os.mkdir(os.path.join(args.config_dir))
-        logging.debug("BAZARR Created data directory")
     except OSError:
-        logging.exception(
-            "BAZARR The configuration directory doesn't exist and Bazarr cannot create it (permission issue?).")
+        print "BAZARR The configuration directory doesn't exist and Bazarr cannot create it (permission issue?)."
         exit(2)
 
 if not os.path.exists(os.path.join(args.config_dir, 'config')):
     os.mkdir(os.path.join(args.config_dir, 'config'))
-    logging.debug("BAZARR Created config folder")
 if not os.path.exists(os.path.join(args.config_dir, 'db')):
     os.mkdir(os.path.join(args.config_dir, 'db'))
-    logging.debug("BAZARR Created db folder")
 if not os.path.exists(os.path.join(args.config_dir, 'log')):
     os.mkdir(os.path.join(args.config_dir, 'log'))
-    logging.debug("BAZARR Created log folder")
 if not os.path.exists(os.path.join(args.config_dir, 'cache')):
     os.mkdir(os.path.join(args.config_dir, 'cache'))
-    logging.debug("BAZARR Created cache folder")
+
+configure_logging(settings.general.getboolean('debug') or args.debug)
+import logging
 
 # create database file
 if not os.path.exists(os.path.join(args.config_dir, 'db', 'bazarr.db')):

bazarr/list_subtitles.py

@@ -12,6 +12,7 @@ import subliminal_patch
 import operator
 from subliminal import core
 from subliminal_patch import search_external_subtitles
+from subzero.language import Language
 from bs4 import UnicodeDammit
 from itertools import islice
 
@@ -64,6 +65,7 @@ def store_subtitles(original_path, reversed_path):
             subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else []
             subtitles = search_external_subtitles(reversed_path, languages=get_language_set(),
                                                   only_one=settings.general.getboolean('single_language'))
+            subtitles = guess_external_subtitles(get_subtitle_destination_folder() or os.path.dirname(file), subtitles)
         except Exception as e:
             logging.exception("BAZARR unable to index external subtitles.")
             pass
@@ -151,14 +153,14 @@ def store_subtitles_movie(original_path, reversed_path):
                 logging.exception(
                     "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(reversed_path)[1], reversed_path))
                 pass
-        
-        dest_folder = get_subtitle_destination_folder() or ''
-        subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else []
+
         brazilian_portuguese = [".pt-br", ".pob", "pb"]
         brazilian_portuguese_forced = [".pt-br.forced", ".pob.forced", "pb.forced"]
         try:
-            subtitles = search_external_subtitles(reversed_path, languages=get_language_set(),
-                                                  only_one=settings.general.getboolean('single_language'))
+            dest_folder = get_subtitle_destination_folder() or ''
+            subliminal_patch.core.CUSTOM_PATHS = [dest_folder] if dest_folder else []
+            subtitles = search_external_subtitles(reversed_path, languages=get_language_set())
+            subtitles = guess_external_subtitles(get_subtitle_destination_folder() or os.path.dirname(reversed_path), subtitles)
         except Exception as e:
             logging.exception("BAZARR unable to index external subtitles.")
             pass
@@ -390,3 +392,31 @@ def get_external_subtitles_path(file, subtitle):
         path = None
     
     return path
+
+
+def guess_external_subtitles(dest_folder, subtitles):
+    for subtitle, language in subtitles.iteritems():
+        if not language:
+            subtitle_path = os.path.join(dest_folder, subtitle)
+            if os.path.exists(subtitle_path) and os.path.splitext(subtitle_path)[1] in core.SUBTITLE_EXTENSIONS:
+                logging.debug("BAZARR falling back to file content analysis to detect language.")
+                detected_language = None
+                with open(subtitle_path, 'r') as f:
+                    text = ' '.join(list(islice(f, 100)))
+                    try:
+                        encoding = UnicodeDammit(text)
+                        text = text.decode(encoding.original_encoding)
+                        detected_language = langdetect.detect(text)
+                    except Exception as e:
+                        logging.exception('BAZARR Error trying to detect language for this subtitles file: ' +
+                                          subtitle_path + ' You should try to delete this subtitles file manually and ask '
+                                                          'Bazarr to download it again.')
+                    else:
+                        if detected_language:
+                            logging.debug("BAZARR external subtitles detected and guessed this language: " + str(
+                                detected_language))
+                            try:
+                                subtitles[subtitle] = Language.rebuild(Language.fromietf(detected_language))
+                            except:
+                                pass
+    return subtitles

bazarr/main.py

@@ -30,12 +30,11 @@ import operator
 from calendar import day_name
 
 from get_args import args
-from logger import configure_logging, empty_log
+from logger import empty_log
 from config import settings, url_sonarr, url_radarr, url_radarr_short, url_sonarr_short, base_url
 
-configure_logging(settings.general.getboolean('debug') or args.debug)
-
 from init import *
+import logging
 from database import database, dict_mapper
 
 from notifier import update_notifier
@@ -152,7 +151,8 @@ def post_get(name, default=''):
 
 @hook('before_request')
 def enable_cors():
-    response.headers['Access-Control-Allow-Origin'] = '*'
+    if response:
+        response.headers['Access-Control-Allow-Origin'] = '*'
 
 
 @route(base_url + 'login')
@@ -184,31 +184,36 @@ def redirect_root():
 @route(base_url + 'shutdown')
 def shutdown():
     try:
-        stop_file = open(os.path.join(args.config_dir, "bazarr.stop"), "w")
-    except Exception as e:
-        logging.error('BAZARR Cannot create bazarr.stop file.')
-    else:
         server.stop()
+    except:
+        logging.error('BAZARR Cannot stop CherryPy.')
+    else:
         database.close()
-        stop_file.write('')
-        stop_file.close()
-        sys.exit(0)
+        try:
+            stop_file = open(os.path.join(args.config_dir, "bazarr.stop"), "w")
+        except Exception as e:
+            logging.error('BAZARR Cannot create bazarr.stop file.')
+        else:
+            stop_file.write('')
+            stop_file.close()
 
 
 @route(base_url + 'restart')
 def restart():
     try:
-        restart_file = open(os.path.join(args.config_dir, "bazarr.restart"), "w")
-    except Exception as e:
-        logging.error('BAZARR Cannot create bazarr.restart file.')
-    else:
-        # print 'Bazarr is being restarted...'
-        logging.info('Bazarr is being restarted...')
         server.stop()
+    except:
+        logging.error('BAZARR Cannot stop CherryPy.')
+    else:
         database.close()
-        restart_file.write('')
-        restart_file.close()
-        sys.exit(0)
+        try:
+            restart_file = open(os.path.join(args.config_dir, "bazarr.restart"), "w")
+        except Exception as e:
+            logging.error('BAZARR Cannot create bazarr.restart file.')
+        else:
+            logging.info('Bazarr is being restarted...')
+            restart_file.write('')
+            restart_file.close()
 
 
 @route(base_url + 'wizard')
@@ -472,10 +477,10 @@ def download_log():
 def image_proxy(url):
     authorize()
     apikey = settings.sonarr.apikey
-    url_image = url_sonarr_short + '/' + url + '?apikey=' + apikey
+    url_image = url_sonarr_short() + '/' + url + '?apikey=' + apikey
     try:
         image_buffer = BytesIO(
-            requests.get(url_sonarr + '/api' + url_image.split(url_sonarr)[1], timeout=15, verify=False).content)
+            requests.get(url_sonarr() + '/api' + url_image.split(url_sonarr())[1], timeout=15, verify=False).content)
     except:
         return None
     else:
@@ -491,13 +496,13 @@ def image_proxy_movies(url):
     authorize()
     apikey = settings.radarr.apikey
     try:
-        url_image = (url_radarr_short + '/' + url + '?apikey=' + apikey).replace('/fanart.jpg', '/banner.jpg')
+        url_image = (url_radarr_short() + '/' + url + '?apikey=' + apikey).replace('/fanart.jpg', '/banner.jpg')
         image_buffer = BytesIO(
-            requests.get(url_radarr + '/api' + url_image.split(url_radarr)[1], timeout=15, verify=False).content)
+            requests.get(url_radarr() + '/api' + url_image.split(url_radarr())[1], timeout=15, verify=False).content)
     except:
-        url_image = url_radarr_short + '/' + url + '?apikey=' + apikey
+        url_image = url_radarr_short() + '/' + url + '?apikey=' + apikey
         image_buffer = BytesIO(
-            requests.get(url_radarr + '/api' + url_image.split(url_radarr)[1], timeout=15, verify=False).content)
+            requests.get(url_radarr() + '/api' + url_image.split(url_radarr())[1], timeout=15, verify=False).content)
     else:
         image_buffer.seek(0)
         bytes = image_buffer.read()
@@ -723,7 +728,7 @@ def episodes(no):
         seasons_list.append(list(season))
 
     return template('episodes', bazarr_version=bazarr_version, no=no, details=series_details,
-                    languages=languages, seasons=seasons_list, url_sonarr_short=url_sonarr_short, base_url=base_url,
+                    languages=languages, seasons=seasons_list, url_sonarr_short=url_sonarr_short(), base_url=base_url,
                     tvdbid=tvdbid, number=number, current_port=settings.general.port)
 
 
@@ -858,7 +863,7 @@ def movie(no):
     languages = database.execute("SELECT code2, name FROM table_settings_languages WHERE enabled=1")
 
     return template('movie', bazarr_version=bazarr_version, no=no, details=movies_details,
-                    languages=languages, url_radarr_short=url_radarr_short, base_url=base_url, tmdbid=tmdbid,
+                    languages=languages, url_radarr_short=url_radarr_short(), base_url=base_url, tmdbid=tmdbid,
                     current_port=settings.general.port)
 
 
@@ -2025,7 +2030,7 @@ def perform_manual_upload_subtitle_movie():
 
 
 def configured():
-    System.update({System.configured: 1}).execute()
+    database.execute("UPDATE system SET configured = 1")
 
 
 @route(base_url + 'api/series/wanted')

bazarr/utils.py

@@ -84,7 +84,7 @@ def cache_maintenance():
 def get_sonarr_version():
     use_sonarr = settings.general.getboolean('use_sonarr')
     apikey_sonarr = settings.sonarr.apikey
-    sv = url_sonarr + "/api/system/status?apikey=" + apikey_sonarr
+    sv = url_sonarr() + "/api/system/status?apikey=" + apikey_sonarr
     sonarr_version = ''
     if use_sonarr:
         try:
@@ -115,7 +115,7 @@ def get_sonarr_platform():
 def get_radarr_version():
     use_radarr = settings.general.getboolean('use_radarr')
     apikey_radarr = settings.radarr.apikey
-    rv = url_radarr + "/api/system/status?apikey=" + apikey_radarr
+    rv = url_radarr() + "/api/system/status?apikey=" + apikey_radarr
     radarr_version = ''
     if use_radarr:
         try:

libs/sqlite3worker.py

@@ -137,7 +137,11 @@ class Sqlite3Worker(threading.Thread):
         else:
             try:
                 self.sqlite3_cursor.execute(query, values)
+                if query.lower().strip().startswith(("insert", "update")):
+                    self.results[token] = self.sqlite3_cursor.rowcount
             except sqlite3.Error as err:
+                self.results[token] = (
+                        "Query returned error: %s: %s: %s" % (query, values, err))
                 LOGGER.error(
                     "Query returned error: %s: %s: %s", query, values, err)
 
@@ -196,7 +200,7 @@ class Sqlite3Worker(threading.Thread):
         token = str(uuid.uuid4())
         # If it's a select we queue it up with a token to mark the results
         # into the output queue so we know what results are ours.
-        if query.lower().strip().startswith("select"):
+        if query.lower().strip().startswith(("select", "insert", "update")):
             self.sql_queue.put((token, query, values, only_one), timeout=5)
             return self.query_results(token)
         else:

views/movie.tpl

@@ -126,7 +126,7 @@
 										%>
 										%if subs_languages is not None:
 										<button class="manual_search ui button" data-tooltip="Manually Search For Subtitles" data-inverted="" data-moviePath="{{details['path']}}" data-scenename="{{details['sceneName']}}" data-language="{{subs_languages_list}}" data-hi="{{details['hearing_impaired']}}" data-forced="{{details['forced']}}" data-movie_title="{{details['title']}}" data-radarrId="{{details['radarrId']}}"><i class="ui inverted large compact user icon"></i></button>
-										<button class="manual_upload ui button" data-tooltip="Upload Subtitle File" data-inverted="" data-moviePath="{{details['path']}}" data-scenename="{{details['sceneName']}}" data-language="{{subs_languages_list}}" data-hi="{{details['hearing_impaired']}}" data-movie_title="{{details['forced']}}" data-radarrId="{{details['title']}}"><i class="ui inverted large compact cloud upload icon"></i></button>
+										<button class="manual_upload ui button" data-tooltip="Upload Subtitle File" data-inverted="" data-moviePath="{{details['path']}}" data-scenename="{{details['sceneName']}}" data-language="{{subs_languages_list}}" data-hi="{{details['hearing_impaired']}}" data-movie_title="{{details['title']}}" data-radarrId="{{details['radarrId']}}"><i class="ui inverted large compact cloud upload icon"></i></button>
 										%end
 										<button id="config" class="ui button" data-tooltip="Edit Movie" data-inverted="" data-tmdbid="{{details['tmdbId']}}" data-title="{{details['title']}}" data-poster="{{details['poster']}}" data-audio="{{details['audio_language']}}" data-languages="{{!subs_languages_list}}" data-hearing-impaired="{{details['hearing_impaired']}}" data-forced="{{details['forced']}}"><i class="ui inverted large compact configure icon"></i></button>
 									</div>

views/wizard_subtitles.tpl

@@ -105,9 +105,9 @@
                                         <option value="">Languages</option>
                                         %enabled_languages = []
                                         %for language in settings_languages:
-                                        <option value="{{language.code2}}">{{language.name}}</option>
-                                        %if language.enabled == True:
-                                        %	enabled_languages.append(str(language.code2))
+                                        <option value="{{language['code2']}}">{{language['name']}}</option>
+                                        %if language['enabled'] == True:
+                                        %	enabled_languages.append(str(language['code2']))
                                         %end
                                         %end
                                     </select>