Merge pull request #388 from morpheus65535/updater

Updater rework to support update from releases instead of using git.

bazarr/check_update.py

@@ -4,9 +4,11 @@ import logging
 import sqlite3
 import json
 import requests
+import tarfile
 
 from get_args import args
-from config import settings
+from config import settings, bazarr_url
+from queueconfig import notifications
 
 if not args.no_update:
     import git
@@ -33,21 +35,100 @@ def gitconfig():
 
 
 def check_and_apply_update():
-    gitconfig()
     check_releases()
-    branch = settings.general.branch
-    g = git.cmd.Git(current_working_directory)
-    g.fetch('origin')
-    result = g.diff('--shortstat', 'origin/' + branch)
-    if len(result) == 0:
-        logging.info('BAZARR No new version of Bazarr available.')
+    if not args.release_update:
+        gitconfig()
+        branch = settings.general.branch
+        g = git.cmd.Git(current_working_directory)
+        g.fetch('origin')
+        result = g.diff('--shortstat', 'origin/' + branch)
+        if len(result) == 0:
+            notifications.write(msg='No new version of Bazarr available.', queue='check_update')
+            logging.info('BAZARR No new version of Bazarr available.')
+        else:
+            g.reset('--hard', 'HEAD')
+            g.checkout(branch)
+            g.reset('--hard', 'origin/' + branch)
+            g.pull()
+            logging.info('BAZARR Updated to latest version. Restart required. ' + result)
+            updated()
     else:
-        g.reset('--hard', 'HEAD')
-        g.checkout(branch)
-        g.reset('--hard', 'origin/' + branch)
-        g.pull()
-        logging.info('BAZARR Updated to latest version. Restart required. ' + result)
-        updated()
+        url = 'https://api.github.com/repos/morpheus65535/bazarr/releases'
+        releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list)
+        
+        if releases is None:
+            notifications.write(msg='Could not get releases from GitHub.',
+                                queue='check_update', type='warning')
+            logging.warn('BAZARR Could not get releases from GitHub.')
+            return
+        else:
+            release = releases[0]
+        latest_release = release['tag_name']
+        
+        if ('v' + os.environ["BAZARR_VERSION"]) != latest_release and settings.general.branch == 'master':
+            update_from_source()
+        elif settings.general.branch != 'master':
+            notifications.write(msg="Can't update development branch from source", queue='check_update')  # fixme
+            logging.info("BAZARR Can't update development branch from source")  # fixme
+        else:
+            notifications.write(msg='Bazarr is up to date', queue='check_update')
+            logging.info('BAZARR is up to date')
+
+
+def update_from_source():
+    tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch)
+    update_dir = os.path.join(os.path.dirname(__file__), '..', 'update')
+    
+    logging.info('BAZARR Downloading update from: ' + tar_download_url)
+    notifications.write(msg='Downloading update from: ' + tar_download_url, queue='check_update')
+    data = request_content(tar_download_url)
+    
+    if not data:
+        logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url)
+        notifications.write(msg=("Unable to retrieve new version from '%s', can't update", tar_download_url),
+                            type='error', queue='check_update')
+        return
+    
+    download_name = settings.general.branch + '-github'
+    tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name)
+    
+    # Save tar to disk
+    with open(tar_download_path, 'wb') as f:
+        f.write(data)
+    
+    # Extract the tar to update folder
+    logging.info('BAZARR Extracting file: ' + tar_download_path)
+    notifications.write(msg='Extracting file: ' + tar_download_path, queue='check_update')
+    tar = tarfile.open(tar_download_path)
+    tar.extractall(update_dir)
+    tar.close()
+    
+    # Delete the tar.gz
+    logging.info('BAZARR Deleting file: ' + tar_download_path)
+    notifications.write(msg='Deleting file: ' + tar_download_path, queue='check_update')
+    os.remove(tar_download_path)
+    
+    # Find update dir name
+    update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))]
+    if len(update_dir_contents) != 1:
+        logging.error("BAZARR Invalid update data, update failed: " + str(update_dir_contents))
+        notifications.write(msg="BAZARR Invalid update data, update failed: " + str(update_dir_contents),
+                            type='error', queue='check_update')
+        return
+    
+    content_dir = os.path.join(update_dir, update_dir_contents[0])
+    
+    # walk temp folder and move files to main folder
+    for dirname, dirnames, filenames in os.walk(content_dir):
+        dirname = dirname[len(content_dir) + 1:]
+        for curfile in filenames:
+            old_path = os.path.join(content_dir, dirname, curfile)
+            new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile)
+            
+            if os.path.isfile(new_path):
+                os.remove(new_path)
+            os.renames(old_path, new_path)
+    updated()
 
 
 def check_releases():
@@ -71,9 +152,158 @@ def check_releases():
             json.dump(releases, f)
 
 
-def updated():
-    conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30)
-    c = conn.cursor()
-    c.execute("UPDATE system SET updated = 1")
-    conn.commit()
-    c.close()
+class FakeLock(object):
+    """
+    If no locking or request throttling is needed, use this
+    """
+    
+    def __enter__(self):
+        """
+        Do nothing on enter
+        """
+        pass
+    
+    def __exit__(self, type, value, traceback):
+        """
+        Do nothing on exit
+        """
+        pass
+
+
+fake_lock = FakeLock()
+
+
+def request_content(url, **kwargs):
+    """
+    Wrapper for `request_response', which will return the raw content.
+    """
+    
+    response = request_response(url, **kwargs)
+    
+    if response is not None:
+        return response.content
+
+
+def request_response(url, method="get", auto_raise=True,
+                     whitelist_status_code=None, lock=fake_lock, **kwargs):
+    """
+    Convenient wrapper for `requests.get', which will capture the exceptions
+    and log them. On success, the Response object is returned. In case of a
+    exception, None is returned.
+
+    Additionally, there is support for rate limiting. To use this feature,
+    supply a tuple of (lock, request_limit). The lock is used to make sure no
+    other request with the same lock is executed. The request limit is the
+    minimal time between two requests (and so 1/request_limit is the number of
+    requests per seconds).
+    """
+    
+    # Convert whitelist_status_code to a list if needed
+    if whitelist_status_code and type(whitelist_status_code) != list:
+        whitelist_status_code = [whitelist_status_code]
+    
+    # Disable verification of SSL certificates if requested. Note: this could
+    # pose a security issue!
+    kwargs["verify"] = True
+    
+    # Map method to the request.XXX method. This is a simple hack, but it
+    # allows requests to apply more magic per method. See lib/requests/api.py.
+    request_method = getattr(requests, method.lower())
+    
+    try:
+        # Request URL and wait for response
+        with lock:
+            logging.debug(
+                "BAZARR Requesting URL via %s method: %s", method.upper(), url)
+            response = request_method(url, **kwargs)
+        
+        # If status code != OK, then raise exception, except if the status code
+        # is white listed.
+        if whitelist_status_code and auto_raise:
+            if response.status_code not in whitelist_status_code:
+                try:
+                    response.raise_for_status()
+                except:
+                    logging.debug(
+                        "BAZARR Response status code %d is not white "
+                        "listed, raised exception", response.status_code)
+                    raise
+        elif auto_raise:
+            response.raise_for_status()
+        
+        return response
+    except requests.exceptions.SSLError as e:
+        if kwargs["verify"]:
+            logging.error(
+                "BAZARR Unable to connect to remote host because of a SSL error. "
+                "It is likely that your system cannot verify the validity"
+                "of the certificate. The remote certificate is either "
+                "self-signed, or the remote server uses SNI. See the wiki for "
+                "more information on this topic.")
+        else:
+            logging.error(
+                "BAZARR SSL error raised during connection, with certificate "
+                "verification turned off: %s", e)
+    except requests.ConnectionError:
+        logging.error(
+            "BAZARR Unable to connect to remote host. Check if the remote "
+            "host is up and running.")
+    except requests.Timeout:
+        logging.error(
+            "BAZARR Request timed out. The remote host did not respond timely.")
+    except requests.HTTPError as e:
+        if e.response is not None:
+            if e.response.status_code >= 500:
+                cause = "remote server error"
+            elif e.response.status_code >= 400:
+                cause = "local client error"
+            else:
+                # I don't think we will end up here, but for completeness
+                cause = "unknown"
+            
+            logging.error(
+                "BAZARR Request raise HTTP error with status code %d (%s).",
+                e.response.status_code, cause)
+        else:
+            logging.error("BAZARR Request raised HTTP error.")
+    except requests.RequestException as e:
+        logging.error("BAZARR Request raised exception: %s", e)
+
+
+def request_json(url, **kwargs):
+    """
+    Wrapper for `request_response', which will decode the response as JSON
+    object and return the result, if no exceptions are raised.
+
+    As an option, a validator callback can be given, which should return True
+    if the result is valid.
+    """
+    
+    validator = kwargs.pop("validator", None)
+    response = request_response(url, **kwargs)
+    
+    if response is not None:
+        try:
+            result = response.json()
+            
+            if validator and not validator(result):
+                logging.error("BAZARR JSON validation result failed")
+            else:
+                return result
+        except ValueError:
+            logging.error("BAZARR Response returned invalid JSON data")
+
+
+def updated(restart=True):
+    if settings.general.getboolean('update_restart') and restart:
+        try:
+            requests.get(bazarr_url + 'restart')
+        except requests.ConnectionError:
+            logging.info('BAZARR Restart failed, please restart Bazarr manualy')
+            updated(restart=False)
+    else:
+        conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30)
+        c = conn.cursor()
+        c.execute("UPDATE system SET updated = 1")
+        conn.commit()
+        c.close()

bazarr/config.py

@@ -40,6 +40,7 @@ defaults = {
         'chmod': '0640',
         'subfolder': 'current',
         'subfolder_custom': '',
+        'update_restart': 'True',
         'upgrade_subs': 'True',
         'days_to_upgrade_subs': '7',
         'upgrade_manual': 'True',
@@ -118,6 +119,7 @@ settings = simpleconfigparser(defaults=defaults)
 settings.read(os.path.join(args.config_dir, 'config', 'config.ini'))
 
 base_url = settings.general.base_url
+bazarr_url = 'http://localhost:' + (str(args.port) if args.port else settings.general.port) + base_url
 
 # sonarr url
 if settings.sonarr.getboolean('ssl'):

bazarr/get_args.py

@@ -19,6 +19,8 @@ def get_args():
                         help="Disable update functionality (default: False)")
     parser.add_argument('--debug', default=False, type=bool, const=True, metavar="BOOL", nargs="?",
                         help="Enable console debugging (default: False)")
+    parser.add_argument('--release-update', default=False, type=bool, const=True, metavar="BOOL", nargs="?",
+                        help="Enable file based updater (default: False)")
 
     return parser.parse_args()
 

bazarr/main.py

@@ -58,8 +58,6 @@ from get_providers import get_providers, get_providers_auth, list_throttled_prov
 from get_series import *
 from get_episodes import *
 
-if not args.no_update:
-    from check_update import check_and_apply_update
 from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \
     list_missing_subtitles, list_missing_subtitles_movies
 from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \
@@ -1221,6 +1219,11 @@ def save_settings():
         settings_general_automatic = 'False'
     else:
         settings_general_automatic = 'True'
+    settings_general_update_restart = request.forms.get('settings_general_update_restart')
+    if settings_general_update_restart is None:
+        settings_general_update_restart = 'False'
+    else:
+        settings_general_update_restart = 'True'
     settings_general_single_language = request.forms.get('settings_general_single_language')
     if settings_general_single_language is None:
         settings_general_single_language = 'False'
@@ -1304,6 +1307,7 @@ def save_settings():
     settings.general.chmod = text_type(settings_general_chmod)
     settings.general.branch = text_type(settings_general_branch)
     settings.general.auto_update = text_type(settings_general_automatic)
+    settings.general.update_restart = text_type(settings_general_update_restart)
     settings.general.single_language = text_type(settings_general_single_language)
     settings.general.minimum_score = text_type(settings_general_minimum_score)
     settings.general.use_scenename = text_type(settings_general_scenename)
@@ -1561,13 +1565,12 @@ def save_settings():
     conn.commit()
     c.close()
 
+    schedule_update_job()
     sonarr_full_update()
     radarr_full_update()
 
     logging.info('BAZARR Settings saved succesfully.')
 
-    # reschedule full update task according to settings
-    sonarr_full_update()
 
     if ref.find('saved=true') > 0:
         redirect(ref)
@@ -2045,7 +2048,6 @@ def notifications():
 def running_tasks_list():
     return dict(tasks=running_tasks)
 
-
 # Mute DeprecationWarning
 warnings.simplefilter("ignore", DeprecationWarning)
 server = WSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app, handler_class=WebSocketHandler)

bazarr/scheduler.py

@@ -6,7 +6,6 @@ from get_series import update_series
 from config import settings
 from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles
 from get_args import args
-
 if not args.no_update:
     from check_update import check_and_apply_update, check_releases
 else:
@@ -82,18 +81,21 @@ def task_listener(event):
 
 scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED)
 
-if not args.no_update:
-    if settings.general.getboolean('auto_update'):
-        scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
-        misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github')
+def schedule_update_job():
+    if not args.no_update:
+        if settings.general.getboolean('auto_update'):
+            scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
+                              misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True)
+        else:
+            scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr',
+                              name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True)
+            scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
+                              misfire_grace_time=15, id='update_release', name='Update release info', replace_existing=True)
+
     else:
-        scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr',
-                          name='Update bazarr from source on Github')
-        scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
-                          misfire_grace_time=15, id='update_release', name='Update release info')
-else:
-    scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15,
-                      id='update_release', name='Update release info')
+        scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15,
+                          id='update_release', name='Update release info', replace_existing=True)
+
 
 if settings.general.getboolean('use_sonarr'):
     scheduler.add_job(update_series, IntervalTrigger(minutes=1), max_instances=1, coalesce=True, misfire_grace_time=15,
@@ -114,6 +116,7 @@ if settings.general.getboolean('upgrade_subs') and (settings.general.getboolean(
     scheduler.add_job(upgrade_subtitles, IntervalTrigger(hours=12), max_instances=1, coalesce=True,
                       misfire_grace_time=15, id='upgrade_subtitles', name='Upgrade previously downloaded subtitles')
 
+schedule_update_job()
 sonarr_full_update()
 radarr_full_update()
 scheduler.start()

views/menu.tpl

@@ -224,7 +224,7 @@
 			% elif restart_required[0] == '1':
 				<div class='ui center aligned grid'><div class='fifteen wide column'><div class="ui red message">Bazarr need to be restarted to apply changes to general settings. Click <a href=# id="restart_link">here</a> to restart.</div></div></div>
 			% end
-		</div>
+        </div>
     </body>
 </html>
 

views/settings.tpl

@@ -665,7 +665,7 @@
                 <div class="ui dividing header">Updates</div>
                 <div class="twelve wide column">
                     <div class="ui grid">
-                        <div class="middle aligned row">
+                        <div class="middle aligned row" id="div_branch">
                             <div class="right aligned four wide column">
                                 <label>Branch</label>
                             </div>
@@ -703,6 +703,28 @@
                                 </div>
                             </div>
                         </div>
+
+                        <div class="middle aligned row">
+                            <div class="right aligned four wide column">
+                                <label>Restart after update</label>
+                            </div>
+                            <div class="one wide column">
+                                <div id="settings_update_restart" class="ui toggle checkbox"
+                                     data-update-restart={{settings.general.getboolean('update_restart')}}>
+                                    <input name="settings_general_update_restart" type="checkbox">
+                                    <label></label>
+                                </div>
+                            </div>
+                            <div class="collapsed column">
+                                <div class="collapsed center aligned column">
+                                    <div class="ui basic icon"
+                                         data-tooltip="Automatically restart after download and install updates. You will still be able to restart manualy"
+                                         data-inverted="">
+                                        <i class="help circle large icon"></i>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
                     </div>
                 </div>
                 </div>
@@ -2262,8 +2284,11 @@
     });
 
     % from get_args import args
+
     % if args.no_update:
     $("#div_update").hide();
+    % elif args.release_update:
+    $("#div_branch").hide();
     % end
     % import sys
     % if sys.platform.startswith('win'):
@@ -2301,6 +2326,12 @@
                 $("#settings_automatic_div").checkbox('uncheck');
             }
 
+    if ($('#settings_update_restart').data("update-restart") === "True") {
+        $("#settings_update_restart").checkbox('check');
+    } else {
+        $("#settings_update_restart").checkbox('uncheck');
+    }
+
     if ($('#settings_debug').data("debug") === "True") {
                 $("#settings_debug").checkbox('check');
             } else {