From ea90e6dd80eda623b574bccbafe2490a3e98e868 Mon Sep 17 00:00:00 2001 From: Halali Date: Fri, 15 Feb 2019 21:38:10 +0100 Subject: [PATCH 01/38] Initial commit --- bazarr/check_update.py | 434 +++++++++++++++++++++++++++++++++++++---- bazarr/main.py | 2 +- bazarr/scheduler.py | 6 +- 3 files changed, 401 insertions(+), 41 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index c42200936..64a2ef3f2 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -1,53 +1,45 @@ -# coding=utf-8 import os +import platform +import re +import subprocess +import tarfile import logging +import requests import sqlite3 import json -import requests from get_args import args from config import settings -if not args.no_update: - import git +bazarr_version = '7.2.0' -current_working_directory = os.path.dirname(os.path.dirname(__file__)) +LATEST_VERSION = None +INSTALL_TYPE = None +UPDATE_AVAILABLE = None +COMMITS_BEHIND = None +LATEST_RELEASE = None +PREV_RELEASE = bazarr_version -def gitconfig(): - g = git.Repo.init(current_working_directory) - config_read = g.config_reader() - config_write = g.config_writer() +class FakeLock(object): + """ + If no locking or request throttling is needed, use this + """ - try: - username = config_read.get_value("user", "name") - except: - logging.debug('BAZARR Settings git username') - config_write.set_value("user", "name", "Bazarr") + def __enter__(self): + """ + Do nothing on enter + """ + pass - try: - email = config_read.get_value("user", "email") - except: - logging.debug('BAZARR Settings git email') - config_write.set_value("user", "email", "bazarr@fake.email") - - -def check_and_apply_update(): - gitconfig() - check_releases() - branch = settings.general.branch - g = git.cmd.Git(current_working_directory) - g.fetch('origin') - result = g.diff('--shortstat', 'origin/' + branch) - if len(result) == 0: - logging.info('BAZARR No new version of Bazarr available.') - else: - g.reset('--hard', 'HEAD') - g.checkout(branch) - g.reset('--hard', 'origin/' + branch) - g.pull() - logging.info('BAZARR Updated to latest version. Restart required. ' + result) - updated() + def __exit__(self, type, value, traceback): + """ + Do nothing on exit + """ + pass + + +fake_lock = FakeLock() def check_releases(): @@ -71,6 +63,374 @@ def check_releases(): json.dump(releases, f) +def runGit(args): + git_locations = ['git'] + + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') + + output = err = None + + for cur_git in git_locations: + cmd = cur_git + ' ' + args + + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() + + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue + + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break + + return (output, err) + + +def getVersion(): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): + + INSTALL_TYPE = 'git' + output, err = runGit('rev-parse HEAD') + + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + + cur_commit_hash = str(output) + + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None + + if settings.general.branch: + branch_name = settings.general.branch + + else: + remote_branch, err = runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') + remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] + if len(remote_branch) == 2: + remote_name, branch_name = remote_branch + else: + remote_name = branch_name = None + + if not remote_name and settings.general.branch: + logging.error('Could not retrieve remote name from git. Defaulting to origin.') + branch_name = settings.general.branch + + if not branch_name: + logging.error('Could not retrieve branch name from git. Defaulting to master.') + branch_name = 'master' + + return cur_commit_hash, 'origin', branch_name + + else: + INSTALL_TYPE = 'source' + + if CURRENT_VERSION: + return CURRENT_VERSION, 'origin', settings.general.branch + else: + return None, 'origin', settings.general.branch + + +CURRENT_VERSION, GIT_REMOTE, GIT_BRANCH = getVersion() + + +def check_updates(): + check_github() + if not CURRENT_VERSION: + UPDATE_AVAILABLE = None + elif COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ + ('V' + bazarr_version) != LATEST_RELEASE: + UPDATE_AVAILABLE = 'release' + elif COMMITS_BEHIND > 0 and CURRENT_VERSION != LATEST_VERSION: + UPDATE_AVAILABLE = 'commit' + else: + UPDATE_AVAILABLE = False + + +def check_github(): + COMMITS_BEHIND = 0 + + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return CURRENT_VERSION + + LATEST_VERSION = version['sha'] + logging.debug("Latest version is %s", LATEST_VERSION) + + # See how many commits behind we are + if not CURRENT_VERSION: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return LATEST_VERSION + + if LATEST_VERSION == CURRENT_VERSION: + logging.info('Bazarr is up to date') + return LATEST_VERSION + + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (LATEST_VERSION, + CURRENT_VERSION) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return LATEST_VERSION + + try: + COMMITS_BEHIND = int(commits['behind_by']) + logging.debug("In total, %d commits behind", COMMITS_BEHIND) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + COMMITS_BEHIND = 0 + + if COMMITS_BEHIND > 0: + logging.info('New version is available. You are %s commits behind' % COMMITS_BEHIND) + + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return LATEST_VERSION + + if settings.general.branch == 'master': + release = next((r for r in releases if not r['prerelease']), releases[0]) + else: + release = releases[0] + LATEST_RELEASE = release['tag_name'] + + elif COMMITS_BEHIND == 0: + logging.info('Bazarr is up to date') + + return LATEST_VERSION + + +def update(): + if INSTALL_TYPE == 'git': + output, err = runGit('pull ' + 'origin' + ' ' + settings.general.branch) + + if not output: + logging.error('Unable to download latest version') + return + + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) + + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) + + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return + + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) + + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() + + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + + +def checkout_git_branch(): + if INSTALL_TYPE == 'git': + output, err = runGit('fetch origin') + output, err = runGit('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = runGit('pull origin %s' % settings.general.branch) + + +def request_content(url, **kwargs): + """ + Wrapper for `request_response', which will return the raw content. + """ + + response = request_response(url, **kwargs) + + if response is not None: + return response.content + + +def request_response(url, method="get", auto_raise=True, + whitelist_status_code=None, lock=fake_lock, **kwargs): + """ + Convenient wrapper for `requests.get', which will capture the exceptions + and log them. On success, the Response object is returned. In case of a + exception, None is returned. + + Additionally, there is support for rate limiting. To use this feature, + supply a tuple of (lock, request_limit). The lock is used to make sure no + other request with the same lock is executed. The request limit is the + minimal time between two requests (and so 1/request_limit is the number of + requests per seconds). + """ + + # Convert whitelist_status_code to a list if needed + if whitelist_status_code and type(whitelist_status_code) != list: + whitelist_status_code = [whitelist_status_code] + + # Disable verification of SSL certificates if requested. Note: this could + # pose a security issue! + kwargs["verify"] = True + + # Map method to the request.XXX method. This is a simple hack, but it + # allows requests to apply more magic per method. See lib/requests/api.py. + request_method = getattr(requests, method.lower()) + + try: + # Request URL and wait for response + with lock: + logging.debug( + "Requesting URL via %s method: %s", method.upper(), url) + response = request_method(url, **kwargs) + + # If status code != OK, then raise exception, except if the status code + # is white listed. + if whitelist_status_code and auto_raise: + if response.status_code not in whitelist_status_code: + try: + response.raise_for_status() + except: + logging.debug( + "Response status code %d is not white " + "listed, raised exception", response.status_code) + raise + elif auto_raise: + response.raise_for_status() + + return response + except requests.exceptions.SSLError as e: + if kwargs["verify"]: + logging.error( + "Unable to connect to remote host because of a SSL error. " + "It is likely that your system cannot verify the validity" + "of the certificate. The remote certificate is either " + "self-signed, or the remote server uses SNI. See the wiki for " + "more information on this topic.") + else: + logging.error( + "SSL error raised during connection, with certificate " + "verification turned off: %s", e) + except requests.ConnectionError: + logging.error( + "Unable to connect to remote host. Check if the remote " + "host is up and running.") + except requests.Timeout: + logging.error( + "Request timed out. The remote host did not respond timely.") + except requests.HTTPError as e: + if e.response is not None: + if e.response.status_code >= 500: + cause = "remote server error" + elif e.response.status_code >= 400: + cause = "local client error" + else: + # I don't think we will end up here, but for completeness + cause = "unknown" + + logging.error( + "Request raise HTTP error with status code %d (%s).", + e.response.status_code, cause) + + # Debug response + # if bazarr.DEBUG: + # server_message(e.response) + else: + logging.error("Request raised HTTP error.") + except requests.RequestException as e: + logging.error("Request raised exception: %s", e) + + +def request_json(url, **kwargs): + """ + Wrapper for `request_response', which will decode the response as JSON + object and return the result, if no exceptions are raised. + + As an option, a validator callback can be given, which should return True + if the result is valid. + """ + + validator = kwargs.pop("validator", None) + response = request_response(url, **kwargs) + + if response is not None: + try: + result = response.json() + + if validator and not validator(result): + logging.error("JSON validation result failed") + else: + return result + except ValueError: + logging.error("Response returned invalid JSON data") + + # Debug response + # if bazarr.DEBUG: + # server_message(response) + + def updated(): conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) c = conn.cursor() diff --git a/bazarr/main.py b/bazarr/main.py index 07be4b7ec..91fcc45ff 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -58,7 +58,7 @@ from get_series import * from get_episodes import * if not args.no_update: - from check_update import check_and_apply_update + from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 53f09d371..3f099027d 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles from get_args import args if not args.no_update: - from check_update import check_and_apply_update, check_releases + from check_update import check_updates, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -68,10 +68,10 @@ else: if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') From 91747099c41b3220a7887b81641ca7fa13f125b8 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 16 Feb 2019 00:21:31 +0100 Subject: [PATCH 02/38] Continue developing --- bazarr/check_update.py | 437 +++++++++++++++++++++-------------------- bazarr/get_args.py | 2 + bazarr/main.py | 2 +- bazarr/scheduler.py | 8 +- views/menu.tpl | 39 +++- 5 files changed, 264 insertions(+), 224 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 64a2ef3f2..d8be1087c 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -11,14 +11,9 @@ import json from get_args import args from config import settings -bazarr_version = '7.2.0' -LATEST_VERSION = None -INSTALL_TYPE = None -UPDATE_AVAILABLE = None -COMMITS_BEHIND = None -LATEST_RELEASE = None -PREV_RELEASE = bazarr_version +# from main import bazarr_version + class FakeLock(object): @@ -63,243 +58,249 @@ def check_releases(): json.dump(releases, f) -def runGit(args): - git_locations = ['git'] - - if platform.system().lower() == 'darwin': - git_locations.append('/usr/local/git/bin/git') - - output = err = None - - for cur_git in git_locations: - cmd = cur_git + ' ' + args +class Updater(object): + def __init__(self): + self.bazarr_version = '7.2.0' - try: - logging.debug('Trying to execute: "' + cmd + '"') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output, err = p.communicate() - output = output.strip() - - logging.debug('Git output: ' + output) - except OSError: - logging.debug('Command failed: %s', cmd) - continue - - if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) - output = None - elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') - output = None - elif output: - break + self.LATEST_VERSION = '' + self.INSTALL_TYPE = '' + self.UPDATE_AVAILABLE = '' + self.COMMITS_BEHIND = '' + self.LATEST_RELEASE = '' + self.CURRENT_VERSION = '' + self.PREV_RELEASE = self.bazarr_version + self.CURRENT_VERSION, self.GIT_REMOTE, self.GIT_BRANCH = self.getVersion() - return (output, err) - - -def getVersion(): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): - - INSTALL_TYPE = 'git' - output, err = runGit('rev-parse HEAD') - - if not output: - logging.error('Could not find latest installed version.') - cur_commit_hash = None + def runGit(self, args): + git_locations = ['git'] - cur_commit_hash = str(output) + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') - cur_commit_hash = None + output = err = None - if settings.general.branch: - branch_name = settings.general.branch - - else: - remote_branch, err = runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') - remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] - if len(remote_branch) == 2: - remote_name, branch_name = remote_branch - else: - remote_name = branch_name = None + for cur_git in git_locations: + cmd = cur_git + ' ' + args - if not remote_name and settings.general.branch: - logging.error('Could not retrieve remote name from git. Defaulting to origin.') - branch_name = settings.general.branch + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() + + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue - if not branch_name: - logging.error('Could not retrieve branch name from git. Defaulting to master.') - branch_name = 'master' + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break - return cur_commit_hash, 'origin', branch_name + return (output, err) - else: - INSTALL_TYPE = 'source' + def getVersion(self): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: + + self.INSTALL_TYPE = 'git' + output, err = self.runGit('rev-parse HEAD') + + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + + cur_commit_hash = str(output) + + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None + + if settings.general.branch: + branch_name = settings.general.branch + + else: + remote_branch, err = self.runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') + remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] + if len(remote_branch) == 2: + remote_name, branch_name = remote_branch + else: + remote_name = branch_name = None + + if not remote_name and settings.general.branch: + logging.error('Could not retrieve remote name from git. Defaulting to origin.') + branch_name = settings.general.branch + + if not branch_name: + logging.error('Could not retrieve branch name from git. Defaulting to master.') + branch_name = 'master' + + return cur_commit_hash, 'origin', branch_name - if CURRENT_VERSION: - return CURRENT_VERSION, 'origin', settings.general.branch else: - return None, 'origin', settings.general.branch - - -CURRENT_VERSION, GIT_REMOTE, GIT_BRANCH = getVersion() - - -def check_updates(): - check_github() - if not CURRENT_VERSION: - UPDATE_AVAILABLE = None - elif COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ - ('V' + bazarr_version) != LATEST_RELEASE: - UPDATE_AVAILABLE = 'release' - elif COMMITS_BEHIND > 0 and CURRENT_VERSION != LATEST_VERSION: - UPDATE_AVAILABLE = 'commit' - else: - UPDATE_AVAILABLE = False - - -def check_github(): - COMMITS_BEHIND = 0 - - # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - - if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') - return CURRENT_VERSION - - LATEST_VERSION = version['sha'] - logging.debug("Latest version is %s", LATEST_VERSION) - - # See how many commits behind we are - if not CURRENT_VERSION: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') - return LATEST_VERSION - - if LATEST_VERSION == CURRENT_VERSION: - logging.info('Bazarr is up to date') - return LATEST_VERSION - - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (LATEST_VERSION, - CURRENT_VERSION) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - - if commits is None: - logging.warn('Could not get commits behind from GitHub.') - return LATEST_VERSION - - try: - COMMITS_BEHIND = int(commits['behind_by']) - logging.debug("In total, %d commits behind", COMMITS_BEHIND) - except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') - COMMITS_BEHIND = 0 + self.INSTALL_TYPE = 'source' + + if self.CURRENT_VERSION: + return self.CURRENT_VERSION, 'origin', settings.general.branch + else: + return None, 'origin', settings.general.branch - if COMMITS_BEHIND > 0: - logging.info('New version is available. You are %s commits behind' % COMMITS_BEHIND) - - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - - if releases is None: - logging.warn('Could not get releases from GitHub.') - return LATEST_VERSION - - if settings.general.branch == 'master': - release = next((r for r in releases if not r['prerelease']), releases[0]) + def check_updates(self): + self.check_github() + if not self.CURRENT_VERSION: + self.UPDATE_AVAILABLE = None + elif self.COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ + ('v' + self.bazarr_version) != self.LATEST_RELEASE: + self.UPDATE_AVAILABLE = 'release' + elif self.COMMITS_BEHIND > 0 and self.CURRENT_VERSION != self.LATEST_VERSION: + self.UPDATE_AVAILABLE = 'commit' else: - release = releases[0] - LATEST_RELEASE = release['tag_name'] + self.UPDATE_AVAILABLE = False + print self.UPDATE_AVAILABLE - elif COMMITS_BEHIND == 0: - logging.info('Bazarr is up to date') - - return LATEST_VERSION - - -def update(): - if INSTALL_TYPE == 'git': - output, err = runGit('pull ' + 'origin' + ' ' + settings.general.branch) - - if not output: - logging.error('Unable to download latest version') - return + def check_github(self): + self.COMMITS_BEHIND = 0 - for line in output.split('\n'): - - if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) - elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) - - else: - tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) - update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - logging.info('Downloading update from: ' + tar_download_url) - data = request_content(tar_download_url) + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return self.CURRENT_VERSION - if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) - return + self.LATEST_VERSION = version['sha'] + logging.debug("Latest version is %s", self.LATEST_VERSION) - download_name = settings.general.branch + '-github' - tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + # See how many commits behind we are + if not self.CURRENT_VERSION: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return self.LATEST_VERSION - # Save tar to disk - with open(tar_download_path, 'wb') as f: - f.write(data) + if self.LATEST_VERSION == self.CURRENT_VERSION: + logging.info('Bazarr is up to date') + return self.LATEST_VERSION - # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) - tar = tarfile.open(tar_download_path) - tar.extractall(update_dir) - tar.close() + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (self.LATEST_VERSION, + self.CURRENT_VERSION) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) - os.remove(tar_download_path) + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return self.LATEST_VERSION - # Find update dir name - update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] - if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) - return - content_dir = os.path.join(update_dir, update_dir_contents[0]) + try: + self.COMMITS_BEHIND = int(commits['behind_by']) + logging.debug("In total, %d commits behind", self.COMMITS_BEHIND) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + self.COMMITS_BEHIND = 0 - # walk temp folder and move files to main folder - for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) - - -def checkout_git_branch(): - if INSTALL_TYPE == 'git': - output, err = runGit('fetch origin') - output, err = runGit('checkout %s' % settings.general.branch) + if self.COMMITS_BEHIND > 0: + logging.info('New version is available. You are %s commits behind' % self.COMMITS_BEHIND) + + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return self.LATEST_VERSION + + if settings.general.branch == 'master': + release = next((r for r in releases if not r['prerelease']), releases[0]) + else: + release = releases[0] + self.LATEST_RELEASE = release['tag_name'] - if not output: - logging.error('Unable to change git branch.') - return + elif self.COMMITS_BEHIND == 0: + logging.info('Bazarr is up to date') - for line in output.split('\n'): - if line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to checkout from git: ' + line) - logging.info('Output: ' + str(output)) + return self.LATEST_VERSION + + def update(self): + if self.INSTALL_TYPE == 'git' and not args.release_update: + output, err = self.runGit('pull ' + 'origin' + ' ' + settings.general.branch) + + if not output: + logging.error('Unable to download latest version') + return + + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) - output, err = runGit('pull origin %s' % settings.general.branch) + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) + + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return + + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) + + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() + + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + + def checkout_git_branch(self): + if self.INSTALL_TYPE == 'git' and not args.release_update: + output, err = self.runGit('fetch origin') + output, err = self.runGit('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = self.runGit('pull origin %s' % settings.general.branch) def request_content(url, **kwargs): diff --git a/bazarr/get_args.py b/bazarr/get_args.py index 74b68955f..c48f78b26 100644 --- a/bazarr/get_args.py +++ b/bazarr/get_args.py @@ -17,6 +17,8 @@ def get_args(): help="Disable update functionality (default: False)") parser.add_argument('--debug', default=False, type=bool, const=True, metavar="BOOL", nargs="?", help="Enable console debugging (default: False)") + parser.add_argument('--release-update', default=False, type=bool, const=True, metavar="BOOL", nargs="?", + help="Enable file based updater (default: False)") return parser.parse_args() diff --git a/bazarr/main.py b/bazarr/main.py index 91fcc45ff..e7c4facf7 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -58,7 +58,7 @@ from get_series import * from get_episodes import * if not args.no_update: - from check_update import check_updates + from check_update import Updater from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 3f099027d..99fd9b24b 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles from get_args import args if not args.no_update: - from check_update import check_updates, check_releases + from check_update import Updater, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -68,10 +68,10 @@ else: if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') + scheduler.add_job(Updater().check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(Updater().check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/menu.tpl b/views/menu.tpl index 039f75d15..692609418 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -135,7 +135,44 @@ % elif restart_required[0] == '1':
Bazarr need to be restarted to apply changes to general settings. Click here to restart.
% end - + % from check_update import Updater + % if Updater().UPDATE_AVAILABLE is None: +
+
+
+ You are running an unknown version of Bazarr. + Update +
+
+
+ % elif Updater().UPDATE_AVAILABLE == 'release': +
+
+
+ A + new release ({{ Updater().LATEST_RELEASE }}) of Bazarr is available! + Update +
+
+
+ % elif Updater().UPDATE_AVAILABLE == 'commit': +
+
+
+ A + newer version of Bazarr is available!
+ You are {{ Updater().COMMITS_BEHIND }} commit{{ 's' if Updater().COMMITS_BEHIND > 1 else '' }} + behind. + Update +
+
+
+ % end + From 08731e517c5486b69eacb9e18ae9fed4a74a48a3 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 16 Feb 2019 16:58:35 +0100 Subject: [PATCH 03/38] Continue developing --- bazarr/check_update.py | 433 ++++++++++++++++++----------------------- bazarr/main.py | 5 +- bazarr/scheduler.py | 6 +- views/menu.tpl | 37 ---- 4 files changed, 191 insertions(+), 290 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index d8be1087c..78cf77565 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import os import platform import re @@ -12,31 +14,6 @@ from get_args import args from config import settings -# from main import bazarr_version - - - -class FakeLock(object): - """ - If no locking or request throttling is needed, use this - """ - - def __enter__(self): - """ - Do nothing on enter - """ - pass - - def __exit__(self, type, value, traceback): - """ - Do nothing on exit - """ - pass - - -fake_lock = FakeLock() - - def check_releases(): releases = [] url_releases = 'https://api.github.com/repos/morpheus65535/Bazarr/releases' @@ -58,249 +35,209 @@ def check_releases(): json.dump(releases, f) -class Updater(object): - def __init__(self): - self.bazarr_version = '7.2.0' - - self.LATEST_VERSION = '' - self.INSTALL_TYPE = '' - self.UPDATE_AVAILABLE = '' - self.COMMITS_BEHIND = '' - self.LATEST_RELEASE = '' - self.CURRENT_VERSION = '' - self.PREV_RELEASE = self.bazarr_version - self.CURRENT_VERSION, self.GIT_REMOTE, self.GIT_BRANCH = self.getVersion() +def run_git(args): + git_locations = ['git'] - def runGit(self, args): - git_locations = ['git'] - - if platform.system().lower() == 'darwin': - git_locations.append('/usr/local/git/bin/git') - - output = err = None + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') + + output = err = None + + for cur_git in git_locations: + cmd = cur_git + ' ' + args - for cur_git in git_locations: - cmd = cur_git + ' ' + args - - try: - logging.debug('Trying to execute: "' + cmd + '"') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output, err = p.communicate() - output = output.strip() - - logging.debug('Git output: ' + output) - except OSError: - logging.debug('Command failed: %s', cmd) - continue + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() - if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) - output = None - elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') - output = None - elif output: - break + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue - return (output, err) + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break - def getVersion(self): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: - - self.INSTALL_TYPE = 'git' - output, err = self.runGit('rev-parse HEAD') - - if not output: - logging.error('Could not find latest installed version.') - cur_commit_hash = None - - cur_commit_hash = str(output) - - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') - cur_commit_hash = None - - if settings.general.branch: - branch_name = settings.general.branch - - else: - remote_branch, err = self.runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') - remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] - if len(remote_branch) == 2: - remote_name, branch_name = remote_branch - else: - remote_name = branch_name = None - - if not remote_name and settings.general.branch: - logging.error('Could not retrieve remote name from git. Defaulting to origin.') - branch_name = settings.general.branch - - if not branch_name: - logging.error('Could not retrieve branch name from git. Defaulting to master.') - branch_name = 'master' - - return cur_commit_hash, 'origin', branch_name - - else: - self.INSTALL_TYPE = 'source' - - if self.CURRENT_VERSION: - return self.CURRENT_VERSION, 'origin', settings.general.branch - else: - return None, 'origin', settings.general.branch + return output, err + + +def check_updates(): + commits_behind = 0 + current_version = get_version() - def check_updates(self): - self.check_github() - if not self.CURRENT_VERSION: - self.UPDATE_AVAILABLE = None - elif self.COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ - ('v' + self.bazarr_version) != self.LATEST_RELEASE: - self.UPDATE_AVAILABLE = 'release' - elif self.COMMITS_BEHIND > 0 and self.CURRENT_VERSION != self.LATEST_VERSION: - self.UPDATE_AVAILABLE = 'commit' - else: - self.UPDATE_AVAILABLE = False - print self.UPDATE_AVAILABLE + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return current_version + + latest_version = version['sha'] + logging.debug("Latest version is %s", latest_version) + + # See how many commits behind we are + if not current_version: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return latest_version + + if latest_version == current_version: + logging.info('Bazarr is up to date') + return latest_version + + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, + current_version) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return latest_version + + try: + commits_behind = int(commits['behind_by']) + logging.debug("In total, %d commits behind", commits_behind) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + commits_behind = 0 + + if commits_behind > 0: + logging.info('New version is available. You are %s commits behind' % commits_behind) + update() - def check_github(self): - self.COMMITS_BEHIND = 0 + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return latest_version + else: + release = releases[0] + latest_release = release['tag_name'] + + if ('v' + current_version) != latest_release and args.release_update: + update() + + elif commits_behind == 0: + logging.info('Bazarr is up to date') + + return latest_version + + +def get_version(): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): - # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + output, err = run_git('rev-parse HEAD') - if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') - return self.CURRENT_VERSION + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + else: + cur_commit_hash = str(output) - self.LATEST_VERSION = version['sha'] - logging.debug("Latest version is %s", self.LATEST_VERSION) + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None - # See how many commits behind we are - if not self.CURRENT_VERSION: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') - return self.LATEST_VERSION + return cur_commit_hash + + else: + return os.environ["BAZARR_VERSION"] + + +def update(): + if not args.release_update: + output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) - if self.LATEST_VERSION == self.CURRENT_VERSION: - logging.info('Bazarr is up to date') - return self.LATEST_VERSION + if not output: + logging.error('Unable to download latest version') + return - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (self.LATEST_VERSION, - self.CURRENT_VERSION) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) + updated() + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - if commits is None: - logging.warn('Could not get commits behind from GitHub.') - return self.LATEST_VERSION + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) - try: - self.COMMITS_BEHIND = int(commits['behind_by']) - logging.debug("In total, %d commits behind", self.COMMITS_BEHIND) - except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') - self.COMMITS_BEHIND = 0 + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return - if self.COMMITS_BEHIND > 0: - logging.info('New version is available. You are %s commits behind' % self.COMMITS_BEHIND) - - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - - if releases is None: - logging.warn('Could not get releases from GitHub.') - return self.LATEST_VERSION - - if settings.general.branch == 'master': - release = next((r for r in releases if not r['prerelease']), releases[0]) - else: - release = releases[0] - self.LATEST_RELEASE = release['tag_name'] + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - elif self.COMMITS_BEHIND == 0: - logging.info('Bazarr is up to date') + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) - return self.LATEST_VERSION - - def update(self): - if self.INSTALL_TYPE == 'git' and not args.release_update: - output, err = self.runGit('pull ' + 'origin' + ' ' + settings.general.branch) - - if not output: - logging.error('Unable to download latest version') - return - - for line in output.split('\n'): - - if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) - elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() - else: - tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) - update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - - logging.info('Downloading update from: ' + tar_download_url) - data = request_content(tar_download_url) - - if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) - return - - download_name = settings.general.branch + '-github' - tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - - # Save tar to disk - with open(tar_download_path, 'wb') as f: - f.write(data) - - # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) - tar = tarfile.open(tar_download_path) - tar.extractall(update_dir) - tar.close() - - # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) - os.remove(tar_download_path) - - # Find update dir name - update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] - if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) - return - content_dir = os.path.join(update_dir, update_dir_contents[0]) - - # walk temp folder and move files to main folder - for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + updated() + + +class FakeLock(object): + """ + If no locking or request throttling is needed, use this + """ - def checkout_git_branch(self): - if self.INSTALL_TYPE == 'git' and not args.release_update: - output, err = self.runGit('fetch origin') - output, err = self.runGit('checkout %s' % settings.general.branch) - - if not output: - logging.error('Unable to change git branch.') - return - - for line in output.split('\n'): - if line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to checkout from git: ' + line) - logging.info('Output: ' + str(output)) - - output, err = self.runGit('pull origin %s' % settings.general.branch) + def __enter__(self): + """ + Do nothing on enter + """ + pass + + def __exit__(self, type, value, traceback): + """ + Do nothing on exit + """ + pass + + +fake_lock = FakeLock() def request_content(url, **kwargs): diff --git a/bazarr/main.py b/bazarr/main.py index e7c4facf7..fe3685481 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,8 +57,8 @@ from get_providers import get_providers, get_providers_auth from get_series import * from get_episodes import * -if not args.no_update: - from check_update import Updater +# if not args.no_update: +# from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ @@ -76,6 +76,7 @@ gc.enable() update_notifier() os.environ["SZ_USER_AGENT"] = "Bazarr/1" +os.environ["BAZARR_VERSION"] = bazarr_version configure_logging(settings.general.getboolean('debug') or args.debug) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 99fd9b24b..e07478c89 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles from get_args import args if not args.no_update: - from check_update import Updater, check_releases + from check_update import check_updates, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -68,10 +68,10 @@ else: if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(Updater().check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(Updater().check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/menu.tpl b/views/menu.tpl index 692609418..26c3fbdeb 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -135,43 +135,6 @@ % elif restart_required[0] == '1':
Bazarr need to be restarted to apply changes to general settings. Click here to restart.
% end - % from check_update import Updater - % if Updater().UPDATE_AVAILABLE is None: -
-
-
- You are running an unknown version of Bazarr. - Update -
-
-
- % elif Updater().UPDATE_AVAILABLE == 'release': -
-
-
- A - new release ({{ Updater().LATEST_RELEASE }}) of Bazarr is available! - Update -
-
-
- % elif Updater().UPDATE_AVAILABLE == 'commit': -
-
-
- A - newer version of Bazarr is available!
- You are {{ Updater().COMMITS_BEHIND }} commit{{ 's' if Updater().COMMITS_BEHIND > 1 else '' }} - behind. - Update -
-
-
- % end From e21651e198965c9836d725b3a353e862d6e3fd26 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 20:07:21 +0100 Subject: [PATCH 04/38] add restart after update --- bazarr/check_update.py | 6 +++++- bazarr/config.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 78cf77565..f65392231 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -11,7 +11,7 @@ import sqlite3 import json from get_args import args -from config import settings +from config import settings, bazarr_url def check_releases(): @@ -375,3 +375,7 @@ def updated(): c.execute("UPDATE system SET updated = 1") conn.commit() c.close() + try: + requests.get(bazarr_url + 'restart') + except requests.ConnectionError: + logging.info('BAZARR: Restart failed, please restart Bazarr manualy') diff --git a/bazarr/config.py b/bazarr/config.py index 895488068..7e3839f1a 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -91,6 +91,7 @@ settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) base_url = settings.general.base_url +bazarr_url = 'http://localhost:' + (args.port if args.port else settings.general.port) + base_url # sonarr url if settings.sonarr.getboolean('ssl'): From de8cc85424634c6684636596150919c6f2baaa2d Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 22:30:43 +0100 Subject: [PATCH 05/38] Continue developing --- bazarr/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/config.py b/bazarr/config.py index 7e3839f1a..d58e14756 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -91,7 +91,7 @@ settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) base_url = settings.general.base_url -bazarr_url = 'http://localhost:' + (args.port if args.port else settings.general.port) + base_url +bazarr_url = 'http://localhost:' + (str(args.port) if args.port else settings.general.port) + base_url # sonarr url if settings.sonarr.getboolean('ssl'): From 34d33f65fab71233615fe03f537f304494c80d48 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 23:40:10 +0100 Subject: [PATCH 06/38] Continue developing --- bazarr/check_update.py | 129 +++++++++++++++++++++-------------------- views/settings.tpl | 6 +- 2 files changed, 72 insertions(+), 63 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index f65392231..5255c5e6a 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,6 +12,7 @@ import json from get_args import args from config import settings, bazarr_url +from queueconfig import q4ws def check_releases(): @@ -51,17 +52,17 @@ def run_git(args): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() - - logging.debug('Git output: ' + output) + + logging.debug('BAZZAR Git output: ' + output) except OSError: - logging.debug('Command failed: %s', cmd) + logging.debug('BAZZAR Command failed: %s', cmd) continue if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) + logging.debug('BAZZAR Unable to find git with command ' + cmd) output = None elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') + logging.error('BAZZAR Git returned bad info. Are you sure this is a git installation?') output = None elif output: break @@ -74,24 +75,28 @@ def check_updates(): current_version = get_version() # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') + logging.info('BAZZAR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + q4ws.append('BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') + logging.warn( + 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') return current_version latest_version = version['sha'] - logging.debug("Latest version is %s", latest_version) + logging.debug("BAZZAR Latest version is %s", latest_version) # See how many commits behind we are if not current_version: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + q4ws.append('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') + logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return latest_version if latest_version == current_version: - logging.info('Bazarr is up to date') + q4ws.append('BAZZAR is up to date') + logging.info('BAZARR is up to date') return latest_version logging.info('Comparing currently installed version with latest GitHub version') @@ -100,35 +105,40 @@ def check_updates(): commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: - logging.warn('Could not get commits behind from GitHub.') + logging.warn('BAZARR Could not get commits behind from GitHub.') return latest_version try: commits_behind = int(commits['behind_by']) logging.debug("In total, %d commits behind", commits_behind) except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') + logging.info('BAZARR Cannot compare versions. Are you running a local development version?') commits_behind = 0 if commits_behind > 0: - logging.info('New version is available. You are %s commits behind' % commits_behind) - update() + q4ws.append('BAZARR New version is available. You are %s commits behind' % commits_behind) + logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) + if settings.general.auto_update: + update() + else: + updated(restart=False) url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: - logging.warn('Could not get releases from GitHub.') + logging.warn('BAZARR Could not get releases from GitHub.') return latest_version else: release = releases[0] latest_release = release['tag_name'] - - if ('v' + current_version) != latest_release and args.release_update: + + if ('v' + current_version) != latest_release and args.release_update and settings.general.branch == 'master': update() elif commits_behind == 0: - logging.info('Bazarr is up to date') + q4ws.append('BAZZAR is up to date') + logging.info('BAZZAR is up to date') return latest_version @@ -139,13 +149,13 @@ def get_version(): output, err = run_git('rev-parse HEAD') if not output: - logging.error('Could not find latest installed version.') + logging.error('BAZZAR Could not find latest installed version.') cur_commit_hash = None else: cur_commit_hash = str(output) if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') + logging.error('BAZZAR Output does not look like a hash, not using it.') cur_commit_hash = None return cur_commit_hash @@ -159,27 +169,27 @@ def update(): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: - logging.error('Unable to download latest version') + logging.error('BAZZAR Unable to download latest version') return for line in output.split('\n'): if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) + logging.info('BAZZAR No update available, not updating') + logging.info('BAZZAR Output: ' + str(output)) elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) - updated() + logging.error('BAZZAR Unable to update from git: ' + line) + logging.info('BAZZAR Output: ' + str(output)) + updated(restart=True) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - - logging.info('Downloading update from: ' + tar_download_url) + + logging.info('BAZZAR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) return download_name = settings.general.branch + '-github' @@ -190,19 +200,19 @@ def update(): f.write(data) # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) + logging.info('BAZZAR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) + logging.info('BAZZAR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) return content_dir = os.path.join(update_dir, update_dir_contents[0]) @@ -216,7 +226,7 @@ def update(): if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) - updated() + updated(restart=True) class FakeLock(object): @@ -281,7 +291,7 @@ def request_response(url, method="get", auto_raise=True, # Request URL and wait for response with lock: logging.debug( - "Requesting URL via %s method: %s", method.upper(), url) + "BAZZAR Requesting URL via %s method: %s", method.upper(), url) response = request_method(url, **kwargs) # If status code != OK, then raise exception, except if the status code @@ -292,7 +302,7 @@ def request_response(url, method="get", auto_raise=True, response.raise_for_status() except: logging.debug( - "Response status code %d is not white " + "BAZZAR Response status code %d is not white " "listed, raised exception", response.status_code) raise elif auto_raise: @@ -302,22 +312,22 @@ def request_response(url, method="get", auto_raise=True, except requests.exceptions.SSLError as e: if kwargs["verify"]: logging.error( - "Unable to connect to remote host because of a SSL error. " + "BAZZAR Unable to connect to remote host because of a SSL error. " "It is likely that your system cannot verify the validity" "of the certificate. The remote certificate is either " "self-signed, or the remote server uses SNI. See the wiki for " "more information on this topic.") else: logging.error( - "SSL error raised during connection, with certificate " + "BAZZAR SSL error raised during connection, with certificate " "verification turned off: %s", e) except requests.ConnectionError: logging.error( - "Unable to connect to remote host. Check if the remote " + "BAZZAR Unable to connect to remote host. Check if the remote " "host is up and running.") except requests.Timeout: logging.error( - "Request timed out. The remote host did not respond timely.") + "BAZZAR Request timed out. The remote host did not respond timely.") except requests.HTTPError as e: if e.response is not None: if e.response.status_code >= 500: @@ -329,16 +339,12 @@ def request_response(url, method="get", auto_raise=True, cause = "unknown" logging.error( - "Request raise HTTP error with status code %d (%s).", + "BAZZAR Request raise HTTP error with status code %d (%s).", e.response.status_code, cause) - - # Debug response - # if bazarr.DEBUG: - # server_message(e.response) else: - logging.error("Request raised HTTP error.") + logging.error("BAZZAR Request raised HTTP error.") except requests.RequestException as e: - logging.error("Request raised exception: %s", e) + logging.error("BAZZAR Request raised exception: %s", e) def request_json(url, **kwargs): @@ -358,24 +364,23 @@ def request_json(url, **kwargs): result = response.json() if validator and not validator(result): - logging.error("JSON validation result failed") + logging.error("BAZZAR JSON validation result failed") else: return result except ValueError: - logging.error("Response returned invalid JSON data") - - # Debug response - # if bazarr.DEBUG: - # server_message(response) + logging.error("BAZZAR Response returned invalid JSON data") -def updated(): - conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) - c = conn.cursor() - c.execute("UPDATE system SET updated = 1") - conn.commit() - c.close() - try: - requests.get(bazarr_url + 'restart') - except requests.ConnectionError: - logging.info('BAZARR: Restart failed, please restart Bazarr manualy') +def updated(restart=False): + if restart: + try: + requests.get(bazarr_url + 'restart') + except requests.ConnectionError: + logging.info('BAZARR Restart failed, please restart Bazarr manualy') + updated(restart=False) + else: + conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) + c = conn.cursor() + c.execute("UPDATE system SET updated = 1") + conn.commit() + c.close() diff --git a/views/settings.tpl b/views/settings.tpl index cd7b3a669..52f07aad5 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1758,7 +1758,11 @@ }); % from get_args import args - % if args.no_update: + + % + if args.no_update or + args.release_update + : $("#div_update").hide(); % end From e5c2e8026fba9b0a0ea555fd54667a290655da11 Mon Sep 17 00:00:00 2001 From: Halali Date: Mon, 18 Feb 2019 12:44:33 +0100 Subject: [PATCH 07/38] Continue developing --- bazarr/check_update.py | 4 ++-- bazarr/main.py | 20 ++++++++++++++++++++ bazarr/queueconfig.py | 2 ++ views/menu.tpl | 29 ++++++++++++++++++++++++----- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 5255c5e6a..38bcd28cb 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,7 +12,7 @@ import json from get_args import args from config import settings, bazarr_url -from queueconfig import q4ws +from queueconfig import q4ws, q4ws_updater def check_releases(): @@ -137,7 +137,7 @@ def check_updates(): update() elif commits_behind == 0: - q4ws.append('BAZZAR is up to date') + q4ws_updater.append('BAZZAR is up to date') logging.info('BAZZAR is up to date') return latest_version diff --git a/bazarr/main.py b/bazarr/main.py index 7ee3959ae..ac7bc0c8d 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1886,6 +1886,26 @@ def handle_websocket(): break +@route(base_url + 'websocket_updater') +@custom_auth_basic(check_credentials) +def handle_websocket(): + wsock = request.environ.get('wsgi.websocket') + if not wsock: + abort(400, 'Expected WebSocket request.') + + queueconfig.q4ws_updater.clear() + + while True: + try: + if queueconfig.q4ws_updater: + wsock.send(queueconfig.q4ws_updater.popleft()) + gevent.sleep(0.1) + else: + gevent.sleep(0.5) + except WebSocketError: + break + + # Mute DeprecationWarning warnings.simplefilter("ignore", DeprecationWarning) server = WSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app, handler_class=WebSocketHandler) diff --git a/bazarr/queueconfig.py b/bazarr/queueconfig.py index 6795fc125..6d903d7a6 100644 --- a/bazarr/queueconfig.py +++ b/bazarr/queueconfig.py @@ -1,4 +1,6 @@ from collections import deque global q4ws +global q4ws_updater q4ws = deque(maxlen=10) +q4ws_updater = deque(maxlen=2) diff --git a/views/menu.tpl b/views/menu.tpl index 3b9cdedcf..6b750e50d 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -225,14 +225,32 @@ if (location.protocol != 'https:') { var ws = new WebSocket("ws://" + window.location.host + "{{base_url}}websocket"); + var wsupdater = new WebSocket("ws://" + window.location.host + "{{base_url}}websocket_updater"); } else { var ws = new WebSocket("wss://" + window.location.host + "{{base_url}}websocket"); + var wsupdater = new WebSocket("wss://" + window.location.host + "{{base_url}}websocket_updater"); } ws.onmessage = function (evt) { new Noty({ - text: evt.data, - timeout: 3000, + text: evt.data, + timeout: 3000, + progressBar: false, + animation: { + open: null, + close: null + }, + killer: true, + type: 'info', + layout: 'bottomRight', + theme: 'semanticui' + }).show(); + }; + + wsupdater.onmessage = function (evt) { + new Noty({ + text: evt.data, + timeout: false, progressBar: false, animation: { open: null, @@ -240,8 +258,9 @@ }, killer: true, type: 'info', - layout: 'bottomRight', - theme: 'semanticui' + layout: 'bottomLeft', + theme: 'semanticui', + visibilityControl: true }).show(); }; - \ No newline at end of file + From ad2a952e141e5a55a02e6f420b47254f1fd5067f Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 24 Feb 2019 19:31:18 +0100 Subject: [PATCH 08/38] Continue developing --- bazarr/check_update.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 38bcd28cb..39f72b952 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,7 +12,7 @@ import json from get_args import args from config import settings, bazarr_url -from queueconfig import q4ws, q4ws_updater +from queueconfig import notifications def check_releases(): @@ -80,7 +80,6 @@ def check_updates(): version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: - q4ws.append('BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') logging.warn( 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') return current_version @@ -90,12 +89,10 @@ def check_updates(): # See how many commits behind we are if not current_version: - q4ws.append('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return latest_version if latest_version == current_version: - q4ws.append('BAZZAR is up to date') logging.info('BAZARR is up to date') return latest_version @@ -116,7 +113,6 @@ def check_updates(): commits_behind = 0 if commits_behind > 0: - q4ws.append('BAZARR New version is available. You are %s commits behind' % commits_behind) logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) if settings.general.auto_update: update() @@ -137,7 +133,6 @@ def check_updates(): update() elif commits_behind == 0: - q4ws_updater.append('BAZZAR is up to date') logging.info('BAZZAR is up to date') return latest_version From 19aa6af4a341f8478ef5cfd03e0e8b3015c9f15a Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 5 Mar 2019 18:41:06 +0100 Subject: [PATCH 09/38] Continue developing --- bazarr/check_update.py | 136 ++++++++++++++++++++++------------------- bazarr/config.py | 3 +- views/settings.tpl | 28 +++++++++ 3 files changed, 104 insertions(+), 63 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 39f72b952..bf0694821 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -48,7 +48,7 @@ def run_git(args): cmd = cur_git + ' ' + args try: - logging.debug('Trying to execute: "' + cmd + '"') + logging.debug('BAZZAR Trying to execute: "' + cmd + '"') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() @@ -72,74 +72,78 @@ def run_git(args): def check_updates(): commits_behind = 0 - current_version = get_version() - - # Get the latest version available from github - logging.info('BAZZAR Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - - if version is None: - logging.warn( - 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') - return current_version - - latest_version = version['sha'] - logging.debug("BAZZAR Latest version is %s", latest_version) + current_version, source = get_version() + + if source == 'git': + # Get the latest version available from github + logging.info('BAZZAR Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - # See how many commits behind we are - if not current_version: - logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') - return latest_version + if version is None: + logging.warn( + 'BAZZAR Could not get the latest version from GitHub.') + return - if latest_version == current_version: - logging.info('BAZARR is up to date') - return latest_version + latest_version = version['sha'] + logging.debug("BAZZAR Latest version is %s", latest_version) - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, - current_version) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + # See how many commits behind we are + if not current_version: + logging.info( + 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') + return - if commits is None: - logging.warn('BAZARR Could not get commits behind from GitHub.') - return latest_version + if latest_version == current_version: + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') + return - try: - commits_behind = int(commits['behind_by']) - logging.debug("In total, %d commits behind", commits_behind) - except KeyError: - logging.info('BAZARR Cannot compare versions. Are you running a local development version?') - commits_behind = 0 + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, + current_version) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - if commits_behind > 0: - logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) - if settings.general.auto_update: - update() - else: - updated(restart=False) + if commits is None: + logging.warn('BAZARR Could not get commits behind from GitHub.') + return - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + try: + commits_behind = int(commits['behind_by']) + logging.debug("BAZARR In total, %d commits behind", commits_behind) + except KeyError: + logging.info('BAZARR Cannot compare versions. Are you running a local development version?') + commits_behind = 0 - if releases is None: - logging.warn('BAZARR Could not get releases from GitHub.') - return latest_version - else: - release = releases[0] - latest_release = release['tag_name'] + if commits_behind > 0: + logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) + notifications.write(msg='BAZARR New version is available. You are %s commits behind' % commits_behind, + queue='check_update') + update(source, restart=True if settings.general.getboolean('update_restart') else False) - if ('v' + current_version) != latest_release and args.release_update and settings.general.branch == 'master': - update() + else: + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - elif commits_behind == 0: - logging.info('BAZZAR is up to date') + if releases is None: + logging.warn('BAZARR Could not get releases from GitHub.') + return + else: + release = releases[0] + latest_release = release['tag_name'] - return latest_version + if ('v' + current_version) != latest_release and settings.general.branch == 'master': + update(source, restart=True if settings.general.getboolean('update_restart') else False) + elif settings.general.branch != 'master': + notifications.write(msg="BAZZAR Can't update development branch from source", queue='check_update') # fixme + logging.info("BAZZAR Can't update development branch from source") # fixme + else: + notifications.write(msg='BAZZAR is up to date', queue='check_update') + logging.info('BAZZAR is up to date') def get_version(): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: output, err = run_git('rev-parse HEAD') @@ -152,15 +156,15 @@ def get_version(): if not re.match('^[a-z0-9]+$', cur_commit_hash): logging.error('BAZZAR Output does not look like a hash, not using it.') cur_commit_hash = None - - return cur_commit_hash + + return cur_commit_hash, 'git' else: - return os.environ["BAZARR_VERSION"] + return os.environ["BAZARR_VERSION"], 'source' -def update(): - if not args.release_update: +def update(source, restart=True): + if source == 'git': output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: @@ -175,16 +179,19 @@ def update(): elif line.endswith(('Aborting', 'Aborting.')): logging.error('BAZZAR Unable to update from git: ' + line) logging.info('BAZZAR Output: ' + str(output)) - updated(restart=True) + updated(restart) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') logging.info('BAZZAR Downloading update from: ' + tar_download_url) + notifications.write(msg='BAZZAR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) + notifications.write(msg=("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url), + type='error') return download_name = settings.general.branch + '-github' @@ -196,19 +203,24 @@ def update(): # Extract the tar to update folder logging.info('BAZZAR Extracting file: ' + tar_download_path) + notifications.write(msg='BAZZAR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz logging.info('BAZZAR Deleting file: ' + tar_download_path) + notifications.write(msg='BAZZAR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) + notifications.write(msg="BAZZAR Invalid update data, update failed: " + str(update_dir_contents), + type='error') return + content_dir = os.path.join(update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder @@ -221,7 +233,7 @@ def update(): if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) - updated(restart=True) + updated(restart) class FakeLock(object): diff --git a/bazarr/config.py b/bazarr/config.py index d317cec8a..f6df26296 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -37,7 +37,8 @@ defaults = { 'multithreading': 'True', 'chmod': '0640', 'subfolder': 'current', - 'subfolder_custom': '' + 'subfolder_custom': '', + 'update_restart': 'True' }, 'auth': { 'type': 'None', diff --git a/views/settings.tpl b/views/settings.tpl index 478563eda..f3076842f 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -692,6 +692,28 @@ + +
+
+ +
+
+
+ + +
+
+ +
@@ -2011,6 +2033,12 @@ $("#settings_automatic_div").checkbox('uncheck'); } + if ($('#settings_restart_update').data("restart_update") === "True") { + $("#settings_restart_update").checkbox('check'); + } else { + $("#settings_restart_update").checkbox('uncheck'); + } + if ($('#settings_debug').data("debug") === "True") { $("#settings_debug").checkbox('check'); } else { From 7dec74c354a223def37c30d9a29abe0bcde26d57 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 9 Mar 2019 09:37:47 +0100 Subject: [PATCH 10/38] Continue developing --- bazarr/check_update.py | 1 + bazarr/main.py | 23 ----------------------- bazarr/scheduler.py | 6 ++---- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index bf0694821..d0628faa2 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -73,6 +73,7 @@ def run_git(args): def check_updates(): commits_behind = 0 current_version, source = get_version() + check_releases() if source == 'git': # Get the latest version available from github diff --git a/bazarr/main.py b/bazarr/main.py index 9330995d6..c57ed89a4 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,8 +57,6 @@ from get_providers import get_providers, get_providers_auth from get_series import * from get_episodes import * -# if not args.no_update: -# from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ @@ -1910,27 +1908,6 @@ def notifications(): def running_tasks_list(): return dict(tasks=running_tasks) - -@route(base_url + 'websocket_updater') -@custom_auth_basic(check_credentials) -def handle_websocket(): - wsock = request.environ.get('wsgi.websocket') - if not wsock: - abort(400, 'Expected WebSocket request.') - - queueconfig.q4ws_updater.clear() - - while True: - try: - if queueconfig.q4ws_updater: - wsock.send(queueconfig.q4ws_updater.popleft()) - gevent.sleep(0.1) - else: - gevent.sleep(0.5) - except WebSocketError: - break - - # Mute DeprecationWarning warnings.simplefilter("ignore", DeprecationWarning) server = WSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app, handler_class=WebSocketHandler) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index c6b193b13..a717fd723 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -7,10 +7,8 @@ from config import settings from get_subtitle import wanted_search_missing_subtitles from get_args import args -if not args.no_update: - from check_update import check_updates, check_releases -else: - from check_update import check_releases + +from check_update import check_updates, check_releases from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from apscheduler.triggers.cron import CronTrigger From 53ac2bd67a88d9461c6ffb960c1bc2e6fe37f3cf Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 12 Mar 2019 18:12:53 +0100 Subject: [PATCH 11/38] Fix bad code format --- views/settings.tpl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/views/settings.tpl b/views/settings.tpl index f3076842f..486edbac4 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1992,10 +1992,7 @@ % from get_args import args - % - if args.no_update or - args.release_update - : + % if args.no_update or args.release_update: $("#div_update").hide(); % end % import sys From 23e2bec8a0675e5868cdadb481890157585c3c9c Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 12 Mar 2019 21:58:09 +0100 Subject: [PATCH 12/38] Continue developing --- bazarr/check_update.py | 25 ++++++++++++++++++++++--- views/settings.tpl | 5 ++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index d0628faa2..22b743827 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -77,7 +77,7 @@ def check_updates(): if source == 'git': # Get the latest version available from github - logging.info('BAZZAR Retrieving latest version information from GitHub') + logging.debug('BAZZAR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) @@ -91,7 +91,7 @@ def check_updates(): # See how many commits behind we are if not current_version: - logging.info( + logging.warn( 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return @@ -100,7 +100,7 @@ def check_updates(): logging.info('BAZARR is up to date') return - logging.info('Comparing currently installed version with latest GitHub version') + logging.debug('Comparing currently installed version with latest GitHub version') url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, current_version) commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) @@ -121,6 +121,9 @@ def check_updates(): notifications.write(msg='BAZARR New version is available. You are %s commits behind' % commits_behind, queue='check_update') update(source, restart=True if settings.general.getboolean('update_restart') else False) + elif commits_behind is 0: + notifications.write(msg='BAZZAR is up to date', queue='check_update') + logging.info('BAZZAR is up to date') else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' @@ -237,6 +240,22 @@ def update(source, restart=True): updated(restart) +def checkout_git_branch(): + output, err = run_git('fetch %s' % 'origin') + output, err = run_git('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = run_git('pull %s %s' % ('origin', settings.general.branch)) + + class FakeLock(object): """ If no locking or request throttling is needed, use this diff --git a/views/settings.tpl b/views/settings.tpl index 486edbac4..7077a9c74 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -654,7 +654,7 @@
Updates
-
+
@@ -1991,9 +1991,12 @@ }); % from get_args import args + % from check_update import get_version % if args.no_update or args.release_update: $("#div_update").hide(); + % elif get_version()[1] != 'git': + $("#div_branch").hide(); % end % import sys % if sys.platform.startswith('win'): From b6b14d93e7235161956e01930f6d797024cb8b91 Mon Sep 17 00:00:00 2001 From: Halali Date: Wed, 13 Mar 2019 23:02:37 +0100 Subject: [PATCH 13/38] Continue developing --- bazarr/scheduler.py | 5 +++-- views/settings.tpl | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index a717fd723..9bf8f5f97 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -83,12 +83,13 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) if not args.no_update: if settings.general.getboolean('auto_update'): scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') else: scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', - name='Update bazarr from source on Github') + name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') + else: scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/settings.tpl b/views/settings.tpl index 7077a9c74..8c8f3c90a 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -1993,9 +1993,9 @@ % from get_args import args % from check_update import get_version - % if args.no_update or args.release_update: + % if args.no_update: $("#div_update").hide(); - % elif get_version()[1] != 'git': + % elif get_version()[1] != 'git' or args.release_update: $("#div_branch").hide(); % end % import sys From 8aff2c17fa98ce855065d9d981a45f1eedee92d1 Mon Sep 17 00:00:00 2001 From: Halali Date: Thu, 14 Mar 2019 18:01:33 +0100 Subject: [PATCH 14/38] Add missed code for save settings --- bazarr/main.py | 6 ++++++ views/settings.tpl | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bazarr/main.py b/bazarr/main.py index d7c612ece..3a602d3ea 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1139,6 +1139,11 @@ def save_settings(): settings_general_automatic = 'False' else: settings_general_automatic = 'True' + settings_general_update_restart = request.forms.get('settings_general_update_restart') + if settings_general_update_restart is None: + settings_general_update_restart = 'False' + else: + settings_general_update_restart = 'True' settings_general_single_language = request.forms.get('settings_general_single_language') if settings_general_single_language is None: settings_general_single_language = 'False' @@ -1201,6 +1206,7 @@ def save_settings(): settings.general.chmod = text_type(settings_general_chmod) settings.general.branch = text_type(settings_general_branch) settings.general.auto_update = text_type(settings_general_automatic) + settings.general.update_restart = text_type(settings_general_update_restart) settings.general.single_language = text_type(settings_general_single_language) settings.general.minimum_score = text_type(settings_general_minimum_score) settings.general.use_scenename = text_type(settings_general_scenename) diff --git a/views/settings.tpl b/views/settings.tpl index 8c8f3c90a..9fc1e714d 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -699,7 +699,7 @@
+ data-update-restart={{settings.general.getboolean('update_restart')}}>
@@ -2033,10 +2033,10 @@ $("#settings_automatic_div").checkbox('uncheck'); } - if ($('#settings_restart_update').data("restart_update") === "True") { - $("#settings_restart_update").checkbox('check'); + if ($('#settings_update_restart').data("update-restart") === "True") { + $("#settings_update_restart").checkbox('check'); } else { - $("#settings_restart_update").checkbox('uncheck'); + $("#settings_update_restart").checkbox('uncheck'); } if ($('#settings_debug').data("debug") === "True") { From e240659b9a175dadf1a0626731fd8397d6d8549b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 14 Mar 2019 20:57:48 -0400 Subject: [PATCH 15/38] Fix typo (BAZZAR vs BAZARR) --- bazarr/check_update.py | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 22b743827..f0b1cdd51 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -48,21 +48,21 @@ def run_git(args): cmd = cur_git + ' ' + args try: - logging.debug('BAZZAR Trying to execute: "' + cmd + '"') + logging.debug('BAZARR Trying to execute: "' + cmd + '"') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() - logging.debug('BAZZAR Git output: ' + output) + logging.debug('BAZARR Git output: ' + output) except OSError: - logging.debug('BAZZAR Command failed: %s', cmd) + logging.debug('BAZARR Command failed: %s', cmd) continue if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('BAZZAR Unable to find git with command ' + cmd) + logging.debug('BAZARR Unable to find git with command ' + cmd) output = None elif 'fatal:' in output or err: - logging.error('BAZZAR Git returned bad info. Are you sure this is a git installation?') + logging.error('BAZARR Git returned bad info. Are you sure this is a git installation?') output = None elif output: break @@ -77,17 +77,17 @@ def check_updates(): if source == 'git': # Get the latest version available from github - logging.debug('BAZZAR Retrieving latest version information from GitHub') + logging.debug('BAZARR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: logging.warn( - 'BAZZAR Could not get the latest version from GitHub.') + 'BAZARR Could not get the latest version from GitHub.') return latest_version = version['sha'] - logging.debug("BAZZAR Latest version is %s", latest_version) + logging.debug("BAZARR Latest version is %s", latest_version) # See how many commits behind we are if not current_version: @@ -122,8 +122,8 @@ def check_updates(): queue='check_update') update(source, restart=True if settings.general.getboolean('update_restart') else False) elif commits_behind is 0: - notifications.write(msg='BAZZAR is up to date', queue='check_update') - logging.info('BAZZAR is up to date') + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' @@ -139,11 +139,11 @@ def check_updates(): if ('v' + current_version) != latest_release and settings.general.branch == 'master': update(source, restart=True if settings.general.getboolean('update_restart') else False) elif settings.general.branch != 'master': - notifications.write(msg="BAZZAR Can't update development branch from source", queue='check_update') # fixme - logging.info("BAZZAR Can't update development branch from source") # fixme + notifications.write(msg="BAZARR Can't update development branch from source", queue='check_update') # fixme + logging.info("BAZARR Can't update development branch from source") # fixme else: - notifications.write(msg='BAZZAR is up to date', queue='check_update') - logging.info('BAZZAR is up to date') + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') def get_version(): @@ -152,13 +152,13 @@ def get_version(): output, err = run_git('rev-parse HEAD') if not output: - logging.error('BAZZAR Could not find latest installed version.') + logging.error('BAZARR Could not find latest installed version.') cur_commit_hash = None else: cur_commit_hash = str(output) if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('BAZZAR Output does not look like a hash, not using it.') + logging.error('BAZARR Output does not look like a hash, not using it.') cur_commit_hash = None return cur_commit_hash, 'git' @@ -172,29 +172,29 @@ def update(source, restart=True): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: - logging.error('BAZZAR Unable to download latest version') + logging.error('BAZARR Unable to download latest version') return for line in output.split('\n'): if 'Already up-to-date.' in line: - logging.info('BAZZAR No update available, not updating') - logging.info('BAZZAR Output: ' + str(output)) + logging.info('BAZARR No update available, not updating') + logging.info('BAZARR Output: ' + str(output)) elif line.endswith(('Aborting', 'Aborting.')): - logging.error('BAZZAR Unable to update from git: ' + line) - logging.info('BAZZAR Output: ' + str(output)) + logging.error('BAZARR Unable to update from git: ' + line) + logging.info('BAZARR Output: ' + str(output)) updated(restart) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - logging.info('BAZZAR Downloading update from: ' + tar_download_url) - notifications.write(msg='BAZZAR Downloading update from: ' + tar_download_url) + logging.info('BAZARR Downloading update from: ' + tar_download_url) + notifications.write(msg='BAZARR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: - logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) - notifications.write(msg=("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url), + logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url) + notifications.write(msg=("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url), type='error') return @@ -206,22 +206,22 @@ def update(source, restart=True): f.write(data) # Extract the tar to update folder - logging.info('BAZZAR Extracting file: ' + tar_download_path) - notifications.write(msg='BAZZAR Extracting file: ' + tar_download_path) + logging.info('BAZARR Extracting file: ' + tar_download_path) + notifications.write(msg='BAZARR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz - logging.info('BAZZAR Deleting file: ' + tar_download_path) - notifications.write(msg='BAZZAR Deleting file: ' + tar_download_path) + logging.info('BAZARR Deleting file: ' + tar_download_path) + notifications.write(msg='BAZARR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: - logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) - notifications.write(msg="BAZZAR Invalid update data, update failed: " + str(update_dir_contents), + logging.error("BAZARR Invalid update data, update failed: " + str(update_dir_contents)) + notifications.write(msg="BAZARR Invalid update data, update failed: " + str(update_dir_contents), type='error') return @@ -318,7 +318,7 @@ def request_response(url, method="get", auto_raise=True, # Request URL and wait for response with lock: logging.debug( - "BAZZAR Requesting URL via %s method: %s", method.upper(), url) + "BAZARR Requesting URL via %s method: %s", method.upper(), url) response = request_method(url, **kwargs) # If status code != OK, then raise exception, except if the status code @@ -329,7 +329,7 @@ def request_response(url, method="get", auto_raise=True, response.raise_for_status() except: logging.debug( - "BAZZAR Response status code %d is not white " + "BAZARR Response status code %d is not white " "listed, raised exception", response.status_code) raise elif auto_raise: @@ -339,22 +339,22 @@ def request_response(url, method="get", auto_raise=True, except requests.exceptions.SSLError as e: if kwargs["verify"]: logging.error( - "BAZZAR Unable to connect to remote host because of a SSL error. " + "BAZARR Unable to connect to remote host because of a SSL error. " "It is likely that your system cannot verify the validity" "of the certificate. The remote certificate is either " "self-signed, or the remote server uses SNI. See the wiki for " "more information on this topic.") else: logging.error( - "BAZZAR SSL error raised during connection, with certificate " + "BAZARR SSL error raised during connection, with certificate " "verification turned off: %s", e) except requests.ConnectionError: logging.error( - "BAZZAR Unable to connect to remote host. Check if the remote " + "BAZARR Unable to connect to remote host. Check if the remote " "host is up and running.") except requests.Timeout: logging.error( - "BAZZAR Request timed out. The remote host did not respond timely.") + "BAZARR Request timed out. The remote host did not respond timely.") except requests.HTTPError as e: if e.response is not None: if e.response.status_code >= 500: @@ -366,12 +366,12 @@ def request_response(url, method="get", auto_raise=True, cause = "unknown" logging.error( - "BAZZAR Request raise HTTP error with status code %d (%s).", + "BAZARR Request raise HTTP error with status code %d (%s).", e.response.status_code, cause) else: - logging.error("BAZZAR Request raised HTTP error.") + logging.error("BAZARR Request raised HTTP error.") except requests.RequestException as e: - logging.error("BAZZAR Request raised exception: %s", e) + logging.error("BAZARR Request raised exception: %s", e) def request_json(url, **kwargs): @@ -391,11 +391,11 @@ def request_json(url, **kwargs): result = response.json() if validator and not validator(result): - logging.error("BAZZAR JSON validation result failed") + logging.error("BAZARR JSON validation result failed") else: return result except ValueError: - logging.error("BAZZAR Response returned invalid JSON data") + logging.error("BAZARR Response returned invalid JSON data") def updated(restart=False): From e6b324b08fc86afce6f86f8dc391e7548ec9ff32 Mon Sep 17 00:00:00 2001 From: Halali Date: Wed, 20 Mar 2019 23:42:19 +0100 Subject: [PATCH 16/38] Some logging improvements --- bazarr/check_update.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index f0b1cdd51..febdfd268 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -82,6 +82,9 @@ def check_updates(): version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: + notifications.write( + msg='BAZARR Could not get the latest version from GitHub.', + queue='check_update', type='warning') logging.warn( 'BAZARR Could not get the latest version from GitHub.') return @@ -91,6 +94,8 @@ def check_updates(): # See how many commits behind we are if not current_version: + notifications.write(msg='BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version', + queue='check_update', type='warning') logging.warn( 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return @@ -106,6 +111,8 @@ def check_updates(): commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: + notifications.write(msg='BAZARR Could not get commits behind from GitHub.', + queue='check_update', type='warning') logging.warn('BAZARR Could not get commits behind from GitHub.') return @@ -113,8 +120,9 @@ def check_updates(): commits_behind = int(commits['behind_by']) logging.debug("BAZARR In total, %d commits behind", commits_behind) except KeyError: + notifications.write(msg='BAZARR Cannot compare versions. Are you running a local development version?', queue='check_update') logging.info('BAZARR Cannot compare versions. Are you running a local development version?') - commits_behind = 0 + return if commits_behind > 0: logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) @@ -130,6 +138,8 @@ def check_updates(): releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: + notifications.write(msg='BAZARR Could not get releases from GitHub.', + queue='check_update', type='warning') logging.warn('BAZARR Could not get releases from GitHub.') return else: @@ -172,6 +182,8 @@ def update(source, restart=True): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: + notifications.write(msg='Unable to download latest version', + queue='check_update', type='error') logging.error('BAZARR Unable to download latest version') return From 570cc3d52851e235e03a4bd95a19d67e171bdb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 20 Mar 2019 19:50:26 -0400 Subject: [PATCH 17/38] Rescheduling of Bazarr update task when saving settings. --- bazarr/main.py | 4 +--- bazarr/scheduler.py | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/bazarr/main.py b/bazarr/main.py index d39a004c2..858554338 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1439,14 +1439,12 @@ def save_settings(): conn.commit() c.close() + schedule_update_job() sonarr_full_update() radarr_full_update() logging.info('BAZARR Settings saved succesfully.') - # reschedule full update task according to settings - sonarr_full_update() - if ref.find('saved=true') > 0: redirect(ref) else: diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 9bf8f5f97..1f1a555c3 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -80,19 +80,21 @@ def task_listener(event): scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) -if not args.no_update: - if settings.general.getboolean('auto_update'): - scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') +def schedule_update_job(): + if not args.no_update: + if settings.general.getboolean('auto_update'): + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) + else: + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) + scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_release', name='Update release info', replace_existing=True) + else: - scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', - name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') - scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_release', name='Update release info') + scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, + id='update_release', name='Update release info', replace_existing=True) -else: - scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, - id='update_release', name='Update release info') if settings.general.getboolean('use_sonarr'): scheduler.add_job(update_series, IntervalTrigger(minutes=1), max_instances=1, coalesce=True, misfire_grace_time=15, @@ -108,6 +110,7 @@ if settings.general.getboolean('use_sonarr') or settings.general.getboolean('use scheduler.add_job(wanted_search_missing_subtitles, IntervalTrigger(hours=3), max_instances=1, coalesce=True, misfire_grace_time=15, id='wanted_search_missing_subtitles', name='Search for wanted subtitles') +schedule_update_job() sonarr_full_update() radarr_full_update() scheduler.start() From 52a37a4988edb3ff63d1102c564313b258bb373e Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 31 Mar 2019 22:10:46 +0200 Subject: [PATCH 18/38] Continue developing --- bazarr/check_update.py | 3 +-- bazarr/main.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index febdfd268..48ff2f529 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -132,7 +132,6 @@ def check_updates(): elif commits_behind is 0: notifications.write(msg='BAZARR is up to date', queue='check_update') logging.info('BAZARR is up to date') - else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) @@ -253,7 +252,7 @@ def update(source, restart=True): def checkout_git_branch(): - output, err = run_git('fetch %s' % 'origin') + output, err = run_git('fetch origin') output, err = run_git('checkout %s' % settings.general.branch) if not output: diff --git a/bazarr/main.py b/bazarr/main.py index fa9bfa624..cca95aae7 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -67,6 +67,7 @@ from notifier import send_notifications, send_notifications_movie from config import settings, url_sonarr, url_radarr, url_radarr_short, url_sonarr_short, base_url from helper import path_replace_movie, get_subtitle_destination_folder from subliminal_patch.extensions import provider_registry as provider_manager +from check_update import checkout_git_branch reload(sys) sys.setdefaultencoding('utf8') @@ -1285,6 +1286,7 @@ def save_settings(): after = (unicode(settings_general_ip), int(settings_general_port), unicode(settings_general_baseurl), unicode(settings_general_pathmapping), unicode(settings_general_use_sonarr), unicode(settings_general_use_radarr), unicode(settings_general_pathmapping_movie)) + updater_before = settings.general.branch settings.general.ip = text_type(settings_general_ip) settings.general.port = text_type(settings_general_port) @@ -1536,6 +1538,9 @@ def save_settings(): radarr_full_update() logging.info('BAZARR Settings saved succesfully.') + + if updater_before != settings_general_branch: + checkout_git_branch() if ref.find('saved=true') > 0: redirect(ref) From 963947d636b8715e17f58be45c99a71fbca546d8 Mon Sep 17 00:00:00 2001 From: Halali Date: Fri, 15 Feb 2019 21:38:10 +0100 Subject: [PATCH 19/38] Initial commit --- bazarr/check_update.py | 434 +++++++++++++++++++++++++++++++++++++---- bazarr/main.py | 2 +- bazarr/scheduler.py | 6 +- 3 files changed, 401 insertions(+), 41 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index c42200936..64a2ef3f2 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -1,53 +1,45 @@ -# coding=utf-8 import os +import platform +import re +import subprocess +import tarfile import logging +import requests import sqlite3 import json -import requests from get_args import args from config import settings -if not args.no_update: - import git +bazarr_version = '7.2.0' -current_working_directory = os.path.dirname(os.path.dirname(__file__)) +LATEST_VERSION = None +INSTALL_TYPE = None +UPDATE_AVAILABLE = None +COMMITS_BEHIND = None +LATEST_RELEASE = None +PREV_RELEASE = bazarr_version -def gitconfig(): - g = git.Repo.init(current_working_directory) - config_read = g.config_reader() - config_write = g.config_writer() +class FakeLock(object): + """ + If no locking or request throttling is needed, use this + """ - try: - username = config_read.get_value("user", "name") - except: - logging.debug('BAZARR Settings git username') - config_write.set_value("user", "name", "Bazarr") + def __enter__(self): + """ + Do nothing on enter + """ + pass - try: - email = config_read.get_value("user", "email") - except: - logging.debug('BAZARR Settings git email') - config_write.set_value("user", "email", "bazarr@fake.email") - - -def check_and_apply_update(): - gitconfig() - check_releases() - branch = settings.general.branch - g = git.cmd.Git(current_working_directory) - g.fetch('origin') - result = g.diff('--shortstat', 'origin/' + branch) - if len(result) == 0: - logging.info('BAZARR No new version of Bazarr available.') - else: - g.reset('--hard', 'HEAD') - g.checkout(branch) - g.reset('--hard', 'origin/' + branch) - g.pull() - logging.info('BAZARR Updated to latest version. Restart required. ' + result) - updated() + def __exit__(self, type, value, traceback): + """ + Do nothing on exit + """ + pass + + +fake_lock = FakeLock() def check_releases(): @@ -71,6 +63,374 @@ def check_releases(): json.dump(releases, f) +def runGit(args): + git_locations = ['git'] + + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') + + output = err = None + + for cur_git in git_locations: + cmd = cur_git + ' ' + args + + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() + + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue + + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break + + return (output, err) + + +def getVersion(): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): + + INSTALL_TYPE = 'git' + output, err = runGit('rev-parse HEAD') + + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + + cur_commit_hash = str(output) + + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None + + if settings.general.branch: + branch_name = settings.general.branch + + else: + remote_branch, err = runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') + remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] + if len(remote_branch) == 2: + remote_name, branch_name = remote_branch + else: + remote_name = branch_name = None + + if not remote_name and settings.general.branch: + logging.error('Could not retrieve remote name from git. Defaulting to origin.') + branch_name = settings.general.branch + + if not branch_name: + logging.error('Could not retrieve branch name from git. Defaulting to master.') + branch_name = 'master' + + return cur_commit_hash, 'origin', branch_name + + else: + INSTALL_TYPE = 'source' + + if CURRENT_VERSION: + return CURRENT_VERSION, 'origin', settings.general.branch + else: + return None, 'origin', settings.general.branch + + +CURRENT_VERSION, GIT_REMOTE, GIT_BRANCH = getVersion() + + +def check_updates(): + check_github() + if not CURRENT_VERSION: + UPDATE_AVAILABLE = None + elif COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ + ('V' + bazarr_version) != LATEST_RELEASE: + UPDATE_AVAILABLE = 'release' + elif COMMITS_BEHIND > 0 and CURRENT_VERSION != LATEST_VERSION: + UPDATE_AVAILABLE = 'commit' + else: + UPDATE_AVAILABLE = False + + +def check_github(): + COMMITS_BEHIND = 0 + + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return CURRENT_VERSION + + LATEST_VERSION = version['sha'] + logging.debug("Latest version is %s", LATEST_VERSION) + + # See how many commits behind we are + if not CURRENT_VERSION: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return LATEST_VERSION + + if LATEST_VERSION == CURRENT_VERSION: + logging.info('Bazarr is up to date') + return LATEST_VERSION + + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (LATEST_VERSION, + CURRENT_VERSION) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return LATEST_VERSION + + try: + COMMITS_BEHIND = int(commits['behind_by']) + logging.debug("In total, %d commits behind", COMMITS_BEHIND) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + COMMITS_BEHIND = 0 + + if COMMITS_BEHIND > 0: + logging.info('New version is available. You are %s commits behind' % COMMITS_BEHIND) + + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return LATEST_VERSION + + if settings.general.branch == 'master': + release = next((r for r in releases if not r['prerelease']), releases[0]) + else: + release = releases[0] + LATEST_RELEASE = release['tag_name'] + + elif COMMITS_BEHIND == 0: + logging.info('Bazarr is up to date') + + return LATEST_VERSION + + +def update(): + if INSTALL_TYPE == 'git': + output, err = runGit('pull ' + 'origin' + ' ' + settings.general.branch) + + if not output: + logging.error('Unable to download latest version') + return + + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) + + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) + + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return + + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) + + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() + + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + + +def checkout_git_branch(): + if INSTALL_TYPE == 'git': + output, err = runGit('fetch origin') + output, err = runGit('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = runGit('pull origin %s' % settings.general.branch) + + +def request_content(url, **kwargs): + """ + Wrapper for `request_response', which will return the raw content. + """ + + response = request_response(url, **kwargs) + + if response is not None: + return response.content + + +def request_response(url, method="get", auto_raise=True, + whitelist_status_code=None, lock=fake_lock, **kwargs): + """ + Convenient wrapper for `requests.get', which will capture the exceptions + and log them. On success, the Response object is returned. In case of a + exception, None is returned. + + Additionally, there is support for rate limiting. To use this feature, + supply a tuple of (lock, request_limit). The lock is used to make sure no + other request with the same lock is executed. The request limit is the + minimal time between two requests (and so 1/request_limit is the number of + requests per seconds). + """ + + # Convert whitelist_status_code to a list if needed + if whitelist_status_code and type(whitelist_status_code) != list: + whitelist_status_code = [whitelist_status_code] + + # Disable verification of SSL certificates if requested. Note: this could + # pose a security issue! + kwargs["verify"] = True + + # Map method to the request.XXX method. This is a simple hack, but it + # allows requests to apply more magic per method. See lib/requests/api.py. + request_method = getattr(requests, method.lower()) + + try: + # Request URL and wait for response + with lock: + logging.debug( + "Requesting URL via %s method: %s", method.upper(), url) + response = request_method(url, **kwargs) + + # If status code != OK, then raise exception, except if the status code + # is white listed. + if whitelist_status_code and auto_raise: + if response.status_code not in whitelist_status_code: + try: + response.raise_for_status() + except: + logging.debug( + "Response status code %d is not white " + "listed, raised exception", response.status_code) + raise + elif auto_raise: + response.raise_for_status() + + return response + except requests.exceptions.SSLError as e: + if kwargs["verify"]: + logging.error( + "Unable to connect to remote host because of a SSL error. " + "It is likely that your system cannot verify the validity" + "of the certificate. The remote certificate is either " + "self-signed, or the remote server uses SNI. See the wiki for " + "more information on this topic.") + else: + logging.error( + "SSL error raised during connection, with certificate " + "verification turned off: %s", e) + except requests.ConnectionError: + logging.error( + "Unable to connect to remote host. Check if the remote " + "host is up and running.") + except requests.Timeout: + logging.error( + "Request timed out. The remote host did not respond timely.") + except requests.HTTPError as e: + if e.response is not None: + if e.response.status_code >= 500: + cause = "remote server error" + elif e.response.status_code >= 400: + cause = "local client error" + else: + # I don't think we will end up here, but for completeness + cause = "unknown" + + logging.error( + "Request raise HTTP error with status code %d (%s).", + e.response.status_code, cause) + + # Debug response + # if bazarr.DEBUG: + # server_message(e.response) + else: + logging.error("Request raised HTTP error.") + except requests.RequestException as e: + logging.error("Request raised exception: %s", e) + + +def request_json(url, **kwargs): + """ + Wrapper for `request_response', which will decode the response as JSON + object and return the result, if no exceptions are raised. + + As an option, a validator callback can be given, which should return True + if the result is valid. + """ + + validator = kwargs.pop("validator", None) + response = request_response(url, **kwargs) + + if response is not None: + try: + result = response.json() + + if validator and not validator(result): + logging.error("JSON validation result failed") + else: + return result + except ValueError: + logging.error("Response returned invalid JSON data") + + # Debug response + # if bazarr.DEBUG: + # server_message(response) + + def updated(): conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) c = conn.cursor() diff --git a/bazarr/main.py b/bazarr/main.py index ea3e4414a..2b0d65a91 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -58,7 +58,7 @@ from get_series import * from get_episodes import * if not args.no_update: - from check_update import check_and_apply_update + from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 9856e57da..ccbf20d07 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles from get_args import args if not args.no_update: - from check_update import check_and_apply_update, check_releases + from check_update import check_updates, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -84,10 +84,10 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') From 7f2a13f702b24b8a5cd107054944e8850f8dd8b1 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 16 Feb 2019 00:21:31 +0100 Subject: [PATCH 20/38] Continue developing --- bazarr/check_update.py | 437 +++++++++++++++++++++-------------------- bazarr/get_args.py | 2 + bazarr/main.py | 2 +- bazarr/scheduler.py | 8 +- views/menu.tpl | 39 +++- 5 files changed, 264 insertions(+), 224 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 64a2ef3f2..d8be1087c 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -11,14 +11,9 @@ import json from get_args import args from config import settings -bazarr_version = '7.2.0' -LATEST_VERSION = None -INSTALL_TYPE = None -UPDATE_AVAILABLE = None -COMMITS_BEHIND = None -LATEST_RELEASE = None -PREV_RELEASE = bazarr_version +# from main import bazarr_version + class FakeLock(object): @@ -63,243 +58,249 @@ def check_releases(): json.dump(releases, f) -def runGit(args): - git_locations = ['git'] - - if platform.system().lower() == 'darwin': - git_locations.append('/usr/local/git/bin/git') - - output = err = None - - for cur_git in git_locations: - cmd = cur_git + ' ' + args +class Updater(object): + def __init__(self): + self.bazarr_version = '7.2.0' - try: - logging.debug('Trying to execute: "' + cmd + '"') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output, err = p.communicate() - output = output.strip() - - logging.debug('Git output: ' + output) - except OSError: - logging.debug('Command failed: %s', cmd) - continue - - if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) - output = None - elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') - output = None - elif output: - break + self.LATEST_VERSION = '' + self.INSTALL_TYPE = '' + self.UPDATE_AVAILABLE = '' + self.COMMITS_BEHIND = '' + self.LATEST_RELEASE = '' + self.CURRENT_VERSION = '' + self.PREV_RELEASE = self.bazarr_version + self.CURRENT_VERSION, self.GIT_REMOTE, self.GIT_BRANCH = self.getVersion() - return (output, err) - - -def getVersion(): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): - - INSTALL_TYPE = 'git' - output, err = runGit('rev-parse HEAD') - - if not output: - logging.error('Could not find latest installed version.') - cur_commit_hash = None + def runGit(self, args): + git_locations = ['git'] - cur_commit_hash = str(output) + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') - cur_commit_hash = None + output = err = None - if settings.general.branch: - branch_name = settings.general.branch - - else: - remote_branch, err = runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') - remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] - if len(remote_branch) == 2: - remote_name, branch_name = remote_branch - else: - remote_name = branch_name = None + for cur_git in git_locations: + cmd = cur_git + ' ' + args - if not remote_name and settings.general.branch: - logging.error('Could not retrieve remote name from git. Defaulting to origin.') - branch_name = settings.general.branch + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() + + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue - if not branch_name: - logging.error('Could not retrieve branch name from git. Defaulting to master.') - branch_name = 'master' + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break - return cur_commit_hash, 'origin', branch_name + return (output, err) - else: - INSTALL_TYPE = 'source' + def getVersion(self): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: + + self.INSTALL_TYPE = 'git' + output, err = self.runGit('rev-parse HEAD') + + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + + cur_commit_hash = str(output) + + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None + + if settings.general.branch: + branch_name = settings.general.branch + + else: + remote_branch, err = self.runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') + remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] + if len(remote_branch) == 2: + remote_name, branch_name = remote_branch + else: + remote_name = branch_name = None + + if not remote_name and settings.general.branch: + logging.error('Could not retrieve remote name from git. Defaulting to origin.') + branch_name = settings.general.branch + + if not branch_name: + logging.error('Could not retrieve branch name from git. Defaulting to master.') + branch_name = 'master' + + return cur_commit_hash, 'origin', branch_name - if CURRENT_VERSION: - return CURRENT_VERSION, 'origin', settings.general.branch else: - return None, 'origin', settings.general.branch - - -CURRENT_VERSION, GIT_REMOTE, GIT_BRANCH = getVersion() - - -def check_updates(): - check_github() - if not CURRENT_VERSION: - UPDATE_AVAILABLE = None - elif COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ - ('V' + bazarr_version) != LATEST_RELEASE: - UPDATE_AVAILABLE = 'release' - elif COMMITS_BEHIND > 0 and CURRENT_VERSION != LATEST_VERSION: - UPDATE_AVAILABLE = 'commit' - else: - UPDATE_AVAILABLE = False - - -def check_github(): - COMMITS_BEHIND = 0 - - # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - - if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') - return CURRENT_VERSION - - LATEST_VERSION = version['sha'] - logging.debug("Latest version is %s", LATEST_VERSION) - - # See how many commits behind we are - if not CURRENT_VERSION: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') - return LATEST_VERSION - - if LATEST_VERSION == CURRENT_VERSION: - logging.info('Bazarr is up to date') - return LATEST_VERSION - - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (LATEST_VERSION, - CURRENT_VERSION) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - - if commits is None: - logging.warn('Could not get commits behind from GitHub.') - return LATEST_VERSION - - try: - COMMITS_BEHIND = int(commits['behind_by']) - logging.debug("In total, %d commits behind", COMMITS_BEHIND) - except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') - COMMITS_BEHIND = 0 + self.INSTALL_TYPE = 'source' + + if self.CURRENT_VERSION: + return self.CURRENT_VERSION, 'origin', settings.general.branch + else: + return None, 'origin', settings.general.branch - if COMMITS_BEHIND > 0: - logging.info('New version is available. You are %s commits behind' % COMMITS_BEHIND) - - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - - if releases is None: - logging.warn('Could not get releases from GitHub.') - return LATEST_VERSION - - if settings.general.branch == 'master': - release = next((r for r in releases if not r['prerelease']), releases[0]) + def check_updates(self): + self.check_github() + if not self.CURRENT_VERSION: + self.UPDATE_AVAILABLE = None + elif self.COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ + ('v' + self.bazarr_version) != self.LATEST_RELEASE: + self.UPDATE_AVAILABLE = 'release' + elif self.COMMITS_BEHIND > 0 and self.CURRENT_VERSION != self.LATEST_VERSION: + self.UPDATE_AVAILABLE = 'commit' else: - release = releases[0] - LATEST_RELEASE = release['tag_name'] + self.UPDATE_AVAILABLE = False + print self.UPDATE_AVAILABLE - elif COMMITS_BEHIND == 0: - logging.info('Bazarr is up to date') - - return LATEST_VERSION - - -def update(): - if INSTALL_TYPE == 'git': - output, err = runGit('pull ' + 'origin' + ' ' + settings.general.branch) - - if not output: - logging.error('Unable to download latest version') - return + def check_github(self): + self.COMMITS_BEHIND = 0 - for line in output.split('\n'): - - if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) - elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) - - else: - tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) - update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - logging.info('Downloading update from: ' + tar_download_url) - data = request_content(tar_download_url) + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return self.CURRENT_VERSION - if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) - return + self.LATEST_VERSION = version['sha'] + logging.debug("Latest version is %s", self.LATEST_VERSION) - download_name = settings.general.branch + '-github' - tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + # See how many commits behind we are + if not self.CURRENT_VERSION: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return self.LATEST_VERSION - # Save tar to disk - with open(tar_download_path, 'wb') as f: - f.write(data) + if self.LATEST_VERSION == self.CURRENT_VERSION: + logging.info('Bazarr is up to date') + return self.LATEST_VERSION - # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) - tar = tarfile.open(tar_download_path) - tar.extractall(update_dir) - tar.close() + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (self.LATEST_VERSION, + self.CURRENT_VERSION) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) - os.remove(tar_download_path) + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return self.LATEST_VERSION - # Find update dir name - update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] - if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) - return - content_dir = os.path.join(update_dir, update_dir_contents[0]) + try: + self.COMMITS_BEHIND = int(commits['behind_by']) + logging.debug("In total, %d commits behind", self.COMMITS_BEHIND) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + self.COMMITS_BEHIND = 0 - # walk temp folder and move files to main folder - for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) - - -def checkout_git_branch(): - if INSTALL_TYPE == 'git': - output, err = runGit('fetch origin') - output, err = runGit('checkout %s' % settings.general.branch) + if self.COMMITS_BEHIND > 0: + logging.info('New version is available. You are %s commits behind' % self.COMMITS_BEHIND) + + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return self.LATEST_VERSION + + if settings.general.branch == 'master': + release = next((r for r in releases if not r['prerelease']), releases[0]) + else: + release = releases[0] + self.LATEST_RELEASE = release['tag_name'] - if not output: - logging.error('Unable to change git branch.') - return + elif self.COMMITS_BEHIND == 0: + logging.info('Bazarr is up to date') - for line in output.split('\n'): - if line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to checkout from git: ' + line) - logging.info('Output: ' + str(output)) + return self.LATEST_VERSION + + def update(self): + if self.INSTALL_TYPE == 'git' and not args.release_update: + output, err = self.runGit('pull ' + 'origin' + ' ' + settings.general.branch) + + if not output: + logging.error('Unable to download latest version') + return + + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) - output, err = runGit('pull origin %s' % settings.general.branch) + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') + + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) + + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return + + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) + + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) + + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() + + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + + def checkout_git_branch(self): + if self.INSTALL_TYPE == 'git' and not args.release_update: + output, err = self.runGit('fetch origin') + output, err = self.runGit('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = self.runGit('pull origin %s' % settings.general.branch) def request_content(url, **kwargs): diff --git a/bazarr/get_args.py b/bazarr/get_args.py index b55a2cf49..2f4bec9fe 100644 --- a/bazarr/get_args.py +++ b/bazarr/get_args.py @@ -19,6 +19,8 @@ def get_args(): help="Disable update functionality (default: False)") parser.add_argument('--debug', default=False, type=bool, const=True, metavar="BOOL", nargs="?", help="Enable console debugging (default: False)") + parser.add_argument('--release-update', default=False, type=bool, const=True, metavar="BOOL", nargs="?", + help="Enable file based updater (default: False)") return parser.parse_args() diff --git a/bazarr/main.py b/bazarr/main.py index 2b0d65a91..2efa50f6b 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -58,7 +58,7 @@ from get_series import * from get_episodes import * if not args.no_update: - from check_update import check_updates + from check_update import Updater from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index ccbf20d07..c07bbbe42 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles from get_args import args if not args.no_update: - from check_update import check_updates, check_releases + from check_update import Updater, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -84,10 +84,10 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') + scheduler.add_job(Updater().check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(Updater().check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/menu.tpl b/views/menu.tpl index 4dfdc9f1b..02cfe3176 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -148,7 +148,44 @@ % elif restart_required[0] == '1':
Bazarr need to be restarted to apply changes to general settings. Click here to restart.
% end -
+ % from check_update import Updater + % if Updater().UPDATE_AVAILABLE is None: +
+
+
+ You are running an unknown version of Bazarr. + Update +
+
+
+ % elif Updater().UPDATE_AVAILABLE == 'release': +
+
+
+ A + new release ({{ Updater().LATEST_RELEASE }}) of Bazarr is available! + Update +
+
+
+ % elif Updater().UPDATE_AVAILABLE == 'commit': +
+
+
+ A + newer version of Bazarr is available!
+ You are {{ Updater().COMMITS_BEHIND }} commit{{ 's' if Updater().COMMITS_BEHIND > 1 else '' }} + behind. + Update +
+
+
+ % end +
From a26f64efc5afcbc99b16689480cdef368ef4a196 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 16 Feb 2019 16:58:35 +0100 Subject: [PATCH 21/38] Continue developing --- bazarr/check_update.py | 433 ++++++++++++++++++----------------------- bazarr/main.py | 4 +- bazarr/scheduler.py | 6 +- views/menu.tpl | 37 ---- 4 files changed, 190 insertions(+), 290 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index d8be1087c..78cf77565 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import os import platform import re @@ -12,31 +14,6 @@ from get_args import args from config import settings -# from main import bazarr_version - - - -class FakeLock(object): - """ - If no locking or request throttling is needed, use this - """ - - def __enter__(self): - """ - Do nothing on enter - """ - pass - - def __exit__(self, type, value, traceback): - """ - Do nothing on exit - """ - pass - - -fake_lock = FakeLock() - - def check_releases(): releases = [] url_releases = 'https://api.github.com/repos/morpheus65535/Bazarr/releases' @@ -58,249 +35,209 @@ def check_releases(): json.dump(releases, f) -class Updater(object): - def __init__(self): - self.bazarr_version = '7.2.0' - - self.LATEST_VERSION = '' - self.INSTALL_TYPE = '' - self.UPDATE_AVAILABLE = '' - self.COMMITS_BEHIND = '' - self.LATEST_RELEASE = '' - self.CURRENT_VERSION = '' - self.PREV_RELEASE = self.bazarr_version - self.CURRENT_VERSION, self.GIT_REMOTE, self.GIT_BRANCH = self.getVersion() +def run_git(args): + git_locations = ['git'] - def runGit(self, args): - git_locations = ['git'] - - if platform.system().lower() == 'darwin': - git_locations.append('/usr/local/git/bin/git') - - output = err = None + if platform.system().lower() == 'darwin': + git_locations.append('/usr/local/git/bin/git') + + output = err = None + + for cur_git in git_locations: + cmd = cur_git + ' ' + args - for cur_git in git_locations: - cmd = cur_git + ' ' + args - - try: - logging.debug('Trying to execute: "' + cmd + '"') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output, err = p.communicate() - output = output.strip() - - logging.debug('Git output: ' + output) - except OSError: - logging.debug('Command failed: %s', cmd) - continue + try: + logging.debug('Trying to execute: "' + cmd + '"') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + output, err = p.communicate() + output = output.strip() - if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) - output = None - elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') - output = None - elif output: - break + logging.debug('Git output: ' + output) + except OSError: + logging.debug('Command failed: %s', cmd) + continue - return (output, err) + if 'not found' in output or "not recognized as an internal or external command" in output: + logging.debug('Unable to find git with command ' + cmd) + output = None + elif 'fatal:' in output or err: + logging.error('Git returned bad info. Are you sure this is a git installation?') + output = None + elif output: + break - def getVersion(self): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: - - self.INSTALL_TYPE = 'git' - output, err = self.runGit('rev-parse HEAD') - - if not output: - logging.error('Could not find latest installed version.') - cur_commit_hash = None - - cur_commit_hash = str(output) - - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') - cur_commit_hash = None - - if settings.general.branch: - branch_name = settings.general.branch - - else: - remote_branch, err = self.runGit('rev-parse --abbrev-ref --symbolic-full-name @{u}') - remote_branch = remote_branch.rsplit('/', 1) if remote_branch else [] - if len(remote_branch) == 2: - remote_name, branch_name = remote_branch - else: - remote_name = branch_name = None - - if not remote_name and settings.general.branch: - logging.error('Could not retrieve remote name from git. Defaulting to origin.') - branch_name = settings.general.branch - - if not branch_name: - logging.error('Could not retrieve branch name from git. Defaulting to master.') - branch_name = 'master' - - return cur_commit_hash, 'origin', branch_name - - else: - self.INSTALL_TYPE = 'source' - - if self.CURRENT_VERSION: - return self.CURRENT_VERSION, 'origin', settings.general.branch - else: - return None, 'origin', settings.general.branch + return output, err + + +def check_updates(): + commits_behind = 0 + current_version = get_version() - def check_updates(self): - self.check_github() - if not self.CURRENT_VERSION: - self.UPDATE_AVAILABLE = None - elif self.COMMITS_BEHIND > 0 and settings.general.branch in ('master') and \ - ('v' + self.bazarr_version) != self.LATEST_RELEASE: - self.UPDATE_AVAILABLE = 'release' - elif self.COMMITS_BEHIND > 0 and self.CURRENT_VERSION != self.LATEST_VERSION: - self.UPDATE_AVAILABLE = 'commit' - else: - self.UPDATE_AVAILABLE = False - print self.UPDATE_AVAILABLE + # Get the latest version available from github + logging.info('Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + + if version is None: + logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + return current_version + + latest_version = version['sha'] + logging.debug("Latest version is %s", latest_version) + + # See how many commits behind we are + if not current_version: + logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + return latest_version + + if latest_version == current_version: + logging.info('Bazarr is up to date') + return latest_version + + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, + current_version) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + + if commits is None: + logging.warn('Could not get commits behind from GitHub.') + return latest_version + + try: + commits_behind = int(commits['behind_by']) + logging.debug("In total, %d commits behind", commits_behind) + except KeyError: + logging.info('Cannot compare versions. Are you running a local development version?') + commits_behind = 0 + + if commits_behind > 0: + logging.info('New version is available. You are %s commits behind' % commits_behind) + update() - def check_github(self): - self.COMMITS_BEHIND = 0 + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + + if releases is None: + logging.warn('Could not get releases from GitHub.') + return latest_version + else: + release = releases[0] + latest_release = release['tag_name'] + + if ('v' + current_version) != latest_release and args.release_update: + update() + + elif commits_behind == 0: + logging.info('Bazarr is up to date') + + return latest_version + + +def get_version(): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): - # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) + output, err = run_git('rev-parse HEAD') - if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') - return self.CURRENT_VERSION + if not output: + logging.error('Could not find latest installed version.') + cur_commit_hash = None + else: + cur_commit_hash = str(output) - self.LATEST_VERSION = version['sha'] - logging.debug("Latest version is %s", self.LATEST_VERSION) + if not re.match('^[a-z0-9]+$', cur_commit_hash): + logging.error('Output does not look like a hash, not using it.') + cur_commit_hash = None - # See how many commits behind we are - if not self.CURRENT_VERSION: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') - return self.LATEST_VERSION + return cur_commit_hash + + else: + return os.environ["BAZARR_VERSION"] + + +def update(): + if not args.release_update: + output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) - if self.LATEST_VERSION == self.CURRENT_VERSION: - logging.info('Bazarr is up to date') - return self.LATEST_VERSION + if not output: + logging.error('Unable to download latest version') + return - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (self.LATEST_VERSION, - self.CURRENT_VERSION) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + for line in output.split('\n'): + + if 'Already up-to-date.' in line: + logging.info('No update available, not updating') + logging.info('Output: ' + str(output)) + elif line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to update from git: ' + line) + logging.info('Output: ' + str(output)) + updated() + else: + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - if commits is None: - logging.warn('Could not get commits behind from GitHub.') - return self.LATEST_VERSION + logging.info('Downloading update from: ' + tar_download_url) + data = request_content(tar_download_url) - try: - self.COMMITS_BEHIND = int(commits['behind_by']) - logging.debug("In total, %d commits behind", self.COMMITS_BEHIND) - except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') - self.COMMITS_BEHIND = 0 + if not data: + logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + return - if self.COMMITS_BEHIND > 0: - logging.info('New version is available. You are %s commits behind' % self.COMMITS_BEHIND) - - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - - if releases is None: - logging.warn('Could not get releases from GitHub.') - return self.LATEST_VERSION - - if settings.general.branch == 'master': - release = next((r for r in releases if not r['prerelease']), releases[0]) - else: - release = releases[0] - self.LATEST_RELEASE = release['tag_name'] + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - elif self.COMMITS_BEHIND == 0: - logging.info('Bazarr is up to date') + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) - return self.LATEST_VERSION - - def update(self): - if self.INSTALL_TYPE == 'git' and not args.release_update: - output, err = self.runGit('pull ' + 'origin' + ' ' + settings.general.branch) - - if not output: - logging.error('Unable to download latest version') - return - - for line in output.split('\n'): - - if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) - elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) + # Extract the tar to update folder + logging.info('Extracting file: ' + tar_download_path) + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() - else: - tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) - update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - - logging.info('Downloading update from: ' + tar_download_url) - data = request_content(tar_download_url) - - if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) - return - - download_name = settings.general.branch + '-github' - tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - - # Save tar to disk - with open(tar_download_path, 'wb') as f: - f.write(data) - - # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) - tar = tarfile.open(tar_download_path) - tar.extractall(update_dir) - tar.close() - - # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) - os.remove(tar_download_path) - - # Find update dir name - update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] - if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) - return - content_dir = os.path.join(update_dir, update_dir_contents[0]) - - # walk temp folder and move files to main folder - for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) + # Delete the tar.gz + logging.info('Deleting file: ' + tar_download_path) + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + return + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + updated() + + +class FakeLock(object): + """ + If no locking or request throttling is needed, use this + """ - def checkout_git_branch(self): - if self.INSTALL_TYPE == 'git' and not args.release_update: - output, err = self.runGit('fetch origin') - output, err = self.runGit('checkout %s' % settings.general.branch) - - if not output: - logging.error('Unable to change git branch.') - return - - for line in output.split('\n'): - if line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to checkout from git: ' + line) - logging.info('Output: ' + str(output)) - - output, err = self.runGit('pull origin %s' % settings.general.branch) + def __enter__(self): + """ + Do nothing on enter + """ + pass + + def __exit__(self, type, value, traceback): + """ + Do nothing on exit + """ + pass + + +fake_lock = FakeLock() def request_content(url, **kwargs): diff --git a/bazarr/main.py b/bazarr/main.py index 2efa50f6b..72d497f9e 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,8 +57,8 @@ from get_providers import get_providers, get_providers_auth, list_throttled_prov from get_series import * from get_episodes import * -if not args.no_update: - from check_update import Updater +# if not args.no_update: +# from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index c07bbbe42..2fd547745 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -8,7 +8,7 @@ from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles from get_args import args if not args.no_update: - from check_update import Updater, check_releases + from check_update import check_updates, check_releases else: from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler @@ -84,10 +84,10 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(Updater().check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') else: - scheduler.add_job(Updater().check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/menu.tpl b/views/menu.tpl index 02cfe3176..ae06f38f8 100644 --- a/views/menu.tpl +++ b/views/menu.tpl @@ -148,43 +148,6 @@ % elif restart_required[0] == '1':
Bazarr need to be restarted to apply changes to general settings. Click here to restart.
% end - % from check_update import Updater - % if Updater().UPDATE_AVAILABLE is None: -
-
-
- You are running an unknown version of Bazarr. - Update -
-
-
- % elif Updater().UPDATE_AVAILABLE == 'release': -
-
-
- A - new release ({{ Updater().LATEST_RELEASE }}) of Bazarr is available! - Update -
-
-
- % elif Updater().UPDATE_AVAILABLE == 'commit': -
-
-
- A - newer version of Bazarr is available!
- You are {{ Updater().COMMITS_BEHIND }} commit{{ 's' if Updater().COMMITS_BEHIND > 1 else '' }} - behind. - Update -
-
-
- % end
From 65729064f4e9e72a18493d287cf7f22b5f13a268 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 20:07:21 +0100 Subject: [PATCH 22/38] add restart after update --- bazarr/check_update.py | 6 +++++- bazarr/config.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 78cf77565..f65392231 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -11,7 +11,7 @@ import sqlite3 import json from get_args import args -from config import settings +from config import settings, bazarr_url def check_releases(): @@ -375,3 +375,7 @@ def updated(): c.execute("UPDATE system SET updated = 1") conn.commit() c.close() + try: + requests.get(bazarr_url + 'restart') + except requests.ConnectionError: + logging.info('BAZARR: Restart failed, please restart Bazarr manualy') diff --git a/bazarr/config.py b/bazarr/config.py index 2d3405f04..240ceb2d1 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -104,6 +104,7 @@ settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) base_url = settings.general.base_url +bazarr_url = 'http://localhost:' + (args.port if args.port else settings.general.port) + base_url # sonarr url if settings.sonarr.getboolean('ssl'): From 87ff6fb5cc548a87acfe0af80d3f3e2b51b41568 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 22:30:43 +0100 Subject: [PATCH 23/38] Continue developing --- bazarr/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/config.py b/bazarr/config.py index 240ceb2d1..ce8a90e04 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -104,7 +104,7 @@ settings = simpleconfigparser(defaults=defaults) settings.read(os.path.join(args.config_dir, 'config', 'config.ini')) base_url = settings.general.base_url -bazarr_url = 'http://localhost:' + (args.port if args.port else settings.general.port) + base_url +bazarr_url = 'http://localhost:' + (str(args.port) if args.port else settings.general.port) + base_url # sonarr url if settings.sonarr.getboolean('ssl'): From b76acacbf03b00caa5828a2821537a590fe93e12 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 17 Feb 2019 23:40:10 +0100 Subject: [PATCH 24/38] Continue developing --- bazarr/check_update.py | 129 +++++++++++++++++++++-------------------- views/settings.tpl | 6 +- 2 files changed, 72 insertions(+), 63 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index f65392231..5255c5e6a 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,6 +12,7 @@ import json from get_args import args from config import settings, bazarr_url +from queueconfig import q4ws def check_releases(): @@ -51,17 +52,17 @@ def run_git(args): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() - - logging.debug('Git output: ' + output) + + logging.debug('BAZZAR Git output: ' + output) except OSError: - logging.debug('Command failed: %s', cmd) + logging.debug('BAZZAR Command failed: %s', cmd) continue if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('Unable to find git with command ' + cmd) + logging.debug('BAZZAR Unable to find git with command ' + cmd) output = None elif 'fatal:' in output or err: - logging.error('Git returned bad info. Are you sure this is a git installation?') + logging.error('BAZZAR Git returned bad info. Are you sure this is a git installation?') output = None elif output: break @@ -74,24 +75,28 @@ def check_updates(): current_version = get_version() # Get the latest version available from github - logging.info('Retrieving latest version information from GitHub') + logging.info('BAZZAR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: - logging.warn('Could not get the latest version from GitHub. Are you running a local development version?') + q4ws.append('BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') + logging.warn( + 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') return current_version latest_version = version['sha'] - logging.debug("Latest version is %s", latest_version) + logging.debug("BAZZAR Latest version is %s", latest_version) # See how many commits behind we are if not current_version: - logging.info('You are running an unknown version of Bazarr. Run the updater to identify your version') + q4ws.append('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') + logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return latest_version if latest_version == current_version: - logging.info('Bazarr is up to date') + q4ws.append('BAZZAR is up to date') + logging.info('BAZARR is up to date') return latest_version logging.info('Comparing currently installed version with latest GitHub version') @@ -100,35 +105,40 @@ def check_updates(): commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: - logging.warn('Could not get commits behind from GitHub.') + logging.warn('BAZARR Could not get commits behind from GitHub.') return latest_version try: commits_behind = int(commits['behind_by']) logging.debug("In total, %d commits behind", commits_behind) except KeyError: - logging.info('Cannot compare versions. Are you running a local development version?') + logging.info('BAZARR Cannot compare versions. Are you running a local development version?') commits_behind = 0 if commits_behind > 0: - logging.info('New version is available. You are %s commits behind' % commits_behind) - update() + q4ws.append('BAZARR New version is available. You are %s commits behind' % commits_behind) + logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) + if settings.general.auto_update: + update() + else: + updated(restart=False) url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: - logging.warn('Could not get releases from GitHub.') + logging.warn('BAZARR Could not get releases from GitHub.') return latest_version else: release = releases[0] latest_release = release['tag_name'] - - if ('v' + current_version) != latest_release and args.release_update: + + if ('v' + current_version) != latest_release and args.release_update and settings.general.branch == 'master': update() elif commits_behind == 0: - logging.info('Bazarr is up to date') + q4ws.append('BAZZAR is up to date') + logging.info('BAZZAR is up to date') return latest_version @@ -139,13 +149,13 @@ def get_version(): output, err = run_git('rev-parse HEAD') if not output: - logging.error('Could not find latest installed version.') + logging.error('BAZZAR Could not find latest installed version.') cur_commit_hash = None else: cur_commit_hash = str(output) if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('Output does not look like a hash, not using it.') + logging.error('BAZZAR Output does not look like a hash, not using it.') cur_commit_hash = None return cur_commit_hash @@ -159,27 +169,27 @@ def update(): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: - logging.error('Unable to download latest version') + logging.error('BAZZAR Unable to download latest version') return for line in output.split('\n'): if 'Already up-to-date.' in line: - logging.info('No update available, not updating') - logging.info('Output: ' + str(output)) + logging.info('BAZZAR No update available, not updating') + logging.info('BAZZAR Output: ' + str(output)) elif line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to update from git: ' + line) - logging.info('Output: ' + str(output)) - updated() + logging.error('BAZZAR Unable to update from git: ' + line) + logging.info('BAZZAR Output: ' + str(output)) + updated(restart=True) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - - logging.info('Downloading update from: ' + tar_download_url) + + logging.info('BAZZAR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: - logging.error("Unable to retrieve new version from '%s', can't update", tar_download_url) + logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) return download_name = settings.general.branch + '-github' @@ -190,19 +200,19 @@ def update(): f.write(data) # Extract the tar to update folder - logging.info('Extracting file: ' + tar_download_path) + logging.info('BAZZAR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz - logging.info('Deleting file: ' + tar_download_path) + logging.info('BAZZAR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: - logging.error("Invalid update data, update failed: " + str(update_dir_contents)) + logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) return content_dir = os.path.join(update_dir, update_dir_contents[0]) @@ -216,7 +226,7 @@ def update(): if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) - updated() + updated(restart=True) class FakeLock(object): @@ -281,7 +291,7 @@ def request_response(url, method="get", auto_raise=True, # Request URL and wait for response with lock: logging.debug( - "Requesting URL via %s method: %s", method.upper(), url) + "BAZZAR Requesting URL via %s method: %s", method.upper(), url) response = request_method(url, **kwargs) # If status code != OK, then raise exception, except if the status code @@ -292,7 +302,7 @@ def request_response(url, method="get", auto_raise=True, response.raise_for_status() except: logging.debug( - "Response status code %d is not white " + "BAZZAR Response status code %d is not white " "listed, raised exception", response.status_code) raise elif auto_raise: @@ -302,22 +312,22 @@ def request_response(url, method="get", auto_raise=True, except requests.exceptions.SSLError as e: if kwargs["verify"]: logging.error( - "Unable to connect to remote host because of a SSL error. " + "BAZZAR Unable to connect to remote host because of a SSL error. " "It is likely that your system cannot verify the validity" "of the certificate. The remote certificate is either " "self-signed, or the remote server uses SNI. See the wiki for " "more information on this topic.") else: logging.error( - "SSL error raised during connection, with certificate " + "BAZZAR SSL error raised during connection, with certificate " "verification turned off: %s", e) except requests.ConnectionError: logging.error( - "Unable to connect to remote host. Check if the remote " + "BAZZAR Unable to connect to remote host. Check if the remote " "host is up and running.") except requests.Timeout: logging.error( - "Request timed out. The remote host did not respond timely.") + "BAZZAR Request timed out. The remote host did not respond timely.") except requests.HTTPError as e: if e.response is not None: if e.response.status_code >= 500: @@ -329,16 +339,12 @@ def request_response(url, method="get", auto_raise=True, cause = "unknown" logging.error( - "Request raise HTTP error with status code %d (%s).", + "BAZZAR Request raise HTTP error with status code %d (%s).", e.response.status_code, cause) - - # Debug response - # if bazarr.DEBUG: - # server_message(e.response) else: - logging.error("Request raised HTTP error.") + logging.error("BAZZAR Request raised HTTP error.") except requests.RequestException as e: - logging.error("Request raised exception: %s", e) + logging.error("BAZZAR Request raised exception: %s", e) def request_json(url, **kwargs): @@ -358,24 +364,23 @@ def request_json(url, **kwargs): result = response.json() if validator and not validator(result): - logging.error("JSON validation result failed") + logging.error("BAZZAR JSON validation result failed") else: return result except ValueError: - logging.error("Response returned invalid JSON data") - - # Debug response - # if bazarr.DEBUG: - # server_message(response) + logging.error("BAZZAR Response returned invalid JSON data") -def updated(): - conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) - c = conn.cursor() - c.execute("UPDATE system SET updated = 1") - conn.commit() - c.close() - try: - requests.get(bazarr_url + 'restart') - except requests.ConnectionError: - logging.info('BAZARR: Restart failed, please restart Bazarr manualy') +def updated(restart=False): + if restart: + try: + requests.get(bazarr_url + 'restart') + except requests.ConnectionError: + logging.info('BAZARR Restart failed, please restart Bazarr manualy') + updated(restart=False) + else: + conn = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'), timeout=30) + c = conn.cursor() + c.execute("UPDATE system SET updated = 1") + conn.commit() + c.close() diff --git a/views/settings.tpl b/views/settings.tpl index 29072aeff..af008f369 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -2037,7 +2037,11 @@ }); % from get_args import args - % if args.no_update: + + % + if args.no_update or + args.release_update + : $("#div_update").hide(); % end % import sys From 49b884cde78f28b5afb0813637789cea4dd6d44f Mon Sep 17 00:00:00 2001 From: Halali Date: Mon, 18 Feb 2019 12:44:33 +0100 Subject: [PATCH 25/38] Continue developing --- bazarr/check_update.py | 4 ++-- bazarr/main.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 5255c5e6a..38bcd28cb 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,7 +12,7 @@ import json from get_args import args from config import settings, bazarr_url -from queueconfig import q4ws +from queueconfig import q4ws, q4ws_updater def check_releases(): @@ -137,7 +137,7 @@ def check_updates(): update() elif commits_behind == 0: - q4ws.append('BAZZAR is up to date') + q4ws_updater.append('BAZZAR is up to date') logging.info('BAZZAR is up to date') return latest_version diff --git a/bazarr/main.py b/bazarr/main.py index 72d497f9e..601156c0f 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -2010,6 +2010,26 @@ def running_tasks_list(): return dict(tasks=running_tasks) +@route(base_url + 'websocket_updater') +@custom_auth_basic(check_credentials) +def handle_websocket(): + wsock = request.environ.get('wsgi.websocket') + if not wsock: + abort(400, 'Expected WebSocket request.') + + queueconfig.q4ws_updater.clear() + + while True: + try: + if queueconfig.q4ws_updater: + wsock.send(queueconfig.q4ws_updater.popleft()) + gevent.sleep(0.1) + else: + gevent.sleep(0.5) + except WebSocketError: + break + + # Mute DeprecationWarning warnings.simplefilter("ignore", DeprecationWarning) server = WSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app, handler_class=WebSocketHandler) From 6fd700772949ffe04bca927d3408742bec60b69c Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 24 Feb 2019 19:31:18 +0100 Subject: [PATCH 26/38] Continue developing --- bazarr/check_update.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 38bcd28cb..39f72b952 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -12,7 +12,7 @@ import json from get_args import args from config import settings, bazarr_url -from queueconfig import q4ws, q4ws_updater +from queueconfig import notifications def check_releases(): @@ -80,7 +80,6 @@ def check_updates(): version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: - q4ws.append('BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') logging.warn( 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') return current_version @@ -90,12 +89,10 @@ def check_updates(): # See how many commits behind we are if not current_version: - q4ws.append('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return latest_version if latest_version == current_version: - q4ws.append('BAZZAR is up to date') logging.info('BAZARR is up to date') return latest_version @@ -116,7 +113,6 @@ def check_updates(): commits_behind = 0 if commits_behind > 0: - q4ws.append('BAZARR New version is available. You are %s commits behind' % commits_behind) logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) if settings.general.auto_update: update() @@ -137,7 +133,6 @@ def check_updates(): update() elif commits_behind == 0: - q4ws_updater.append('BAZZAR is up to date') logging.info('BAZZAR is up to date') return latest_version From 5b1414f81f3643e267f381f44fbb55b40c7113f8 Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 5 Mar 2019 18:41:06 +0100 Subject: [PATCH 27/38] Continue developing --- bazarr/check_update.py | 136 ++++++++++++++++++++++------------------- bazarr/config.py | 4 +- views/settings.tpl | 28 +++++++++ 3 files changed, 105 insertions(+), 63 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 39f72b952..bf0694821 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -48,7 +48,7 @@ def run_git(args): cmd = cur_git + ' ' + args try: - logging.debug('Trying to execute: "' + cmd + '"') + logging.debug('BAZZAR Trying to execute: "' + cmd + '"') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() @@ -72,74 +72,78 @@ def run_git(args): def check_updates(): commits_behind = 0 - current_version = get_version() - - # Get the latest version available from github - logging.info('BAZZAR Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - - if version is None: - logging.warn( - 'BAZZAR Could not get the latest version from GitHub. Are you running a local development version?') - return current_version - - latest_version = version['sha'] - logging.debug("BAZZAR Latest version is %s", latest_version) + current_version, source = get_version() + + if source == 'git': + # Get the latest version available from github + logging.info('BAZZAR Retrieving latest version information from GitHub') + url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch + version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - # See how many commits behind we are - if not current_version: - logging.info('BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') - return latest_version + if version is None: + logging.warn( + 'BAZZAR Could not get the latest version from GitHub.') + return - if latest_version == current_version: - logging.info('BAZARR is up to date') - return latest_version + latest_version = version['sha'] + logging.debug("BAZZAR Latest version is %s", latest_version) - logging.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, - current_version) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + # See how many commits behind we are + if not current_version: + logging.info( + 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') + return - if commits is None: - logging.warn('BAZARR Could not get commits behind from GitHub.') - return latest_version + if latest_version == current_version: + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') + return - try: - commits_behind = int(commits['behind_by']) - logging.debug("In total, %d commits behind", commits_behind) - except KeyError: - logging.info('BAZARR Cannot compare versions. Are you running a local development version?') - commits_behind = 0 + logging.info('Comparing currently installed version with latest GitHub version') + url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, + current_version) + commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - if commits_behind > 0: - logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) - if settings.general.auto_update: - update() - else: - updated(restart=False) + if commits is None: + logging.warn('BAZARR Could not get commits behind from GitHub.') + return - url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' - releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) + try: + commits_behind = int(commits['behind_by']) + logging.debug("BAZARR In total, %d commits behind", commits_behind) + except KeyError: + logging.info('BAZARR Cannot compare versions. Are you running a local development version?') + commits_behind = 0 - if releases is None: - logging.warn('BAZARR Could not get releases from GitHub.') - return latest_version - else: - release = releases[0] - latest_release = release['tag_name'] + if commits_behind > 0: + logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) + notifications.write(msg='BAZARR New version is available. You are %s commits behind' % commits_behind, + queue='check_update') + update(source, restart=True if settings.general.getboolean('update_restart') else False) - if ('v' + current_version) != latest_release and args.release_update and settings.general.branch == 'master': - update() + else: + url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' + releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - elif commits_behind == 0: - logging.info('BAZZAR is up to date') + if releases is None: + logging.warn('BAZARR Could not get releases from GitHub.') + return + else: + release = releases[0] + latest_release = release['tag_name'] - return latest_version + if ('v' + current_version) != latest_release and settings.general.branch == 'master': + update(source, restart=True if settings.general.getboolean('update_restart') else False) + elif settings.general.branch != 'master': + notifications.write(msg="BAZZAR Can't update development branch from source", queue='check_update') # fixme + logging.info("BAZZAR Can't update development branch from source") # fixme + else: + notifications.write(msg='BAZZAR is up to date', queue='check_update') + logging.info('BAZZAR is up to date') def get_version(): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')): + if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: output, err = run_git('rev-parse HEAD') @@ -152,15 +156,15 @@ def get_version(): if not re.match('^[a-z0-9]+$', cur_commit_hash): logging.error('BAZZAR Output does not look like a hash, not using it.') cur_commit_hash = None - - return cur_commit_hash + + return cur_commit_hash, 'git' else: - return os.environ["BAZARR_VERSION"] + return os.environ["BAZARR_VERSION"], 'source' -def update(): - if not args.release_update: +def update(source, restart=True): + if source == 'git': output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: @@ -175,16 +179,19 @@ def update(): elif line.endswith(('Aborting', 'Aborting.')): logging.error('BAZZAR Unable to update from git: ' + line) logging.info('BAZZAR Output: ' + str(output)) - updated(restart=True) + updated(restart) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') logging.info('BAZZAR Downloading update from: ' + tar_download_url) + notifications.write(msg='BAZZAR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) + notifications.write(msg=("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url), + type='error') return download_name = settings.general.branch + '-github' @@ -196,19 +203,24 @@ def update(): # Extract the tar to update folder logging.info('BAZZAR Extracting file: ' + tar_download_path) + notifications.write(msg='BAZZAR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz logging.info('BAZZAR Deleting file: ' + tar_download_path) + notifications.write(msg='BAZZAR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) + notifications.write(msg="BAZZAR Invalid update data, update failed: " + str(update_dir_contents), + type='error') return + content_dir = os.path.join(update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder @@ -221,7 +233,7 @@ def update(): if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) - updated(restart=True) + updated(restart) class FakeLock(object): diff --git a/bazarr/config.py b/bazarr/config.py index ce8a90e04..42a263011 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -41,7 +41,9 @@ defaults = { 'subfolder_custom': '', 'upgrade_subs': 'True', 'days_to_upgrade_subs': '7', - 'upgrade_manual': 'True' + 'upgrade_manual': 'True', + 'subfolder_custom': '', + 'update_restart': 'True' }, 'auth': { 'type': 'None', diff --git a/views/settings.tpl b/views/settings.tpl index af008f369..68a169072 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -703,6 +703,28 @@
+ +
+
+ +
+
+
+ + +
+
+ +
@@ -2080,6 +2102,12 @@ $("#settings_automatic_div").checkbox('uncheck'); } + if ($('#settings_restart_update').data("restart_update") === "True") { + $("#settings_restart_update").checkbox('check'); + } else { + $("#settings_restart_update").checkbox('uncheck'); + } + if ($('#settings_debug').data("debug") === "True") { $("#settings_debug").checkbox('check'); } else { From fa1ea0eea7d8eb4233e50da1e6e58fe3de246f46 Mon Sep 17 00:00:00 2001 From: Halali Date: Sat, 9 Mar 2019 09:37:47 +0100 Subject: [PATCH 28/38] Continue developing --- bazarr/check_update.py | 1 + bazarr/main.py | 23 ----------------------- bazarr/scheduler.py | 6 ++---- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index bf0694821..d0628faa2 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -73,6 +73,7 @@ def run_git(args): def check_updates(): commits_behind = 0 current_version, source = get_version() + check_releases() if source == 'git': # Get the latest version available from github diff --git a/bazarr/main.py b/bazarr/main.py index 601156c0f..8553db57a 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,8 +57,6 @@ from get_providers import get_providers, get_providers_auth, list_throttled_prov from get_series import * from get_episodes import * -# if not args.no_update: -# from check_update import check_updates from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ @@ -2009,27 +2007,6 @@ def notifications(): def running_tasks_list(): return dict(tasks=running_tasks) - -@route(base_url + 'websocket_updater') -@custom_auth_basic(check_credentials) -def handle_websocket(): - wsock = request.environ.get('wsgi.websocket') - if not wsock: - abort(400, 'Expected WebSocket request.') - - queueconfig.q4ws_updater.clear() - - while True: - try: - if queueconfig.q4ws_updater: - wsock.send(queueconfig.q4ws_updater.popleft()) - gevent.sleep(0.1) - else: - gevent.sleep(0.5) - except WebSocketError: - break - - # Mute DeprecationWarning warnings.simplefilter("ignore", DeprecationWarning) server = WSGIServer((str(settings.general.ip), (int(args.port) if args.port else int(settings.general.port))), app, handler_class=WebSocketHandler) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 2fd547745..f23e47af4 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -7,10 +7,8 @@ from config import settings from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles from get_args import args -if not args.no_update: - from check_update import check_updates, check_releases -else: - from check_update import check_releases + +from check_update import check_updates, check_releases from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from apscheduler.triggers.cron import CronTrigger From b5b212ad7649162f91267980a637ebaf776ffe9e Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 12 Mar 2019 18:12:53 +0100 Subject: [PATCH 29/38] Fix bad code format --- views/settings.tpl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/views/settings.tpl b/views/settings.tpl index 68a169072..be2853768 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -2060,10 +2060,7 @@ % from get_args import args - % - if args.no_update or - args.release_update - : + % if args.no_update or args.release_update: $("#div_update").hide(); % end % import sys From 8b5a04cb279be45a53f86a1bb7ed7f875b9e55a1 Mon Sep 17 00:00:00 2001 From: Halali Date: Tue, 12 Mar 2019 21:58:09 +0100 Subject: [PATCH 30/38] Continue developing --- bazarr/check_update.py | 25 ++++++++++++++++++++++--- views/settings.tpl | 5 ++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index d0628faa2..22b743827 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -77,7 +77,7 @@ def check_updates(): if source == 'git': # Get the latest version available from github - logging.info('BAZZAR Retrieving latest version information from GitHub') + logging.debug('BAZZAR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) @@ -91,7 +91,7 @@ def check_updates(): # See how many commits behind we are if not current_version: - logging.info( + logging.warn( 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return @@ -100,7 +100,7 @@ def check_updates(): logging.info('BAZARR is up to date') return - logging.info('Comparing currently installed version with latest GitHub version') + logging.debug('Comparing currently installed version with latest GitHub version') url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, current_version) commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) @@ -121,6 +121,9 @@ def check_updates(): notifications.write(msg='BAZARR New version is available. You are %s commits behind' % commits_behind, queue='check_update') update(source, restart=True if settings.general.getboolean('update_restart') else False) + elif commits_behind is 0: + notifications.write(msg='BAZZAR is up to date', queue='check_update') + logging.info('BAZZAR is up to date') else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' @@ -237,6 +240,22 @@ def update(source, restart=True): updated(restart) +def checkout_git_branch(): + output, err = run_git('fetch %s' % 'origin') + output, err = run_git('checkout %s' % settings.general.branch) + + if not output: + logging.error('Unable to change git branch.') + return + + for line in output.split('\n'): + if line.endswith(('Aborting', 'Aborting.')): + logging.error('Unable to checkout from git: ' + line) + logging.info('Output: ' + str(output)) + + output, err = run_git('pull %s %s' % ('origin', settings.general.branch)) + + class FakeLock(object): """ If no locking or request throttling is needed, use this diff --git a/views/settings.tpl b/views/settings.tpl index be2853768..3df9f406f 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -665,7 +665,7 @@
Updates
-
+
@@ -2059,9 +2059,12 @@ }); % from get_args import args + % from check_update import get_version % if args.no_update or args.release_update: $("#div_update").hide(); + % elif get_version()[1] != 'git': + $("#div_branch").hide(); % end % import sys % if sys.platform.startswith('win'): From 7d5bc71b325070feecd1fe708a76eae13b356484 Mon Sep 17 00:00:00 2001 From: Halali Date: Wed, 13 Mar 2019 23:02:37 +0100 Subject: [PATCH 31/38] Continue developing --- bazarr/scheduler.py | 5 +++-- views/settings.tpl | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index f23e47af4..74226d3a2 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -83,12 +83,13 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) if not args.no_update: if settings.general.getboolean('auto_update'): scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github') + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') else: scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', - name='Update bazarr from source on Github') + name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') + else: scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info') diff --git a/views/settings.tpl b/views/settings.tpl index 3df9f406f..34a8bf8aa 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -2061,9 +2061,9 @@ % from get_args import args % from check_update import get_version - % if args.no_update or args.release_update: + % if args.no_update: $("#div_update").hide(); - % elif get_version()[1] != 'git': + % elif get_version()[1] != 'git' or args.release_update: $("#div_branch").hide(); % end % import sys From caf4ede82af844a8dec106dfde582290c8b3ec06 Mon Sep 17 00:00:00 2001 From: Halali Date: Thu, 14 Mar 2019 18:01:33 +0100 Subject: [PATCH 32/38] Add missed code for save settings --- bazarr/main.py | 6 ++++++ views/settings.tpl | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bazarr/main.py b/bazarr/main.py index 8553db57a..13f27f491 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1216,6 +1216,11 @@ def save_settings(): settings_general_automatic = 'False' else: settings_general_automatic = 'True' + settings_general_update_restart = request.forms.get('settings_general_update_restart') + if settings_general_update_restart is None: + settings_general_update_restart = 'False' + else: + settings_general_update_restart = 'True' settings_general_single_language = request.forms.get('settings_general_single_language') if settings_general_single_language is None: settings_general_single_language = 'False' @@ -1290,6 +1295,7 @@ def save_settings(): settings.general.chmod = text_type(settings_general_chmod) settings.general.branch = text_type(settings_general_branch) settings.general.auto_update = text_type(settings_general_automatic) + settings.general.update_restart = text_type(settings_general_update_restart) settings.general.single_language = text_type(settings_general_single_language) settings.general.minimum_score = text_type(settings_general_minimum_score) settings.general.use_scenename = text_type(settings_general_scenename) diff --git a/views/settings.tpl b/views/settings.tpl index 34a8bf8aa..ab95eb0d9 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -710,7 +710,7 @@
+ data-update-restart={{settings.general.getboolean('update_restart')}}>
@@ -2102,10 +2102,10 @@ $("#settings_automatic_div").checkbox('uncheck'); } - if ($('#settings_restart_update').data("restart_update") === "True") { - $("#settings_restart_update").checkbox('check'); + if ($('#settings_update_restart').data("update-restart") === "True") { + $("#settings_update_restart").checkbox('check'); } else { - $("#settings_restart_update").checkbox('uncheck'); + $("#settings_update_restart").checkbox('uncheck'); } if ($('#settings_debug').data("debug") === "True") { From b3a9563245ca9917646095a923412be01eddddbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 14 Mar 2019 20:57:48 -0400 Subject: [PATCH 33/38] Fix typo (BAZZAR vs BAZARR) --- bazarr/check_update.py | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 22b743827..f0b1cdd51 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -48,21 +48,21 @@ def run_git(args): cmd = cur_git + ' ' + args try: - logging.debug('BAZZAR Trying to execute: "' + cmd + '"') + logging.debug('BAZARR Trying to execute: "' + cmd + '"') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) output, err = p.communicate() output = output.strip() - logging.debug('BAZZAR Git output: ' + output) + logging.debug('BAZARR Git output: ' + output) except OSError: - logging.debug('BAZZAR Command failed: %s', cmd) + logging.debug('BAZARR Command failed: %s', cmd) continue if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('BAZZAR Unable to find git with command ' + cmd) + logging.debug('BAZARR Unable to find git with command ' + cmd) output = None elif 'fatal:' in output or err: - logging.error('BAZZAR Git returned bad info. Are you sure this is a git installation?') + logging.error('BAZARR Git returned bad info. Are you sure this is a git installation?') output = None elif output: break @@ -77,17 +77,17 @@ def check_updates(): if source == 'git': # Get the latest version available from github - logging.debug('BAZZAR Retrieving latest version information from GitHub') + logging.debug('BAZARR Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: logging.warn( - 'BAZZAR Could not get the latest version from GitHub.') + 'BAZARR Could not get the latest version from GitHub.') return latest_version = version['sha'] - logging.debug("BAZZAR Latest version is %s", latest_version) + logging.debug("BAZARR Latest version is %s", latest_version) # See how many commits behind we are if not current_version: @@ -122,8 +122,8 @@ def check_updates(): queue='check_update') update(source, restart=True if settings.general.getboolean('update_restart') else False) elif commits_behind is 0: - notifications.write(msg='BAZZAR is up to date', queue='check_update') - logging.info('BAZZAR is up to date') + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' @@ -139,11 +139,11 @@ def check_updates(): if ('v' + current_version) != latest_release and settings.general.branch == 'master': update(source, restart=True if settings.general.getboolean('update_restart') else False) elif settings.general.branch != 'master': - notifications.write(msg="BAZZAR Can't update development branch from source", queue='check_update') # fixme - logging.info("BAZZAR Can't update development branch from source") # fixme + notifications.write(msg="BAZARR Can't update development branch from source", queue='check_update') # fixme + logging.info("BAZARR Can't update development branch from source") # fixme else: - notifications.write(msg='BAZZAR is up to date', queue='check_update') - logging.info('BAZZAR is up to date') + notifications.write(msg='BAZARR is up to date', queue='check_update') + logging.info('BAZARR is up to date') def get_version(): @@ -152,13 +152,13 @@ def get_version(): output, err = run_git('rev-parse HEAD') if not output: - logging.error('BAZZAR Could not find latest installed version.') + logging.error('BAZARR Could not find latest installed version.') cur_commit_hash = None else: cur_commit_hash = str(output) if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('BAZZAR Output does not look like a hash, not using it.') + logging.error('BAZARR Output does not look like a hash, not using it.') cur_commit_hash = None return cur_commit_hash, 'git' @@ -172,29 +172,29 @@ def update(source, restart=True): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: - logging.error('BAZZAR Unable to download latest version') + logging.error('BAZARR Unable to download latest version') return for line in output.split('\n'): if 'Already up-to-date.' in line: - logging.info('BAZZAR No update available, not updating') - logging.info('BAZZAR Output: ' + str(output)) + logging.info('BAZARR No update available, not updating') + logging.info('BAZARR Output: ' + str(output)) elif line.endswith(('Aborting', 'Aborting.')): - logging.error('BAZZAR Unable to update from git: ' + line) - logging.info('BAZZAR Output: ' + str(output)) + logging.error('BAZARR Unable to update from git: ' + line) + logging.info('BAZARR Output: ' + str(output)) updated(restart) else: tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - logging.info('BAZZAR Downloading update from: ' + tar_download_url) - notifications.write(msg='BAZZAR Downloading update from: ' + tar_download_url) + logging.info('BAZARR Downloading update from: ' + tar_download_url) + notifications.write(msg='BAZARR Downloading update from: ' + tar_download_url) data = request_content(tar_download_url) if not data: - logging.error("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url) - notifications.write(msg=("BAZZAR Unable to retrieve new version from '%s', can't update", tar_download_url), + logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url) + notifications.write(msg=("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url), type='error') return @@ -206,22 +206,22 @@ def update(source, restart=True): f.write(data) # Extract the tar to update folder - logging.info('BAZZAR Extracting file: ' + tar_download_path) - notifications.write(msg='BAZZAR Extracting file: ' + tar_download_path) + logging.info('BAZARR Extracting file: ' + tar_download_path) + notifications.write(msg='BAZARR Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz - logging.info('BAZZAR Deleting file: ' + tar_download_path) - notifications.write(msg='BAZZAR Deleting file: ' + tar_download_path) + logging.info('BAZARR Deleting file: ' + tar_download_path) + notifications.write(msg='BAZARR Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: - logging.error("BAZZAR Invalid update data, update failed: " + str(update_dir_contents)) - notifications.write(msg="BAZZAR Invalid update data, update failed: " + str(update_dir_contents), + logging.error("BAZARR Invalid update data, update failed: " + str(update_dir_contents)) + notifications.write(msg="BAZARR Invalid update data, update failed: " + str(update_dir_contents), type='error') return @@ -318,7 +318,7 @@ def request_response(url, method="get", auto_raise=True, # Request URL and wait for response with lock: logging.debug( - "BAZZAR Requesting URL via %s method: %s", method.upper(), url) + "BAZARR Requesting URL via %s method: %s", method.upper(), url) response = request_method(url, **kwargs) # If status code != OK, then raise exception, except if the status code @@ -329,7 +329,7 @@ def request_response(url, method="get", auto_raise=True, response.raise_for_status() except: logging.debug( - "BAZZAR Response status code %d is not white " + "BAZARR Response status code %d is not white " "listed, raised exception", response.status_code) raise elif auto_raise: @@ -339,22 +339,22 @@ def request_response(url, method="get", auto_raise=True, except requests.exceptions.SSLError as e: if kwargs["verify"]: logging.error( - "BAZZAR Unable to connect to remote host because of a SSL error. " + "BAZARR Unable to connect to remote host because of a SSL error. " "It is likely that your system cannot verify the validity" "of the certificate. The remote certificate is either " "self-signed, or the remote server uses SNI. See the wiki for " "more information on this topic.") else: logging.error( - "BAZZAR SSL error raised during connection, with certificate " + "BAZARR SSL error raised during connection, with certificate " "verification turned off: %s", e) except requests.ConnectionError: logging.error( - "BAZZAR Unable to connect to remote host. Check if the remote " + "BAZARR Unable to connect to remote host. Check if the remote " "host is up and running.") except requests.Timeout: logging.error( - "BAZZAR Request timed out. The remote host did not respond timely.") + "BAZARR Request timed out. The remote host did not respond timely.") except requests.HTTPError as e: if e.response is not None: if e.response.status_code >= 500: @@ -366,12 +366,12 @@ def request_response(url, method="get", auto_raise=True, cause = "unknown" logging.error( - "BAZZAR Request raise HTTP error with status code %d (%s).", + "BAZARR Request raise HTTP error with status code %d (%s).", e.response.status_code, cause) else: - logging.error("BAZZAR Request raised HTTP error.") + logging.error("BAZARR Request raised HTTP error.") except requests.RequestException as e: - logging.error("BAZZAR Request raised exception: %s", e) + logging.error("BAZARR Request raised exception: %s", e) def request_json(url, **kwargs): @@ -391,11 +391,11 @@ def request_json(url, **kwargs): result = response.json() if validator and not validator(result): - logging.error("BAZZAR JSON validation result failed") + logging.error("BAZARR JSON validation result failed") else: return result except ValueError: - logging.error("BAZZAR Response returned invalid JSON data") + logging.error("BAZARR Response returned invalid JSON data") def updated(restart=False): From dd2cdd6c2955203472ec019ba4700b1d6d76e98b Mon Sep 17 00:00:00 2001 From: Halali Date: Wed, 20 Mar 2019 23:42:19 +0100 Subject: [PATCH 34/38] Some logging improvements --- bazarr/check_update.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index f0b1cdd51..febdfd268 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -82,6 +82,9 @@ def check_updates(): version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: + notifications.write( + msg='BAZARR Could not get the latest version from GitHub.', + queue='check_update', type='warning') logging.warn( 'BAZARR Could not get the latest version from GitHub.') return @@ -91,6 +94,8 @@ def check_updates(): # See how many commits behind we are if not current_version: + notifications.write(msg='BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version', + queue='check_update', type='warning') logging.warn( 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') return @@ -106,6 +111,8 @@ def check_updates(): commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: + notifications.write(msg='BAZARR Could not get commits behind from GitHub.', + queue='check_update', type='warning') logging.warn('BAZARR Could not get commits behind from GitHub.') return @@ -113,8 +120,9 @@ def check_updates(): commits_behind = int(commits['behind_by']) logging.debug("BAZARR In total, %d commits behind", commits_behind) except KeyError: + notifications.write(msg='BAZARR Cannot compare versions. Are you running a local development version?', queue='check_update') logging.info('BAZARR Cannot compare versions. Are you running a local development version?') - commits_behind = 0 + return if commits_behind > 0: logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) @@ -130,6 +138,8 @@ def check_updates(): releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: + notifications.write(msg='BAZARR Could not get releases from GitHub.', + queue='check_update', type='warning') logging.warn('BAZARR Could not get releases from GitHub.') return else: @@ -172,6 +182,8 @@ def update(source, restart=True): output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) if not output: + notifications.write(msg='Unable to download latest version', + queue='check_update', type='error') logging.error('BAZARR Unable to download latest version') return From cb7e35a992b4ba18ac40b58fa35a65a90ba7443e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 20 Mar 2019 19:50:26 -0400 Subject: [PATCH 35/38] Rescheduling of Bazarr update task when saving settings. --- bazarr/main.py | 4 +--- bazarr/scheduler.py | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/bazarr/main.py b/bazarr/main.py index 13f27f491..9ce773e21 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -1531,14 +1531,12 @@ def save_settings(): conn.commit() c.close() + schedule_update_job() sonarr_full_update() radarr_full_update() logging.info('BAZARR Settings saved succesfully.') - # reschedule full update task according to settings - sonarr_full_update() - if ref.find('saved=true') > 0: redirect(ref) else: diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 74226d3a2..8c8ba94d7 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -80,19 +80,21 @@ def task_listener(event): scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) -if not args.no_update: - if settings.general.getboolean('auto_update'): - scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') +def schedule_update_job(): + if not args.no_update: + if settings.general.getboolean('auto_update'): + scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) + else: + scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) + scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + misfire_grace_time=15, id='update_release', name='Update release info', replace_existing=True) + else: - scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', - name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github') - scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, - misfire_grace_time=15, id='update_release', name='Update release info') + scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, + id='update_release', name='Update release info', replace_existing=True) -else: - scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, - id='update_release', name='Update release info') if settings.general.getboolean('use_sonarr'): scheduler.add_job(update_series, IntervalTrigger(minutes=1), max_instances=1, coalesce=True, misfire_grace_time=15, @@ -112,6 +114,7 @@ if settings.general.getboolean('upgrade_subs'): scheduler.add_job(upgrade_subtitles, IntervalTrigger(hours=12), max_instances=1, coalesce=True, misfire_grace_time=15, id='upgrade_subtitles', name='Upgrade previously downloaded subtitles') +schedule_update_job() sonarr_full_update() radarr_full_update() scheduler.start() From 4eabaa869ec8ceaf38bf0d42c19f9b59dd6b4c77 Mon Sep 17 00:00:00 2001 From: Halali Date: Sun, 31 Mar 2019 22:10:46 +0200 Subject: [PATCH 36/38] Continue developing --- bazarr/check_update.py | 3 +-- bazarr/main.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index febdfd268..48ff2f529 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -132,7 +132,6 @@ def check_updates(): elif commits_behind is 0: notifications.write(msg='BAZARR is up to date', queue='check_update') logging.info('BAZARR is up to date') - else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) @@ -253,7 +252,7 @@ def update(source, restart=True): def checkout_git_branch(): - output, err = run_git('fetch %s' % 'origin') + output, err = run_git('fetch origin') output, err = run_git('checkout %s' % settings.general.branch) if not output: diff --git a/bazarr/main.py b/bazarr/main.py index 9ce773e21..42039f035 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -67,6 +67,7 @@ from notifier import send_notifications, send_notifications_movie from config import settings, url_sonarr, url_radarr, url_radarr_short, url_sonarr_short, base_url from helper import path_replace_movie, get_subtitle_destination_folder from subliminal_patch.extensions import provider_registry as provider_manager +from check_update import checkout_git_branch reload(sys) sys.setdefaultencoding('utf8') @@ -1285,6 +1286,7 @@ def save_settings(): after = (unicode(settings_general_ip), int(settings_general_port), unicode(settings_general_baseurl), unicode(settings_general_pathmapping), unicode(settings_general_use_sonarr), unicode(settings_general_use_radarr), unicode(settings_general_pathmapping_movie)) + updater_before = settings.general.branch settings.general.ip = text_type(settings_general_ip) settings.general.port = text_type(settings_general_port) @@ -1536,6 +1538,9 @@ def save_settings(): radarr_full_update() logging.info('BAZARR Settings saved succesfully.') + + if updater_before != settings_general_branch: + checkout_git_branch() if ref.find('saved=true') > 0: redirect(ref) From 1a0e316dfc6191e42190aad339f64838d1b56521 Mon Sep 17 00:00:00 2001 From: Halali Date: Wed, 3 Apr 2019 19:08:43 +0200 Subject: [PATCH 37/38] Continue developing --- bazarr/check_update.py | 339 ++++++++++++++--------------------------- bazarr/main.py | 8 +- bazarr/scheduler.py | 11 +- views/settings.tpl | 3 +- 4 files changed, 122 insertions(+), 239 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 48ff2f529..17b9fc00d 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -1,141 +1,61 @@ # coding=utf-8 - import os -import platform -import re -import subprocess -import tarfile import logging -import requests import sqlite3 import json +import requests +import tarfile from get_args import args from config import settings, bazarr_url from queueconfig import notifications +if not args.no_update: + import git -def check_releases(): - releases = [] - url_releases = 'https://api.github.com/repos/morpheus65535/Bazarr/releases' - try: - r = requests.get(url_releases, timeout=15) - r.raise_for_status() - except requests.exceptions.HTTPError as errh: - logging.exception("Error trying to get releases from Github. Http error.") - except requests.exceptions.ConnectionError as errc: - logging.exception("Error trying to get releases from Github. Connection Error.") - except requests.exceptions.Timeout as errt: - logging.exception("Error trying to get releases from Github. Timeout Error.") - except requests.exceptions.RequestException as err: - logging.exception("Error trying to get releases from Github.") - else: - for release in r.json(): - releases.append([release['name'], release['body']]) - with open(os.path.join(args.config_dir, 'config', 'releases.txt'), 'w') as f: - json.dump(releases, f) +current_working_directory = os.path.dirname(os.path.dirname(__file__)) -def run_git(args): - git_locations = ['git'] - - if platform.system().lower() == 'darwin': - git_locations.append('/usr/local/git/bin/git') +def gitconfig(): + g = git.Repo.init(current_working_directory) + config_read = g.config_reader() + config_write = g.config_writer() - output = err = None - - for cur_git in git_locations: - cmd = cur_git + ' ' + args - - try: - logging.debug('BAZARR Trying to execute: "' + cmd + '"') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - output, err = p.communicate() - output = output.strip() - - logging.debug('BAZARR Git output: ' + output) - except OSError: - logging.debug('BAZARR Command failed: %s', cmd) - continue - - if 'not found' in output or "not recognized as an internal or external command" in output: - logging.debug('BAZARR Unable to find git with command ' + cmd) - output = None - elif 'fatal:' in output or err: - logging.error('BAZARR Git returned bad info. Are you sure this is a git installation?') - output = None - elif output: - break + try: + username = config_read.get_value("user", "name") + except: + logging.debug('BAZARR Settings git username') + config_write.set_value("user", "name", "Bazarr") - return output, err + try: + email = config_read.get_value("user", "email") + except: + logging.debug('BAZARR Settings git email') + config_write.set_value("user", "email", "bazarr@fake.email") -def check_updates(): - commits_behind = 0 - current_version, source = get_version() +def check_and_apply_update(): check_releases() - - if source == 'git': - # Get the latest version available from github - logging.debug('BAZARR Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/morpheus65535/bazarr/commits/%s' % settings.general.branch - version = request_json(url, timeout=20, validator=lambda x: type(x) == dict) - - if version is None: - notifications.write( - msg='BAZARR Could not get the latest version from GitHub.', - queue='check_update', type='warning') - logging.warn( - 'BAZARR Could not get the latest version from GitHub.') - return - - latest_version = version['sha'] - logging.debug("BAZARR Latest version is %s", latest_version) - - # See how many commits behind we are - if not current_version: - notifications.write(msg='BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version', - queue='check_update', type='warning') - logging.warn( - 'BAZARR You are running an unknown version of Bazarr. Run the updater to identify your version') - return - - if latest_version == current_version: - notifications.write(msg='BAZARR is up to date', queue='check_update') - logging.info('BAZARR is up to date') - return - - logging.debug('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/morpheus65535/bazarr/compare/%s...%s' % (latest_version, - current_version) - commits = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - - if commits is None: - notifications.write(msg='BAZARR Could not get commits behind from GitHub.', - queue='check_update', type='warning') - logging.warn('BAZARR Could not get commits behind from GitHub.') - return - - try: - commits_behind = int(commits['behind_by']) - logging.debug("BAZARR In total, %d commits behind", commits_behind) - except KeyError: - notifications.write(msg='BAZARR Cannot compare versions. Are you running a local development version?', queue='check_update') - logging.info('BAZARR Cannot compare versions. Are you running a local development version?') - return - - if commits_behind > 0: - logging.info('BAZARR New version is available. You are %s commits behind' % commits_behind) - notifications.write(msg='BAZARR New version is available. You are %s commits behind' % commits_behind, - queue='check_update') - update(source, restart=True if settings.general.getboolean('update_restart') else False) - elif commits_behind is 0: - notifications.write(msg='BAZARR is up to date', queue='check_update') - logging.info('BAZARR is up to date') + if not args.release_update: + gitconfig() + branch = settings.general.branch + g = git.cmd.Git(current_working_directory) + g.fetch('origin') + result = g.diff('--shortstat', 'origin/' + branch) + if len(result) == 0: + notifications.write(msg='BAZARR No new version of Bazarr available.', queue='check_update') + logging.info('BAZARR No new version of Bazarr available.') + else: + g.reset('--hard', 'HEAD') + g.checkout(branch) + g.reset('--hard', 'origin/' + branch) + g.pull() + logging.info('BAZARR Updated to latest version. Restart required. ' + result) + updated() else: url = 'https://api.github.com/repos/morpheus65535/bazarr/releases' releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) - + if releases is None: notifications.write(msg='BAZARR Could not get releases from GitHub.', queue='check_update', type='warning') @@ -144,9 +64,9 @@ def check_updates(): else: release = releases[0] latest_release = release['tag_name'] - - if ('v' + current_version) != latest_release and settings.general.branch == 'master': - update(source, restart=True if settings.general.getboolean('update_restart') else False) + + if ('v' + os.environ["BAZARR_VERSION"]) != latest_release and settings.general.branch == 'master': + update_from_source() elif settings.general.branch != 'master': notifications.write(msg="BAZARR Can't update development branch from source", queue='check_update') # fixme logging.info("BAZARR Can't update development branch from source") # fixme @@ -155,116 +75,81 @@ def check_updates(): logging.info('BAZARR is up to date') -def get_version(): - if os.path.isdir(os.path.join(os.path.dirname(__file__), '..', '.git')) and not args.release_update: - - output, err = run_git('rev-parse HEAD') - - if not output: - logging.error('BAZARR Could not find latest installed version.') - cur_commit_hash = None - else: - cur_commit_hash = str(output) - - if not re.match('^[a-z0-9]+$', cur_commit_hash): - logging.error('BAZARR Output does not look like a hash, not using it.') - cur_commit_hash = None - - return cur_commit_hash, 'git' +def update_from_source(): + tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) + update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - else: - return os.environ["BAZARR_VERSION"], 'source' - - -def update(source, restart=True): - if source == 'git': - output, err = run_git('pull ' + 'origin' + ' ' + settings.general.branch) - - if not output: - notifications.write(msg='Unable to download latest version', - queue='check_update', type='error') - logging.error('BAZARR Unable to download latest version') - return - - for line in output.split('\n'): - - if 'Already up-to-date.' in line: - logging.info('BAZARR No update available, not updating') - logging.info('BAZARR Output: ' + str(output)) - elif line.endswith(('Aborting', 'Aborting.')): - logging.error('BAZARR Unable to update from git: ' + line) - logging.info('BAZARR Output: ' + str(output)) - updated(restart) - else: - tar_download_url = 'https://github.com/morpheus65535/bazarr/tarball/{}'.format(settings.general.branch) - update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') - - logging.info('BAZARR Downloading update from: ' + tar_download_url) - notifications.write(msg='BAZARR Downloading update from: ' + tar_download_url) - data = request_content(tar_download_url) - - if not data: - logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url) - notifications.write(msg=("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url), - type='error') - return - - download_name = settings.general.branch + '-github' - tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - - # Save tar to disk - with open(tar_download_path, 'wb') as f: - f.write(data) - - # Extract the tar to update folder - logging.info('BAZARR Extracting file: ' + tar_download_path) - notifications.write(msg='BAZARR Extracting file: ' + tar_download_path) - tar = tarfile.open(tar_download_path) - tar.extractall(update_dir) - tar.close() - - # Delete the tar.gz - logging.info('BAZARR Deleting file: ' + tar_download_path) - notifications.write(msg='BAZARR Deleting file: ' + tar_download_path) - os.remove(tar_download_path) - - # Find update dir name - update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] - if len(update_dir_contents) != 1: - logging.error("BAZARR Invalid update data, update failed: " + str(update_dir_contents)) - notifications.write(msg="BAZARR Invalid update data, update failed: " + str(update_dir_contents), - type='error') - return - - content_dir = os.path.join(update_dir, update_dir_contents[0]) - - # walk temp folder and move files to main folder - for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir) + 1:] - for curfile in filenames: - old_path = os.path.join(content_dir, dirname, curfile) - new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) - - if os.path.isfile(new_path): - os.remove(new_path) - os.renames(old_path, new_path) - updated(restart) - - -def checkout_git_branch(): - output, err = run_git('fetch origin') - output, err = run_git('checkout %s' % settings.general.branch) + logging.info('BAZARR Downloading update from: ' + tar_download_url) + notifications.write(msg='BAZARR Downloading update from: ' + tar_download_url, queue='check_update') + data = request_content(tar_download_url) - if not output: - logging.error('Unable to change git branch.') + if not data: + logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url) + notifications.write(msg=("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url), + type='error', queue='check_update') return - for line in output.split('\n'): - if line.endswith(('Aborting', 'Aborting.')): - logging.error('Unable to checkout from git: ' + line) - logging.info('Output: ' + str(output)) + download_name = settings.general.branch + '-github' + tar_download_path = os.path.join(os.path.dirname(__file__), '..', download_name) - output, err = run_git('pull %s %s' % ('origin', settings.general.branch)) + # Save tar to disk + with open(tar_download_path, 'wb') as f: + f.write(data) + + # Extract the tar to update folder + logging.info('BAZARR Extracting file: ' + tar_download_path) + notifications.write(msg='BAZARR Extracting file: ' + tar_download_path, queue='check_update') + tar = tarfile.open(tar_download_path) + tar.extractall(update_dir) + tar.close() + + # Delete the tar.gz + logging.info('BAZARR Deleting file: ' + tar_download_path) + notifications.write(msg='BAZARR Deleting file: ' + tar_download_path, queue='check_update') + os.remove(tar_download_path) + + # Find update dir name + update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] + if len(update_dir_contents) != 1: + logging.error("BAZARR Invalid update data, update failed: " + str(update_dir_contents)) + notifications.write(msg="BAZARR Invalid update data, update failed: " + str(update_dir_contents), + type='error', queue='check_update') + return + + content_dir = os.path.join(update_dir, update_dir_contents[0]) + + # walk temp folder and move files to main folder + for dirname, dirnames, filenames in os.walk(content_dir): + dirname = dirname[len(content_dir) + 1:] + for curfile in filenames: + old_path = os.path.join(content_dir, dirname, curfile) + new_path = os.path.join(os.path.dirname(__file__), '..', dirname, curfile) + + if os.path.isfile(new_path): + os.remove(new_path) + os.renames(old_path, new_path) + updated() + + +def check_releases(): + releases = [] + url_releases = 'https://api.github.com/repos/morpheus65535/Bazarr/releases' + try: + r = requests.get(url_releases, timeout=15) + r.raise_for_status() + except requests.exceptions.HTTPError as errh: + logging.exception("Error trying to get releases from Github. Http error.") + except requests.exceptions.ConnectionError as errc: + logging.exception("Error trying to get releases from Github. Connection Error.") + except requests.exceptions.Timeout as errt: + logging.exception("Error trying to get releases from Github. Timeout Error.") + except requests.exceptions.RequestException as err: + logging.exception("Error trying to get releases from Github.") + else: + for release in r.json(): + releases.append([release['name'], release['body']]) + with open(os.path.join(args.config_dir, 'config', 'releases.txt'), 'w') as f: + json.dump(releases, f) class FakeLock(object): @@ -409,8 +294,8 @@ def request_json(url, **kwargs): logging.error("BAZARR Response returned invalid JSON data") -def updated(restart=False): - if restart: +def updated(restart=True): + if settings.general.getboolean('update_restart') and restart: try: requests.get(bazarr_url + 'restart') except requests.ConnectionError: diff --git a/bazarr/main.py b/bazarr/main.py index 42039f035..dd21c7028 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,6 +57,8 @@ from get_providers import get_providers, get_providers_auth, list_throttled_prov from get_series import * from get_episodes import * +if not args.no_update: + from check_update import check_and_apply_update from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ @@ -67,7 +69,6 @@ from notifier import send_notifications, send_notifications_movie from config import settings, url_sonarr, url_radarr, url_radarr_short, url_sonarr_short, base_url from helper import path_replace_movie, get_subtitle_destination_folder from subliminal_patch.extensions import provider_registry as provider_manager -from check_update import checkout_git_branch reload(sys) sys.setdefaultencoding('utf8') @@ -1286,7 +1287,6 @@ def save_settings(): after = (unicode(settings_general_ip), int(settings_general_port), unicode(settings_general_baseurl), unicode(settings_general_pathmapping), unicode(settings_general_use_sonarr), unicode(settings_general_use_radarr), unicode(settings_general_pathmapping_movie)) - updater_before = settings.general.branch settings.general.ip = text_type(settings_general_ip) settings.general.port = text_type(settings_general_port) @@ -1538,9 +1538,7 @@ def save_settings(): radarr_full_update() logging.info('BAZARR Settings saved succesfully.') - - if updater_before != settings_general_branch: - checkout_git_branch() + if ref.find('saved=true') > 0: redirect(ref) diff --git a/bazarr/scheduler.py b/bazarr/scheduler.py index 8c8ba94d7..fe1bbf138 100644 --- a/bazarr/scheduler.py +++ b/bazarr/scheduler.py @@ -6,9 +6,10 @@ from get_series import update_series from config import settings from get_subtitle import wanted_search_missing_subtitles, upgrade_subtitles from get_args import args - - -from check_update import check_updates, check_releases +if not args.no_update: + from check_update import check_and_apply_update, check_releases +else: + from check_update import check_releases from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from apscheduler.triggers.cron import CronTrigger @@ -83,10 +84,10 @@ scheduler.add_listener(task_listener, EVENT_JOB_SUBMITTED | EVENT_JOB_EXECUTED) def schedule_update_job(): if not args.no_update: if settings.general.getboolean('auto_update'): - scheduler.add_job(check_updates, IntervalTrigger(hours=6), max_instances=1, coalesce=True, + scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) else: - scheduler.add_job(check_updates, CronTrigger(year='2100'), hour=4, id='update_bazarr', + scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr', name='Update bazarr from source on Github' if not args.release_update else 'Update bazarr from release on Github', replace_existing=True) scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update release info', replace_existing=True) diff --git a/views/settings.tpl b/views/settings.tpl index ab95eb0d9..5d1940f93 100644 --- a/views/settings.tpl +++ b/views/settings.tpl @@ -2059,11 +2059,10 @@ }); % from get_args import args - % from check_update import get_version % if args.no_update: $("#div_update").hide(); - % elif get_version()[1] != 'git' or args.release_update: + % elif args.release_update: $("#div_branch").hide(); % end % import sys From f932a523782d310d01c523403ec8a0e2a6ef90a8 Mon Sep 17 00:00:00 2001 From: halali Date: Mon, 20 May 2019 22:55:57 +0200 Subject: [PATCH 38/38] Apply suggestions from code review Co-Authored-By: morpheus65535 --- bazarr/check_update.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bazarr/check_update.py b/bazarr/check_update.py index 17b9fc00d..930bfda1a 100644 --- a/bazarr/check_update.py +++ b/bazarr/check_update.py @@ -43,7 +43,7 @@ def check_and_apply_update(): g.fetch('origin') result = g.diff('--shortstat', 'origin/' + branch) if len(result) == 0: - notifications.write(msg='BAZARR No new version of Bazarr available.', queue='check_update') + notifications.write(msg='No new version of Bazarr available.', queue='check_update') logging.info('BAZARR No new version of Bazarr available.') else: g.reset('--hard', 'HEAD') @@ -57,7 +57,7 @@ def check_and_apply_update(): releases = request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == list) if releases is None: - notifications.write(msg='BAZARR Could not get releases from GitHub.', + notifications.write(msg='Could not get releases from GitHub.', queue='check_update', type='warning') logging.warn('BAZARR Could not get releases from GitHub.') return @@ -68,10 +68,10 @@ def check_and_apply_update(): if ('v' + os.environ["BAZARR_VERSION"]) != latest_release and settings.general.branch == 'master': update_from_source() elif settings.general.branch != 'master': - notifications.write(msg="BAZARR Can't update development branch from source", queue='check_update') # fixme + notifications.write(msg="Can't update development branch from source", queue='check_update') # fixme logging.info("BAZARR Can't update development branch from source") # fixme else: - notifications.write(msg='BAZARR is up to date', queue='check_update') + notifications.write(msg='Bazarr is up to date', queue='check_update') logging.info('BAZARR is up to date') @@ -80,12 +80,12 @@ def update_from_source(): update_dir = os.path.join(os.path.dirname(__file__), '..', 'update') logging.info('BAZARR Downloading update from: ' + tar_download_url) - notifications.write(msg='BAZARR Downloading update from: ' + tar_download_url, queue='check_update') + notifications.write(msg='Downloading update from: ' + tar_download_url, queue='check_update') data = request_content(tar_download_url) if not data: logging.error("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url) - notifications.write(msg=("BAZARR Unable to retrieve new version from '%s', can't update", tar_download_url), + notifications.write(msg=("Unable to retrieve new version from '%s', can't update", tar_download_url), type='error', queue='check_update') return @@ -98,14 +98,14 @@ def update_from_source(): # Extract the tar to update folder logging.info('BAZARR Extracting file: ' + tar_download_path) - notifications.write(msg='BAZARR Extracting file: ' + tar_download_path, queue='check_update') + notifications.write(msg='Extracting file: ' + tar_download_path, queue='check_update') tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz logging.info('BAZARR Deleting file: ' + tar_download_path) - notifications.write(msg='BAZARR Deleting file: ' + tar_download_path, queue='check_update') + notifications.write(msg='Deleting file: ' + tar_download_path, queue='check_update') os.remove(tar_download_path) # Find update dir name