From 41e04023d932a9468e53b58ac2e19a6cbf050a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 29 Aug 2019 11:24:34 -0400 Subject: [PATCH 1/2] Fix for upgrade that is not taking into account the monitored status when download only monitored is enabled. --- bazarr/get_subtitle.py | 16 +++++++++++++--- bazarr/main.py | 22 ++++++++++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 4a027fa24..b5117f255 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -978,7 +978,17 @@ def upgrade_subtitles(): days_to_upgrade_subs = settings.general.days_to_upgrade_subs minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) - datetime(1970, 1, 1)).total_seconds() - + + if settings.sonarr.getboolean('only_monitored'): + series_monitored_only_query_string = ' AND table_episodes.monitored = "True"' + else: + series_monitored_only_query_string = "" + + if settings.radarr.getboolean('only_monitored'): + movies_monitored_only_query_string = ' AND table_movies.monitored = "True"' + else: + movies_monitored_only_query_string = "" + if settings.general.getboolean('upgrade_manual'): query_actions = [1, 2, 3] else: @@ -994,7 +1004,7 @@ def upgrade_subtitles(): INNER JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND timestamp > ? AND - score is not null + score is not null""" + series_monitored_only_query_string + """ GROUP BY table_history.video_path, table_history.language""", (minimum_timestamp,)).fetchall() movies_list = c.execute("""SELECT table_history_movie.video_path, table_history_movie.language, @@ -1004,7 +1014,7 @@ def upgrade_subtitles(): FROM table_history_movie INNER JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND timestamp > ? AND - score is not null + score is not null""" + movies_monitored_only_query_string + """ GROUP BY table_history_movie.video_path, table_history_movie.language""", (minimum_timestamp,)).fetchall() db.close() diff --git a/bazarr/main.py b/bazarr/main.py index 999482bc7..da8146f09 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -998,7 +998,12 @@ def historyseries(): days_to_upgrade_subs = settings.general.days_to_upgrade_subs minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) - datetime(1970, 1, 1)).total_seconds() - + + if settings.sonarr.getboolean('only_monitored'): + series_monitored_only_query_string = ' AND table_episodes.monitored = "True"' + else: + series_monitored_only_query_string = "" + if settings.general.getboolean('upgrade_manual'): query_actions = [1, 2, 3] else: @@ -1006,8 +1011,10 @@ def historyseries(): upgradable_episodes = c.execute("""SELECT video_path, MAX(timestamp), score FROM table_history + INNER JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND - timestamp > ? AND score is not null + timestamp > ? AND score is not null""" + + series_monitored_only_query_string + """ GROUP BY table_history.video_path, table_history.language""", (minimum_timestamp,)).fetchall() for upgradable_episode in upgradable_episodes: @@ -1071,7 +1078,12 @@ def historymovies(): days_to_upgrade_subs = settings.general.days_to_upgrade_subs minimum_timestamp = ((datetime.now() - timedelta(days=int(days_to_upgrade_subs))) - datetime(1970, 1, 1)).total_seconds() - + + if settings.radarr.getboolean('only_monitored'): + movies_monitored_only_query_string = ' AND table_movies.monitored = "True"' + else: + movies_monitored_only_query_string = "" + if settings.general.getboolean('upgrade_manual'): query_actions = [1, 2, 3] else: @@ -1079,8 +1091,10 @@ def historymovies(): upgradable_movies = c.execute("""SELECT video_path, MAX(timestamp), score FROM table_history_movie + INNER JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId WHERE action IN (""" + ','.join(map(str, query_actions)) + """) AND - timestamp > ? AND score is not null + timestamp > ? AND score is not null""" + + movies_monitored_only_query_string + """ GROUP BY video_path, language""", (minimum_timestamp,)).fetchall() for upgradable_movie in upgradable_movies: From d5c0061cc9ab6be6fa662ddd6ab29482e37879ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Mon, 2 Sep 2019 23:22:38 -0400 Subject: [PATCH 2/2] Google Analytics implementation to get usage data in order to better understand how Bazarr is used. --- bazarr/analytics.py | 52 ++++++++++++++++++++++++++++++++++++++ bazarr/config.py | 3 +++ bazarr/get_subtitle.py | 6 +++++ bazarr/main.py | 29 ++++++++------------- bazarr/utils.py | 32 ++++++++++++++++++++++- libs/pyga/entities.py | 12 ++++++--- libs/pyga/requests.py | 29 +++++++++++++-------- libs/pyga/utils.py | 21 ++++++++++----- libs/version.txt | 1 + views/settings_general.tpl | 30 ++++++++++++++++++++++ 10 files changed, 175 insertions(+), 40 deletions(-) create mode 100644 bazarr/analytics.py diff --git a/bazarr/analytics.py b/bazarr/analytics.py new file mode 100644 index 000000000..b533cfb60 --- /dev/null +++ b/bazarr/analytics.py @@ -0,0 +1,52 @@ +# coding=utf-8 + +import cPickle as pickle +import base64 +import random +import platform +import os + +from pyga.requests import Event, Page, Tracker, Session, Visitor, Config +from pyga.entities import CustomVariable + +from get_args import args +from config import settings +from utils import get_sonarr_version, get_radarr_version + +sonarr_version = get_sonarr_version() +radarr_version = get_radarr_version() + + +def track_event(category=None, action=None, label=None): + if not settings.analytics.getboolean('enabled'): + print "no analytics" + return + + anonymousConfig = Config() + anonymousConfig.anonimize_ip_address = True + + tracker = Tracker('UA-138214134-3', 'none', conf=anonymousConfig) + + try: + visitor = pickle.loads(base64.b64decode(settings.analytics.visitor)) + except: + visitor = Visitor() + unique_id = long(random.getrandbits(32)) + visitor.unique_id = unique_id + + session = Session() + event = Event(category=category, action=action, label=label, value=1) + path = u"/" + u"/".join([category, action, label]) + page = Page(path.lower()) + + tracker.add_custom_variable(CustomVariable(index=1, name='BazarrVersion', value=os.environ["BAZARR_VERSION"], scope=1)) + tracker.add_custom_variable(CustomVariable(index=2, name='PythonVersion', value=platform.python_version(), scope=1)) + tracker.add_custom_variable(CustomVariable(index=3, name='SonarrVersion', value=sonarr_version, scope=1)) + tracker.add_custom_variable(CustomVariable(index=4, name='RadarrVersion', value=radarr_version, scope=1)) + tracker.add_custom_variable(CustomVariable(index=5, name='OSVersion', value=platform.platform(), scope=1)) + tracker.track_event(event, session, visitor) + tracker.track_pageview(page, session, visitor) + + settings.analytics.visitor = base64.b64encode(pickle.dumps(visitor)) + with open(os.path.join(args.config_dir, 'config', 'config.ini'), 'w+') as handle: + settings.write(handle) diff --git a/bazarr/config.py b/bazarr/config.py index eaf765dae..14efb97b3 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -122,6 +122,9 @@ defaults = { }, 'betaseries': { 'token': '' + }, + 'analytics': { + 'enabled': 'True' } } diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index b5117f255..b5d70a9cb 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -34,6 +34,8 @@ from get_args import args from queueconfig import notifications from pyprobe.pyprobe import VideoFileParser +from analytics import track_event + def get_video(path, title, sceneName, use_scenename, providers=None, media_type="movie"): """ @@ -261,6 +263,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce reversed_path = path_replace_reverse(path) else: reversed_path = path_replace_reverse_movie(path) + + track_event(category=downloaded_provider, action=action, label=downloaded_language) return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, subtitle.language.forced @@ -478,6 +482,8 @@ def manual_download_subtitle(path, language, hi, forced, subtitle, provider, pro reversed_path = path_replace_reverse(path) else: reversed_path = path_replace_reverse_movie(path) + + track_event(category=downloaded_provider, action="downloaded", label=downloaded_language) return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, subtitle.language.forced else: diff --git a/bazarr/main.py b/bazarr/main.py index da8146f09..a18e59c41 100644 --- a/bazarr/main.py +++ b/bazarr/main.py @@ -57,12 +57,13 @@ from get_languages import load_language_in_db, language_from_alpha3 from get_providers import get_providers, get_providers_auth, list_throttled_providers from get_series import * from get_episodes import * +from get_movies import * from list_subtitles import store_subtitles, store_subtitles_movie, series_scan_subtitles, movies_scan_subtitles, \ list_missing_subtitles, list_missing_subtitles_movies from get_subtitle import download_subtitle, series_download_subtitles, movies_download_subtitles, \ manual_search, manual_download_subtitle, manual_upload_subtitle -from utils import history_log, history_log_movie +from utils import history_log, history_log_movie, get_sonarr_version, get_radarr_version from scheduler import * from notifier import send_notifications, send_notifications_movie from config import settings, url_sonarr, url_radarr, url_radarr_short, url_sonarr_short, base_url @@ -1260,6 +1261,11 @@ def save_settings(): settings_general_update_restart = 'False' else: settings_general_update_restart = 'True' + settings_analytics_enabled = request.forms.get('settings_analytics_enabled') + if settings_analytics_enabled is None: + settings_analytics_enabled = 'False' + else: + settings_analytics_enabled = 'True' settings_general_single_language = request.forms.get('settings_general_single_language') if settings_general_single_language is None: settings_general_single_language = 'False' @@ -1349,6 +1355,7 @@ def save_settings(): settings.general.branch = text_type(settings_general_branch) settings.general.auto_update = text_type(settings_general_automatic) settings.general.update_restart = text_type(settings_general_update_restart) + settings.analytics.enabled = text_type(settings_analytics_enabled) settings.general.single_language = text_type(settings_general_single_language) settings.general.minimum_score = text_type(settings_general_minimum_score) settings.general.use_scenename = text_type(settings_general_scenename) @@ -1738,25 +1745,9 @@ def system(): logging.exception( 'BAZARR cannot parse releases caching file: ' + os.path.join(args.config_dir, 'config', 'releases.txt')) - use_sonarr = settings.general.getboolean('use_sonarr') - apikey_sonarr = settings.sonarr.apikey - sv = url_sonarr + "/api/system/status?apikey=" + apikey_sonarr - sonarr_version = '' - if use_sonarr: - try: - sonarr_version = requests.get(sv, timeout=15, verify=False).json()['version'] - except: - pass + sonarr_version = get_sonarr_version() - use_radarr = settings.general.getboolean('use_radarr') - apikey_radarr = settings.radarr.apikey - rv = url_radarr + "/api/system/status?apikey=" + apikey_radarr - radarr_version = '' - if use_radarr: - try: - radarr_version = requests.get(rv, timeout=15, verify=False).json()['version'] - except: - pass + radarr_version = get_radarr_version() page_size = int(settings.general.page_size) diff --git a/bazarr/utils.py b/bazarr/utils.py index 868d3b807..4392ba0e3 100644 --- a/bazarr/utils.py +++ b/bazarr/utils.py @@ -6,9 +6,11 @@ import time import platform import sys import logging +import requests from whichcraft import which from get_args import args +from config import settings, url_sonarr, url_radarr from subliminal import region as subliminal_cache_region import datetime @@ -92,4 +94,32 @@ def cache_maintenance(): # archive cache for fn in glob.iglob(os.path.join(args.config_dir, "*.archive")): - remove_expired(fn, pack_cache_validity) \ No newline at end of file + remove_expired(fn, pack_cache_validity) + + +def get_sonarr_version(): + use_sonarr = settings.general.getboolean('use_sonarr') + apikey_sonarr = settings.sonarr.apikey + sv = url_sonarr + "/api/system/status?apikey=" + apikey_sonarr + sonarr_version = '' + if use_sonarr: + try: + sonarr_version = requests.get(sv, timeout=15, verify=False).json()['version'] + except: + sonarr_version = '' + + return sonarr_version + + +def get_radarr_version(): + use_radarr = settings.general.getboolean('use_radarr') + apikey_radarr = settings.radarr.apikey + rv = url_radarr + "/api/system/status?apikey=" + apikey_radarr + radarr_version = '' + if use_radarr: + try: + radarr_version = requests.get(rv, timeout=15, verify=False).json()['version'] + except: + radarr_version = '' + + return radarr_version diff --git a/libs/pyga/entities.py b/libs/pyga/entities.py index 130c8db46..2c049427f 100644 --- a/libs/pyga/entities.py +++ b/libs/pyga/entities.py @@ -2,9 +2,15 @@ from datetime import datetime from operator import itemgetter -import six from pyga import utils from pyga import exceptions +try: + from urlparse import urlparse + from urllib import unquote_plus +except ImportError as e: + from urllib.parse import urlparse + from urllib.parse import unquote_plus + __author__ = "Arun KR (kra3) " __license__ = "Simplified BSD" @@ -95,7 +101,7 @@ class Campaign(object): @staticmethod def create_from_referrer(url): obj = Campaign(Campaign.TYPE_REFERRAL) - parse_rslt = six.moves.urllib.parse.urlparse(url) + parse_rslt = urlparse(url) obj.source = parse_rslt.netloc obj.content = parse_rslt.path return obj @@ -114,7 +120,7 @@ class Campaign(object): key, val = param.split('=') try: - setattr(self, self.UTMZ_PARAM_MAP[key], six.moves.urllib.parse.unquote_plus(val)) + setattr(self, self.UTMZ_PARAM_MAP[key], unquote_plus(val)) except KeyError: continue diff --git a/libs/pyga/requests.py b/libs/pyga/requests.py index f0a97b929..72e4bc953 100644 --- a/libs/pyga/requests.py +++ b/libs/pyga/requests.py @@ -5,11 +5,18 @@ import calendar from math import floor from pyga.entities import Campaign, CustomVariable, Event, Item, Page, Session, SocialInteraction, Transaction, Visitor import pyga.utils as utils -import six - -__author__ = "Arun KR (kra3) " +try: + from urllib import urlencode + from urllib2 import Request as urllib_request + from urllib2 import urlopen +except ImportError as e: + from urllib.parse import urlencode + from urllib.request import Request as urllib_request + from urllib.request import urlopen + +__author__ = "Arun KR (kra3) +
Analytics
+
+
+
+
+ +
+
+
+ + +
+
+ +
+
+
+