diff --git a/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs b/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs index ad9e5b8e0..2565e5095 100644 --- a/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs +++ b/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs @@ -108,7 +108,7 @@ namespace NzbDrone.Common.Instrumentation Layout = "${message}" }; - var loggingRule = new LoggingRule("*", updateClient ? LogLevel.Trace : LogLevel.Warn, target); + var loggingRule = new LoggingRule("*", updateClient ? LogLevel.Trace : LogLevel.Debug, target); LogManager.Configuration.AddTarget("sentryTarget", target); LogManager.Configuration.LoggingRules.Add(loggingRule); @@ -117,7 +117,7 @@ namespace NzbDrone.Common.Instrumentation LogManager.Configuration.LoggingRules.Insert(0, loggingRuleSentry); } - private static void RegisterDebugger() + private static void RegisterDebugger() { DebuggerTarget target = new DebuggerTarget(); target.Name = "debuggerLogger"; diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs b/src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs deleted file mode 100644 index 989f10bbb..000000000 --- a/src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Collections.Generic; -using SharpRaven.Data; - -namespace NzbDrone.Common.Instrumentation.Sentry -{ - public class LidarrJsonPacketFactory : IJsonPacketFactory - { - private readonly SentryPacketCleanser _cleanser; - - public LidarrJsonPacketFactory() - { - _cleanser = new SentryPacketCleanser(); - } - - private static string ShortenPath(string path) - { - - if (string.IsNullOrWhiteSpace(path)) - { - return null; - } - - var index = path.IndexOf("\\src\\", StringComparison.Ordinal); - - if (index <= 0) - { - return path; - } - - return path.Substring(index + "\\src".Length); - } - - public JsonPacket Create(string project, SentryEvent @event) - { - var packet = new LidarrSentryPacket(project, @event); - - try - { - foreach (var exception in packet.Exceptions) - { - foreach (var frame in exception.Stacktrace.Frames) - { - frame.Filename = ShortenPath(frame.Filename); - } - } - - _cleanser.CleansePacket(packet); - } - - catch (Exception) - { - - } - - return packet; - } - - [Obsolete] - public JsonPacket Create(string project, SentryMessage message, ErrorLevel level = ErrorLevel.Info, IDictionary tags = null, - string[] fingerprint = null, object extra = null) - { - throw new NotImplementedException(); - } - - [Obsolete] - public JsonPacket Create(string project, Exception exception, SentryMessage message = null, ErrorLevel level = ErrorLevel.Error, - IDictionary tags = null, string[] fingerprint = null, object extra = null) - { - throw new NotImplementedException(); - } - } -} diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/LidarrSentryPacket.cs b/src/NzbDrone.Common/Instrumentation/Sentry/LidarrSentryPacket.cs deleted file mode 100644 index e9b4d36a2..000000000 --- a/src/NzbDrone.Common/Instrumentation/Sentry/LidarrSentryPacket.cs +++ /dev/null @@ -1,29 +0,0 @@ -using Newtonsoft.Json; -using SharpRaven.Data; - -namespace NzbDrone.Common.Instrumentation.Sentry -{ - public class LidarrSentryPacket : JsonPacket - { - private readonly JsonSerializerSettings _setting; - - public LidarrSentryPacket(string project, SentryEvent @event) : - base(project, @event) - { - _setting = new JsonSerializerSettings - { - DefaultValueHandling = DefaultValueHandling.Ignore - }; - } - - public override string ToString(Formatting formatting) - { - return JsonConvert.SerializeObject(this, formatting, _setting); - } - - public override string ToString() - { - return ToString(Formatting.None); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/MachineNameUserFactory.cs b/src/NzbDrone.Common/Instrumentation/Sentry/MachineNameUserFactory.cs deleted file mode 100644 index 59e892542..000000000 --- a/src/NzbDrone.Common/Instrumentation/Sentry/MachineNameUserFactory.cs +++ /dev/null @@ -1,12 +0,0 @@ -using SharpRaven.Data; - -namespace NzbDrone.Common.Instrumentation.Sentry -{ - public class MachineNameUserFactory : ISentryUserFactory - { - public SentryUser Create() - { - return new SentryUser(HashUtil.AnonymousToken()); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SentryCleanser.cs b/src/NzbDrone.Common/Instrumentation/Sentry/SentryCleanser.cs new file mode 100644 index 000000000..96a6df420 --- /dev/null +++ b/src/NzbDrone.Common/Instrumentation/Sentry/SentryCleanser.cs @@ -0,0 +1,80 @@ +using System; +using System.Linq; +using NzbDrone.Common.EnvironmentInfo; +using Sentry; +using Sentry.Protocol; + +namespace NzbDrone.Common.Instrumentation.Sentry +{ + public static class SentryCleanser + { + public static SentryEvent CleanseEvent(SentryEvent sentryEvent) + { + try + { + sentryEvent.Message = CleanseLogMessage.Cleanse(sentryEvent.Message); + + if (sentryEvent.Fingerprint != null) + { + var fingerprint = sentryEvent.Fingerprint.Select(x => CleanseLogMessage.Cleanse(x)).ToList(); + sentryEvent.SetFingerprint(fingerprint); + } + + if (sentryEvent.Extra != null) + { + var extras = sentryEvent.Extra.ToDictionary(x => x.Key, y => (object)CleanseLogMessage.Cleanse((string)y.Value)); + sentryEvent.SetExtras(extras); + } + + foreach (var exception in sentryEvent.SentryExceptions) + { + foreach (var frame in exception.Stacktrace.Frames) + { + frame.FileName = ShortenPath(frame.FileName); + } + } + } + catch (Exception) + { + + } + + return sentryEvent; + } + + public static Breadcrumb CleanseBreadcrumb(Breadcrumb b) + { + try + { + var message = CleanseLogMessage.Cleanse(b.Message); + var data = b.Data?.ToDictionary(x => x.Key, y => CleanseLogMessage.Cleanse(y.Value)); + return new Breadcrumb(message, b.Type, data, b.Category, b.Level); + } + catch(Exception) + { + + } + + return b; + } + + private static string ShortenPath(string path) + { + + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + var rootDir = OsInfo.IsWindows ? "\\src\\" : "/src/"; + var index = path.IndexOf(rootDir, StringComparison.Ordinal); + + if (index <= 0) + { + return path; + } + + return path.Substring(index + rootDir.Length - 1); + } + } +} diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs b/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs deleted file mode 100644 index 950dbdda8..000000000 --- a/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Newtonsoft.Json.Linq; - -namespace NzbDrone.Common.Instrumentation.Sentry -{ - public class SentryPacketCleanser - { - public void CleansePacket(LidarrSentryPacket packet) - { - packet.Message = CleanseLogMessage.Cleanse(packet.Message); - - if (packet.Fingerprint != null) - { - for (var i = 0; i < packet.Fingerprint.Length; i++) - { - packet.Fingerprint[i] = CleanseLogMessage.Cleanse(packet.Fingerprint[i]); - } - } - - if (packet.Extra != null) - { - var target = JObject.FromObject(packet.Extra); - new CleansingJsonVisitor().Visit(target); - packet.Extra = target; - } - } - } -} diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SentryTarget.cs b/src/NzbDrone.Common/Instrumentation/Sentry/SentryTarget.cs index 6d66f5e02..f88173afe 100644 --- a/src/NzbDrone.Common/Instrumentation/Sentry/SentryTarget.cs +++ b/src/NzbDrone.Common/Instrumentation/Sentry/SentryTarget.cs @@ -4,51 +4,109 @@ using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading; +using System.Data.SQLite; using NLog; using NLog.Common; using NLog.Targets; using NzbDrone.Common.EnvironmentInfo; -using SharpRaven; -using SharpRaven.Data; +using System.Globalization; +using Sentry; +using Sentry.Protocol; +using NzbDrone.Common.Extensions; namespace NzbDrone.Common.Instrumentation.Sentry { [Target("Sentry")] public class SentryTarget : TargetWithLayout { - private readonly RavenClient _client; + // don't report uninformative SQLite exceptions + // busy/locked are benign https://forums.sonarr.tv/t/owin-sqlite-error-5-database-is-locked/5423/11 + // The others will be user configuration problems and silt up Sentry + private static readonly HashSet FilteredSQLiteErrors = new HashSet { + SQLiteErrorCode.Busy, + SQLiteErrorCode.Locked, + SQLiteErrorCode.Perm, + SQLiteErrorCode.ReadOnly, + SQLiteErrorCode.IoErr, + SQLiteErrorCode.Corrupt, + SQLiteErrorCode.Full, + SQLiteErrorCode.CantOpen, + SQLiteErrorCode.Auth + }; + + // use string and not Type so we don't need a reference to the project + // where these are defined + private static readonly HashSet FilteredExceptionTypeNames = new HashSet { + // UnauthorizedAccessExceptions will just be user configuration issues + "UnauthorizedAccessException", + // Filter out people stuck in boot loops + "CorruptDatabaseException" + }; + + private static readonly List FilteredExceptionMessages = new List { + // Swallow the many, many exceptions flowing through from Jackett + "Jackett.Common.IndexerException", + // Fix openflixr being stupid with permissions + "openflixr" + }; + + private static readonly IDictionary LoggingLevelMap = new Dictionary + { + {LogLevel.Debug, SentryLevel.Debug}, + {LogLevel.Error, SentryLevel.Error}, + {LogLevel.Fatal, SentryLevel.Fatal}, + {LogLevel.Info, SentryLevel.Info}, + {LogLevel.Trace, SentryLevel.Debug}, + {LogLevel.Warn, SentryLevel.Warning}, + }; - private static readonly IDictionary LoggingLevelMap = new Dictionary + private static readonly IDictionary BreadcrumbLevelMap = new Dictionary { - {LogLevel.Debug, ErrorLevel.Debug}, - {LogLevel.Error, ErrorLevel.Error}, - {LogLevel.Fatal, ErrorLevel.Fatal}, - {LogLevel.Info, ErrorLevel.Info}, - {LogLevel.Trace, ErrorLevel.Debug}, - {LogLevel.Warn, ErrorLevel.Warning}, + {LogLevel.Debug, BreadcrumbLevel.Debug}, + {LogLevel.Error, BreadcrumbLevel.Error}, + {LogLevel.Fatal, BreadcrumbLevel.Critical}, + {LogLevel.Info, BreadcrumbLevel.Info}, + {LogLevel.Trace, BreadcrumbLevel.Debug}, + {LogLevel.Warn, BreadcrumbLevel.Warning}, }; + private readonly IDisposable _sdk; + private bool _disposed; + private readonly SentryDebounce _debounce; private bool _unauthorized; - + public bool FilterEvents { get; set; } + public SentryTarget(string dsn) { - _client = new RavenClient(new Dsn(dsn), new LidarrJsonPacketFactory(), new SentryRequestFactory(), new MachineNameUserFactory()) - { - Compression = true, - Environment = RuntimeInfo.IsProduction ? "production" : "development", - Release = BuildInfo.Release, - ErrorOnCapture = OnError - }; - - - _client.Tags.Add("osfamily", OsInfo.Os.ToString()); - _client.Tags.Add("runtime", PlatformInfo.PlatformName); - _client.Tags.Add("culture", Thread.CurrentThread.CurrentCulture.Name); - _client.Tags.Add("branch", BuildInfo.Branch); - _client.Tags.Add("version", BuildInfo.Version.ToString()); - + _sdk = SentrySdk.Init(o => + { + o.Dsn = new Dsn(dsn); + o.AttachStacktrace = true; + o.MaxBreadcrumbs = 200; + o.SendDefaultPii = true; + o.Debug = false; + o.DiagnosticsLevel = SentryLevel.Debug; + o.Environment = RuntimeInfo.IsProduction ? "production" : "development"; + o.Release = BuildInfo.Release; + o.BeforeSend = x => SentryCleanser.CleanseEvent(x); + o.BeforeBreadcrumb = x => SentryCleanser.CleanseBreadcrumb(x); + }); + + SentrySdk.ConfigureScope(scope => + { + scope.User = new User { + Username = HashUtil.AnonymousToken() + }; + + scope.SetTag("osfamily", OsInfo.Os.ToString()); + scope.SetTag("runtime", PlatformInfo.PlatformName); + scope.SetTag("culture", Thread.CurrentThread.CurrentCulture.Name); + scope.SetTag("branch", BuildInfo.Branch); + scope.SetTag("version", BuildInfo.Version.ToString()); + }); + _debounce = new SentryDebounce(); } @@ -109,6 +167,25 @@ namespace NzbDrone.Common.Instrumentation.Sentry if (logEvent.Level >= LogLevel.Error && logEvent.Exception != null) { + if (FilterEvents) + { + var sqlEx = logEvent.Exception as SQLiteException; + if (sqlEx != null && FilteredSQLiteErrors.Contains(sqlEx.ResultCode)) + { + return false; + } + + if (FilteredExceptionTypeNames.Contains(logEvent.Exception.GetType().Name)) + { + return false; + } + + if (FilteredExceptionMessages.Any(x => logEvent.Exception.Message.Contains(x))) + { + return false; + } + } + return true; } @@ -125,6 +202,8 @@ namespace NzbDrone.Common.Instrumentation.Sentry try { + SentrySdk.AddBreadcrumb(logEvent.FormattedMessage, logEvent.LoggerName, level: BreadcrumbLevelMap[logEvent.Level]); + // don't report non-critical events without exceptions if (!IsSentryMessage(logEvent)) { @@ -137,9 +216,8 @@ namespace NzbDrone.Common.Instrumentation.Sentry return; } - var extras = logEvent.Properties.ToDictionary(x => x.Key.ToString(), x => x.Value.ToString()); + var extras = logEvent.Properties.ToDictionary(x => x.Key.ToString(), x => (object)x.Value.ToString()); extras.Remove("Sentry"); - _client.Logger = logEvent.LoggerName; if (logEvent.Exception != null) { @@ -149,54 +227,95 @@ namespace NzbDrone.Common.Instrumentation.Sentry } } - var sentryMessage = new SentryMessage(logEvent.Message, logEvent.Parameters); - var sentryEvent = new SentryEvent(logEvent.Exception) { Level = LoggingLevelMap[logEvent.Level], - Message = sentryMessage, - Extra = extras, - Fingerprint = - { + Logger = logEvent.LoggerName, + Message = logEvent.FormattedMessage, + }; + + var sentryFingerprint = new List { logEvent.Level.ToString(), logEvent.LoggerName, logEvent.Message - } }; - - // Fix openflixr being stupid with permissions - var serverName = sentryEvent.Contexts.Device.Name.ToLower(); - - if (serverName == "openflixr") - { - return; - } + + sentryEvent.SetExtras(extras); if (logEvent.Exception != null) { - sentryEvent.Fingerprint.Add(logEvent.Exception.GetType().FullName); + sentryFingerprint.Add(logEvent.Exception.GetType().FullName); + sentryFingerprint.Add(logEvent.Exception.TargetSite.ToString()); + + // only try to use the exeception message to fingerprint if there's no inner + // exception and the message is short, otherwise we're in danger of getting a + // stacktrace which will break the grouping + if (logEvent.Exception.InnerException == null) + { + string message = null; + + // bodge to try to get the exception message in English + // https://stackoverflow.com/questions/209133/exception-messages-in-english + // There may still be some localization but this is better than nothing. + var t = new Thread(() => { + message = logEvent.Exception?.Message; + }); + t.CurrentCulture = CultureInfo.InvariantCulture; + t.CurrentUICulture = CultureInfo.InvariantCulture; + t.Start(); + t.Join(); + + if (message.IsNotNullOrWhiteSpace() && message.Length < 200) + { + sentryFingerprint.Add(message); + } + } } if (logEvent.Properties.ContainsKey("Sentry")) { - sentryEvent.Fingerprint.Clear(); - Array.ForEach((string[])logEvent.Properties["Sentry"], sentryEvent.Fingerprint.Add); + sentryFingerprint = ((string[])logEvent.Properties["Sentry"]).ToList(); } + + sentryEvent.SetFingerprint(sentryFingerprint); + // this can't be in the constructor as at that point OsInfo won't have + // populated these values yet var osName = Environment.GetEnvironmentVariable("OS_NAME"); var osVersion = Environment.GetEnvironmentVariable("OS_VERSION"); var runTimeVersion = Environment.GetEnvironmentVariable("RUNTIME_VERSION"); - sentryEvent.Tags.Add("os_name", osName); - sentryEvent.Tags.Add("os_version", $"{osName} {osVersion}"); - sentryEvent.Tags.Add("runtime_version", $"{PlatformInfo.PlatformName} {runTimeVersion}"); + sentryEvent.SetTag("os_name", osName); + sentryEvent.SetTag("os_version", $"{osName} {osVersion}"); + sentryEvent.SetTag("runtime_version", $"{PlatformInfo.PlatformName} {runTimeVersion}"); - _client.Capture(sentryEvent); + SentrySdk.CaptureEvent(sentryEvent); } catch (Exception e) { OnError(e); } } + + // https://stackoverflow.com/questions/2496311/implementing-idisposable-on-a-subclass-when-the-parent-also-implements-idisposab + protected override void Dispose(bool disposing) + { + // Only do something if we're not already disposed + if (_disposed) + { + // If disposing == true, we're being called from a call to base.Dispose(). In this case, we Dispose() our logger + // If we're being called from a finalizer, our logger will get finalized as well, so no need to do anything. + if (disposing) + { + _sdk?.Dispose(); + } + // Flag us as disposed. This allows us to handle multiple calls to Dispose() as well as ObjectDisposedException + _disposed = true; + } + + // This should always be safe to call multiple times! + // We could include it in the check for disposed above, but I left it out to demonstrate that it's safe + base.Dispose(disposing); + } } } diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 6764dc70e..f7a3ea5e6 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -54,8 +54,17 @@ ..\packages\DotNet4.SocksProxy.1.3.4.0\lib\net40\Org.Mentalis.dll True - - ..\packages\SharpRaven.2.4.0\lib\net45\SharpRaven.dll + + ..\packages\Sentry.1.1.2\lib\net461\Sentry.dll + + + ..\packages\Sentry.Protocol.1.0.4\lib\net46\Sentry.Protocol.dll + + + ..\packages\Sentry.PlatformAbstractions.1.0.0\lib\net45\Sentry.PlatformAbstractions.dll + + + ..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll ..\packages\DotNet4.SocksProxy.1.3.4.0\lib\net40\SocksWebProxy.dll @@ -69,6 +78,9 @@ + + ..\Libraries\Sqlite\System.Data.SQLite.dll + @@ -193,11 +205,8 @@ - - - - + @@ -244,6 +253,12 @@ + + + libsqlite3.0.dylib + PreserveNewest + + {74420a79-cc16-442c-8b1e-7c1b913844f0} @@ -262,4 +277,4 @@ --> - \ No newline at end of file + diff --git a/src/NzbDrone.Common/packages.config b/src/NzbDrone.Common/packages.config index d2e511546..0b18b0846 100644 --- a/src/NzbDrone.Common/packages.config +++ b/src/NzbDrone.Common/packages.config @@ -4,5 +4,8 @@ - - \ No newline at end of file + + + + + diff --git a/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs b/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs index ec5f46545..d7323af4d 100644 --- a/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs +++ b/src/NzbDrone.Core/Configuration/ConfigFileProvider.cs @@ -34,6 +34,7 @@ namespace NzbDrone.Core.Configuration bool AnalyticsEnabled { get; } string LogLevel { get; } string ConsoleLogLevel { get; } + bool FilterSentryEvents { get; } string Branch { get; } string ApiKey { get; } string SslCertHash { get; } @@ -181,6 +182,7 @@ namespace NzbDrone.Core.Configuration public string LogLevel => GetValue("LogLevel", "Info"); public string ConsoleLogLevel => GetValue("ConsoleLogLevel", string.Empty, persist: false); + public bool FilterSentryEvents => GetValueBoolean("FilterSentryEvents", true, persist: false); public string SslCertHash => GetValue("SslCertHash", ""); diff --git a/src/NzbDrone.Core/Instrumentation/ReconfigureLogging.cs b/src/NzbDrone.Core/Instrumentation/ReconfigureLogging.cs index 9c99dfb02..e4cd4c927 100644 --- a/src/NzbDrone.Core/Instrumentation/ReconfigureLogging.cs +++ b/src/NzbDrone.Core/Instrumentation/ReconfigureLogging.cs @@ -3,6 +3,7 @@ using System.Linq; using NLog; using NLog.Config; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Instrumentation.Sentry; using NzbDrone.Core.Configuration; using NzbDrone.Core.Configuration.Events; using NzbDrone.Core.Messaging.Events; @@ -40,6 +41,9 @@ namespace NzbDrone.Core.Instrumentation SetMinimumLogLevel(rules, "appFileDebug", minimumLogLevel <= LogLevel.Debug ? LogLevel.Debug : LogLevel.Off); SetMinimumLogLevel(rules, "appFileTrace", minimumLogLevel <= LogLevel.Trace ? LogLevel.Trace : LogLevel.Off); + // Sentry filtering + LogManager.Configuration.FindTargetByName("sentryTarget").FilterEvents = _configFileProvider.FilterSentryEvents; + LogManager.ReconfigExistingLoggers(); }