From 0807fcc6168c270902887a76da61aa27df6baebd Mon Sep 17 00:00:00 2001 From: Qstick Date: Sat, 2 Sep 2017 06:39:21 -0400 Subject: [PATCH] Cleanse Log Exception Data (#83) --- .../Extensions/ExceptionExtensions.cs | 55 +++++++++++ .../Instrumentation/CleansingJsonVisitor.cs | 47 +++++++++ .../Instrumentation/NzbDroneLogger.cs | 13 +-- ...tFactory.cs => LidarrJsonPacketFactory.cs} | 17 +++- ...rSentryPacket.cs => LidarrSentryPacket.cs} | 0 .../Sentry/SentryPacketCleanser.cs | 31 ++++++ src/NzbDrone.Common/NzbDrone.Common.csproj | 8 +- src/NzbDrone.Common/Serializer/JsonVisitor.cs | 95 +++++++++++++++++++ src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 7 +- .../Newznab/NewznabCapabilitiesProvider.cs | 3 +- src/NzbDrone.Core/Indexers/RssParser.cs | 11 ++- .../TorrentRss/TorrentRssSettingsDetector.cs | 30 +++--- 12 files changed, 283 insertions(+), 34 deletions(-) create mode 100644 src/NzbDrone.Common/Extensions/ExceptionExtensions.cs create mode 100644 src/NzbDrone.Common/Instrumentation/CleansingJsonVisitor.cs rename src/NzbDrone.Common/Instrumentation/Sentry/{SonarrJsonPacketFactory.cs => LidarrJsonPacketFactory.cs} (85%) rename src/NzbDrone.Common/Instrumentation/Sentry/{SonarrSentryPacket.cs => LidarrSentryPacket.cs} (100%) create mode 100644 src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs create mode 100644 src/NzbDrone.Common/Serializer/JsonVisitor.cs diff --git a/src/NzbDrone.Common/Extensions/ExceptionExtensions.cs b/src/NzbDrone.Common/Extensions/ExceptionExtensions.cs new file mode 100644 index 000000000..c719cef3e --- /dev/null +++ b/src/NzbDrone.Common/Extensions/ExceptionExtensions.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Common.Extensions +{ + public static class ExceptionExtensions + { + public static T WithData(this T ex, string key, string value) where T : Exception + { + ex.AddData(key, value); + + return ex; + } + public static T WithData(this T ex, string key, int value) where T : Exception + { + ex.AddData(key, value.ToString()); + + return ex; + } + + public static T WithData(this T ex, string key, Http.HttpUri value) where T : Exception + { + ex.AddData(key, value.ToString()); + + return ex; + } + + + public static T WithData(this T ex, Http.HttpResponse response, int maxSampleLength = 512) where T : Exception + { + if (response == null || response.Content == null) return ex; + + var contentSample = response.Content.Substring(0, Math.Min(response.Content.Length, 512)); + + if (response.Headers != null) + { + ex.AddData("ContentType", response.Headers.ContentType ?? string.Empty); + } + ex.AddData("ContentLength", response.Content.Length.ToString()); + ex.AddData("ContentSample", contentSample); + + return ex; + } + + + private static void AddData(this Exception ex, string key, string value) + { + if (value.IsNullOrWhiteSpace()) return; + + ex.Data[key] = value; + } + } +} diff --git a/src/NzbDrone.Common/Instrumentation/CleansingJsonVisitor.cs b/src/NzbDrone.Common/Instrumentation/CleansingJsonVisitor.cs new file mode 100644 index 000000000..f33f4587b --- /dev/null +++ b/src/NzbDrone.Common/Instrumentation/CleansingJsonVisitor.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; +using NzbDrone.Common.Serializer; + +namespace NzbDrone.Common.Instrumentation +{ + public class CleansingJsonVisitor : JsonVisitor + { + public override void Visit(JArray json) + { + for (var i = 0; i < json.Count; i++) + { + if (json[i].Type == JTokenType.String) + { + var text = json[i].Value(); + json[i] = new JValue(CleanseLogMessage.Cleanse(text)); + } + } + foreach (JToken token in json) + { + Visit(token); + } + } + + public override void Visit(JProperty property) + { + if (property.Value.Type == JTokenType.String) + { + property.Value = CleanseValue(property.Value as JValue); + } + else + { + base.Visit(property); + } + } + + private JValue CleanseValue(JValue value) + { + var text = value.Value(); + var cleansed = CleanseLogMessage.Cleanse(text); + return new JValue(cleansed); + } + } +} diff --git a/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs b/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs index b9c94be9b..0845caeaa 100644 --- a/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs +++ b/src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs @@ -40,9 +40,7 @@ namespace NzbDrone.Common.Instrumentation RegisterDebugger(); } - // Need to register for Sentry service (sonarr) - // Exceptron is being considered by Radarr - // RegisterSentry(updateApp); + RegisterSentry(updateApp); if (updateApp) { @@ -89,22 +87,21 @@ namespace NzbDrone.Common.Instrumentation private static void RegisterSentry(bool updateClient) { - // TODO Enable above when we recieve sentry service account. string dsn; if (updateClient) { dsn = RuntimeInfo.IsProduction - ? "https://b85aa82c65b84b0e99e3b7c281438357:392b5bc007974147a922c5d841c47cf9@sentry.lidarr.audio/11" - : "https://6168f0946aba4e60ac23e469ac08eac5:bd59e8454ccc454ea27a90cff1f814ca@sentry.lidarr.audio/9"; + ? "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545" + : "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545"; } else { dsn = RuntimeInfo.IsProduction - ? "https://3e8a38b1a4df4de8b0453a724f5a1139:5a708dd75c724b32ae5128b6a895650f@sentry.lidarr.audio/8" - : "https://4ee3580e01d8407c96a7430fbc953512:5f2d07227a0b4fde99dea07041a3ff93@sentry.lidarr.audio/10"; + ? "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545" + : "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545"; } var target = new SentryTarget(dsn) diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SonarrJsonPacketFactory.cs b/src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs similarity index 85% rename from src/NzbDrone.Common/Instrumentation/Sentry/SonarrJsonPacketFactory.cs rename to src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs index 9966b1e95..989f10bbb 100644 --- a/src/NzbDrone.Common/Instrumentation/Sentry/SonarrJsonPacketFactory.cs +++ b/src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using SharpRaven.Data; @@ -6,7 +6,14 @@ namespace NzbDrone.Common.Instrumentation.Sentry { public class LidarrJsonPacketFactory : IJsonPacketFactory { - private static string ShortenPath(string path) + private readonly SentryPacketCleanser _cleanser; + + public LidarrJsonPacketFactory() + { + _cleanser = new SentryPacketCleanser(); + } + + private static string ShortenPath(string path) { if (string.IsNullOrWhiteSpace(path)) @@ -37,7 +44,10 @@ namespace NzbDrone.Common.Instrumentation.Sentry frame.Filename = ShortenPath(frame.Filename); } } + + _cleanser.CleansePacket(packet); } + catch (Exception) { @@ -46,7 +56,6 @@ namespace NzbDrone.Common.Instrumentation.Sentry return packet; } - [Obsolete] public JsonPacket Create(string project, SentryMessage message, ErrorLevel level = ErrorLevel.Info, IDictionary tags = null, string[] fingerprint = null, object extra = null) @@ -61,4 +70,4 @@ namespace NzbDrone.Common.Instrumentation.Sentry throw new NotImplementedException(); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SonarrSentryPacket.cs b/src/NzbDrone.Common/Instrumentation/Sentry/LidarrSentryPacket.cs similarity index 100% rename from src/NzbDrone.Common/Instrumentation/Sentry/SonarrSentryPacket.cs rename to src/NzbDrone.Common/Instrumentation/Sentry/LidarrSentryPacket.cs diff --git a/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs b/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs new file mode 100644 index 000000000..950dbdda8 --- /dev/null +++ b/src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; + +namespace NzbDrone.Common.Instrumentation.Sentry +{ + public class SentryPacketCleanser + { + public void CleansePacket(LidarrSentryPacket packet) + { + packet.Message = CleanseLogMessage.Cleanse(packet.Message); + + if (packet.Fingerprint != null) + { + for (var i = 0; i < packet.Fingerprint.Length; i++) + { + packet.Fingerprint[i] = CleanseLogMessage.Cleanse(packet.Fingerprint[i]); + } + } + + if (packet.Extra != null) + { + var target = JObject.FromObject(packet.Extra); + new CleansingJsonVisitor().Visit(target); + packet.Extra = target; + } + } + } +} diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 874650375..611cdc4cc 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -136,6 +136,7 @@ + @@ -175,6 +176,7 @@ + @@ -182,10 +184,11 @@ + - - + + @@ -206,6 +209,7 @@ + diff --git a/src/NzbDrone.Common/Serializer/JsonVisitor.cs b/src/NzbDrone.Common/Serializer/JsonVisitor.cs new file mode 100644 index 000000000..87fdeeeec --- /dev/null +++ b/src/NzbDrone.Common/Serializer/JsonVisitor.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; + +namespace NzbDrone.Common.Serializer +{ + + public class JsonVisitor + { + protected void Dispatch(JToken json) + { + switch (json.Type) + { + case JTokenType.Object: + Visit(json as JObject); + break; + + case JTokenType.Array: + Visit(json as JArray); + break; + + case JTokenType.Raw: + Visit(json as JRaw); + break; + + case JTokenType.Constructor: + Visit(json as JConstructor); + break; + + case JTokenType.Property: + Visit(json as JProperty); + break; + + case JTokenType.Comment: + case JTokenType.Integer: + case JTokenType.Float: + case JTokenType.String: + case JTokenType.Boolean: + case JTokenType.Null: + case JTokenType.Undefined: + case JTokenType.Date: + case JTokenType.Bytes: + case JTokenType.Guid: + case JTokenType.Uri: + case JTokenType.TimeSpan: + Visit(json as JValue); + break; + + default: + break; + } + } + + public virtual void Visit(JToken json) + { + Dispatch(json); + } + + public virtual void Visit(JContainer json) + { + Dispatch(json); + } + + public virtual void Visit(JArray json) + { + foreach (JToken token in json) + { + Visit(token); + } + } + public virtual void Visit(JConstructor json) + { + } + + public virtual void Visit(JObject json) + { + foreach (JProperty property in json.Properties()) + { + Visit(property); + } + } + + public virtual void Visit(JProperty property) + { + Visit(property.Value); + } + + public virtual void Visit(JValue value) + { + + } + } +} diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index e1349665e..bc76f3f83 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -212,6 +212,7 @@ namespace NzbDrone.Core.Indexers catch (CloudFlareCaptchaException ex) { _indexerStatusService.RecordFailure(Definition.Id); + ex.WithData("FeedUrl", url); if (ex.IsExpired) { _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this); @@ -226,11 +227,11 @@ namespace NzbDrone.Core.Indexers _indexerStatusService.RecordFailure(Definition.Id); _logger.Warn(ex, "{0}", url); } - catch (Exception feedEx) + catch (Exception ex) { _indexerStatusService.RecordFailure(Definition.Id); - feedEx.Data.Add("FeedUrl", url); - _logger.Error(feedEx, "An error occurred while processing feed. {0}", url); + ex.WithData("FeedUrl", url); + _logger.Error(ex, "An error occurred while processing feed. {0}", url); } return CleanupReleases(releases); diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs index 401ea2284..e9397e4c7 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Net; using System.Xml; @@ -69,6 +69,7 @@ namespace NzbDrone.Core.Indexers.Newznab catch (XmlException ex) { _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}.", indexerSettings.Url); + ex.WithData(response); throw; } catch (Exception ex) diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 1cd46e754..fb7e60d0e 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -65,12 +65,14 @@ namespace NzbDrone.Core.Indexers } catch (UnsupportedFeedException itemEx) { - itemEx.Data.Add("Item", item.Title()); + itemEx.WithData("FeedUrl", indexerResponse.Request.Url); + itemEx.WithData("ItemTitle", item.Title()); throw; } catch (Exception itemEx) { - itemEx.Data.Add("Item", item.Title()); + itemEx.WithData("FeedUrl", indexerResponse.Request.Url); + itemEx.WithData("ItemTitle", item.Title()); _logger.Error(itemEx, "An error occurred while processing feed item from {0}", indexerResponse.Request.Url); } } @@ -95,8 +97,7 @@ namespace NzbDrone.Core.Indexers var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512)); _logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample); - ex.Data.Add("ContentLength", indexerResponse.Content.Length); - ex.Data.Add("ContentSample", contentSample); + ex.WithData(indexerResponse.HttpResponse); throw; } diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs index c20c09b2a..8142ac892 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.IO; using System.Text.RegularExpressions; @@ -40,22 +40,30 @@ namespace NzbDrone.Core.Indexers.TorrentRss { _logger.Debug("Evaluating TorrentRss feed '{0}'", indexerSettings.BaseUrl); - var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings }; - var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First(); - - HttpResponse httpResponse = null; try { - httpResponse = _httpClient.Execute(request.HttpRequest); + var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings }; + var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First(); + + HttpResponse httpResponse = null; + try + { + httpResponse = _httpClient.Execute(request.HttpRequest); + } + catch (Exception ex) + { + _logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message)); + return null; + } + + var indexerResponse = new IndexerResponse(request, httpResponse); + return GetParserSettings(indexerResponse, indexerSettings); } catch (Exception ex) { - _logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message)); - return null; + ex.WithData("FeedUrl", indexerSettings.BaseUrl); + throw; } - - var indexerResponse = new IndexerResponse(request, httpResponse); - return GetParserSettings(indexerResponse, indexerSettings); } private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)