From cb00644a09d58d11f1e5fddac7d4cab6d902d9ee Mon Sep 17 00:00:00 2001 From: Keivan Beigi Date: Wed, 15 Oct 2014 19:10:25 -0700 Subject: [PATCH 1/2] Use raw sql to write logs to db. --- .../DatabaseTargetFixture.cs | 14 ++++ .../Datastore/ConnectionStringFactory.cs | 1 + .../Instrumentation/DatabaseTarget.cs | 69 +++++++++++-------- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs b/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs index f5d99e564..c0cc34b57 100644 --- a/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs +++ b/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs @@ -1,6 +1,7 @@ using System; using System.Diagnostics; using FluentAssertions; +using Marr.Data; using NLog; using NUnit.Framework; using NzbDrone.Common.Instrumentation; @@ -65,6 +66,19 @@ namespace NzbDrone.Core.Test.InstrumentationTests VerifyLog(StoredModel, LogLevel.Info); } + + [Test] + [Explicit] + public void perf_test() + { + MapRepository.Instance.EnableTraceLogging = false; + for (int i = 0; i < 1000; i++) + { + _logger.Info(Guid.NewGuid()); + } + MapRepository.Instance.EnableTraceLogging = true; + } + [Test] public void write_log_exception() { diff --git a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs index 312df4cc8..680376c34 100644 --- a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs +++ b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs @@ -30,6 +30,7 @@ namespace NzbDrone.Core.Datastore connectionBuilder.CacheSize = (int)-10.Megabytes(); connectionBuilder.DateTimeKind = DateTimeKind.Utc; connectionBuilder.JournalMode = SQLiteJournalModeEnum.Wal; + connectionBuilder.Pooling = true; return connectionBuilder.ConnectionString; } diff --git a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs index f2e14f7ea..cc0fc0ccb 100644 --- a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs +++ b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs @@ -1,4 +1,5 @@ using System; +using System.Data; using System.Data.SQLite; using NLog.Common; using NLog.Config; @@ -6,6 +7,7 @@ using NLog; using NLog.Layouts; using NLog.Targets; using NzbDrone.Common.Instrumentation; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; @@ -14,11 +16,14 @@ namespace NzbDrone.Core.Instrumentation public class DatabaseTarget : TargetWithLayout, IHandle { - private readonly ILogRepository _repository; + private readonly SQLiteConnection _connection; - public DatabaseTarget(ILogRepository repository) + const string INSERT_COMMAND = "INSERT INTO [Logs]([Message],[Time],[Logger],[Method],[Exception],[ExceptionType],[Level]) " + + "VALUES(@Message,@Time,@Logger,@Method,@Exception,@ExceptionType,@Level)"; + + public DatabaseTarget(IConnectionStringFactory connectionStringFactory) { - _repository = repository; + _connection = new SQLiteConnection(connectionStringFactory.LogDbConnectionString).OpenAndReturn(); } public void Register() @@ -51,40 +56,50 @@ namespace NzbDrone.Core.Instrumentation protected override void Write(LogEventInfo logEvent) { - var log = new Log(); - log.Time = logEvent.TimeStamp; - log.Message = CleanseLogMessage.Cleanse(logEvent.FormattedMessage); - log.Method = Layout.Render(logEvent); - - log.Logger = logEvent.LoggerName; - - if (log.Logger.StartsWith("NzbDrone.")) + try { - log.Logger = log.Logger.Remove(0, 9); - } + var log = new Log(); + log.Time = logEvent.TimeStamp; + log.Message = CleanseLogMessage.Cleanse(logEvent.FormattedMessage); + log.Method = Layout.Render(logEvent); - if (logEvent.Exception != null) - { - if (String.IsNullOrWhiteSpace(log.Message)) + log.Logger = logEvent.LoggerName; + + if (log.Logger.StartsWith("NzbDrone.")) { - log.Message = logEvent.Exception.Message; + log.Logger = log.Logger.Remove(0, 9); } - else + + if (logEvent.Exception != null) { - log.Message += ": " + logEvent.Exception.Message; + if (String.IsNullOrWhiteSpace(log.Message)) + { + log.Message = logEvent.Exception.Message; + } + else + { + log.Message += ": " + logEvent.Exception.Message; + } + + + log.Exception = logEvent.Exception.ToString(); + log.ExceptionType = logEvent.Exception.GetType().ToString(); } - log.Exception = logEvent.Exception.ToString(); - log.ExceptionType = logEvent.Exception.GetType().ToString(); - } + log.Level = logEvent.Level.Name; - - log.Level = logEvent.Level.Name; + var sqlCommand = new SQLiteCommand(INSERT_COMMAND, _connection); - try - { - _repository.Insert(log); + sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); + sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time }); + sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); + sqlCommand.Parameters.Add(new SQLiteParameter("Method", DbType.String) { Value = log.Method }); + sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); + sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); + sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); + + sqlCommand.ExecuteNonQuery(); } catch (SQLiteException ex) { From d10cdf9c87e631bcdaca9f2efbe0ba07f4a43aae Mon Sep 17 00:00:00 2001 From: Keivan Beigi Date: Wed, 15 Oct 2014 19:23:10 -0700 Subject: [PATCH 2/2] removed method column from log db. --- .../InstrumentationTests/DatabaseTargetFixture.cs | 1 - .../Migration/064_add_remove_method_from_logs.cs | 14 ++++++++++++++ .../Instrumentation/DatabaseTarget.cs | 8 ++------ src/NzbDrone.Core/Instrumentation/Log.cs | 2 -- src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 5 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/064_add_remove_method_from_logs.cs diff --git a/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs b/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs index c0cc34b57..ea4a63bae 100644 --- a/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs +++ b/src/NzbDrone.Core.Test/InstrumentationTests/DatabaseTargetFixture.cs @@ -133,7 +133,6 @@ namespace NzbDrone.Core.Test.InstrumentationTests logItem.Time.Should().BeWithin(TimeSpan.FromSeconds(2)); logItem.Logger.Should().Be(this.GetType().Name); logItem.Level.Should().Be(level.Name); - logItem.Method.Should().Be(new StackTrace().GetFrame(1).GetMethod().Name); _logger.Name.Should().EndWith(logItem.Logger); } } diff --git a/src/NzbDrone.Core/Datastore/Migration/064_add_remove_method_from_logs.cs b/src/NzbDrone.Core/Datastore/Migration/064_add_remove_method_from_logs.cs new file mode 100644 index 000000000..2fd04ea97 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/064_add_remove_method_from_logs.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(64)] + public class remove_method_from_logs : NzbDroneMigrationBase + { + protected override void LogDbUpgrade() + { + Delete.Column("Method").FromTable("Logs"); + } + } +} diff --git a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs index cc0fc0ccb..4c9ed9eda 100644 --- a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs +++ b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs @@ -18,8 +18,8 @@ namespace NzbDrone.Core.Instrumentation { private readonly SQLiteConnection _connection; - const string INSERT_COMMAND = "INSERT INTO [Logs]([Message],[Time],[Logger],[Method],[Exception],[ExceptionType],[Level]) " + - "VALUES(@Message,@Time,@Logger,@Method,@Exception,@ExceptionType,@Level)"; + const string INSERT_COMMAND = "INSERT INTO [Logs]([Message],[Time],[Logger],[Exception],[ExceptionType],[Level]) " + + "VALUES(@Message,@Time,@Logger,@Exception,@ExceptionType,@Level)"; public DatabaseTarget(IConnectionStringFactory connectionStringFactory) { @@ -28,8 +28,6 @@ namespace NzbDrone.Core.Instrumentation public void Register() { - Layout = new SimpleLayout("${callsite:className=false:fileName=false:includeSourcePath=false:methodName=true}"); - Rule = new LoggingRule("*", LogLevel.Info, this); LogManager.Configuration.AddTarget("DbLogger", this); @@ -61,7 +59,6 @@ namespace NzbDrone.Core.Instrumentation var log = new Log(); log.Time = logEvent.TimeStamp; log.Message = CleanseLogMessage.Cleanse(logEvent.FormattedMessage); - log.Method = Layout.Render(logEvent); log.Logger = logEvent.LoggerName; @@ -94,7 +91,6 @@ namespace NzbDrone.Core.Instrumentation sqlCommand.Parameters.Add(new SQLiteParameter("Message", DbType.String) { Value = log.Message }); sqlCommand.Parameters.Add(new SQLiteParameter("Time", DbType.DateTime) { Value = log.Time }); sqlCommand.Parameters.Add(new SQLiteParameter("Logger", DbType.String) { Value = log.Logger }); - sqlCommand.Parameters.Add(new SQLiteParameter("Method", DbType.String) { Value = log.Method }); sqlCommand.Parameters.Add(new SQLiteParameter("Exception", DbType.String) { Value = log.Exception }); sqlCommand.Parameters.Add(new SQLiteParameter("ExceptionType", DbType.String) { Value = log.ExceptionType }); sqlCommand.Parameters.Add(new SQLiteParameter("Level", DbType.String) { Value = log.Level }); diff --git a/src/NzbDrone.Core/Instrumentation/Log.cs b/src/NzbDrone.Core/Instrumentation/Log.cs index 58119c84c..67cd0c1f9 100644 --- a/src/NzbDrone.Core/Instrumentation/Log.cs +++ b/src/NzbDrone.Core/Instrumentation/Log.cs @@ -13,8 +13,6 @@ namespace NzbDrone.Core.Instrumentation public string Logger { get; set; } - public string Method { get; set; } - public string Exception { get; set; } public string ExceptionType { get; set; } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 56b13d5fe..d617dda58 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -225,6 +225,7 @@ +