From 32a6c9fe2a00671d76ccfd66cc701db449277670 Mon Sep 17 00:00:00 2001 From: ta264 Date: Thu, 14 May 2020 20:56:26 +0100 Subject: [PATCH] Revert "Revert "Fixed: Rename more than 999 movies in one go"" This reverts commit c0b80696bccb2d6c39ce5bcd17baf0540af0fa37. --- .../Datastore/TableMapperFixture.cs | 4 +- .../Datastore/WhereBuilderFixture.cs | 106 ++++++----- .../Blacklisting/BlacklistRepository.cs | 20 +- .../Datastore/BasicRepository.cs | 73 +++----- .../Datastore/Extensions/BuilderExtensions.cs | 119 ++++++++---- .../Datastore/Extensions/MappingExtensions.cs | 47 +++++ .../Extensions/SqlMapperExtensions.cs | 176 ++++++++++++++++++ src/NzbDrone.Core/Datastore/LazyLoaded.cs | 139 ++++++++++++++ src/NzbDrone.Core/Datastore/SqlBuilder.cs | 168 +++++++++++++++++ src/NzbDrone.Core/Datastore/TableMapper.cs | 100 +++++----- src/NzbDrone.Core/Datastore/WhereBuilder.cs | 68 ++++--- .../History/HistoryRepository.cs | 26 +-- src/NzbDrone.Core/Movies/MovieRepository.cs | 42 ++--- .../Profiles/ProfileRepository.cs | 5 +- src/NzbDrone.Core/Radarr.Core.csproj | 1 - .../ThingiProvider/ProviderRepository.cs | 7 +- 16 files changed, 839 insertions(+), 262 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Extensions/MappingExtensions.cs create mode 100644 src/NzbDrone.Core/Datastore/Extensions/SqlMapperExtensions.cs create mode 100644 src/NzbDrone.Core/Datastore/LazyLoaded.cs create mode 100644 src/NzbDrone.Core/Datastore/SqlBuilder.cs diff --git a/src/NzbDrone.Core.Test/Datastore/TableMapperFixture.cs b/src/NzbDrone.Core.Test/Datastore/TableMapperFixture.cs index 492e0a754..436344e2d 100644 --- a/src/NzbDrone.Core.Test/Datastore/TableMapperFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/TableMapperFixture.cs @@ -45,7 +45,7 @@ namespace NzbDrone.Core.Test.Datastore { var properties = typeof(TypeWithAllMappableProperties).GetProperties(); properties.Should().NotBeEmpty(); - properties.Should().OnlyContain(c => ColumnMapper.IsMappableProperty(c)); + properties.Should().OnlyContain(c => c.IsMappableProperty()); } [Test] @@ -53,7 +53,7 @@ namespace NzbDrone.Core.Test.Datastore { var properties = typeof(TypeWithNoMappableProperties).GetProperties(); properties.Should().NotBeEmpty(); - properties.Should().NotContain(c => ColumnMapper.IsMappableProperty(c)); + properties.Should().NotContain(c => c.IsMappableProperty()); } } } diff --git a/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs b/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs index e954bdfde..1d6b6cb3f 100644 --- a/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/WhereBuilderFixture.cs @@ -24,7 +24,7 @@ namespace NzbDrone.Core.Test.Datastore private WhereBuilder Where(Expression> filter) { - return new WhereBuilder(filter, true); + return new WhereBuilder(filter, true, 0); } [Test] @@ -32,9 +32,8 @@ namespace NzbDrone.Core.Test.Datastore { _subject = Where(x => x.Id == 10); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = @{name})"); - _subject.Parameters.Get(name).Should().Be(10); + _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(10); } [Test] @@ -43,44 +42,71 @@ namespace NzbDrone.Core.Test.Datastore var id = 10; _subject = Where(x => x.Id == id); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = @{name})"); - _subject.Parameters.Get(name).Should().Be(id); + _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(id); + } + + [Test] + public void where_equal_property() + { + var movie = new Movie { Id = 10 }; + _subject = Where(x => x.Id == movie.Id); + + _subject.Parameters.ParameterNames.Should().HaveCount(1); + _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(movie.Id); + } + + [Test] + public void where_equal_joined_property() + { + _subject = Where(x => x.Profile.Id == 1); + + _subject.Parameters.ParameterNames.Should().HaveCount(1); + _subject.ToString().Should().Be($"(\"Profiles\".\"Id\" = @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(1); } [Test] public void where_throws_without_concrete_condition_if_requiresConcreteCondition() { - var movie = new Movie(); - Expression> filter = (x) => x.Id == movie.Id; - _subject = new WhereBuilder(filter, true); + Expression> filter = (x, y) => x.Id == y.Id; + _subject = new WhereBuilder(filter, true, 0); Assert.Throws(() => _subject.ToString()); } [Test] public void where_allows_abstract_condition_if_not_requiresConcreteCondition() { - var movie = new Movie(); - Expression> filter = (x) => x.Id == movie.Id; - _subject = new WhereBuilder(filter, false); + Expression> filter = (x, y) => x.Id == y.Id; + _subject = new WhereBuilder(filter, false, 0); _subject.ToString().Should().Be($"(\"Movies\".\"Id\" = \"Movies\".\"Id\")"); } [Test] public void where_string_is_null() { - _subject = Where(x => x.ImdbId == null); + _subject = Where(x => x.CleanTitle == null); - _subject.ToString().Should().Be($"(\"Movies\".\"ImdbId\" IS NULL)"); + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" IS NULL)"); } [Test] public void where_string_is_null_value() { - string imdb = null; - _subject = Where(x => x.ImdbId == imdb); + string cleanTitle = null; + _subject = Where(x => x.CleanTitle == cleanTitle); + + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" IS NULL)"); + } + + [Test] + public void where_equal_null_property() + { + var movie = new Movie { CleanTitle = null }; + _subject = Where(x => x.CleanTitle == movie.CleanTitle); - _subject.ToString().Should().Be($"(\"Movies\".\"ImdbId\" IS NULL)"); + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" IS NULL)"); } [Test] @@ -89,9 +115,8 @@ namespace NzbDrone.Core.Test.Datastore var test = "small"; _subject = Where(x => x.CleanTitle.Contains(test)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE '%' || @{name} || '%')"); - _subject.Parameters.Get(name).Should().Be(test); + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE '%' || @Clause1_P1 || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); } [Test] @@ -100,9 +125,8 @@ namespace NzbDrone.Core.Test.Datastore var test = "small"; _subject = Where(x => test.Contains(x.CleanTitle)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(@{name} LIKE '%' || \"Movies\".\"CleanTitle\" || '%')"); - _subject.Parameters.Get(name).Should().Be(test); + _subject.ToString().Should().Be($"(@Clause1_P1 LIKE '%' || \"Movies\".\"CleanTitle\" || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); } [Test] @@ -111,9 +135,8 @@ namespace NzbDrone.Core.Test.Datastore var test = "small"; _subject = Where(x => x.CleanTitle.StartsWith(test)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE @{name} || '%')"); - _subject.Parameters.Get(name).Should().Be(test); + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE @Clause1_P1 || '%')"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); } [Test] @@ -122,9 +145,8 @@ namespace NzbDrone.Core.Test.Datastore var test = "small"; _subject = Where(x => x.CleanTitle.EndsWith(test)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE '%' || @{name})"); - _subject.Parameters.Get(name).Should().Be(test); + _subject.ToString().Should().Be($"(\"Movies\".\"CleanTitle\" LIKE '%' || @Clause1_P1)"); + _subject.Parameters.Get("Clause1_P1").Should().Be(test); } [Test] @@ -133,11 +155,9 @@ namespace NzbDrone.Core.Test.Datastore var list = new List { 1, 2, 3 }; _subject = Where(x => list.Contains(x.Id)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Id\" IN @{name})"); + _subject.ToString().Should().Be($"(\"Movies\".\"Id\" IN (1, 2, 3))"); - var param = _subject.Parameters.Get>(name); - param.Should().BeEquivalentTo(list); + _subject.Parameters.ParameterNames.Should().BeEmpty(); } [Test] @@ -146,37 +166,33 @@ namespace NzbDrone.Core.Test.Datastore var list = new List { 1, 2, 3 }; _subject = Where(x => x.CleanTitle == "test" && list.Contains(x.Id)); - var names = _subject.Parameters.ParameterNames.ToList(); - _subject.ToString().Should().Be($"((\"Movies\".\"CleanTitle\" = @{names[0]}) AND (\"Movies\".\"Id\" IN @{names[1]}))"); + _subject.ToString().Should().Be($"((\"Movies\".\"CleanTitle\" = @Clause1_P1) AND (\"Movies\".\"Id\" IN (1, 2, 3)))"); } [Test] public void enum_as_int() { - _subject = Where(x => x.Status == MovieStatusType.Released); + _subject = Where(x => x.Status == MovieStatusType.Announced); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Status\" = @{name})"); + _subject.ToString().Should().Be($"(\"Movies\".\"Status\" = @Clause1_P1)"); } [Test] public void enum_in_list() { - var allowed = new List { MovieStatusType.InCinemas, MovieStatusType.Released }; + var allowed = new List { MovieStatusType.Announced, MovieStatusType.InCinemas }; _subject = Where(x => allowed.Contains(x.Status)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Status\" IN @{name})"); + _subject.ToString().Should().Be($"(\"Movies\".\"Status\" IN @Clause1_P1)"); } [Test] public void enum_in_array() { - var allowed = new MovieStatusType[] { MovieStatusType.InCinemas, MovieStatusType.Released }; + var allowed = new MovieStatusType[] { MovieStatusType.Announced, MovieStatusType.InCinemas }; _subject = Where(x => allowed.Contains(x.Status)); - var name = _subject.Parameters.ParameterNames.First(); - _subject.ToString().Should().Be($"(\"Movies\".\"Status\" IN @{name})"); + _subject.ToString().Should().Be($"(\"Movies\".\"Status\" IN @Clause1_P1)"); } } } diff --git a/src/NzbDrone.Core/Blacklisting/BlacklistRepository.cs b/src/NzbDrone.Core/Blacklisting/BlacklistRepository.cs index 4f4f4e063..eb479eb76 100644 --- a/src/NzbDrone.Core/Blacklisting/BlacklistRepository.cs +++ b/src/NzbDrone.Core/Blacklisting/BlacklistRepository.cs @@ -1,6 +1,4 @@ using System.Collections.Generic; -using System.Linq; -using Dapper; using NzbDrone.Core.Datastore; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Movies; @@ -36,23 +34,11 @@ namespace NzbDrone.Core.Blacklisting return Query(x => x.MovieId == movieId); } - private IEnumerable SelectJoined(SqlBuilder.Template sql) - { - using (var conn = _database.OpenConnection()) - { - return conn.Query( - sql.RawSql, - (bl, movie) => + protected override SqlBuilder PagedBuilder() => new SqlBuilder().Join((b, m) => b.MovieId == m.Id); + protected override IEnumerable PagedQuery(SqlBuilder sql) => _database.QueryJoined(sql, (bl, movie) => { bl.Movie = movie; return bl; - }, - sql.Parameters) - .ToList(); - } - } - - protected override SqlBuilder PagedBuilder() => new SqlBuilder().Join((b, m) => b.MovieId == m.Id); - protected override IEnumerable PagedSelector(SqlBuilder.Template sql) => SelectJoined(sql); + }); } } diff --git a/src/NzbDrone.Core/Datastore/BasicRepository.cs b/src/NzbDrone.Core/Datastore/BasicRepository.cs index d73160107..3f2b02a35 100644 --- a/src/NzbDrone.Core/Datastore/BasicRepository.cs +++ b/src/NzbDrone.Core/Datastore/BasicRepository.cs @@ -47,8 +47,6 @@ namespace NzbDrone.Core.Datastore protected readonly IDatabase _database; protected readonly string _table; - protected string _selectTemplate; - protected string _deleteTemplate; public BasicRepository(IDatabase database, IEventAggregator eventAggregator) { @@ -62,42 +60,19 @@ namespace NzbDrone.Core.Datastore var excluded = TableMapping.Mapper.ExcludeProperties(type).Select(x => x.Name).ToList(); excluded.Add(_keyProperty.Name); - _properties = type.GetProperties().Where(x => !excluded.Contains(x.Name)).ToList(); + _properties = type.GetProperties().Where(x => x.IsMappableProperty() && !excluded.Contains(x.Name)).ToList(); _insertSql = GetInsertSql(); _updateSql = GetUpdateSql(_properties); - - _selectTemplate = $"SELECT /**select**/ FROM {_table} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**orderby**/"; - _deleteTemplate = $"DELETE FROM {_table} /**where**/"; - } - - protected virtual SqlBuilder BuilderBase() => new SqlBuilder(); - protected virtual SqlBuilder Builder() => BuilderBase().SelectAll(); - - protected virtual IEnumerable GetResults(SqlBuilder.Template sql) - { - using (var conn = _database.OpenConnection()) - { - return conn.Query(sql.RawSql, sql.Parameters); - } } - protected List Query(Expression> where) - { - return Query(Builder().Where(where)); - } + protected virtual SqlBuilder Builder() => new SqlBuilder(); - protected List Query(SqlBuilder builder) - { - return Query(builder, GetResults); - } + protected virtual List Query(SqlBuilder builder) => _database.Query(builder).ToList(); - protected List Query(SqlBuilder builder, Func> queryFunc) - { - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); + protected List Query(Expression> where) => Query(Builder().Where(where)); - return queryFunc(sql).ToList(); - } + protected virtual List QueryDistinct(SqlBuilder builder) => _database.QueryDistinct(builder).ToList(); public int Count() { @@ -197,6 +172,7 @@ namespace NzbDrone.Core.Datastore private TModel Insert(IDbConnection connection, IDbTransaction transaction, TModel model) { + SqlBuilderExtensions.LogQuery(_insertSql, model); var multi = connection.QueryMultiple(_insertSql, model, transaction); var id = (int)multi.Read().First().id; _keyProperty.SetValue(model, id); @@ -262,7 +238,7 @@ namespace NzbDrone.Core.Datastore protected void Delete(SqlBuilder builder) { - var sql = builder.AddTemplate(_deleteTemplate).LogQuery(); + var sql = builder.AddDeleteTemplate(typeof(TModel)).LogQuery(); using (var conn = _database.OpenConnection()) { @@ -368,7 +344,7 @@ namespace NzbDrone.Core.Datastore private string GetUpdateSql(List propertiesToUpdate) { var sb = new StringBuilder(); - sb.AppendFormat("update {0} set ", _table); + sb.AppendFormat("UPDATE {0} SET ", _table); for (var i = 0; i < propertiesToUpdate.Count; i++) { @@ -380,7 +356,7 @@ namespace NzbDrone.Core.Datastore } } - sb.Append($" where \"{_keyProperty.Name}\" = @{_keyProperty.Name}"); + sb.Append($" WHERE \"{_keyProperty.Name}\" = @{_keyProperty.Name}"); return sb.ToString(); } @@ -389,6 +365,8 @@ namespace NzbDrone.Core.Datastore { var sql = propertiesToUpdate == _properties ? _updateSql : GetUpdateSql(propertiesToUpdate); + SqlBuilderExtensions.LogQuery(sql, model); + connection.Execute(sql, model, transaction: transaction); } @@ -396,15 +374,20 @@ namespace NzbDrone.Core.Datastore { var sql = propertiesToUpdate == _properties ? _updateSql : GetUpdateSql(propertiesToUpdate); + foreach (var model in models) + { + SqlBuilderExtensions.LogQuery(sql, model); + } + connection.Execute(sql, models, transaction: transaction); } - protected virtual SqlBuilder PagedBuilder() => BuilderBase(); - protected virtual IEnumerable PagedSelector(SqlBuilder.Template sql) => GetResults(sql); + protected virtual SqlBuilder PagedBuilder() => Builder(); + protected virtual IEnumerable PagedQuery(SqlBuilder sql) => Query(sql); public virtual PagingSpec GetPaged(PagingSpec pagingSpec) { - pagingSpec.Records = GetPagedRecords(PagedBuilder().SelectAll(), pagingSpec, PagedSelector); + pagingSpec.Records = GetPagedRecords(PagedBuilder(), pagingSpec, PagedQuery); pagingSpec.TotalRecords = GetPagedRecordCount(PagedBuilder().SelectCount(), pagingSpec); return pagingSpec; @@ -420,7 +403,7 @@ namespace NzbDrone.Core.Datastore } } - protected List GetPagedRecords(SqlBuilder builder, PagingSpec pagingSpec, Func> queryFunc) + protected List GetPagedRecords(SqlBuilder builder, PagingSpec pagingSpec, Func> queryFunc) { AddFilters(builder, pagingSpec); @@ -428,16 +411,22 @@ namespace NzbDrone.Core.Datastore var pagingOffset = (pagingSpec.Page - 1) * pagingSpec.PageSize; builder.OrderBy($"{pagingSpec.SortKey} {sortDirection} LIMIT {pagingSpec.PageSize} OFFSET {pagingOffset}"); - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); - - return queryFunc(sql).ToList(); + return queryFunc(builder).ToList(); } - protected int GetPagedRecordCount(SqlBuilder builder, PagingSpec pagingSpec) + protected int GetPagedRecordCount(SqlBuilder builder, PagingSpec pagingSpec, string template = null) { AddFilters(builder, pagingSpec); - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); + SqlBuilder.Template sql; + if (template != null) + { + sql = builder.AddTemplate(template).LogQuery(); + } + else + { + sql = builder.AddPageCountTemplate(typeof(TModel)); + } using (var conn = _database.OpenConnection()) { diff --git a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs index 8136ff427..cba52d2cb 100644 --- a/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs +++ b/src/NzbDrone.Core/Datastore/Extensions/BuilderExtensions.cs @@ -14,12 +14,18 @@ namespace NzbDrone.Core.Datastore { public static class SqlBuilderExtensions { - public static bool LogSql { get; set; } private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(SqlBuilderExtensions)); - public static SqlBuilder SelectAll(this SqlBuilder builder) + public static bool LogSql { get; set; } + + public static SqlBuilder Select(this SqlBuilder builder, params Type[] types) + { + return builder.Select(types.Select(x => TableMapping.Mapper.TableNameMapping(x) + ".*").Join(", ")); + } + + public static SqlBuilder SelectDistinct(this SqlBuilder builder, params Type[] types) { - return builder.Select("*"); + return builder.Select("DISTINCT " + types.Select(x => TableMapping.Mapper.TableNameMapping(x) + ".*").Join(", ")); } public static SqlBuilder SelectCount(this SqlBuilder builder) @@ -27,23 +33,30 @@ namespace NzbDrone.Core.Datastore return builder.Select("COUNT(*)"); } + public static SqlBuilder SelectCountDistinct(this SqlBuilder builder, Expression> property) + { + var table = TableMapping.Mapper.TableNameMapping(typeof(TModel)); + var propName = property.GetMemberName().Name; + return builder.Select($"COUNT(DISTINCT \"{table}\".\"{propName}\")"); + } + public static SqlBuilder Where(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, true); + var wb = new WhereBuilder(filter, true, builder.Sequence); return builder.Where(wb.ToString(), wb.Parameters); } public static SqlBuilder OrWhere(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, true); + var wb = new WhereBuilder(filter, true, builder.Sequence); return builder.OrWhere(wb.ToString(), wb.Parameters); } public static SqlBuilder Join(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, false); + var wb = new WhereBuilder(filter, false, builder.Sequence); var rightTable = TableMapping.Mapper.TableNameMapping(typeof(TRight)); @@ -52,39 +65,74 @@ namespace NzbDrone.Core.Datastore public static SqlBuilder LeftJoin(this SqlBuilder builder, Expression> filter) { - var wb = new WhereBuilder(filter, false); + var wb = new WhereBuilder(filter, false, builder.Sequence); var rightTable = TableMapping.Mapper.TableNameMapping(typeof(TRight)); return builder.LeftJoin($"{rightTable} ON {wb.ToString()}"); } + public static SqlBuilder GroupBy(this SqlBuilder builder, Expression> property) + { + var table = TableMapping.Mapper.TableNameMapping(typeof(TModel)); + var propName = property.GetMemberName().Name; + return builder.GroupBy($"{table}.{propName}"); + } + + public static SqlBuilder.Template AddSelectTemplate(this SqlBuilder builder, Type type) + { + return builder.AddTemplate(TableMapping.Mapper.SelectTemplate(type)).LogQuery(); + } + + public static SqlBuilder.Template AddPageCountTemplate(this SqlBuilder builder, Type type) + { + return builder.AddTemplate(TableMapping.Mapper.PageCountTemplate(type)).LogQuery(); + } + + public static SqlBuilder.Template AddDeleteTemplate(this SqlBuilder builder, Type type) + { + return builder.AddTemplate(TableMapping.Mapper.DeleteTemplate(type)).LogQuery(); + } + public static SqlBuilder.Template LogQuery(this SqlBuilder.Template template) { if (LogSql) { - var sb = new StringBuilder(); - sb.AppendLine(); - sb.AppendLine("==== Begin Query Trace ===="); - sb.AppendLine(); - sb.AppendLine("QUERY TEXT:"); - sb.AppendLine(template.RawSql); - sb.AppendLine(); - sb.AppendLine("PARAMETERS:"); - foreach (var p in ((DynamicParameters)template.Parameters).ToDictionary()) - { - object val = (p.Value is string) ? string.Format("\"{0}\"", p.Value) : p.Value; - sb.AppendFormat("{0} = [{1}]", p.Key, val.ToJson() ?? "NULL").AppendLine(); - } + LogQuery(template.RawSql, (DynamicParameters)template.Parameters); + } - sb.AppendLine(); - sb.AppendLine("==== End Query Trace ===="); - sb.AppendLine(); + return template; + } - Logger.Trace(sb.ToString()); + public static void LogQuery(string sql, object parameters) + { + if (LogSql) + { + LogQuery(sql, new DynamicParameters(parameters)); } + } - return template; + private static void LogQuery(string sql, DynamicParameters parameters) + { + var sb = new StringBuilder(); + sb.AppendLine(); + sb.AppendLine("==== Begin Query Trace ===="); + sb.AppendLine(); + sb.AppendLine("QUERY TEXT:"); + sb.AppendLine(sql); + sb.AppendLine(); + sb.AppendLine("PARAMETERS:"); + foreach (var p in parameters.ToDictionary()) + { + var val = (p.Value is string) ? string.Format("\"{0}\"", p.Value) : p.Value; + sb.AppendFormat("{0} = [{1}]", p.Key, val.ToJson() ?? "NULL").AppendLine(); + } + + sb.AppendLine(); + sb.AppendLine("==== End Query Trace ===="); + sb.AppendLine(); + + Logger.Trace(sb.ToString()); } private static Dictionary ToDictionary(this DynamicParameters dynamicParams) @@ -99,32 +147,21 @@ namespace NzbDrone.Core.Datastore } var templates = dynamicParams.GetType().GetField("templates", BindingFlags.NonPublic | BindingFlags.Instance); - if (templates != null) + if (templates != null && templates.GetValue(dynamicParams) is List list) { - var list = templates.GetValue(dynamicParams) as List; - if (list != null) + foreach (var objProps in list.Select(obj => obj.GetPropertyValuePairs().ToList())) { - foreach (var objProps in list.Select(obj => obj.GetPropertyValuePairs().ToList())) - { - objProps.ForEach(p => argsDictionary.Add(p.Key, p.Value)); - } + objProps.ForEach(p => argsDictionary.Add(p.Key, p.Value)); } } return argsDictionary; } - private static Dictionary GetPropertyValuePairs(this object obj, string[] hidden = null) + private static Dictionary GetPropertyValuePairs(this object obj) { var type = obj.GetType(); - var pairs = hidden == null - ? type.GetProperties() - .DistinctBy(propertyInfo => propertyInfo.Name) - .ToDictionary( - propertyInfo => propertyInfo.Name, - propertyInfo => propertyInfo.GetValue(obj, null)) - : type.GetProperties() - .Where(it => !hidden.Contains(it.Name)) + var pairs = type.GetProperties().Where(x => x.IsMappableProperty()) .DistinctBy(propertyInfo => propertyInfo.Name) .ToDictionary( propertyInfo => propertyInfo.Name, diff --git a/src/NzbDrone.Core/Datastore/Extensions/MappingExtensions.cs b/src/NzbDrone.Core/Datastore/Extensions/MappingExtensions.cs new file mode 100644 index 000000000..7be628d96 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Extensions/MappingExtensions.cs @@ -0,0 +1,47 @@ +using System; +using System.Linq.Expressions; +using System.Reflection; +using Dapper; +using NzbDrone.Common.Reflection; + +namespace NzbDrone.Core.Datastore +{ + public static class MappingExtensions + { + public static PropertyInfo GetMemberName(this Expression> member) + { + if (!(member.Body is MemberExpression memberExpression)) + { + memberExpression = (member.Body as UnaryExpression).Operand as MemberExpression; + } + + return (PropertyInfo)memberExpression.Member; + } + + public static bool IsMappableProperty(this MemberInfo memberInfo) + { + var propertyInfo = memberInfo as PropertyInfo; + + if (propertyInfo == null) + { + return false; + } + + if (!propertyInfo.IsReadable() || !propertyInfo.IsWritable()) + { + return false; + } + + // This is a bit of a hack but is the only way to see if a type has a handler set in Dapper +#pragma warning disable 618 + SqlMapper.LookupDbType(propertyInfo.PropertyType, "", false, out var handler); +#pragma warning restore 618 + if (propertyInfo.PropertyType.IsSimpleType() || handler != null) + { + return true; + } + + return false; + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Extensions/SqlMapperExtensions.cs b/src/NzbDrone.Core/Datastore/Extensions/SqlMapperExtensions.cs new file mode 100644 index 000000000..801961c30 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Extensions/SqlMapperExtensions.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Data; +using Dapper; + +namespace NzbDrone.Core.Datastore +{ + public static class SqlMapperExtensions + { + public static IEnumerable Query(this IDatabase db, string sql, object param = null) + { + using (var conn = db.OpenConnection()) + { + var items = SqlMapper.Query(conn, sql, param); + if (TableMapping.Mapper.LazyLoadList.TryGetValue(typeof(T), out var lazyProperties)) + { + foreach (var item in items) + { + ApplyLazyLoad(db, item, lazyProperties); + } + } + + return items; + } + } + + public static IEnumerable Query(this IDatabase db, string sql, Func map, object param = null, IDbTransaction transaction = null, bool buffered = true, string splitOn = "Id", int? commandTimeout = null, CommandType? commandType = null) + { + TReturn MapWithLazy(TFirst first, TSecond second) + { + ApplyLazyLoad(db, first); + ApplyLazyLoad(db, second); + return map(first, second); + } + + IEnumerable result = null; + using (var conn = db.OpenConnection()) + { + result = SqlMapper.Query(conn, sql, MapWithLazy, param, transaction, buffered, splitOn, commandTimeout, commandType); + } + + return result; + } + + public static IEnumerable Query(this IDatabase db, string sql, Func map, object param = null, IDbTransaction transaction = null, bool buffered = true, string splitOn = "Id", int? commandTimeout = null, CommandType? commandType = null) + { + TReturn MapWithLazy(TFirst first, TSecond second, TThird third) + { + ApplyLazyLoad(db, first); + ApplyLazyLoad(db, second); + ApplyLazyLoad(db, third); + return map(first, second, third); + } + + IEnumerable result = null; + using (var conn = db.OpenConnection()) + { + result = SqlMapper.Query(conn, sql, MapWithLazy, param, transaction, buffered, splitOn, commandTimeout, commandType); + } + + return result; + } + + public static IEnumerable Query(this IDatabase db, string sql, Func map, object param = null, IDbTransaction transaction = null, bool buffered = true, string splitOn = "Id", int? commandTimeout = null, CommandType? commandType = null) + { + TReturn MapWithLazy(TFirst first, TSecond second, TThird third, TFourth fourth) + { + ApplyLazyLoad(db, first); + ApplyLazyLoad(db, second); + ApplyLazyLoad(db, third); + ApplyLazyLoad(db, fourth); + return map(first, second, third, fourth); + } + + IEnumerable result = null; + using (var conn = db.OpenConnection()) + { + result = SqlMapper.Query(conn, sql, MapWithLazy, param, transaction, buffered, splitOn, commandTimeout, commandType); + } + + return result; + } + + public static IEnumerable Query(this IDatabase db, string sql, Func map, object param = null, IDbTransaction transaction = null, bool buffered = true, string splitOn = "Id", int? commandTimeout = null, CommandType? commandType = null) + { + TReturn MapWithLazy(TFirst first, TSecond second, TThird third, TFourth fourth, TFifth fifth) + { + ApplyLazyLoad(db, first); + ApplyLazyLoad(db, second); + ApplyLazyLoad(db, third); + ApplyLazyLoad(db, fourth); + ApplyLazyLoad(db, fifth); + return map(first, second, third, fourth, fifth); + } + + IEnumerable result = null; + using (var conn = db.OpenConnection()) + { + result = SqlMapper.Query(conn, sql, MapWithLazy, param, transaction, buffered, splitOn, commandTimeout, commandType); + } + + return result; + } + + public static IEnumerable Query(this IDatabase db, SqlBuilder builder) + { + var type = typeof(T); + var sql = builder.Select(type).AddSelectTemplate(type); + + return db.Query(sql.RawSql, sql.Parameters); + } + + public static IEnumerable QueryDistinct(this IDatabase db, SqlBuilder builder) + { + var type = typeof(T); + var sql = builder.SelectDistinct(type).AddSelectTemplate(type); + + return db.Query(sql.RawSql, sql.Parameters); + } + + public static IEnumerable QueryJoined(this IDatabase db, SqlBuilder builder, Func mapper) + { + var type = typeof(T); + var sql = builder.Select(type, typeof(T2)).AddSelectTemplate(type); + + return db.Query(sql.RawSql, mapper, sql.Parameters); + } + + public static IEnumerable QueryJoined(this IDatabase db, SqlBuilder builder, Func mapper) + { + var type = typeof(T); + var sql = builder.Select(type, typeof(T2), typeof(T3)).AddSelectTemplate(type); + + return db.Query(sql.RawSql, mapper, sql.Parameters); + } + + public static IEnumerable QueryJoined(this IDatabase db, SqlBuilder builder, Func mapper) + { + var type = typeof(T); + var sql = builder.Select(type, typeof(T2), typeof(T3), typeof(T4)).AddSelectTemplate(type); + + return db.Query(sql.RawSql, mapper, sql.Parameters); + } + + public static IEnumerable QueryJoined(this IDatabase db, SqlBuilder builder, Func mapper) + { + var type = typeof(T); + var sql = builder.Select(type, typeof(T2), typeof(T3), typeof(T4), typeof(T5)).AddSelectTemplate(type); + + return db.Query(sql.RawSql, mapper, sql.Parameters); + } + + private static void ApplyLazyLoad(IDatabase db, TModel model) + { + if (TableMapping.Mapper.LazyLoadList.TryGetValue(typeof(TModel), out var lazyProperties)) + { + ApplyLazyLoad(db, model, lazyProperties); + } + } + + private static void ApplyLazyLoad(IDatabase db, TModel model, List lazyProperties) + { + if (model == null) + { + return; + } + + foreach (var lazyProperty in lazyProperties) + { + var lazy = (ILazyLoaded)lazyProperty.LazyLoad.Clone(); + lazy.Prepare(db, model); + lazyProperty.Property.SetValue(model, lazy); + } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/LazyLoaded.cs b/src/NzbDrone.Core/Datastore/LazyLoaded.cs new file mode 100644 index 000000000..91ff44c82 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/LazyLoaded.cs @@ -0,0 +1,139 @@ +using System; +using NLog; +using NzbDrone.Common.Instrumentation; + +namespace NzbDrone.Core.Datastore +{ + public interface ILazyLoaded : ICloneable + { + bool IsLoaded { get; } + void Prepare(IDatabase database, object parent); + void LazyLoad(); + } + + /// + /// Allows a field to be lazy loaded. + /// + /// + public class LazyLoaded : ILazyLoaded + { + protected TChild _value; + + public LazyLoaded() + { + } + + public LazyLoaded(TChild val) + { + _value = val; + IsLoaded = true; + } + + public TChild Value + { + get + { + LazyLoad(); + return _value; + } + } + + public bool IsLoaded { get; protected set; } + + public static implicit operator LazyLoaded(TChild val) + { + return new LazyLoaded(val); + } + + public static implicit operator TChild(LazyLoaded lazy) + { + return lazy.Value; + } + + public virtual void Prepare(IDatabase database, object parent) + { + } + + public virtual void LazyLoad() + { + } + + public object Clone() + { + return MemberwiseClone(); + } + + public bool ShouldSerializeValue() + { + return IsLoaded; + } + } + + /// + /// This is the lazy loading proxy. + /// + /// The parent entity that contains the lazy loaded entity. + /// The child entity that is being lazy loaded. + internal class LazyLoaded : LazyLoaded + { + private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(LazyLoaded)); + + private readonly Func _query; + private readonly Func _condition; + + private IDatabase _database; + private TParent _parent; + + public LazyLoaded(TChild val) + : base(val) + { + _value = val; + IsLoaded = true; + } + + internal LazyLoaded(Func query, Func condition = null) + { + _query = query; + _condition = condition; + } + + public static implicit operator LazyLoaded(TChild val) + { + return new LazyLoaded(val); + } + + public static implicit operator TChild(LazyLoaded lazy) + { + return lazy.Value; + } + + public override void Prepare(IDatabase database, object parent) + { + _database = database; + _parent = (TParent)parent; + } + + public override void LazyLoad() + { + if (!IsLoaded) + { + if (_condition != null && _condition(_parent)) + { + if (SqlBuilderExtensions.LogSql) + { + Logger.Trace($"Lazy loading {typeof(TChild)} for {typeof(TParent)}"); + Logger.Trace("StackTrace: '{0}'", Environment.StackTrace); + } + + _value = _query(_database, _parent); + } + else + { + _value = default; + } + + IsLoaded = true; + } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/SqlBuilder.cs b/src/NzbDrone.Core/Datastore/SqlBuilder.cs new file mode 100644 index 000000000..e686d4852 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/SqlBuilder.cs @@ -0,0 +1,168 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Dapper; + +namespace NzbDrone.Core.Datastore +{ + public class SqlBuilder + { + private readonly Dictionary _data = new Dictionary(); + + public int Sequence { get; private set; } + + public Template AddTemplate(string sql, dynamic parameters = null) => + new Template(this, sql, parameters); + + public SqlBuilder Intersect(string sql, dynamic parameters = null) => + AddClause("intersect", sql, parameters, "\nINTERSECT\n ", "\n ", "\n", false); + + public SqlBuilder InnerJoin(string sql, dynamic parameters = null) => + AddClause("innerjoin", sql, parameters, "\nINNER JOIN ", "\nINNER JOIN ", "\n", false); + + public SqlBuilder LeftJoin(string sql, dynamic parameters = null) => + AddClause("leftjoin", sql, parameters, "\nLEFT JOIN ", "\nLEFT JOIN ", "\n", false); + + public SqlBuilder RightJoin(string sql, dynamic parameters = null) => + AddClause("rightjoin", sql, parameters, "\nRIGHT JOIN ", "\nRIGHT JOIN ", "\n", false); + + public SqlBuilder Where(string sql, dynamic parameters = null) => + AddClause("where", sql, parameters, " AND ", "WHERE ", "\n", false); + + public SqlBuilder OrWhere(string sql, dynamic parameters = null) => + AddClause("where", sql, parameters, " OR ", "WHERE ", "\n", true); + + public SqlBuilder OrderBy(string sql, dynamic parameters = null) => + AddClause("orderby", sql, parameters, " , ", "ORDER BY ", "\n", false); + + public SqlBuilder Select(string sql, dynamic parameters = null) => + AddClause("select", sql, parameters, " , ", "", "\n", false); + + public SqlBuilder AddParameters(dynamic parameters) => + AddClause("--parameters", "", parameters, "", "", "", false); + + public SqlBuilder Join(string sql, dynamic parameters = null) => + AddClause("join", sql, parameters, "\nJOIN ", "\nJOIN ", "\n", false); + + public SqlBuilder GroupBy(string sql, dynamic parameters = null) => + AddClause("groupby", sql, parameters, " , ", "\nGROUP BY ", "\n", false); + + public SqlBuilder Having(string sql, dynamic parameters = null) => + AddClause("having", sql, parameters, "\nAND ", "HAVING ", "\n", false); + + protected SqlBuilder AddClause(string name, string sql, object parameters, string joiner, string prefix = "", string postfix = "", bool isInclusive = false) + { + if (!_data.TryGetValue(name, out var clauses)) + { + clauses = new Clauses(joiner, prefix, postfix); + _data[name] = clauses; + } + + clauses.Add(new Clause { Sql = sql, Parameters = parameters, IsInclusive = isInclusive }); + Sequence++; + return this; + } + + public class Template + { + private static readonly Regex _regex = new Regex(@"\/\*\*.+?\*\*\/", RegexOptions.Compiled | RegexOptions.Multiline); + + private readonly string _sql; + private readonly SqlBuilder _builder; + private readonly object _initParams; + + private int _dataSeq = -1; // Unresolved + private string _rawSql; + private object _parameters; + + public Template(SqlBuilder builder, string sql, dynamic parameters) + { + _initParams = parameters; + _sql = sql; + _builder = builder; + } + + public string RawSql + { + get + { + ResolveSql(); + return _rawSql; + } + } + + public object Parameters + { + get + { + ResolveSql(); + return _parameters; + } + } + + private void ResolveSql() + { + if (_dataSeq != _builder.Sequence) + { + var p = new DynamicParameters(_initParams); + + _rawSql = _sql; + + foreach (var pair in _builder._data) + { + _rawSql = _rawSql.Replace("/**" + pair.Key + "**/", pair.Value.ResolveClauses(p)); + } + + _parameters = p; + + // replace all that is left with empty + _rawSql = _regex.Replace(_rawSql, ""); + + _dataSeq = _builder.Sequence; + } + } + } + + private class Clause + { + public string Sql { get; set; } + public object Parameters { get; set; } + public bool IsInclusive { get; set; } + } + + private class Clauses : List + { + private readonly string _joiner; + private readonly string _prefix; + private readonly string _postfix; + + public Clauses(string joiner, string prefix = "", string postfix = "") + { + _joiner = joiner; + _prefix = prefix; + _postfix = postfix; + } + + public string ResolveClauses(DynamicParameters p) + { + foreach (var item in this) + { + p.AddDynamicParams(item.Parameters); + } + + return this.Any(a => a.IsInclusive) + ? _prefix + + string.Join(_joiner, + this.Where(a => !a.IsInclusive) + .Select(c => c.Sql) + .Union(new[] + { + " ( " + + string.Join(" OR ", this.Where(a => a.IsInclusive).Select(c => c.Sql).ToArray()) + + " ) " + }).ToArray()) + _postfix + : _prefix + string.Join(_joiner, this.Select(c => c.Sql).ToArray()) + _postfix; + } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapper.cs b/src/NzbDrone.Core/Datastore/TableMapper.cs index 8a8e19b73..0dbfca678 100644 --- a/src/NzbDrone.Core/Datastore/TableMapper.cs +++ b/src/NzbDrone.Core/Datastore/TableMapper.cs @@ -3,48 +3,36 @@ using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; -using Dapper; -using NzbDrone.Common.Reflection; namespace NzbDrone.Core.Datastore { - public static class MappingExtensions - { - public static PropertyInfo GetMemberName(this Expression> member) - { - var memberExpression = member.Body as MemberExpression; - if (memberExpression == null) - { - memberExpression = (member.Body as UnaryExpression).Operand as MemberExpression; - } - - return (PropertyInfo)memberExpression.Member; - } - } - public class TableMapper { public TableMapper() { IgnoreList = new Dictionary>(); + LazyLoadList = new Dictionary>(); TableMap = new Dictionary(); } public Dictionary> IgnoreList { get; set; } + public Dictionary> LazyLoadList { get; set; } public Dictionary TableMap { get; set; } public ColumnMapper Entity(string tableName) + where TEntity : ModelBase { - TableMap.Add(typeof(TEntity), tableName); + var type = typeof(TEntity); + TableMap.Add(type, tableName); - if (IgnoreList.TryGetValue(typeof(TEntity), out var list)) + if (IgnoreList.TryGetValue(type, out var list)) { - return new ColumnMapper(list); + return new ColumnMapper(list, LazyLoadList[type]); } - list = new List(); - IgnoreList[typeof(TEntity)] = list; - return new ColumnMapper(list); + IgnoreList[type] = new List(); + LazyLoadList[type] = new List(); + return new ColumnMapper(IgnoreList[type], LazyLoadList[type]); } public List ExcludeProperties(Type x) @@ -56,21 +44,44 @@ namespace NzbDrone.Core.Datastore { return TableMap.ContainsKey(x) ? TableMap[x] : null; } + + public string SelectTemplate(Type x) + { + return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; + } + + public string DeleteTemplate(Type x) + { + return $"DELETE FROM {TableMap[x]} /**where**/"; + } + + public string PageCountTemplate(Type x) + { + return $"SELECT /**select**/ FROM {TableMap[x]} /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/"; + } + } + + public class LazyLoadedProperty + { + public PropertyInfo Property { get; set; } + public ILazyLoaded LazyLoad { get; set; } } public class ColumnMapper + where T : ModelBase { private readonly List _ignoreList; + private readonly List _lazyLoadList; - public ColumnMapper(List ignoreList) + public ColumnMapper(List ignoreList, List lazyLoadList) { _ignoreList = ignoreList; + _lazyLoadList = lazyLoadList; } public ColumnMapper AutoMapPropertiesWhere(Func predicate) { - Type entityType = typeof(T); - var properties = entityType.GetProperties(); + var properties = typeof(T).GetProperties(); _ignoreList.AddRange(properties.Where(x => !predicate(x))); return this; @@ -78,7 +89,7 @@ namespace NzbDrone.Core.Datastore public ColumnMapper RegisterModel() { - return AutoMapPropertiesWhere(IsMappableProperty); + return AutoMapPropertiesWhere(x => x.IsMappableProperty()); } public ColumnMapper Ignore(Expression> property) @@ -87,30 +98,31 @@ namespace NzbDrone.Core.Datastore return this; } - public static bool IsMappableProperty(MemberInfo memberInfo) + public ColumnMapper LazyLoad(Expression>> property, Func query, Func condition) { - var propertyInfo = memberInfo as PropertyInfo; + var lazyLoad = new LazyLoaded(query, condition); - if (propertyInfo == null) + var item = new LazyLoadedProperty { - return false; - } + Property = property.GetMemberName(), + LazyLoad = lazyLoad + }; - if (!propertyInfo.IsReadable() || !propertyInfo.IsWritable()) - { - return false; - } + _lazyLoadList.Add(item); - // This is a bit of a hack but is the only way to see if a type has a handler set in Dapper -#pragma warning disable 618 - SqlMapper.LookupDbType(propertyInfo.PropertyType, "", false, out var handler); -#pragma warning restore 618 - if (propertyInfo.PropertyType.IsSimpleType() || handler != null) - { - return true; - } + return this; + } - return false; + public ColumnMapper HasOne(Expression>> portalExpression, Func childIdSelector) + where TChild : ModelBase + { + return LazyLoad(portalExpression, + (db, parent) => + { + var id = childIdSelector(parent); + return db.Query(new SqlBuilder().Where(x => x.Id == id)).SingleOrDefault(); + }, + parent => childIdSelector(parent) > 0); } } } diff --git a/src/NzbDrone.Core/Datastore/WhereBuilder.cs b/src/NzbDrone.Core/Datastore/WhereBuilder.cs index 24b3b17e3..e42d0e990 100644 --- a/src/NzbDrone.Core/Datastore/WhereBuilder.cs +++ b/src/NzbDrone.Core/Datastore/WhereBuilder.cs @@ -19,9 +19,9 @@ namespace NzbDrone.Core.Datastore private int _paramCount = 0; private bool _gotConcreteValue = false; - public WhereBuilder(Expression filter, bool requireConcreteValue) + public WhereBuilder(Expression filter, bool requireConcreteValue, int seq) { - _paramNamePrefix = Guid.NewGuid().ToString().Replace("-", "_"); + _paramNamePrefix = string.Format("Clause{0}", seq + 1); _requireConcreteValue = requireConcreteValue; _sb = new StringBuilder(); @@ -87,16 +87,16 @@ namespace NzbDrone.Core.Datastore protected override Expression VisitMemberAccess(MemberExpression expression) { - var tableName = expression != null ? TableMapping.Mapper.TableNameMapping(expression.Expression.Type) : null; + var tableName = expression?.Expression?.Type != null ? TableMapping.Mapper.TableNameMapping(expression.Expression.Type) : null; + var gotValue = TryGetRightValue(expression, out var value); - if (tableName != null) + // Only use the SQL condition if the expression didn't resolve to an actual value + if (tableName != null && !gotValue) { _sb.Append($"\"{tableName}\".\"{expression.Member.Name}\""); } else { - var value = GetRightValue(expression); - if (value != null) { // string is IEnumerable but we don't want to pick up that case @@ -138,33 +138,43 @@ namespace NzbDrone.Core.Datastore private bool TryGetConstantValue(Expression expression, out object result) { + result = null; + if (expression is ConstantExpression constExp) { result = constExp.Value; return true; } - result = null; return false; } private bool TryGetPropertyValue(MemberExpression expression, out object result) { + result = null; + if (expression.Expression is MemberExpression nested) { // Value is passed in as a property on a parent entity - var container = (nested.Expression as ConstantExpression).Value; + var container = (nested.Expression as ConstantExpression)?.Value; + + if (container == null) + { + return false; + } + var entity = GetFieldValue(container, nested.Member); result = GetFieldValue(entity, expression.Member); return true; } - result = null; return false; } private bool TryGetVariableValue(MemberExpression expression, out object result) { + result = null; + // Value is passed in as a variable if (expression.Expression is ConstantExpression nested) { @@ -172,30 +182,31 @@ namespace NzbDrone.Core.Datastore return true; } - result = null; return false; } - private object GetRightValue(Expression expression) + private bool TryGetRightValue(Expression expression, out object value) { - if (TryGetConstantValue(expression, out var constValue)) + value = null; + + if (TryGetConstantValue(expression, out value)) { - return constValue; + return true; } var memberExp = expression as MemberExpression; - if (TryGetPropertyValue(memberExp, out var propValue)) + if (TryGetPropertyValue(memberExp, out value)) { - return propValue; + return true; } - if (TryGetVariableValue(memberExp, out var variableValue)) + if (TryGetVariableValue(memberExp, out value)) { - return variableValue; + return true; } - return null; + return false; } private object GetFieldValue(object entity, MemberInfo member) @@ -224,8 +235,8 @@ namespace NzbDrone.Core.Datastore if (expression.NodeType == ExpressionType.MemberAccess && expression is MemberExpression member && - TryGetVariableValue(member, out var variableResult) && - variableResult == null) + ((TryGetPropertyValue(member, out var result) && result == null) || + (TryGetVariableValue(member, out result) && result == null))) { return true; } @@ -264,7 +275,7 @@ namespace NzbDrone.Core.Datastore { var list = expression.Object; - if (list != null && list.Type == typeof(string)) + if (list != null && (list.Type == typeof(string) || list.Type == typeof(List))) { ParseStringContains(expression); return; @@ -304,7 +315,20 @@ namespace NzbDrone.Core.Datastore _sb.Append(" IN "); - Visit(list); + // hardcode the integer list if it exists to bypass parameter limit + if (item.Type == typeof(int) && TryGetRightValue(list, out var value)) + { + var items = (IEnumerable)value; + _sb.Append("("); + _sb.Append(string.Join(", ", items)); + _sb.Append(")"); + + _gotConcreteValue = true; + } + else + { + Visit(list); + } _sb.Append(")"); } diff --git a/src/NzbDrone.Core/History/HistoryRepository.cs b/src/NzbDrone.Core/History/HistoryRepository.cs index d3c9da069..b1237e9df 100644 --- a/src/NzbDrone.Core/History/HistoryRepository.cs +++ b/src/NzbDrone.Core/History/HistoryRepository.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Dapper; using NzbDrone.Core.Datastore; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Movies; @@ -74,28 +73,17 @@ namespace NzbDrone.Core.History Delete(c => c.MovieId == movieId); } - private IEnumerable SelectJoined(SqlBuilder.Template sql) - { - using (var conn = _database.OpenConnection()) - { - return conn.Query( - sql.RawSql, - (hist, movie, profile) => - { - hist.Movie = movie; - hist.Movie.Profile = profile; - return hist; - }, - sql.Parameters) - .ToList(); - } - } - protected override SqlBuilder PagedBuilder() => new SqlBuilder() .Join((h, m) => h.MovieId == m.Id) .Join((m, p) => m.ProfileId == p.Id); - protected override IEnumerable PagedSelector(SqlBuilder.Template sql) => SelectJoined(sql); + protected override IEnumerable PagedQuery(SqlBuilder sql) => + _database.QueryJoined(sql, (hist, movie, profile) => + { + hist.Movie = movie; + hist.Movie.Profile = profile; + return hist; + }); public MovieHistory MostRecentForMovie(int movieId) { diff --git a/src/NzbDrone.Core/Movies/MovieRepository.cs b/src/NzbDrone.Core/Movies/MovieRepository.cs index dedcbf4dd..03f55de70 100644 --- a/src/NzbDrone.Core/Movies/MovieRepository.cs +++ b/src/NzbDrone.Core/Movies/MovieRepository.cs @@ -40,7 +40,7 @@ namespace NzbDrone.Core.Movies _profileRepository = profileRepository; } - protected override SqlBuilder BuilderBase() => new SqlBuilder() + protected override SqlBuilder Builder() => new SqlBuilder() .Join((m, p) => m.ProfileId == p.Id) .LeftJoin((m, t) => m.Id == t.MovieId) .LeftJoin((m, f) => m.Id == f.MovieId); @@ -65,40 +65,33 @@ namespace NzbDrone.Core.Movies return movieEntry; } - protected override IEnumerable GetResults(SqlBuilder.Template sql) + protected override List Query(SqlBuilder builder) { var movieDictionary = new Dictionary(); - using (var conn = _database.OpenConnection()) - { - conn.Query( - sql.RawSql, - (movie, profile, altTitle, file) => Map(movieDictionary, movie, profile, altTitle, file), - sql.Parameters); - } + _ = _database.QueryJoined( + builder, + (movie, profile, altTitle, file) => Map(movieDictionary, movie, profile, altTitle, file)); - return movieDictionary.Values; + return movieDictionary.Values.ToList(); } public override IEnumerable All() { // the skips the join on profile and populates manually // to avoid repeatedly deserializing the same profile - var noProfileTemplate = $"SELECT /**select**/ FROM {_table} /**leftjoin**/ /**where**/ /**orderby**/"; - var sql = Builder().AddTemplate(noProfileTemplate).LogQuery(); + var builder = new SqlBuilder() + .LeftJoin((m, t) => m.Id == t.MovieId) + .LeftJoin((m, f) => m.Id == f.MovieId); var movieDictionary = new Dictionary(); var profiles = _profileRepository.All().ToDictionary(x => x.Id); - using (var conn = _database.OpenConnection()) - { - conn.Query( - sql.RawSql, - (movie, altTitle, file) => Map(movieDictionary, movie, profiles[movie.ProfileId], altTitle, file), - sql.Parameters); - } + _ = _database.QueryJoined( + builder, + (movie, altTitle, file) => Map(movieDictionary, movie, profiles[movie.ProfileId], altTitle, file)); - return movieDictionary.Values; + return movieDictionary.Values.ToList(); } public bool MoviePathExists(string path) @@ -163,23 +156,24 @@ namespace NzbDrone.Core.Movies return Query(builder); } - public SqlBuilder MoviesWithoutFilesBuilder() => BuilderBase().Where(x => x.MovieFileId == 0); + public SqlBuilder MoviesWithoutFilesBuilder() => Builder() + .Where(x => x.MovieFileId == 0); public PagingSpec MoviesWithoutFiles(PagingSpec pagingSpec) { - pagingSpec.Records = GetPagedRecords(MoviesWithoutFilesBuilder().SelectAll(), pagingSpec, PagedSelector); + pagingSpec.Records = GetPagedRecords(MoviesWithoutFilesBuilder(), pagingSpec, PagedQuery); pagingSpec.TotalRecords = GetPagedRecordCount(MoviesWithoutFilesBuilder().SelectCount(), pagingSpec); return pagingSpec; } - public SqlBuilder MoviesWhereCutoffUnmetBuilder(List qualitiesBelowCutoff) => BuilderBase() + public SqlBuilder MoviesWhereCutoffUnmetBuilder(List qualitiesBelowCutoff) => Builder() .Where(x => x.MovieFileId != 0) .Where(BuildQualityCutoffWhereClause(qualitiesBelowCutoff)); public PagingSpec MoviesWhereCutoffUnmet(PagingSpec pagingSpec, List qualitiesBelowCutoff) { - pagingSpec.Records = GetPagedRecords(MoviesWhereCutoffUnmetBuilder(qualitiesBelowCutoff).SelectAll(), pagingSpec, PagedSelector); + pagingSpec.Records = GetPagedRecords(MoviesWhereCutoffUnmetBuilder(qualitiesBelowCutoff), pagingSpec, PagedQuery); pagingSpec.TotalRecords = GetPagedRecordCount(MoviesWhereCutoffUnmetBuilder(qualitiesBelowCutoff).SelectCount(), pagingSpec); return pagingSpec; diff --git a/src/NzbDrone.Core/Profiles/ProfileRepository.cs b/src/NzbDrone.Core/Profiles/ProfileRepository.cs index ed5616da8..8d312a53c 100644 --- a/src/NzbDrone.Core/Profiles/ProfileRepository.cs +++ b/src/NzbDrone.Core/Profiles/ProfileRepository.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using System.Linq; -using Dapper; using NzbDrone.Core.CustomFormats; using NzbDrone.Core.Datastore; using NzbDrone.Core.Messaging.Events; @@ -24,11 +23,11 @@ namespace NzbDrone.Core.Profiles _customFormatService = customFormatService; } - protected override IEnumerable GetResults(SqlBuilder.Template sql) + protected override List Query(SqlBuilder builder) { var cfs = _customFormatService.All().ToDictionary(c => c.Id); - var profiles = base.GetResults(sql); + var profiles = base.Query(builder); // Do the conversions from Id to full CustomFormat object here instead of in // CustomFormatIntConverter to remove need to for a static property containing diff --git a/src/NzbDrone.Core/Radarr.Core.csproj b/src/NzbDrone.Core/Radarr.Core.csproj index ecfd8a5b7..6f9c408d4 100644 --- a/src/NzbDrone.Core/Radarr.Core.csproj +++ b/src/NzbDrone.Core/Radarr.Core.csproj @@ -4,7 +4,6 @@ - diff --git a/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs b/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs index c78b29d7d..66bbe559a 100644 --- a/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs +++ b/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs @@ -16,15 +16,18 @@ namespace NzbDrone.Core.ThingiProvider { } - protected override IEnumerable GetResults(SqlBuilder.Template sql) + protected override List Query(SqlBuilder builder) { + var type = typeof(TProviderDefinition); + var sql = builder.Select(type).AddSelectTemplate(type); + var results = new List(); using (var conn = _database.OpenConnection()) using (var reader = conn.ExecuteReader(sql.RawSql, sql.Parameters)) { var parser = reader.GetRowParser(typeof(TProviderDefinition)); - var settingsIndex = reader.GetOrdinal("Settings"); + var settingsIndex = reader.GetOrdinal(nameof(ProviderDefinition.Settings)); var serializerSettings = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; while (reader.Read())