From 82cbfb37411d27c515ea5d1a6904d465ce7cf6cf Mon Sep 17 00:00:00 2001 From: Robert Dailey Date: Sat, 9 Sep 2023 22:45:46 -0500 Subject: [PATCH] refactor: Separate JSON loading from CFs --- .../CustomFormat/CustomFormatAutofacModule.cs | 1 - .../CustomFormat/Guide/CustomFormatLoader.cs | 48 ++----- .../CustomFormat/Guide/CustomFormatParser.cs | 18 --- .../CustomFormat/Guide/ICustomFormatParser.cs | 8 -- src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs | 55 ++++++++ .../TrashLibAutofacModule.cs | 2 + ...s => CustomFormatLoaderIntegrationTest.cs} | 4 +- .../Guide/CustomFormatParserTest.cs | 131 ------------------ .../Json/BulkJsonLoaderTest.cs | 37 +++++ 9 files changed, 109 insertions(+), 195 deletions(-) delete mode 100644 src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatParser.cs delete mode 100644 src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/ICustomFormatParser.cs create mode 100644 src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs rename src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/{CustomFormatLoaderTest.cs => CustomFormatLoaderIntegrationTest.cs} (89%) delete mode 100644 src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatParserTest.cs create mode 100644 src/tests/Recyclarr.TrashLib.Tests/Json/BulkJsonLoaderTest.cs diff --git a/src/Recyclarr.Cli/Pipelines/CustomFormat/CustomFormatAutofacModule.cs b/src/Recyclarr.Cli/Pipelines/CustomFormat/CustomFormatAutofacModule.cs index b58407f5..ab0df5a8 100644 --- a/src/Recyclarr.Cli/Pipelines/CustomFormat/CustomFormatAutofacModule.cs +++ b/src/Recyclarr.Cli/Pipelines/CustomFormat/CustomFormatAutofacModule.cs @@ -18,7 +18,6 @@ public class CustomFormatAutofacModule : Module builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); - builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType(); diff --git a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatLoader.cs b/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatLoader.cs index eab8d4a6..d85f0e08 100644 --- a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatLoader.cs +++ b/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatLoader.cs @@ -1,23 +1,19 @@ using System.IO.Abstractions; using System.Reactive.Linq; -using System.Reactive.Threading.Tasks; -using Newtonsoft.Json; -using Recyclarr.Common; using Recyclarr.Common.Extensions; +using Recyclarr.TrashLib.Json; using Recyclarr.TrashLib.Models; namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide; public class CustomFormatLoader : ICustomFormatLoader { - private readonly ILogger _log; - private readonly ICustomFormatParser _parser; + private readonly BulkJsonLoader _loader; private readonly ICustomFormatCategoryParser _categoryParser; - public CustomFormatLoader(ILogger log, ICustomFormatParser parser, ICustomFormatCategoryParser categoryParser) + public CustomFormatLoader(BulkJsonLoader loader, ICustomFormatCategoryParser categoryParser) { - _log = log; - _parser = parser; + _loader = loader; _categoryParser = categoryParser; } @@ -26,34 +22,16 @@ public class CustomFormatLoader : ICustomFormatLoader IFileInfo collectionOfCustomFormats) { var categories = _categoryParser.Parse(collectionOfCustomFormats).AsReadOnly(); - var jsonFiles = JsonUtils.GetJsonFilesInDirectories(jsonPaths, _log); - return jsonFiles.ToObservable() - .Select(x => Observable.Defer(() => LoadJsonFromFile(x, categories))) - .Merge(8) - .NotNull() - .ToEnumerable() - .ToList(); - } + return _loader.LoadAllFilesAtPaths(jsonPaths, x => x.Select(cf => + { + var matchingCategory = categories.FirstOrDefault( + y => y.CfName.EqualsIgnoreCase(cf.Obj.Name) || y.CfAnchor.EqualsIgnoreCase(cf.File.Name)); - private IObservable LoadJsonFromFile( - IFileInfo file, - IReadOnlyCollection categories) - { - return Observable.Using(file.OpenText, x => x.ReadToEndAsync().ToObservable()) - .Select(x => - { - var cf = _parser.ParseCustomFormatData(x, file.Name); - var matchingCategory = categories.FirstOrDefault(y => - { - var fileName = Path.GetFileNameWithoutExtension(cf.FileName); - return y.CfName.EqualsIgnoreCase(cf.Name) || y.CfAnchor.EqualsIgnoreCase(fileName); - }); - return cf with {Category = matchingCategory?.CategoryName}; - }) - .Catch((JsonException e) => + return cf.Obj with { - _log.Warning("Failed to parse JSON file: {File} ({Reason})", file.Name, e.Message); - return Observable.Empty(); - }); + Category = matchingCategory?.CategoryName, + FileName = cf.File.Name + }; + })); } } diff --git a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatParser.cs b/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatParser.cs deleted file mode 100644 index dc908ef0..00000000 --- a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatParser.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Newtonsoft.Json; -using Recyclarr.TrashLib.Models; - -namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide; - -public class CustomFormatParser : ICustomFormatParser -{ - public CustomFormatData ParseCustomFormatData(string guideData, string fileName) - { - var cf = JsonConvert.DeserializeObject(guideData); - if (cf is null) - { - throw new JsonSerializationException($"Unable to parse JSON at file {fileName}"); - } - - return cf with {FileName = fileName}; - } -} diff --git a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/ICustomFormatParser.cs b/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/ICustomFormatParser.cs deleted file mode 100644 index e38fabc7..00000000 --- a/src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/ICustomFormatParser.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Recyclarr.TrashLib.Models; - -namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide; - -public interface ICustomFormatParser -{ - CustomFormatData ParseCustomFormatData(string guideData, string fileName); -} diff --git a/src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs b/src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs new file mode 100644 index 00000000..eabba8f2 --- /dev/null +++ b/src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs @@ -0,0 +1,55 @@ +using System.IO.Abstractions; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using Newtonsoft.Json; +using Recyclarr.Common; + +namespace Recyclarr.TrashLib.Json; + +public record LoadedJsonObject(IFileInfo File, T Obj); + +public class BulkJsonLoader +{ + private readonly ILogger _log; + + public BulkJsonLoader(ILogger log) + { + _log = log; + } + + public ICollection LoadAllFilesAtPaths( + IEnumerable jsonPaths, + Func>, IObservable>? extra = null) + { + var jsonFiles = JsonUtils.GetJsonFilesInDirectories(jsonPaths, _log); + var observable = jsonFiles.ToObservable() + .Select(x => Observable.Defer(() => LoadJsonFromFile(x))) + .Merge(8); + + var convertedObservable = extra?.Invoke(observable) ?? observable.Select(x => x.Obj); + + return convertedObservable.ToEnumerable().ToList(); + } + + private static T ParseJson(string guideData, string fileName) + { + var obj = JsonConvert.DeserializeObject(guideData, GlobalJsonSerializerSettings.Guide); + if (obj is null) + { + throw new JsonSerializationException($"Unable to parse JSON at file {fileName}"); + } + + return obj; + } + + private IObservable> LoadJsonFromFile(IFileInfo file) + { + return Observable.Using(file.OpenText, x => x.ReadToEndAsync().ToObservable()) + .Select(x => new LoadedJsonObject(file, ParseJson(x, file.Name))) + .Catch((JsonException e) => + { + _log.Warning("Failed to parse JSON file: {File} ({Reason})", file.Name, e.Message); + return Observable.Empty>(); + }); + } +} diff --git a/src/Recyclarr.TrashLib/TrashLibAutofacModule.cs b/src/Recyclarr.TrashLib/TrashLibAutofacModule.cs index 433ff482..fa506de2 100644 --- a/src/Recyclarr.TrashLib/TrashLibAutofacModule.cs +++ b/src/Recyclarr.TrashLib/TrashLibAutofacModule.cs @@ -9,6 +9,7 @@ using Recyclarr.TrashLib.ApiServices; using Recyclarr.TrashLib.Compatibility; using Recyclarr.TrashLib.Config; using Recyclarr.TrashLib.Http; +using Recyclarr.TrashLib.Json; using Recyclarr.TrashLib.Repo; using Recyclarr.TrashLib.Repo.VersionControl; using Recyclarr.TrashLib.Startup; @@ -37,6 +38,7 @@ public class TrashLibAutofacModule : Module builder.RegisterModule(); builder.RegisterType().As(); builder.RegisterType().As().SingleInstance(); + builder.RegisterType(); var mapperAssemblies = new List {ThisAssembly}; if (AdditionalMapperProfileAssembly is not null) diff --git a/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderTest.cs b/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderIntegrationTest.cs similarity index 89% rename from src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderTest.cs rename to src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderIntegrationTest.cs index 7966accd..2df24084 100644 --- a/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderTest.cs +++ b/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderIntegrationTest.cs @@ -8,12 +8,12 @@ namespace Recyclarr.Cli.Tests.Pipelines.CustomFormat.Guide; [TestFixture] [Parallelizable(ParallelScope.All)] -public class CustomFormatLoaderTest : CliIntegrationFixture +public class CustomFormatLoaderIntegrationTest : CliIntegrationFixture { [Test] public void Get_custom_format_json_works() { - var sut = Resolve(); + var sut = Resolve(); Fs.AddFile("first.json", new MockFileData("{'name':'first','trash_id':'1'}")); Fs.AddFile("second.json", new MockFileData("{'name':'second','trash_id':'2'}")); Fs.AddFile("collection_of_cfs.md", new MockFileData("")); diff --git a/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatParserTest.cs b/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatParserTest.cs deleted file mode 100644 index 7a97b5e6..00000000 --- a/src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatParserTest.cs +++ /dev/null @@ -1,131 +0,0 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Recyclarr.Cli.Pipelines.CustomFormat.Guide; -using Recyclarr.Common.Extensions; -using Recyclarr.TrashLib.Json; -using Recyclarr.TrashLib.Models; - -namespace Recyclarr.Cli.Tests.Pipelines.CustomFormat.Guide; - -[TestFixture] -[Parallelizable(ParallelScope.All)] -public class CustomFormatParserTest -{ - [Test, AutoMockData] - public void Deserialize_works(CustomFormatParser sut) - { - const string jsonData = - """ - { - "trash_id": "90cedc1fea7ea5d11298bebd3d1d3223", - "trash_scores": { - "default": -10000, - }, - "name": "EVO (no WEBDL)", - "includeCustomFormatWhenRenaming": false, - "specifications": [ - { - "name": "EVO", - "implementation": "ReleaseTitleSpecification", - "negate": false, - "required": true, - "fields": [{ - "value": "\\bEVO(TGX)?\\b" - }] - }, - { - "name": "WEBDL", - "implementation": "SourceSpecification", - "negate": true, - "required": true, - "fields": { - "value": 7 - } - }, - { - "name": "WEBRIP", - "implementation": "SourceSpecification", - "negate": true, - "required": true, - "fields": { - "value": 8 - } - } - ] - } - """; - - var result = sut.ParseCustomFormatData(jsonData, "file.json"); - - result.Should().BeEquivalentTo(new CustomFormatData - { - FileName = "file.json", - TrashId = "90cedc1fea7ea5d11298bebd3d1d3223", - TrashScores = {["default"] = -10000}, - Name = "EVO (no WEBDL)", - IncludeCustomFormatWhenRenaming = false, - Specifications = new[] - { - new CustomFormatSpecificationData - { - Name = "EVO", - Implementation = "ReleaseTitleSpecification", - Negate = false, - Required = true, - Fields = new[] - { - new CustomFormatFieldData - { - Value = "\\bEVO(TGX)?\\b" - } - } - }, - new CustomFormatSpecificationData - { - Name = "WEBDL", - Implementation = "SourceSpecification", - Negate = true, - Required = true, - Fields = new[] - { - new CustomFormatFieldData - { - Value = 7 - } - } - }, - new CustomFormatSpecificationData - { - Name = "WEBRIP", - Implementation = "SourceSpecification", - Negate = true, - Required = true, - Fields = new[] - { - new CustomFormatFieldData - { - Value = 8 - } - } - } - } - }); - } - - [Test, AutoMockData] - public void Serialize_skips_trash_properties(CustomFormatParser sut) - { - var cf = new CustomFormatData - { - FileName = "file.json", - TrashId = "90cedc1fea7ea5d11298bebd3d1d3223", - TrashScores = {["default"] = -10000}, - Name = "EVO (no WEBDL)", - IncludeCustomFormatWhenRenaming = false - }; - - var json = JObject.FromObject(cf, JsonSerializer.Create(GlobalJsonSerializerSettings.Services)); - - json.Children().Should().NotContain(x => x.Name.ContainsIgnoreCase("trash")); - } -} diff --git a/src/tests/Recyclarr.TrashLib.Tests/Json/BulkJsonLoaderTest.cs b/src/tests/Recyclarr.TrashLib.Tests/Json/BulkJsonLoaderTest.cs new file mode 100644 index 00000000..06d4420d --- /dev/null +++ b/src/tests/Recyclarr.TrashLib.Tests/Json/BulkJsonLoaderTest.cs @@ -0,0 +1,37 @@ +using System.Diagnostics.CodeAnalysis; +using System.IO.Abstractions; +using Recyclarr.TrashLib.Json; + +namespace Recyclarr.TrashLib.Tests.Json; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class BulkJsonLoaderTest +{ + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private sealed record TestJsonObject(string TrashId, int TrashScore, string Name); + + [Test, AutoMockData] + public void Deserialize_works( + [Frozen(Matching.ImplementedInterfaces)] MockFileSystem fs, + BulkJsonLoader sut) + { + var jsonData = + """ + { + "trash_id": "90cedc1fea7ea5d11298bebd3d1d3223", + "trash_score": "-10000", + "name": "TheName" + } + """; + + fs.AddFile(fs.CurrentDirectory().File("file.json"), new MockFileData(jsonData)); + + var result = sut.LoadAllFilesAtPaths(new[] {fs.CurrentDirectory()}); + + result.Should().BeEquivalentTo(new[] + { + new TestJsonObject("90cedc1fea7ea5d11298bebd3d1d3223", -10000, "TheName") + }); + } +}