open-source-mirrors
/
recyclarr
mirror of https://github.com/recyclarr/recyclarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: Separate JSON loading from CFs

Browse Source
json-serializing-nullable-fields-issue
Robert Dailey 1 year ago
parent 016bcb6624
commit 82cbfb3741
9 changed files with 109 additions and 195 deletions
Show Stats Download Patch File Download Diff File

1
src/Recyclarr.Cli/Pipelines/CustomFormat/CustomFormatAutofacModule.cs
Unescape View File

@ -18,7 +18,6 @@ public class CustomFormatAutofacModule : Module
builder.RegisterType<CustomFormatService>().As<ICustomFormatService>();
builder.RegisterType<CachePersister>().As<ICachePersister>();
builder.RegisterType<CustomFormatLoader>().As<ICustomFormatLoader>();
builder.RegisterType<CustomFormatParser>().As<ICustomFormatParser>();
builder.RegisterType<CustomFormatCategoryParser>().As<ICustomFormatCategoryParser>();
builder.RegisterType<CustomFormatDataLister>();

48
src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatLoader.cs
Unescape View File

@ -1,23 +1,19 @@
using System.IO.Abstractions;
using System.Reactive.Linq;
using System.Reactive.Threading.Tasks;
using Newtonsoft.Json;
using Recyclarr.Common;
using Recyclarr.Common.Extensions;
using Recyclarr.TrashLib.Json;
using Recyclarr.TrashLib.Models;
namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide;
public class CustomFormatLoader : ICustomFormatLoader
{
private readonly ILogger _log;
private readonly ICustomFormatParser _parser;
private readonly BulkJsonLoader _loader;
private readonly ICustomFormatCategoryParser _categoryParser;
public CustomFormatLoader(ILogger log, ICustomFormatParser parser, ICustomFormatCategoryParser categoryParser)
public CustomFormatLoader(BulkJsonLoader loader, ICustomFormatCategoryParser categoryParser)
{
_log = log;
_parser = parser;
_loader = loader;
_categoryParser = categoryParser;
}
@ -26,34 +22,16 @@ public class CustomFormatLoader : ICustomFormatLoader
IFileInfo collectionOfCustomFormats)
{
var categories = _categoryParser.Parse(collectionOfCustomFormats).AsReadOnly();
var jsonFiles = JsonUtils.GetJsonFilesInDirectories(jsonPaths, _log);
return jsonFiles.ToObservable()
.Select(x => Observable.Defer(() => LoadJsonFromFile(x, categories)))
.Merge(8)
.NotNull()
.ToEnumerable()
.ToList();
}
return _loader.LoadAllFilesAtPaths<CustomFormatData>(jsonPaths, x => x.Select(cf =>
{
var matchingCategory = categories.FirstOrDefault(
y => y.CfName.EqualsIgnoreCase(cf.Obj.Name) || y.CfAnchor.EqualsIgnoreCase(cf.File.Name));
private IObservable<CustomFormatData?> LoadJsonFromFile(
IFileInfo file,
IReadOnlyCollection<CustomFormatCategoryItem> categories)
{
return Observable.Using(file.OpenText, x => x.ReadToEndAsync().ToObservable())
.Select(x =>
{
var cf = _parser.ParseCustomFormatData(x, file.Name);
var matchingCategory = categories.FirstOrDefault(y =>
{
var fileName = Path.GetFileNameWithoutExtension(cf.FileName);
return y.CfName.EqualsIgnoreCase(cf.Name) || y.CfAnchor.EqualsIgnoreCase(fileName);
});
return cf with {Category = matchingCategory?.CategoryName};
})
.Catch((JsonException e) =>
return cf.Obj with
{
_log.Warning("Failed to parse JSON file: {File} ({Reason})", file.Name, e.Message);
return Observable.Empty<CustomFormatData>();
});
Category = matchingCategory?.CategoryName,
FileName = cf.File.Name
};
}));
}
}

18
src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/CustomFormatParser.cs
Unescape View File

@ -1,18 +0,0 @@
using Newtonsoft.Json;
using Recyclarr.TrashLib.Models;
namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide;
public class CustomFormatParser : ICustomFormatParser
{
public CustomFormatData ParseCustomFormatData(string guideData, string fileName)
{
var cf = JsonConvert.DeserializeObject<CustomFormatData>(guideData);
if (cf is null)
{
throw new JsonSerializationException($"Unable to parse JSON at file {fileName}");
}
return cf with {FileName = fileName};
}
}

8
src/Recyclarr.Cli/Pipelines/CustomFormat/Guide/ICustomFormatParser.cs
Unescape View File

@ -1,8 +0,0 @@
using Recyclarr.TrashLib.Models;
namespace Recyclarr.Cli.Pipelines.CustomFormat.Guide;
public interface ICustomFormatParser
{
CustomFormatData ParseCustomFormatData(string guideData, string fileName);
}

55
src/Recyclarr.TrashLib/Json/BulkJsonLoader.cs
Unescape View File

@ -0,0 +1,55 @@
using System.IO.Abstractions;
using System.Reactive.Linq;
using System.Reactive.Threading.Tasks;
using Newtonsoft.Json;
using Recyclarr.Common;
namespace Recyclarr.TrashLib.Json;
public record LoadedJsonObject<T>(IFileInfo File, T Obj);
public class BulkJsonLoader
{
private readonly ILogger _log;
public BulkJsonLoader(ILogger log)
{
_log = log;
}
public ICollection<T> LoadAllFilesAtPaths<T>(
IEnumerable<IDirectoryInfo> jsonPaths,
Func<IObservable<LoadedJsonObject<T>>, IObservable<T>>? extra = null)
{
var jsonFiles = JsonUtils.GetJsonFilesInDirectories(jsonPaths, _log);
var observable = jsonFiles.ToObservable()
.Select(x => Observable.Defer(() => LoadJsonFromFile<T>(x)))
.Merge(8);
var convertedObservable = extra?.Invoke(observable) ?? observable.Select(x => x.Obj);
return convertedObservable.ToEnumerable().ToList();
}
private static T ParseJson<T>(string guideData, string fileName)
{
var obj = JsonConvert.DeserializeObject<T>(guideData, GlobalJsonSerializerSettings.Guide);
if (obj is null)
{
throw new JsonSerializationException($"Unable to parse JSON at file {fileName}");
}
return obj;
}
private IObservable<LoadedJsonObject<T>> LoadJsonFromFile<T>(IFileInfo file)
{
return Observable.Using(file.OpenText, x => x.ReadToEndAsync().ToObservable())
.Select(x => new LoadedJsonObject<T>(file, ParseJson<T>(x, file.Name)))
.Catch((JsonException e) =>
{
_log.Warning("Failed to parse JSON file: {File} ({Reason})", file.Name, e.Message);
return Observable.Empty<LoadedJsonObject<T>>();
});
}
}

2
src/Recyclarr.TrashLib/TrashLibAutofacModule.cs
Unescape View File

@ -9,6 +9,7 @@ using Recyclarr.TrashLib.ApiServices;
using Recyclarr.TrashLib.Compatibility;
using Recyclarr.TrashLib.Config;
using Recyclarr.TrashLib.Http;
using Recyclarr.TrashLib.Json;
using Recyclarr.TrashLib.Repo;
using Recyclarr.TrashLib.Repo.VersionControl;
using Recyclarr.TrashLib.Startup;
@ -37,6 +38,7 @@ public class TrashLibAutofacModule : Module
builder.RegisterModule<ConfigAutofacModule>();
builder.RegisterType<ServiceRequestBuilder>().As<IServiceRequestBuilder>();
builder.RegisterType<FlurlClientFactory>().As<IFlurlClientFactory>().SingleInstance();
builder.RegisterType<BulkJsonLoader>();
var mapperAssemblies = new List<Assembly> {ThisAssembly};
if (AdditionalMapperProfileAssembly is not null)

4
src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderTest.cs → src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatLoaderIntegrationTest.cs
Unescape View File

@ -8,12 +8,12 @@ namespace Recyclarr.Cli.Tests.Pipelines.CustomFormat.Guide;
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class CustomFormatLoaderTest : CliIntegrationFixture
public class CustomFormatLoaderIntegrationTest : CliIntegrationFixture
{
[Test]
public void Get_custom_format_json_works()
{
var sut = Resolve<ICustomFormatLoader>();
var sut = Resolve<CustomFormatLoader>();
Fs.AddFile("first.json", new MockFileData("{'name':'first','trash_id':'1'}"));
Fs.AddFile("second.json", new MockFileData("{'name':'second','trash_id':'2'}"));
Fs.AddFile("collection_of_cfs.md", new MockFileData(""));

131
src/tests/Recyclarr.Cli.Tests/Pipelines/CustomFormat/Guide/CustomFormatParserTest.cs
Unescape View File

@ -1,131 +0,0 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Recyclarr.Cli.Pipelines.CustomFormat.Guide;
using Recyclarr.Common.Extensions;
using Recyclarr.TrashLib.Json;
using Recyclarr.TrashLib.Models;
namespace Recyclarr.Cli.Tests.Pipelines.CustomFormat.Guide;
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class CustomFormatParserTest
{
[Test, AutoMockData]
public void Deserialize_works(CustomFormatParser sut)
{
const string jsonData =
"""
{
"trash_id": "90cedc1fea7ea5d11298bebd3d1d3223",
"trash_scores": {
"default": -10000,
},
"name": "EVO (no WEBDL)",
"includeCustomFormatWhenRenaming": false,
"specifications": [
{
"name": "EVO",
"implementation": "ReleaseTitleSpecification",
"negate": false,
"required": true,
"fields": [{
"value": "\\bEVO(TGX)?\\b"
}]
},
{
"name": "WEBDL",
"implementation": "SourceSpecification",
"negate": true,
"required": true,
"fields": {
"value": 7
}
},
{
"name": "WEBRIP",
"implementation": "SourceSpecification",
"negate": true,
"required": true,
"fields": {
"value": 8
}
}
]
}
""";
var result = sut.ParseCustomFormatData(jsonData, "file.json");
result.Should().BeEquivalentTo(new CustomFormatData
{
FileName = "file.json",
TrashId = "90cedc1fea7ea5d11298bebd3d1d3223",
TrashScores = {["default"] = -10000},
Name = "EVO (no WEBDL)",
IncludeCustomFormatWhenRenaming = false,
Specifications = new[]
{
new CustomFormatSpecificationData
{
Name = "EVO",
Implementation = "ReleaseTitleSpecification",
Negate = false,
Required = true,
Fields = new[]
{
new CustomFormatFieldData
{
Value = "\\bEVO(TGX)?\\b"
}
}
},
new CustomFormatSpecificationData
{
Name = "WEBDL",
Implementation = "SourceSpecification",
Negate = true,
Required = true,
Fields = new[]
{
new CustomFormatFieldData
{
Value = 7
}
}
},
new CustomFormatSpecificationData
{
Name = "WEBRIP",
Implementation = "SourceSpecification",
Negate = true,
Required = true,
Fields = new[]
{
new CustomFormatFieldData
{
Value = 8
}
}
}
}
});
}
[Test, AutoMockData]
public void Serialize_skips_trash_properties(CustomFormatParser sut)
{
var cf = new CustomFormatData
{
FileName = "file.json",
TrashId = "90cedc1fea7ea5d11298bebd3d1d3223",
TrashScores = {["default"] = -10000},
Name = "EVO (no WEBDL)",
IncludeCustomFormatWhenRenaming = false
};
var json = JObject.FromObject(cf, JsonSerializer.Create(GlobalJsonSerializerSettings.Services));
json.Children<JProperty>().Should().NotContain(x => x.Name.ContainsIgnoreCase("trash"));
}
}

37
src/tests/Recyclarr.TrashLib.Tests/Json/BulkJsonLoaderTest.cs
Unescape View File

@ -0,0 +1,37 @@
using System.Diagnostics.CodeAnalysis;
using System.IO.Abstractions;
using Recyclarr.TrashLib.Json;
namespace Recyclarr.TrashLib.Tests.Json;
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class BulkJsonLoaderTest
{
[SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")]
private sealed record TestJsonObject(string TrashId, int TrashScore, string Name);
[Test, AutoMockData]
public void Deserialize_works(
[Frozen(Matching.ImplementedInterfaces)] MockFileSystem fs,
BulkJsonLoader sut)
{
var jsonData =
"""
{
"trash_id": "90cedc1fea7ea5d11298bebd3d1d3223",
"trash_score": "-10000",
"name": "TheName"
}
""";
fs.AddFile(fs.CurrentDirectory().File("file.json"), new MockFileData(jsonData));
var result = sut.LoadAllFilesAtPaths<TestJsonObject>(new[] {fs.CurrentDirectory()});
result.Should().BeEquivalentTo(new[]
{
new TestJsonObject("90cedc1fea7ea5d11298bebd3d1d3223", -10000, "TheName")
});
}
}
Write Preview
Loading…
Cancel
Save