From df203e3b4622eea528550bf4bdbd4caf356a9a83 Mon Sep 17 00:00:00 2001 From: Robert Dailey Date: Sun, 30 May 2021 17:25:46 -0500 Subject: [PATCH] feat(radarr): support new custom format guide structure Custom formats are now a list of JSON files in the github repository. Support for parsing the custom format markdown data has been removed. --- CHANGELOG.md | 5 + src/.editorconfig | 11 +- .../GithubCustomFormatJsonRequesterTest.cs | 32 ++++ .../Processors/Data/CF_Markdown1.md | 71 --------- .../Data/ImportableCustomFormat1.json | 1 + .../Data/ImportableCustomFormat2.json | 1 + .../CustomFormat/Processors/Data/NoScore.json | 4 + .../Processors/Data/WontBeInConfig.json | 5 + .../Processors/GuideProcessorTest.cs | 48 +++--- .../GuideSteps/CustomFormatStepTest.cs | 110 +++++-------- src/Trash.Tests/Trash.Tests.csproj | 4 - src/Trash/CompositionRoot.cs | 2 +- .../CustomFormat/Guide/CustomFormatData.cs | 8 - .../Guide/CustomFormatGuideParser.cs | 149 ------------------ .../Guide/GithubCustomFormatJsonRequester.cs | 57 +++++++ .../Guide/ICustomFormatGuideParser.cs | 11 -- .../CustomFormat/Guide/IRadarrGuideService.cs | 10 ++ .../Radarr/CustomFormat/Guide/ParserState.cs | 26 --- .../CustomFormat/Processors/GuideProcessor.cs | 17 +- .../Processors/GuideSteps/CustomFormatStep.cs | 14 +- .../GuideSteps/ICustomFormatStep.cs | 3 +- 21 files changed, 207 insertions(+), 382 deletions(-) create mode 100644 src/Trash.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs delete mode 100644 src/Trash.Tests/Radarr/CustomFormat/Processors/Data/CF_Markdown1.md create mode 100644 src/Trash.Tests/Radarr/CustomFormat/Processors/Data/NoScore.json create mode 100644 src/Trash.Tests/Radarr/CustomFormat/Processors/Data/WontBeInConfig.json delete mode 100644 src/Trash/Radarr/CustomFormat/Guide/CustomFormatData.cs delete mode 100644 src/Trash/Radarr/CustomFormat/Guide/CustomFormatGuideParser.cs create mode 100644 src/Trash/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs delete mode 100644 src/Trash/Radarr/CustomFormat/Guide/ICustomFormatGuideParser.cs create mode 100644 src/Trash/Radarr/CustomFormat/Guide/IRadarrGuideService.cs delete mode 100644 src/Trash/Radarr/CustomFormat/Guide/ParserState.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b754030..93a86d77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Support the new custom format structure in the guide: JSON files are parsed directly now. Trash + Updater no longer parses the markdown file. + ## [1.5.1] - 2021-05-26 ### Changed diff --git a/src/.editorconfig b/src/.editorconfig index bac8b5a3..c9ef6644 100644 --- a/src/.editorconfig +++ b/src/.editorconfig @@ -3589,6 +3589,15 @@ resharper_xunit_xunit_test_with_console_output_highlighting = warning indent_style = space indent_size = 2 +[*.{xml,xsd}] +indent_style = space +indent_size = 2 + +[*.json] +indent_style = space +indent_size = 2 +ij_javascript_array_initializer_right_brace_on_new_line = false + [{*.overridetasks,*.targets,*.tasks}] indent_style = space indent_size = 2 @@ -3601,7 +3610,7 @@ indent_size = 2 indent_style = space indent_size = 2 -[*.{appxmanifest,asax,ascx,aspx,axaml,build,cg,cginc,compute,cs,cshtml,dtd,hlsl,hlsli,hlslinc,master,nuspec,paml,razor,resw,resx,skin,usf,ush,vb,xaml,xamlx,xoml,xsd}] +[*.{appxmanifest,asax,ascx,aspx,axaml,build,cg,cginc,compute,cs,cshtml,dtd,hlsl,hlsli,hlslinc,master,nuspec,paml,razor,resw,resx,skin,usf,ush,vb,xaml,xamlx,xoml}] indent_style = space indent_size = 4 tab_width = 4 diff --git a/src/Trash.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs b/src/Trash.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs new file mode 100644 index 00000000..2865e494 --- /dev/null +++ b/src/Trash.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs @@ -0,0 +1,32 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using Trash.Radarr.CustomFormat.Guide; + +namespace Trash.Tests.Radarr.CustomFormat.Guide +{ + [TestFixture] + [Parallelizable(ParallelScope.All)] + public class GithubCustomFormatJsonRequesterTest + { + [Test] + public async Task Requesting_json_from_github_works() + { + var requester = new GithubCustomFormatJsonRequester(); + + var jsonList = (await requester.GetCustomFormatJson()).ToList(); + + Action act = () => JObject.Parse(jsonList.First()); + + // As of the time this test was written, there are around 58 custom format JSON files. + // This number can fluctuate over time, but I'm only interested in verifying we get a handful + // of files in the response. + jsonList.Should().HaveCountGreaterOrEqualTo(5); + + act.Should().NotThrow(); + } + } +} diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/CF_Markdown1.md b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/CF_Markdown1.md deleted file mode 100644 index 63957a59..00000000 --- a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/CF_Markdown1.md +++ /dev/null @@ -1,71 +0,0 @@ - -Score [480] - -??? example "json" - - ```json - { - "trash_id": "4eb3c272d48db8ab43c2c85283b69744", - "name": "DTS-HD/DTS:X", - "includeCustomFormatWhenRenaming": false, - "specifications": [{ - "name": "dts.?(hd|es|x(?!\\d))", - "implementation": "ReleaseTitleSpecification", - "negate": false, - "required": false, - "fields": { - "value": "dts.?(hd|es|x(?!\\d))" - } - }] - } - ``` - -[TOP](#index) - ------- - -### Surround Sound - ->If you prefer all kind of surround sounds - -!!! warning - - Don't use this Custom Format in combination with the `Audio Advanced` CF if you want to fine tune your audio formats or else it will add up the scores. - - -Score [500] - -??? example "json" - - ```json - { - "trash_id": "43bb5f09c79641e7a22e48d440bd8868", - "name": "Surround Sound", - "includeCustomFormatWhenRenaming": false, - "specifications": [{ - "name": "dts\\-?(hd|x)|truehd|atmos|dd(\\+|p)(5|7)", - "implementation": "ReleaseTitleSpecification", - "negate": false, - "required": false, - "fields": { - "value": "dts\\-?(hd|x)|truehd|atmos|dd(\\+|p)(5|7)" - } - }] - } - ``` - - ```json - { - "trash_id": "abc", - "name": "No Score" - } - ``` - - ```json - { - "trash_id": "xyz", - "name": "One that won't be in config" - } - ``` - - diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat1.json b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat1.json index ba54edce..043a8f51 100644 --- a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat1.json +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat1.json @@ -1,5 +1,6 @@ { "trash_id": "43bb5f09c79641e7a22e48d440bd8868", + "trash_score": 500, "name": "Surround Sound", "includeCustomFormatWhenRenaming": false, "specifications": [{ diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat2.json b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat2.json index 708e8155..24031e9f 100644 --- a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat2.json +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/ImportableCustomFormat2.json @@ -1,5 +1,6 @@ { "trash_id": "4eb3c272d48db8ab43c2c85283b69744", + "trash_score": 480, "name": "DTS-HD/DTS:X", "includeCustomFormatWhenRenaming": false, "specifications": [{ diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/NoScore.json b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/NoScore.json new file mode 100644 index 00000000..030d948e --- /dev/null +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/NoScore.json @@ -0,0 +1,4 @@ +{ + "trash_id": "abc", + "name": "No Score" +} diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/WontBeInConfig.json b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/WontBeInConfig.json new file mode 100644 index 00000000..28fd9373 --- /dev/null +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/Data/WontBeInConfig.json @@ -0,0 +1,5 @@ +{ + "trash_id": "xyz", + "trash_score": -100, + "name": "One that won't be in config" +} diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs b/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs index ce4f8fc9..077c542b 100644 --- a/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs @@ -51,16 +51,27 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors [Test] [SuppressMessage("Maintainability", "CA1506", Justification = "Designed to be a high-level integration test")] - public void Guide_processor_behaves_as_expected_with_normal_markdown() + public void Guide_processor_behaves_as_expected_with_normal_guide_data() { var ctx = new Context(); var guideProcessor = - new GuideProcessor(ctx.Logger, new CustomFormatGuideParser(ctx.Logger), + new GuideProcessor(ctx.Logger, new GithubCustomFormatJsonRequester(), () => new TestGuideProcessorSteps()); // simulate guide data using var testHttp = new HttpTest(); - testHttp.RespondWith(ctx.Data.ReadData("CF_Markdown1.md")); + testHttp.RespondWithJson(new[] + { + new {name = "ImportableCustomFormat1.json", type = "file", download_url = "http://not_real/file.json"}, + new {name = "ImportableCustomFormat2.json", type = "file", download_url = "http://not_real/file.json"}, + new {name = "NoScore.json", type = "file", download_url = "http://not_real/file.json"}, + new {name = "WontBeInConfig.json", type = "file", download_url = "http://not_real/file.json"} + }); + + testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat1.json")); + testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat2.json")); + testHttp.RespondWithJson(ctx.ReadJson("NoScore.json")); + testHttp.RespondWithJson(ctx.ReadJson("WontBeInConfig.json")); // Simulate user config in YAML var config = new List @@ -102,24 +113,23 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors new("No Score", "abc", JObject.FromObject(new {name = "No Score"})) }; - guideProcessor.ProcessedCustomFormats.Should().BeEquivalentTo(expectedProcessedCustomFormatData, - op => op.Using(new JsonEquivalencyStep())); + guideProcessor.ProcessedCustomFormats.Should() + .BeEquivalentTo(expectedProcessedCustomFormatData, op => op.Using(new JsonEquivalencyStep())); - guideProcessor.ConfigData.Should().BeEquivalentTo(new List - { - new() - { - CustomFormats = expectedProcessedCustomFormatData, - QualityProfiles = config[0].QualityProfiles - }, - new() + guideProcessor.ConfigData.Should() + .BeEquivalentTo(new List { - CustomFormats = expectedProcessedCustomFormatData.GetRange(2, 1), - QualityProfiles = config[1].QualityProfiles - } - }, op => op - .Using(jctx => jctx.Subject.Should().BeEquivalentTo(jctx.Expectation)) - .WhenTypeIs()); + new() + { + CustomFormats = expectedProcessedCustomFormatData, + QualityProfiles = config[0].QualityProfiles + }, + new() + { + CustomFormats = expectedProcessedCustomFormatData.GetRange(2, 1), + QualityProfiles = config[1].QualityProfiles + } + }, op => op.Using(new JsonEquivalencyStep())); guideProcessor.CustomFormatsWithoutScore.Should() .Equal(new List<(string name, string trashId, string profileName)> diff --git a/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStepTest.cs b/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStepTest.cs index 19f2daf1..6cd3ff61 100644 --- a/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStepTest.cs +++ b/src/Trash.Tests/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStepTest.cs @@ -6,7 +6,6 @@ using Newtonsoft.Json.Linq; using NUnit.Framework; using TestLibrary.FluentAssertions; using Trash.Radarr; -using Trash.Radarr.CustomFormat.Guide; using Trash.Radarr.CustomFormat.Models; using Trash.Radarr.CustomFormat.Models.Cache; using Trash.Radarr.CustomFormat.Processors.GuideSteps; @@ -19,34 +18,23 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps { private class Context { - public List TestGuideData { get; } = new() + public List TestGuideData { get; } = new() { - new CustomFormatData + JsonConvert.SerializeObject(new { - Score = 100, - Json = JsonConvert.SerializeObject(new - { - trash_id = "id1", - name = "name1" - }, Formatting.Indented) - }, - new CustomFormatData + trash_id = "id1", + name = "name1" + }, Formatting.Indented), + JsonConvert.SerializeObject(new { - Score = 200, - Json = JsonConvert.SerializeObject(new - { - trash_id = "id2", - name = "name2" - }, Formatting.Indented) - }, - new CustomFormatData + trash_id = "id2", + name = "name2" + }, Formatting.Indented), + JsonConvert.SerializeObject(new { - Json = JsonConvert.SerializeObject(new - { - trash_id = "id3", - name = "name3" - }, Formatting.Indented) - } + trash_id = "id3", + name = "name3" + }, Formatting.Indented) }; } @@ -69,17 +57,13 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps } }; - var testGuideData = new List + var testGuideData = new List { - new() + JsonConvert.SerializeObject(new { - Score = 100, - Json = JsonConvert.SerializeObject(new - { - trash_id = "id1", - name = variableCfName - }, Formatting.Indented) - } + trash_id = "id1", + name = variableCfName + }, Formatting.Indented) }; var processor = new CustomFormatStep(); @@ -92,7 +76,6 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps { new(variableCfName, "id1", JObject.FromObject(new {name = variableCfName})) { - Score = 100, CacheEntry = testCache.TrashIdMappings[0] } }, @@ -102,12 +85,9 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Cache_entry_is_not_set_when_id_is_different() { - var guideData = new List + var guideData = new List { - new() - { - Json = @"{'name': 'name1', 'trash_id': 'id1'}" - } + @"{'name': 'name1', 'trash_id': 'id1'}" }; var testConfig = new List @@ -157,14 +137,8 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps processor.DeletedCustomFormatsInCache.Should().BeEmpty(); processor.ProcessedCustomFormats.Should().BeEquivalentTo(new List { - new("name1", "id1", JObject.FromObject(new {name = "name1"})) - { - Score = 100 - }, - new("name3", "id3", JObject.FromObject(new {name = "name3"})) - { - Score = null - } + new("name1", "id1", JObject.FromObject(new {name = "name1"})) {Score = null}, + new("name3", "id3", JObject.FromObject(new {name = "name3"})) {Score = null} }, op => op.Using(new JsonEquivalencyStep())); } @@ -187,8 +161,8 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps processor.DeletedCustomFormatsInCache.Should().BeEmpty(); processor.ProcessedCustomFormats.Should().BeEquivalentTo(new List { - new("name1", "id1", JObject.FromObject(new {name = "name1"})) {Score = 100}, - new("name2", "id2", JObject.FromObject(new {name = "name2"})) {Score = 200}, + new("name1", "id1", JObject.FromObject(new {name = "name1"})) {Score = null}, + new("name2", "id2", JObject.FromObject(new {name = "name2"})) {Score = null}, new("name3", "id3", JObject.FromObject(new {name = "name3"})) {Score = null} }, op => op.Using(new JsonEquivalencyStep())); @@ -197,12 +171,9 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Custom_format_is_deleted_if_in_config_and_cache_but_not_in_guide() { - var guideData = new List + var guideData = new List { - new() - { - Json = @"{'name': 'name1', 'trash_id': 'id1'}" - } + @"{'name': 'name1', 'trash_id': 'id1'}" }; var testConfig = new List @@ -237,9 +208,9 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps TrashIdMappings = new List {new("id1", "3D", 9)} }; - var guideCfs = new List + var guideCfs = new List { - new() {Json = "{'name': '3D', 'trash_id': 'id1'}"} + "{'name': '3D', 'trash_id': 'id1'}" }; var processor = new CustomFormatStep(); @@ -254,12 +225,9 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Custom_format_name_in_cache_is_updated_if_renamed_in_guide_and_config() { - var guideData = new List + var guideData = new List { - new() - { - Json = @"{'name': 'name2', 'trash_id': 'id1'}" - } + @"{'name': 'name2', 'trash_id': 'id1'}" }; var testConfig = new List @@ -286,10 +254,10 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Duplicates_are_recorded_and_removed_from_processed_custom_formats_list() { - var guideData = new List + var guideData = new List { - new() {Json = @"{'name': 'name1', 'trash_id': 'id1'}"}, - new() {Json = @"{'name': 'name1', 'trash_id': 'id2'}"} + @"{'name': 'name1', 'trash_id': 'id1'}", + @"{'name': 'name1', 'trash_id': 'id2'}" }; var testConfig = new List @@ -329,7 +297,7 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps processor.DeletedCustomFormatsInCache.Should().BeEmpty(); processor.ProcessedCustomFormats.Should().BeEquivalentTo(new List { - new("name1", "id1", JObject.FromObject(new {name = "name1"})) {Score = 100} + new("name1", "id1", JObject.FromObject(new {name = "name1"})) }, op => op.Using(new JsonEquivalencyStep())); } @@ -337,10 +305,10 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Match_custom_format_using_trash_id() { - var guideData = new List + var guideData = new List { - new() {Json = @"{'name': 'name1', 'trash_id': 'id1'}"}, - new() {Json = @"{'name': 'name2', 'trash_id': 'id2'}"} + @"{'name': 'name1', 'trash_id': 'id1'}", + @"{'name': 'name2', 'trash_id': 'id2'}" }; var testConfig = new List @@ -382,9 +350,9 @@ namespace Trash.Tests.Radarr.CustomFormat.Processors.GuideSteps [Test] public void Score_from_json_takes_precedence_over_score_from_guide() { - var guideData = new List + var guideData = new List { - new() {Json = @"{'name': 'name1', 'trash_id': 'id1', 'trash_score': 100}"} + @"{'name': 'name1', 'trash_id': 'id1', 'trash_score': 100}" }; var testConfig = new List diff --git a/src/Trash.Tests/Trash.Tests.csproj b/src/Trash.Tests/Trash.Tests.csproj index 3e880dbd..15dfca36 100644 --- a/src/Trash.Tests/Trash.Tests.csproj +++ b/src/Trash.Tests/Trash.Tests.csproj @@ -7,8 +7,4 @@ - - - - diff --git a/src/Trash/CompositionRoot.cs b/src/Trash/CompositionRoot.cs index 63888a28..0002453a 100644 --- a/src/Trash/CompositionRoot.cs +++ b/src/Trash/CompositionRoot.cs @@ -74,7 +74,7 @@ namespace Trash // Custom Format Support builder.RegisterType().As(); - builder.RegisterType().As(); + builder.RegisterType().As(); builder.RegisterType().As(); // Guide Processor diff --git a/src/Trash/Radarr/CustomFormat/Guide/CustomFormatData.cs b/src/Trash/Radarr/CustomFormat/Guide/CustomFormatData.cs deleted file mode 100644 index ed61f9ab..00000000 --- a/src/Trash/Radarr/CustomFormat/Guide/CustomFormatData.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Trash.Radarr.CustomFormat.Guide -{ - public class CustomFormatData - { - public int? Score { get; set; } = null; - public string Json { get; set; } = ""; - } -} diff --git a/src/Trash/Radarr/CustomFormat/Guide/CustomFormatGuideParser.cs b/src/Trash/Radarr/CustomFormat/Guide/CustomFormatGuideParser.cs deleted file mode 100644 index 0cf7e99e..00000000 --- a/src/Trash/Radarr/CustomFormat/Guide/CustomFormatGuideParser.cs +++ /dev/null @@ -1,149 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Flurl.Http; -using Serilog; -using Trash.Extensions; - -namespace Trash.Radarr.CustomFormat.Guide -{ - public class CustomFormatGuideParser : ICustomFormatGuideParser - { - private readonly Regex _regexFence = BuildRegex(@"(\s*)```(json)?"); - private readonly Regex _regexPotentialScore = BuildRegex(@"\[(-?[\d]+)\]"); - private readonly Regex _regexScore = BuildRegex(@"score.*?\[(-?[\d]+)\]"); - - public CustomFormatGuideParser(ILogger logger) - { - Log = logger; - } - - private ILogger Log { get; } - - public async Task GetMarkdownData() - { - return await - "https://raw.githubusercontent.com/TRaSH-/Guides/master/docs/Radarr/V3/Radarr-collection-of-custom-formats.md" - .GetStringAsync(); - } - - public IList ParseMarkdown(string markdown) - { - var state = new ParserState(); - - var reader = new StringReader(markdown); - for (var line = reader.ReadLine(); line != null; line = reader.ReadLine()) - { - state.LineNumber++; - if (string.IsNullOrEmpty(line)) - { - continue; - } - - // Always check if we're starting a fenced code block. Whether we are inside one or not greatly affects - // the logic we use. - if (_regexFence.Match(line, out Match match)) - { - ProcessCodeBlockBoundary(match.Groups, state); - continue; - } - - if (state.CodeBlockIndentation != null) - { - InsideFence_ParseMarkdown(line, state); - } - else - { - OutsideFence_ParseMarkdown(line, state); - } - } - - return state.Results; - } - - private static Regex BuildRegex(string regex) - { - return new(regex, RegexOptions.Compiled | RegexOptions.IgnoreCase); - } - - private void OutsideFence_ParseMarkdown(string line, ParserState state) - { - // ReSharper disable once InlineOutVariableDeclaration - Match match; - - if (_regexScore.Match(line, out match)) - { - state.Score = int.Parse(match.Groups[1].Value); - } - else if (_regexPotentialScore.Match(line, out match)) - { - Log.Warning("Found a potential score on line #{Line} that will be ignored because the " + - "word 'score' is missing (This is probably a bug in the guide itself): {ScoreMatch}", - state.LineNumber, match.Groups[0].Value); - } - } - - private void ProcessCodeBlockBoundary(GroupCollection groups, ParserState state) - { - if (groups[2].Value == "json") - { - state.CodeBlockIndentation = groups[1].Value; - } - else - { - // Record previously captured JSON data since we're leaving the code block - var json = state.JsonStream.ToString(); - if (!string.IsNullOrEmpty(json)) - { - state.Results.Add(new CustomFormatData {Json = json, Score = state.Score}); - } - - state.ResetParserState(); - } - } - - private static void InsideFence_ParseMarkdown(string line, ParserState state) - { - state.JsonStream.WriteLine(line[state.CodeBlockIndentation!.Length..]); - } - - // private void OutsideFence_ParseMarkdown(string line, RadarrParserState state) - // { - // // ReSharper disable once InlineOutVariableDeclaration - // Match match; - // - // // Header Processing. Never do any additional processing to headers, so return after processing it - // if (_regexHeader.Match(line, out match)) - // { - // OutsideFence_ParseHeader(state, match); - // // return here if we add more logic below - // } - // } - - // private void OutsideFence_ParseHeader(RadarrParserState state, Match match) - // { - // var headerDepth = match.Groups[1].Length; - // var headerText = match.Groups[2].Value; - // - // var stack = state.HeaderStack; - // while (stack.Count > 0 && stack.Peek().Item1 >= headerDepth) - // { - // stack.Pop(); - // } - // - // if (headerDepth == 0) - // { - // return; - // } - // - // if (state.HeaderStack.TryPeek(out var header)) - // { - // headerText = $"{header.Item2}|{headerText}"; - // } - // - // Log.Debug("> Process Header: {HeaderPath}", headerText); - // state.HeaderStack.Push((headerDepth, headerText)); - // } - } -} diff --git a/src/Trash/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs b/src/Trash/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs new file mode 100644 index 00000000..2e9e17ad --- /dev/null +++ b/src/Trash/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs @@ -0,0 +1,57 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Flurl.Http; +using Flurl.Http.Configuration; +using JetBrains.Annotations; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace Trash.Radarr.CustomFormat.Guide +{ + public class GithubCustomFormatJsonRequester : IRadarrGuideService + { + private readonly ISerializer _flurlSerializer; + + public GithubCustomFormatJsonRequester() + { + // In addition to setting the naming strategy, this also serves as a mechanism to avoid inheriting the + // global Flurl serializer setting: MissingMemberHandling. We do not want missing members to error out + // since we're only deserializing a subset of the github response object. + _flurlSerializer = new NewtonsoftJsonSerializer(new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new SnakeCaseNamingStrategy() + } + }); + } + + public async Task> GetCustomFormatJson() + { + var response = await "https://api.github.com/repos/TRaSH-/Guides/contents/docs/json/radarr" + .WithHeader("User-Agent", "Trash Updater") + .ConfigureRequest(settings => settings.JsonSerializer = _flurlSerializer) + .GetJsonAsync>(); + + var tasks = response + .Where(o => o.Type == "file" && o.Name.EndsWith(".json")) + .Select(o => DownloadJsonContents(o.DownloadUrl)); + + return await Task.WhenAll(tasks); + } + + private async Task DownloadJsonContents(string jsonUrl) + { + return await jsonUrl.GetStringAsync(); + } + + [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] + private record RepoContentEntry + { + public string Name { get; init; } = default!; + public string Type { get; init; } = default!; + public string DownloadUrl { get; init; } = default!; + } + } +} diff --git a/src/Trash/Radarr/CustomFormat/Guide/ICustomFormatGuideParser.cs b/src/Trash/Radarr/CustomFormat/Guide/ICustomFormatGuideParser.cs deleted file mode 100644 index 59f42dc7..00000000 --- a/src/Trash/Radarr/CustomFormat/Guide/ICustomFormatGuideParser.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Trash.Radarr.CustomFormat.Guide -{ - public interface ICustomFormatGuideParser - { - Task GetMarkdownData(); - IList ParseMarkdown(string markdown); - } -} diff --git a/src/Trash/Radarr/CustomFormat/Guide/IRadarrGuideService.cs b/src/Trash/Radarr/CustomFormat/Guide/IRadarrGuideService.cs new file mode 100644 index 00000000..02d305b0 --- /dev/null +++ b/src/Trash/Radarr/CustomFormat/Guide/IRadarrGuideService.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Trash.Radarr.CustomFormat.Guide +{ + public interface IRadarrGuideService + { + Task> GetCustomFormatJson(); + } +} diff --git a/src/Trash/Radarr/CustomFormat/Guide/ParserState.cs b/src/Trash/Radarr/CustomFormat/Guide/ParserState.cs deleted file mode 100644 index 9b125413..00000000 --- a/src/Trash/Radarr/CustomFormat/Guide/ParserState.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Collections.Generic; -using System.IO; - -namespace Trash.Radarr.CustomFormat.Guide -{ - public class ParserState - { - public ParserState() - { - ResetParserState(); - } - - public int? Score { get; set; } - public string? CodeBlockIndentation { get; set; } - public int LineNumber { get; set; } - public List Results { get; } = new(); - public StringWriter JsonStream { get; } = new(); - - public void ResetParserState() - { - CodeBlockIndentation = null; - JsonStream.GetStringBuilder().Clear(); - Score = null; - } - } -} diff --git a/src/Trash/Radarr/CustomFormat/Processors/GuideProcessor.cs b/src/Trash/Radarr/CustomFormat/Processors/GuideProcessor.cs index 32035df8..24e087a4 100644 --- a/src/Trash/Radarr/CustomFormat/Processors/GuideProcessor.cs +++ b/src/Trash/Radarr/CustomFormat/Processors/GuideProcessor.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Serilog; using Trash.Radarr.CustomFormat.Guide; @@ -18,15 +19,14 @@ namespace Trash.Radarr.CustomFormat.Processors internal class GuideProcessor : IGuideProcessor { - private readonly ICustomFormatGuideParser _guideParser; + private readonly IRadarrGuideService _guideService; private readonly Func _stepsFactory; - private IList? _guideData; + private IList? _guideCustomFormatJson; private IGuideProcessorSteps _steps; - public GuideProcessor(ILogger log, ICustomFormatGuideParser guideParser, - Func stepsFactory) + public GuideProcessor(ILogger log, IRadarrGuideService guideService, Func stepsFactory) { - _guideParser = guideParser; + _guideService = guideService; _stepsFactory = stepsFactory; Log = log; _steps = stepsFactory(); @@ -60,16 +60,15 @@ namespace Trash.Radarr.CustomFormat.Processors public async Task BuildGuideData(IReadOnlyList config, CustomFormatCache? cache) { - if (_guideData == null) + if (_guideCustomFormatJson == null) { Log.Debug("Requesting and parsing guide markdown"); - var markdownData = await _guideParser.GetMarkdownData(); - _guideData = _guideParser.ParseMarkdown(markdownData); + _guideCustomFormatJson = (await _guideService.GetCustomFormatJson()).ToList(); } // Step 1: Process and filter the custom formats from the guide. // Custom formats in the guide not mentioned in the config are filtered out. - _steps.CustomFormat.Process(_guideData, config, cache); + _steps.CustomFormat.Process(_guideCustomFormatJson, config, cache); // todo: Process cache entries that do not exist in the guide. Those should be deleted // This might get taken care of when we rebuild the cache based on what is actually updated when diff --git a/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStep.cs b/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStep.cs index 2e1ee5d9..2106d1be 100644 --- a/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStep.cs +++ b/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/CustomFormatStep.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Linq; using Newtonsoft.Json.Linq; using Trash.Extensions; -using Trash.Radarr.CustomFormat.Guide; using Trash.Radarr.CustomFormat.Models; using Trash.Radarr.CustomFormat.Models.Cache; @@ -18,7 +17,7 @@ namespace Trash.Radarr.CustomFormat.Processors.GuideSteps public Dictionary> DuplicatedCustomFormats { get; private set; } = new(); - public void Process(IEnumerable customFormatGuideData, + public void Process(IEnumerable customFormatGuideData, IReadOnlyCollection config, CustomFormatCache? cache) { var processedCfs = customFormatGuideData @@ -93,23 +92,18 @@ namespace Trash.Radarr.CustomFormat.Processors.GuideSteps ProcessedCustomFormats.RemoveAll(cf => DuplicatedCustomFormats.ContainsKey(cf.Name)); } - private static ProcessedCustomFormatData ProcessCustomFormatData(CustomFormatData guideData, - CustomFormatCache? cache) + private static ProcessedCustomFormatData ProcessCustomFormatData(string guideData, CustomFormatCache? cache) { - JObject obj = JObject.Parse(guideData.Json); + JObject obj = JObject.Parse(guideData); var name = (string) obj["name"]; var trashId = (string) obj["trash_id"]; - int? finalScore; + int? finalScore = null; if (obj.TryGetValue("trash_score", out var score)) { finalScore = (int) score; obj.Property("trash_score").Remove(); } - else - { - finalScore = guideData.Score; - } // Remove trash_id, it's metadata that is not meant for Radarr itself // Radarr supposedly drops this anyway, but I prefer it to be removed by TrashUpdater diff --git a/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/ICustomFormatStep.cs b/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/ICustomFormatStep.cs index 4bc3db36..e3dcacb7 100644 --- a/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/ICustomFormatStep.cs +++ b/src/Trash/Radarr/CustomFormat/Processors/GuideSteps/ICustomFormatStep.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Trash.Radarr.CustomFormat.Guide; using Trash.Radarr.CustomFormat.Models; using Trash.Radarr.CustomFormat.Models.Cache; @@ -12,7 +11,7 @@ namespace Trash.Radarr.CustomFormat.Processors.GuideSteps List<(string, string)> CustomFormatsWithOutdatedNames { get; } Dictionary> DuplicatedCustomFormats { get; } - void Process(IEnumerable customFormatGuideData, + void Process(IEnumerable customFormatGuideData, IReadOnlyCollection config, CustomFormatCache? cache); } }