diff --git a/CHANGELOG.md b/CHANGELOG.md index c283d2a7..e63f7607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Directly use the Trash Guides git repository to avoid getting HTTP 403 - rate limit reached error + in github. + ## [1.6.1] - 2021-05-31 ### Changed diff --git a/src/Trash.Tests/Config/ConfigurationLoaderTest.cs b/src/Trash.Tests/Config/ConfigurationLoaderTest.cs index 035aeb01..6340a2c6 100644 --- a/src/Trash.Tests/Config/ConfigurationLoaderTest.cs +++ b/src/Trash.Tests/Config/ConfigurationLoaderTest.cs @@ -60,7 +60,9 @@ namespace Trash.Tests.Config // .Returns(t => Substitute.For(new[] {(Type)t[0]}, Array.Empty())); var actualActiveConfigs = new List(); +#pragma warning disable NS1004 provider.ActiveConfiguration = Arg.Do(a => actualActiveConfigs.Add(a)); +#pragma warning restore NS1004 var validator = Substitute.For>(); var loader = diff --git a/src/Trash.Tests/Trash.Tests.csproj b/src/Trash.Tests/Trash.Tests.csproj index 1492c910..15dfca36 100644 --- a/src/Trash.Tests/Trash.Tests.csproj +++ b/src/Trash.Tests/Trash.Tests.csproj @@ -7,8 +7,4 @@ - - - - diff --git a/src/Trash/AppPaths.cs b/src/Trash/AppPaths.cs index a8bb2476..b65001ed 100644 --- a/src/Trash/AppPaths.cs +++ b/src/Trash/AppPaths.cs @@ -11,5 +11,7 @@ namespace Trash public static string DefaultConfigPath { get; } = Path.Combine(AppContext.BaseDirectory, "trash.yml"); public static string LogDirectory { get; } = Path.Combine(AppDataPath, "logs"); + + public static string RepoDirectory { get; } = Path.Combine(AppDataPath, "repo"); } } diff --git a/src/Trash/CompositionRoot.cs b/src/Trash/CompositionRoot.cs index 41e678a7..6b0396c5 100644 --- a/src/Trash/CompositionRoot.cs +++ b/src/Trash/CompositionRoot.cs @@ -12,6 +12,7 @@ using Trash.Config; using TrashLib.Cache; using TrashLib.Config; using TrashLib.Radarr; +using TrashLib.Radarr.Config; using TrashLib.Sonarr; using YamlDotNet.Serialization; @@ -44,8 +45,8 @@ namespace Trash { builder.RegisterModule(); - builder.RegisterType() - .As(); + builder.RegisterType().As(); + builder.RegisterType().As(); builder.RegisterGeneric(typeof(ConfigurationLoader<>)) .WithProperty(new AutowiringParameter()) diff --git a/src/Trash/ResourcePaths.cs b/src/Trash/ResourcePaths.cs new file mode 100644 index 00000000..62e0f99f --- /dev/null +++ b/src/Trash/ResourcePaths.cs @@ -0,0 +1,9 @@ +using TrashLib.Radarr.Config; + +namespace Trash +{ + public class ResourcePaths : IResourcePaths + { + public string RepoPath => AppPaths.RepoDirectory; + } +} diff --git a/src/TrashLib.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs b/src/TrashLib.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs deleted file mode 100644 index 2aa9a90a..00000000 --- a/src/TrashLib.Tests/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequesterTest.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using FluentAssertions; -using Newtonsoft.Json.Linq; -using NUnit.Framework; -using TrashLib.Radarr.CustomFormat.Guide; - -namespace TrashLib.Tests.Radarr.CustomFormat.Guide -{ - [TestFixture] - [Parallelizable(ParallelScope.All)] - public class GithubCustomFormatJsonRequesterTest - { - [Test] - public async Task Requesting_json_from_github_works() - { - var requester = new GithubCustomFormatJsonRequester(); - - var jsonList = (await requester.GetCustomFormatJson()).ToList(); - - Action act = () => JObject.Parse(jsonList.First()); - - // As of the time this test was written, there are around 58 custom format JSON files. - // This number can fluctuate over time, but I'm only interested in verifying we get a handful - // of files in the response. - jsonList.Should().HaveCountGreaterOrEqualTo(5); - - act.Should().NotThrow(); - } - } -} diff --git a/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs b/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs index 3d5298ba..05a8efda 100644 --- a/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs +++ b/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs @@ -1,9 +1,10 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; using Common; using FluentAssertions; -using Flurl.Http.Testing; using Newtonsoft.Json.Linq; +using NSubstitute; using NUnit.Framework; using Serilog; using TestLibrary.FluentAssertions; @@ -43,37 +44,27 @@ namespace TrashLib.Tests.Radarr.CustomFormat.Processors public ILogger Logger { get; } public ResourceDataReader Data { get; } - public JObject ReadJson(string jsonFile) - { - var jsonData = Data.ReadData(jsonFile); - return JObject.Parse(jsonData); - } + public string ReadText(string textFile) => Data.ReadData(textFile); + public JObject ReadJson(string jsonFile) => JObject.Parse(ReadText(jsonFile)); } [Test] [SuppressMessage("Maintainability", "CA1506", Justification = "Designed to be a high-level integration test")] - public void Guide_processor_behaves_as_expected_with_normal_guide_data() + public async Task Guide_processor_behaves_as_expected_with_normal_guide_data() { var ctx = new Context(); - var guideProcessor = - new GuideProcessor(ctx.Logger, new GithubCustomFormatJsonRequester(), - () => new TestGuideProcessorSteps()); + var guideService = Substitute.For(); + var guideProcessor = new GuideProcessor(ctx.Logger, guideService, () => new TestGuideProcessorSteps()); // simulate guide data - using var testHttp = new HttpTest(); - testHttp.RespondWithJson(new[] + guideService.GetCustomFormatJsonAsync().Returns(new[] { - new {name = "ImportableCustomFormat1.json", type = "file", download_url = "http://not_real/file.json"}, - new {name = "ImportableCustomFormat2.json", type = "file", download_url = "http://not_real/file.json"}, - new {name = "NoScore.json", type = "file", download_url = "http://not_real/file.json"}, - new {name = "WontBeInConfig.json", type = "file", download_url = "http://not_real/file.json"} + ctx.ReadText("ImportableCustomFormat1.json"), + ctx.ReadText("ImportableCustomFormat2.json"), + ctx.ReadText("NoScore.json"), + ctx.ReadText("WontBeInConfig.json") }); - testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat1.json")); - testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat2.json")); - testHttp.RespondWithJson(ctx.ReadJson("NoScore.json")); - testHttp.RespondWithJson(ctx.ReadJson("WontBeInConfig.json")); - // Simulate user config in YAML var config = new List { @@ -97,7 +88,7 @@ namespace TrashLib.Tests.Radarr.CustomFormat.Processors } }; - guideProcessor.BuildGuideData(config, null); + await guideProcessor.BuildGuideDataAsync(config, null); var expectedProcessedCustomFormatData = new List { diff --git a/src/TrashLib/Radarr/Config/IResourcePaths.cs b/src/TrashLib/Radarr/Config/IResourcePaths.cs new file mode 100644 index 00000000..a1f51c8b --- /dev/null +++ b/src/TrashLib/Radarr/Config/IResourcePaths.cs @@ -0,0 +1,7 @@ +namespace TrashLib.Radarr.Config +{ + public interface IResourcePaths + { + string RepoPath { get; } + } +} diff --git a/src/TrashLib/Radarr/CustomFormat/CustomFormatUpdater.cs b/src/TrashLib/Radarr/CustomFormat/CustomFormatUpdater.cs index babc10cb..64247388 100644 --- a/src/TrashLib/Radarr/CustomFormat/CustomFormatUpdater.cs +++ b/src/TrashLib/Radarr/CustomFormat/CustomFormatUpdater.cs @@ -33,7 +33,7 @@ namespace TrashLib.Radarr.CustomFormat { _cache.Load(); - await _guideProcessor.BuildGuideData(config.CustomFormats, _cache.CfCache); + await _guideProcessor.BuildGuideDataAsync(config.CustomFormats, _cache.CfCache); if (!ValidateGuideDataAndCheckShouldProceed(config)) { diff --git a/src/TrashLib/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs b/src/TrashLib/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs deleted file mode 100644 index e2c66ef9..00000000 --- a/src/TrashLib/Radarr/CustomFormat/Guide/GithubCustomFormatJsonRequester.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Flurl.Http; -using Flurl.Http.Configuration; -using JetBrains.Annotations; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace TrashLib.Radarr.CustomFormat.Guide -{ - internal class GithubCustomFormatJsonRequester : IRadarrGuideService - { - private readonly ISerializer _flurlSerializer; - - public GithubCustomFormatJsonRequester() - { - // In addition to setting the naming strategy, this also serves as a mechanism to avoid inheriting the - // global Flurl serializer setting: MissingMemberHandling. We do not want missing members to error out - // since we're only deserializing a subset of the github response object. - _flurlSerializer = new NewtonsoftJsonSerializer(new JsonSerializerSettings - { - ContractResolver = new DefaultContractResolver - { - NamingStrategy = new SnakeCaseNamingStrategy() - } - }); - } - - public async Task> GetCustomFormatJson() - { - var response = await "https://api.github.com/repos/TRaSH-/Guides/contents/docs/json/radarr" - .WithHeader("User-Agent", "Trash Updater") - .ConfigureRequest(settings => settings.JsonSerializer = _flurlSerializer) - .GetJsonAsync>(); - - var tasks = response - .Where(o => o.Type == "file" && o.Name.EndsWith(".json")) - .Select(o => DownloadJsonContents(o.DownloadUrl)); - - return await Task.WhenAll(tasks); - } - - private async Task DownloadJsonContents(string jsonUrl) - { - return await jsonUrl.GetStringAsync(); - } - - [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] - private record RepoContentEntry - { - public string Name { get; init; } = default!; - public string Type { get; init; } = default!; - public string DownloadUrl { get; init; } = default!; - } - } -} diff --git a/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs b/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs index f9be9b12..71dbee07 100644 --- a/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs +++ b/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs @@ -5,6 +5,6 @@ namespace TrashLib.Radarr.CustomFormat.Guide { public interface IRadarrGuideService { - Task> GetCustomFormatJson(); + Task> GetCustomFormatJsonAsync(); } } diff --git a/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs b/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs new file mode 100644 index 00000000..9960ccd0 --- /dev/null +++ b/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using System.Threading.Tasks; +using LibGit2Sharp; +using TrashLib.Radarr.Config; + +namespace TrashLib.Radarr.CustomFormat.Guide +{ + internal class LocalRepoCustomFormatJsonParser : IRadarrGuideService + { + private readonly IFileSystem _fileSystem; + private readonly string _repoPath; + + public LocalRepoCustomFormatJsonParser(IFileSystem fileSystem, IResourcePaths paths) + { + _fileSystem = fileSystem; + _repoPath = paths.RepoPath; + } + + public async Task> GetCustomFormatJsonAsync() + { + await Task.Run(() => + { + if (!Repository.IsValid(_repoPath)) + { + if (_fileSystem.Directory.Exists(_repoPath)) + { + _fileSystem.Directory.Delete(_repoPath, true); + } + + Repository.Clone("https://github.com/TRaSH-/Guides.git", _repoPath, new CloneOptions + { + RecurseSubmodules = false + }); + } + + using var repo = new Repository(_repoPath); + Commands.Checkout(repo, "master", new CheckoutOptions + { + CheckoutModifiers = CheckoutModifiers.Force + }); + + var origin = repo.Network.Remotes["origin"]; + Commands.Fetch(repo, origin.Name, origin.FetchRefSpecs.Select(s => s.Specification), null, ""); + + repo.Reset(ResetMode.Hard, repo.Branches["origin/master"].Tip); + }); + + var jsonDir = Path.Combine(_repoPath, "docs/json/radarr"); + var tasks = _fileSystem.Directory.GetFiles(jsonDir, "*.json") + .Select(async f => await _fileSystem.File.ReadAllTextAsync(f)); + + return await Task.WhenAll(tasks); + } + } +} diff --git a/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs b/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs index 8e0b86fe..a486816c 100644 --- a/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs +++ b/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs @@ -59,12 +59,12 @@ namespace TrashLib.Radarr.CustomFormat.Processors public Dictionary> DuplicatedCustomFormats => _steps.CustomFormat.DuplicatedCustomFormats; - public async Task BuildGuideData(IReadOnlyList config, CustomFormatCache? cache) + public async Task BuildGuideDataAsync(IReadOnlyList config, CustomFormatCache? cache) { if (_guideCustomFormatJson == null) { - Log.Debug("Requesting and parsing guide markdown"); - _guideCustomFormatJson = (await _guideService.GetCustomFormatJson()).ToList(); + Log.Information("Requesting and parsing guide markdown"); + _guideCustomFormatJson = (await _guideService.GetCustomFormatJsonAsync()).ToList(); } // Step 1: Process and filter the custom formats from the guide. diff --git a/src/TrashLib/Radarr/CustomFormat/Processors/IGuideProcessor.cs b/src/TrashLib/Radarr/CustomFormat/Processors/IGuideProcessor.cs index 66a5b2e5..d3b10fd4 100644 --- a/src/TrashLib/Radarr/CustomFormat/Processors/IGuideProcessor.cs +++ b/src/TrashLib/Radarr/CustomFormat/Processors/IGuideProcessor.cs @@ -17,7 +17,7 @@ namespace TrashLib.Radarr.CustomFormat.Processors List<(string, string)> CustomFormatsWithOutdatedNames { get; } Dictionary> DuplicatedCustomFormats { get; } - Task BuildGuideData(IReadOnlyList config, CustomFormatCache? cache); + Task BuildGuideDataAsync(IReadOnlyList config, CustomFormatCache? cache); void Reset(); } } diff --git a/src/TrashLib/Radarr/CustomFormat/Processors/PersistenceSteps/QualityProfileApiPersistenceStep.cs b/src/TrashLib/Radarr/CustomFormat/Processors/PersistenceSteps/QualityProfileApiPersistenceStep.cs index d567779f..983dcdba 100644 --- a/src/TrashLib/Radarr/CustomFormat/Processors/PersistenceSteps/QualityProfileApiPersistenceStep.cs +++ b/src/TrashLib/Radarr/CustomFormat/Processors/PersistenceSteps/QualityProfileApiPersistenceStep.cs @@ -63,7 +63,7 @@ namespace TrashLib.Radarr.CustomFormat.Processors.PersistenceSteps continue; } - json!["score"] = scoreToUse.Value; + json["score"] = scoreToUse.Value; _updatedScores.GetOrCreate(profileName) .Add(new UpdatedFormatScore((string) json["name"], scoreToUse.Value, reason.Value)); } diff --git a/src/TrashLib/Radarr/RadarrAutofacModule.cs b/src/TrashLib/Radarr/RadarrAutofacModule.cs index 026df56f..29797567 100644 --- a/src/TrashLib/Radarr/RadarrAutofacModule.cs +++ b/src/TrashLib/Radarr/RadarrAutofacModule.cs @@ -37,7 +37,7 @@ namespace TrashLib.Radarr // Custom Format Support builder.RegisterType().As(); - builder.RegisterType().As(); + builder.RegisterType().As(); builder.RegisterType().As(); // Guide Processor diff --git a/src/TrashLib/TrashLib.csproj b/src/TrashLib/TrashLib.csproj index 8730575b..77f0cc69 100644 --- a/src/TrashLib/TrashLib.csproj +++ b/src/TrashLib/TrashLib.csproj @@ -5,6 +5,7 @@ +