fix(radarr): use git repo instead of api for radarr json data

pull/47/head
Robert Dailey 3 years ago
parent 136c7d4999
commit 785da4bd07

@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- Directly use the Trash Guides git repository to avoid getting HTTP 403 - rate limit reached error
in github.
## [1.6.1] - 2021-05-31
### Changed

@ -60,7 +60,9 @@ namespace Trash.Tests.Config
// .Returns(t => Substitute.For(new[] {(Type)t[0]}, Array.Empty<object>()));
var actualActiveConfigs = new List<SonarrConfiguration>();
#pragma warning disable NS1004
provider.ActiveConfiguration = Arg.Do<SonarrConfiguration>(a => actualActiveConfigs.Add(a));
#pragma warning restore NS1004
var validator = Substitute.For<IValidator<SonarrConfiguration>>();
var loader =

@ -7,8 +7,4 @@
<ProjectReference Include="..\TestLibrary\TestLibrary.csproj" />
<ProjectReference Include="..\Trash\Trash.csproj" />
</ItemGroup>
<ItemGroup>
<Folder Include="CreateConfig" />
</ItemGroup>
</Project>

@ -11,5 +11,7 @@ namespace Trash
public static string DefaultConfigPath { get; } = Path.Combine(AppContext.BaseDirectory, "trash.yml");
public static string LogDirectory { get; } = Path.Combine(AppDataPath, "logs");
public static string RepoDirectory { get; } = Path.Combine(AppDataPath, "repo");
}
}

@ -12,6 +12,7 @@ using Trash.Config;
using TrashLib.Cache;
using TrashLib.Config;
using TrashLib.Radarr;
using TrashLib.Radarr.Config;
using TrashLib.Sonarr;
using YamlDotNet.Serialization;
@ -44,8 +45,8 @@ namespace Trash
{
builder.RegisterModule<ConfigAutofacModule>();
builder.RegisterType<ObjectFactory>()
.As<IObjectFactory>();
builder.RegisterType<ObjectFactory>().As<IObjectFactory>();
builder.RegisterType<ResourcePaths>().As<IResourcePaths>();
builder.RegisterGeneric(typeof(ConfigurationLoader<>))
.WithProperty(new AutowiringParameter())

@ -0,0 +1,9 @@
using TrashLib.Radarr.Config;
namespace Trash
{
public class ResourcePaths : IResourcePaths
{
public string RepoPath => AppPaths.RepoDirectory;
}
}

@ -1,32 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
using Newtonsoft.Json.Linq;
using NUnit.Framework;
using TrashLib.Radarr.CustomFormat.Guide;
namespace TrashLib.Tests.Radarr.CustomFormat.Guide
{
[TestFixture]
[Parallelizable(ParallelScope.All)]
public class GithubCustomFormatJsonRequesterTest
{
[Test]
public async Task Requesting_json_from_github_works()
{
var requester = new GithubCustomFormatJsonRequester();
var jsonList = (await requester.GetCustomFormatJson()).ToList();
Action act = () => JObject.Parse(jsonList.First());
// As of the time this test was written, there are around 58 custom format JSON files.
// This number can fluctuate over time, but I'm only interested in verifying we get a handful
// of files in the response.
jsonList.Should().HaveCountGreaterOrEqualTo(5);
act.Should().NotThrow();
}
}
}

@ -1,9 +1,10 @@
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Common;
using FluentAssertions;
using Flurl.Http.Testing;
using Newtonsoft.Json.Linq;
using NSubstitute;
using NUnit.Framework;
using Serilog;
using TestLibrary.FluentAssertions;
@ -43,37 +44,27 @@ namespace TrashLib.Tests.Radarr.CustomFormat.Processors
public ILogger Logger { get; }
public ResourceDataReader Data { get; }
public JObject ReadJson(string jsonFile)
{
var jsonData = Data.ReadData(jsonFile);
return JObject.Parse(jsonData);
}
public string ReadText(string textFile) => Data.ReadData(textFile);
public JObject ReadJson(string jsonFile) => JObject.Parse(ReadText(jsonFile));
}
[Test]
[SuppressMessage("Maintainability", "CA1506", Justification = "Designed to be a high-level integration test")]
public void Guide_processor_behaves_as_expected_with_normal_guide_data()
public async Task Guide_processor_behaves_as_expected_with_normal_guide_data()
{
var ctx = new Context();
var guideProcessor =
new GuideProcessor(ctx.Logger, new GithubCustomFormatJsonRequester(),
() => new TestGuideProcessorSteps());
var guideService = Substitute.For<IRadarrGuideService>();
var guideProcessor = new GuideProcessor(ctx.Logger, guideService, () => new TestGuideProcessorSteps());
// simulate guide data
using var testHttp = new HttpTest();
testHttp.RespondWithJson(new[]
guideService.GetCustomFormatJsonAsync().Returns(new[]
{
new {name = "ImportableCustomFormat1.json", type = "file", download_url = "http://not_real/file.json"},
new {name = "ImportableCustomFormat2.json", type = "file", download_url = "http://not_real/file.json"},
new {name = "NoScore.json", type = "file", download_url = "http://not_real/file.json"},
new {name = "WontBeInConfig.json", type = "file", download_url = "http://not_real/file.json"}
ctx.ReadText("ImportableCustomFormat1.json"),
ctx.ReadText("ImportableCustomFormat2.json"),
ctx.ReadText("NoScore.json"),
ctx.ReadText("WontBeInConfig.json")
});
testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat1.json"));
testHttp.RespondWithJson(ctx.ReadJson("ImportableCustomFormat2.json"));
testHttp.RespondWithJson(ctx.ReadJson("NoScore.json"));
testHttp.RespondWithJson(ctx.ReadJson("WontBeInConfig.json"));
// Simulate user config in YAML
var config = new List<CustomFormatConfig>
{
@ -97,7 +88,7 @@ namespace TrashLib.Tests.Radarr.CustomFormat.Processors
}
};
guideProcessor.BuildGuideData(config, null);
await guideProcessor.BuildGuideDataAsync(config, null);
var expectedProcessedCustomFormatData = new List<ProcessedCustomFormatData>
{

@ -0,0 +1,7 @@
namespace TrashLib.Radarr.Config
{
public interface IResourcePaths
{
string RepoPath { get; }
}
}

@ -33,7 +33,7 @@ namespace TrashLib.Radarr.CustomFormat
{
_cache.Load();
await _guideProcessor.BuildGuideData(config.CustomFormats, _cache.CfCache);
await _guideProcessor.BuildGuideDataAsync(config.CustomFormats, _cache.CfCache);
if (!ValidateGuideDataAndCheckShouldProceed(config))
{

@ -1,57 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Flurl.Http;
using Flurl.Http.Configuration;
using JetBrains.Annotations;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
namespace TrashLib.Radarr.CustomFormat.Guide
{
internal class GithubCustomFormatJsonRequester : IRadarrGuideService
{
private readonly ISerializer _flurlSerializer;
public GithubCustomFormatJsonRequester()
{
// In addition to setting the naming strategy, this also serves as a mechanism to avoid inheriting the
// global Flurl serializer setting: MissingMemberHandling. We do not want missing members to error out
// since we're only deserializing a subset of the github response object.
_flurlSerializer = new NewtonsoftJsonSerializer(new JsonSerializerSettings
{
ContractResolver = new DefaultContractResolver
{
NamingStrategy = new SnakeCaseNamingStrategy()
}
});
}
public async Task<IEnumerable<string>> GetCustomFormatJson()
{
var response = await "https://api.github.com/repos/TRaSH-/Guides/contents/docs/json/radarr"
.WithHeader("User-Agent", "Trash Updater")
.ConfigureRequest(settings => settings.JsonSerializer = _flurlSerializer)
.GetJsonAsync<List<RepoContentEntry>>();
var tasks = response
.Where(o => o.Type == "file" && o.Name.EndsWith(".json"))
.Select(o => DownloadJsonContents(o.DownloadUrl));
return await Task.WhenAll(tasks);
}
private async Task<string> DownloadJsonContents(string jsonUrl)
{
return await jsonUrl.GetStringAsync();
}
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
private record RepoContentEntry
{
public string Name { get; init; } = default!;
public string Type { get; init; } = default!;
public string DownloadUrl { get; init; } = default!;
}
}
}

@ -5,6 +5,6 @@ namespace TrashLib.Radarr.CustomFormat.Guide
{
public interface IRadarrGuideService
{
Task<IEnumerable<string>> GetCustomFormatJson();
Task<IEnumerable<string>> GetCustomFormatJsonAsync();
}
}

@ -0,0 +1,58 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using System.Threading.Tasks;
using LibGit2Sharp;
using TrashLib.Radarr.Config;
namespace TrashLib.Radarr.CustomFormat.Guide
{
internal class LocalRepoCustomFormatJsonParser : IRadarrGuideService
{
private readonly IFileSystem _fileSystem;
private readonly string _repoPath;
public LocalRepoCustomFormatJsonParser(IFileSystem fileSystem, IResourcePaths paths)
{
_fileSystem = fileSystem;
_repoPath = paths.RepoPath;
}
public async Task<IEnumerable<string>> GetCustomFormatJsonAsync()
{
await Task.Run(() =>
{
if (!Repository.IsValid(_repoPath))
{
if (_fileSystem.Directory.Exists(_repoPath))
{
_fileSystem.Directory.Delete(_repoPath, true);
}
Repository.Clone("https://github.com/TRaSH-/Guides.git", _repoPath, new CloneOptions
{
RecurseSubmodules = false
});
}
using var repo = new Repository(_repoPath);
Commands.Checkout(repo, "master", new CheckoutOptions
{
CheckoutModifiers = CheckoutModifiers.Force
});
var origin = repo.Network.Remotes["origin"];
Commands.Fetch(repo, origin.Name, origin.FetchRefSpecs.Select(s => s.Specification), null, "");
repo.Reset(ResetMode.Hard, repo.Branches["origin/master"].Tip);
});
var jsonDir = Path.Combine(_repoPath, "docs/json/radarr");
var tasks = _fileSystem.Directory.GetFiles(jsonDir, "*.json")
.Select(async f => await _fileSystem.File.ReadAllTextAsync(f));
return await Task.WhenAll(tasks);
}
}
}

@ -59,12 +59,12 @@ namespace TrashLib.Radarr.CustomFormat.Processors
public Dictionary<string, List<ProcessedCustomFormatData>> DuplicatedCustomFormats
=> _steps.CustomFormat.DuplicatedCustomFormats;
public async Task BuildGuideData(IReadOnlyList<CustomFormatConfig> config, CustomFormatCache? cache)
public async Task BuildGuideDataAsync(IReadOnlyList<CustomFormatConfig> config, CustomFormatCache? cache)
{
if (_guideCustomFormatJson == null)
{
Log.Debug("Requesting and parsing guide markdown");
_guideCustomFormatJson = (await _guideService.GetCustomFormatJson()).ToList();
Log.Information("Requesting and parsing guide markdown");
_guideCustomFormatJson = (await _guideService.GetCustomFormatJsonAsync()).ToList();
}
// Step 1: Process and filter the custom formats from the guide.

@ -17,7 +17,7 @@ namespace TrashLib.Radarr.CustomFormat.Processors
List<(string, string)> CustomFormatsWithOutdatedNames { get; }
Dictionary<string, List<ProcessedCustomFormatData>> DuplicatedCustomFormats { get; }
Task BuildGuideData(IReadOnlyList<CustomFormatConfig> config, CustomFormatCache? cache);
Task BuildGuideDataAsync(IReadOnlyList<CustomFormatConfig> config, CustomFormatCache? cache);
void Reset();
}
}

@ -63,7 +63,7 @@ namespace TrashLib.Radarr.CustomFormat.Processors.PersistenceSteps
continue;
}
json!["score"] = scoreToUse.Value;
json["score"] = scoreToUse.Value;
_updatedScores.GetOrCreate(profileName)
.Add(new UpdatedFormatScore((string) json["name"], scoreToUse.Value, reason.Value));
}

@ -37,7 +37,7 @@ namespace TrashLib.Radarr
// Custom Format Support
builder.RegisterType<CustomFormatUpdater>().As<ICustomFormatUpdater>();
builder.RegisterType<GithubCustomFormatJsonRequester>().As<IRadarrGuideService>();
builder.RegisterType<LocalRepoCustomFormatJsonParser>().As<IRadarrGuideService>();
builder.RegisterType<CachePersister>().As<ICachePersister>();
// Guide Processor

@ -5,6 +5,7 @@
<PackageReference Include="FluentValidation" />
<PackageReference Include="Flurl" />
<PackageReference Include="Flurl.Http" />
<PackageReference Include="LibGit2Sharp" Version="0.26.2" />
<PackageReference Include="Serilog" />
<PackageReference Include="System.Data.HashFunction.FNV" />
<PackageReference Include="System.IO.Abstractions" />

Loading…
Cancel
Save