diff --git a/CHANGELOG.md b/CHANGELOG.md index 87b545c5..e0eb6944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,14 +16,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Schema added for `settings.yml`. - Add setting to bypass HTTPS certificate validation (useful for self-signed certificates used with Sonarr and Radarr instances) ([#20]). - -[setref]: https://github.com/rcdailey/trash-updater/wiki/Settings-Reference -[#20]: https://github.com/rcdailey/trash-updater/issues/20 +- A progress bar that is visible when pulling down Custom Formats (Radarr Only). ### Fixed - Remove `System.Reactive.xml` from the published ZIP files. - Fix exception that may occur at startup. +- Sometimes the "Requesting and parsing guide markdown" step would appear stuck and fail after + several minutes. Many changes have been made to try to alleviate this. + +[setref]: https://github.com/rcdailey/trash-updater/wiki/Settings-Reference +[#20]: https://github.com/rcdailey/trash-updater/issues/20 ## [1.6.6] - 2021-10-30 diff --git a/src/Common/ProgressBar.cs b/src/Common/ProgressBar.cs new file mode 100644 index 00000000..513395c7 --- /dev/null +++ b/src/Common/ProgressBar.cs @@ -0,0 +1,113 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Threading; + +namespace Common; + +/// +/// An ASCII progress bar +/// +/// +/// Full credit to: https://gist.github.com/DanielSWolf/0ab6a96899cc5377bf54 +/// +[SuppressMessage("Reliability", "CA2002:Do not lock on objects with weak identity", + Justification = "Code is from third party")] +public sealed class ProgressBar : IDisposable, IProgress +{ + private const int BlockCount = 10; + private readonly TimeSpan _animationInterval = TimeSpan.FromSeconds(1.0 / 8); + private const string Animation = @"|/-\"; + + private readonly Timer _timer; + + private double _currentProgress; + private string _currentText = string.Empty; + private bool _disposed; + private int _animationIndex; + + public ProgressBar() + { + _timer = new Timer(TimerHandler); + + // A progress bar is only for temporary display in a console window. + // If the console output is redirected to a file, draw nothing. + // Otherwise, we'll end up with a lot of garbage in the target file. + if (!Console.IsOutputRedirected) + { + ResetTimer(); + } + } + + public void Report(double value) + { + // Make sure value is in [0..1] range + value = Math.Max(0, Math.Min(1, value)); + Interlocked.Exchange(ref _currentProgress, value); + } + + private void TimerHandler(object? state) + { + lock (_timer) + { + if (_disposed) + { + return; + } + + var progressBlockCount = (int) (_currentProgress * BlockCount); + var percent = (int) (_currentProgress * 100); + var progressBlocks = new string('#', progressBlockCount); + var progressBlocksUnfilled = new string('-', BlockCount - progressBlockCount); + var currentAnimationFrame = Animation[_animationIndex++ % Animation.Length]; + var text = $"[{progressBlocks}{progressBlocksUnfilled}] {percent,3}% {currentAnimationFrame}"; + UpdateText(text); + + ResetTimer(); + } + } + + private void UpdateText(string text) + { + // Get length of common portion + var commonPrefixLength = 0; + var commonLength = Math.Min(_currentText.Length, text.Length); + while (commonPrefixLength < commonLength && text[commonPrefixLength] == _currentText[commonPrefixLength]) + { + commonPrefixLength++; + } + + // Backtrack to the first differing character + var outputBuilder = new StringBuilder(); + outputBuilder.Append('\b', _currentText.Length - commonPrefixLength); + + // Output new suffix + outputBuilder.Append(text.AsSpan(commonPrefixLength)); + + // If the new text is shorter than the old one: delete overlapping characters + var overlapCount = _currentText.Length - text.Length; + if (overlapCount > 0) + { + outputBuilder.Append(' ', overlapCount); + outputBuilder.Append('\b', overlapCount); + } + + Console.Write(outputBuilder); + _currentText = text; + } + + private void ResetTimer() + { + _timer.Change(_animationInterval, TimeSpan.FromMilliseconds(-1)); + } + + public void Dispose() + { + lock (_timer) + { + _disposed = true; + UpdateText(string.Empty); + _timer.Dispose(); + } + } +} diff --git a/src/Trash/CompositionRoot.cs b/src/Trash/CompositionRoot.cs index 673b352c..30a22358 100644 --- a/src/Trash/CompositionRoot.cs +++ b/src/Trash/CompositionRoot.cs @@ -16,6 +16,7 @@ using TrashLib.Radarr; using TrashLib.Radarr.Config; using TrashLib.Sonarr; using TrashLib.Startup; +using VersionControl; using YamlDotNet.Serialization; namespace Trash; @@ -92,6 +93,7 @@ public static class CompositionRoot builder.RegisterModule(); builder.RegisterModule(); + builder.RegisterModule(); builder.Register(_ => AutoMapperConfig.Setup()).SingleInstance(); diff --git a/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs b/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs index 0c8a97e2..9d2d3c8d 100644 --- a/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs +++ b/src/TrashLib.Tests/Radarr/CustomFormat/Processors/GuideProcessorTest.cs @@ -6,7 +6,6 @@ using FluentAssertions; using Newtonsoft.Json.Linq; using NSubstitute; using NUnit.Framework; -using Serilog; using TestLibrary.FluentAssertions; using Trash.TestLibrary; using TrashLib.Radarr.Config; @@ -32,16 +31,9 @@ public class GuideProcessorTest { public Context() { - Logger = new LoggerConfiguration() - .WriteTo.TestCorrelator() - .WriteTo.NUnitOutput() - .MinimumLevel.Debug() - .CreateLogger(); - Data = new ResourceDataReader(typeof(GuideProcessorTest), "Data"); } - public ILogger Logger { get; } public ResourceDataReader Data { get; } public string ReadText(string textFile) => Data.ReadData(textFile); @@ -54,10 +46,10 @@ public class GuideProcessorTest { var ctx = new Context(); var guideService = Substitute.For(); - var guideProcessor = new GuideProcessor(ctx.Logger, guideService, () => new TestGuideProcessorSteps()); + var guideProcessor = new GuideProcessor(guideService, () => new TestGuideProcessorSteps()); // simulate guide data - guideService.GetCustomFormatJsonAsync().Returns(new[] + guideService.GetCustomFormatJson().Returns(new[] { ctx.ReadText("ImportableCustomFormat1.json"), ctx.ReadText("ImportableCustomFormat2.json"), diff --git a/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs b/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs index cddd6762..8aca149a 100644 --- a/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs +++ b/src/TrashLib/Radarr/CustomFormat/Guide/IRadarrGuideService.cs @@ -1,9 +1,8 @@ using System.Collections.Generic; -using System.Threading.Tasks; namespace TrashLib.Radarr.CustomFormat.Guide; public interface IRadarrGuideService { - Task> GetCustomFormatJsonAsync(); + IEnumerable GetCustomFormatJson(); } diff --git a/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs b/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs index 7f0c3bcd..47bc8a11 100644 --- a/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs +++ b/src/TrashLib/Radarr/CustomFormat/Guide/LocalRepoCustomFormatJsonParser.cs @@ -1,82 +1,86 @@ +using System; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; using System.Linq; using System.Threading.Tasks; +using Common; using LibGit2Sharp; using Serilog; using TrashLib.Config.Settings; using TrashLib.Radarr.Config; +using VersionControl; -namespace TrashLib.Radarr.CustomFormat.Guide -{ - internal class LocalRepoCustomFormatJsonParser : IRadarrGuideService - { - private readonly IFileSystem _fileSystem; - private readonly ISettingsProvider _settings; - private readonly ILogger _log; - private readonly string _repoPath; - - public LocalRepoCustomFormatJsonParser( - IFileSystem fileSystem, - IResourcePaths paths, - ISettingsProvider settings, - ILogger log) - { - _fileSystem = fileSystem; - _settings = settings; - _log = log; - _repoPath = paths.RepoPath; - } - - public async Task> GetCustomFormatJsonAsync() - { - CloneOrUpdateGitRepo(); +namespace TrashLib.Radarr.CustomFormat.Guide; - var jsonDir = Path.Combine(_repoPath, "docs/json/radarr"); - var tasks = _fileSystem.Directory.GetFiles(jsonDir, "*.json") - .Select(async f => await _fileSystem.File.ReadAllTextAsync(f)); +internal class LocalRepoCustomFormatJsonParser : IRadarrGuideService +{ + private readonly ILogger _log; + private readonly IFileSystem _fileSystem; + private readonly IGitRepositoryFactory _repositoryFactory; + private readonly IFileUtilities _fileUtils; + private readonly ISettingsProvider _settingsProvider; + private readonly string _repoPath; - return await Task.WhenAll(tasks); - } + public LocalRepoCustomFormatJsonParser( + ILogger log, + IFileSystem fileSystem, + IResourcePaths paths, + IGitRepositoryFactory repositoryFactory, + IFileUtilities fileUtils, + ISettingsProvider settingsProvider) + { + _log = log; + _fileSystem = fileSystem; + _repositoryFactory = repositoryFactory; + _fileUtils = fileUtils; + _settingsProvider = settingsProvider; + _repoPath = paths.RepoPath; + } - private void CloneOrUpdateGitRepo() + public IEnumerable GetCustomFormatJson() + { + // Retry only once if there's a failure. This gives us an opportunity to delete the git repository and start + // fresh. + var exception = CheckoutAndUpdateRepo(); + if (exception is not null) { - var cloneUrl = _settings.Settings.Repository.CloneUrl; + _log.Information("Deleting local git repo and retrying git operation..."); + _fileUtils.DeleteReadOnlyDirectory(_repoPath); - if (!Repository.IsValid(_repoPath)) + exception = CheckoutAndUpdateRepo(); + if (exception is not null) { - if (_fileSystem.Directory.Exists(_repoPath)) - { - _fileSystem.Directory.Delete(_repoPath, true); - } - - Repository.Clone(cloneUrl, _repoPath, new CloneOptions - { - RecurseSubmodules = false - }); + throw exception; } + } - using var repo = new Repository(_repoPath); - Commands.Checkout(repo, "master", new CheckoutOptions - { - CheckoutModifiers = CheckoutModifiers.Force - }); - - var origin = repo.Network.Remotes["origin"]; - if (origin.Url != cloneUrl) - { - _log.Debug( - "Origin's URL ({OriginUrl}) does not match the clone URL from settings ({CloneUrl}) and will be updated", - origin.Url, cloneUrl); + var jsonDir = Path.Combine(_repoPath, "docs/json/radarr"); + var tasks = _fileSystem.Directory.GetFiles(jsonDir, "*.json") + .Select(async f => await _fileSystem.File.ReadAllTextAsync(f)); - repo.Network.Remotes.Update("origin", updater => updater.Url = cloneUrl); - origin = repo.Network.Remotes["origin"]; - } + return Task.WhenAll(tasks).Result; + } - Commands.Fetch(repo, origin.Name, origin.FetchRefSpecs.Select(s => s.Specification), null, ""); + private Exception? CheckoutAndUpdateRepo() + { + var repoSettings = _settingsProvider.Settings.Repository; + var cloneUrl = repoSettings.CloneUrl; + const string branch = "master"; - repo.Reset(ResetMode.Hard, repo.Branches["origin/master"].Tip); + try + { + using var repo = _repositoryFactory.CreateAndCloneIfNeeded(cloneUrl, _repoPath, branch); + repo.ForceCheckout(branch); + repo.Fetch(); + repo.ResetHard($"origin/{branch}"); } + catch (LibGit2SharpException e) + { + _log.Error(e, "An exception occurred during git operations on path: {RepoPath}", _repoPath); + return e; + } + + return null; } } diff --git a/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs b/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs index e33cf6cf..07a9b393 100644 --- a/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs +++ b/src/TrashLib/Radarr/CustomFormat/Processors/GuideProcessor.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Serilog; using TrashLib.Radarr.Config; using TrashLib.Radarr.CustomFormat.Guide; using TrashLib.Radarr.CustomFormat.Models; @@ -25,16 +24,13 @@ internal class GuideProcessor : IGuideProcessor private IList? _guideCustomFormatJson; private IGuideProcessorSteps _steps; - public GuideProcessor(ILogger log, IRadarrGuideService guideService, Func stepsFactory) + public GuideProcessor(IRadarrGuideService guideService, Func stepsFactory) { _guideService = guideService; _stepsFactory = stepsFactory; - Log = log; _steps = stepsFactory(); } - private ILogger Log { get; } - public IReadOnlyCollection ProcessedCustomFormats => _steps.CustomFormat.ProcessedCustomFormats; @@ -59,12 +55,11 @@ internal class GuideProcessor : IGuideProcessor public IDictionary> DuplicatedCustomFormats => _steps.CustomFormat.DuplicatedCustomFormats; - public async Task BuildGuideDataAsync(IReadOnlyCollection config, CustomFormatCache? cache) + public Task BuildGuideDataAsync(IReadOnlyCollection config, CustomFormatCache? cache) { if (_guideCustomFormatJson == null) { - Log.Information("Requesting and parsing guide markdown"); - _guideCustomFormatJson = (await _guideService.GetCustomFormatJsonAsync()).ToList(); + _guideCustomFormatJson = _guideService.GetCustomFormatJson().ToList(); } // Step 1: Process and filter the custom formats from the guide. @@ -84,6 +79,8 @@ internal class GuideProcessor : IGuideProcessor // Score precedence logic is utilized here to decide the CF score per profile (same CF can actually have // different scores depending on which profile it goes into). _steps.QualityProfile.Process(_steps.Config.ConfigData); + + return Task.CompletedTask; } public void Reset() diff --git a/src/VersionControl.Tests/GitRepositoryFactoryTest.cs b/src/VersionControl.Tests/GitRepositoryFactoryTest.cs new file mode 100644 index 00000000..1c1a427b --- /dev/null +++ b/src/VersionControl.Tests/GitRepositoryFactoryTest.cs @@ -0,0 +1,50 @@ +using AutoFixture.NUnit3; +using Common; +using FluentAssertions; +using LibGit2Sharp; +using NSubstitute; +using NUnit.Framework; +using TestLibrary.AutoFixture; +using TestLibrary.NSubstitute; +using VersionControl.Wrappers; + +namespace VersionControl.Tests; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class GitRepositoryFactoryTest +{ + [Test, AutoMockData] + public void Delete_and_clone_when_repo_is_not_valid( + [Frozen] IRepositoryStaticWrapper wrapper, + [Frozen] IFileUtilities fileUtils, + GitRepositoryFactory sut) + { + wrapper.IsValid(Arg.Any()).Returns(false); + + sut.CreateAndCloneIfNeeded("repo_url", "repo_path", "branch"); + + Received.InOrder(() => + { + wrapper.IsValid("repo_path"); + fileUtils.DeleteReadOnlyDirectory("repo_path"); + wrapper.Clone("repo_url", "repo_path", + Verify.That(x => x.BranchName.Should().Be("branch"))); + }); + } + + [Test, AutoMockData] + public void No_delete_and_clone_when_repo_is_valid( + [Frozen] IRepositoryStaticWrapper wrapper, + [Frozen] IFileUtilities fileUtils, + GitRepositoryFactory sut) + { + wrapper.IsValid(Arg.Any()).Returns(true); + + sut.CreateAndCloneIfNeeded("repo_url", "repo_path", "branch"); + + wrapper.Received().IsValid("repo_path"); + fileUtils.DidNotReceiveWithAnyArgs().DeleteReadOnlyDirectory(default!); + wrapper.DidNotReceiveWithAnyArgs().Clone(default!, default!, default!); + } +} diff --git a/src/VersionControl/GitRepository.cs b/src/VersionControl/GitRepository.cs new file mode 100644 index 00000000..8f9e7915 --- /dev/null +++ b/src/VersionControl/GitRepository.cs @@ -0,0 +1,39 @@ +using System.Linq; +using LibGit2Sharp; + +namespace VersionControl; + +public sealed class GitRepository : IGitRepository +{ + private readonly Repository _repo; + + public GitRepository(string repoPath) + { + _repo = new Repository(repoPath); + } + + public void Dispose() + { + _repo.Dispose(); + } + + public void ForceCheckout(string branch) + { + Commands.Checkout(_repo, branch, new CheckoutOptions + { + CheckoutModifiers = CheckoutModifiers.Force + }); + } + + public void Fetch(string remote = "origin") + { + var origin = _repo.Network.Remotes[remote]; + Commands.Fetch(_repo, origin.Name, origin.FetchRefSpecs.Select(s => s.Specification), null, ""); + } + + public void ResetHard(string toBranch) + { + var commit = _repo.Branches[toBranch].Tip; + _repo.Reset(ResetMode.Hard, commit); + } +} diff --git a/src/VersionControl/GitRepositoryFactory.cs b/src/VersionControl/GitRepositoryFactory.cs new file mode 100644 index 00000000..297ca8f7 --- /dev/null +++ b/src/VersionControl/GitRepositoryFactory.cs @@ -0,0 +1,53 @@ +using System; +using Common; +using LibGit2Sharp; +using VersionControl.Wrappers; + +namespace VersionControl; + +public class GitRepositoryFactory : IGitRepositoryFactory +{ + private readonly IFileUtilities _fileUtils; + private readonly IRepositoryStaticWrapper _staticWrapper; + private readonly Func _repoFactory; + + public GitRepositoryFactory( + IFileUtilities fileUtils, + IRepositoryStaticWrapper staticWrapper, + Func repoFactory) + { + _fileUtils = fileUtils; + _staticWrapper = staticWrapper; + _repoFactory = repoFactory; + } + + public IGitRepository CreateAndCloneIfNeeded(string repoUrl, string repoPath, string branch) + { + if (!_staticWrapper.IsValid(repoPath)) + { + DeleteAndCloneRepo(repoUrl, repoPath, branch); + } + + return _repoFactory(repoPath); + } + + private void DeleteAndCloneRepo(string repoUrl, string repoPath, string branch) + { + _fileUtils.DeleteReadOnlyDirectory(repoPath); + + Console.Write("Requesting and parsing guide markdown "); + + using var progress = new ProgressBar(); + _staticWrapper.Clone(repoUrl, repoPath, new CloneOptions + { + RecurseSubmodules = false, + BranchName = branch, + OnTransferProgress = gitProgress => + { + // ReSharper disable once AccessToDisposedClosure + progress.Report((float) gitProgress.ReceivedObjects / gitProgress.TotalObjects); + return true; + } + }); + } +} diff --git a/src/VersionControl/IGitRepository.cs b/src/VersionControl/IGitRepository.cs new file mode 100644 index 00000000..3bb703e6 --- /dev/null +++ b/src/VersionControl/IGitRepository.cs @@ -0,0 +1,10 @@ +using System; + +namespace VersionControl; + +public interface IGitRepository : IDisposable +{ + void ForceCheckout(string branch); + void Fetch(string remote = "origin"); + void ResetHard(string toBranch); +} diff --git a/src/VersionControl/IGitRepositoryFactory.cs b/src/VersionControl/IGitRepositoryFactory.cs new file mode 100644 index 00000000..028454d3 --- /dev/null +++ b/src/VersionControl/IGitRepositoryFactory.cs @@ -0,0 +1,6 @@ +namespace VersionControl; + +public interface IGitRepositoryFactory +{ + IGitRepository CreateAndCloneIfNeeded(string repoUrl, string repoPath, string branch); +} diff --git a/src/VersionControl/VersionControlAutofacModule.cs b/src/VersionControl/VersionControlAutofacModule.cs new file mode 100644 index 00000000..beedf248 --- /dev/null +++ b/src/VersionControl/VersionControlAutofacModule.cs @@ -0,0 +1,15 @@ +using Autofac; +using VersionControl.Wrappers; + +namespace VersionControl; + +public class VersionControlAutofacModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + builder.RegisterType().As(); + builder.RegisterType().As(); + builder.RegisterType().As(); + base.Load(builder); + } +} diff --git a/src/VersionControl/Wrappers/IRepositoryStaticWrapper.cs b/src/VersionControl/Wrappers/IRepositoryStaticWrapper.cs new file mode 100644 index 00000000..31faa559 --- /dev/null +++ b/src/VersionControl/Wrappers/IRepositoryStaticWrapper.cs @@ -0,0 +1,9 @@ +using LibGit2Sharp; + +namespace VersionControl.Wrappers; + +public interface IRepositoryStaticWrapper +{ + string Clone(string sourceUrl, string workdirPath, CloneOptions options); + bool IsValid(string path); +} diff --git a/src/VersionControl/Wrappers/LibGit2SharpRepositoryStaticWrapper.cs b/src/VersionControl/Wrappers/LibGit2SharpRepositoryStaticWrapper.cs new file mode 100644 index 00000000..b523ad9b --- /dev/null +++ b/src/VersionControl/Wrappers/LibGit2SharpRepositoryStaticWrapper.cs @@ -0,0 +1,12 @@ +using LibGit2Sharp; + +namespace VersionControl.Wrappers; + +public class LibGit2SharpRepositoryStaticWrapper : IRepositoryStaticWrapper +{ + public string Clone(string sourceUrl, string workdirPath, CloneOptions options) + => Repository.Clone(sourceUrl, workdirPath, options); + + public bool IsValid(string path) + => Repository.IsValid(path); +}