From 498c019f5df078b9dbb02d174edf0c930824a661 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 10 Jan 2018 10:30:55 +0000 Subject: [PATCH] Moved the update check code from the External azure service into Ombi at /api/v1/update/BRANCH --- src/Ombi.Api.Service/AppVeyorApi.cs | 108 +++++++++ src/Ombi.Api.Service/IAppVeyorApi.cs | 9 + src/Ombi.Api/IOmbiHttpClient.cs | 4 +- src/Ombi.Api/OmbiHttpClient.cs | 14 +- src/Ombi.DependencyInjection/IocExtensions.cs | 4 + .../Jobs/Ombi/OmbiAutomaticUpdater.cs | 14 +- src/Ombi.Schedule/Ombi.Schedule.csproj | 3 + src/Ombi.Schedule/Processor/AppVeyor.cs | 123 +++++++++++ .../Processor/ChangeLogProcessor.cs | 206 ++++++++++++++++++ .../Processor/IChangeLogProcessor.cs | 9 + src/Ombi/Controllers/UpdateController.cs | 31 +++ 11 files changed, 516 insertions(+), 9 deletions(-) create mode 100644 src/Ombi.Api.Service/AppVeyorApi.cs create mode 100644 src/Ombi.Api.Service/IAppVeyorApi.cs create mode 100644 src/Ombi.Schedule/Processor/AppVeyor.cs create mode 100644 src/Ombi.Schedule/Processor/ChangeLogProcessor.cs create mode 100644 src/Ombi.Schedule/Processor/IChangeLogProcessor.cs create mode 100644 src/Ombi/Controllers/UpdateController.cs diff --git a/src/Ombi.Api.Service/AppVeyorApi.cs b/src/Ombi.Api.Service/AppVeyorApi.cs new file mode 100644 index 000000000..c41656509 --- /dev/null +++ b/src/Ombi.Api.Service/AppVeyorApi.cs @@ -0,0 +1,108 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Ombi.Api.Service +{ + public class AppVeyorApi : IAppVeyorApi + { + public const string AppveyorApiUrl = "https://ci.appveyor.com/api"; + + public AppVeyorApi(IApi api) + { + _api = api; + } + + private readonly IApi _api; + public async Task GetProjectHistory(string branchName, int records = 10) + { + var request = new Request($"projects/tidusjar/requestplex/history?recordsNumber={records}&branch={branchName}", AppveyorApiUrl, HttpMethod.Get); + + request.ApplicationJsonContentType(); + + return await _api.Request(request); + } + + public class AppveyorProjects + { + public Project project { get; set; } + public Build[] builds { get; set; } + } + + public class Project + { + public int projectId { get; set; } + public int accountId { get; set; } + public string accountName { get; set; } + public object[] builds { get; set; } + public string name { get; set; } + public string slug { get; set; } + public string repositoryType { get; set; } + public string repositoryScm { get; set; } + public string repositoryName { get; set; } + public bool isPrivate { get; set; } + public bool skipBranchesWithoutAppveyorYml { get; set; } + public bool enableSecureVariablesInPullRequests { get; set; } + public bool enableSecureVariablesInPullRequestsFromSameRepo { get; set; } + public bool enableDeploymentInPullRequests { get; set; } + public bool rollingBuilds { get; set; } + public bool alwaysBuildClosedPullRequests { get; set; } + public string tags { get; set; } + public Securitydescriptor securityDescriptor { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + } + + public class Securitydescriptor + { + public Accessrightdefinition[] accessRightDefinitions { get; set; } + public Roleace[] roleAces { get; set; } + } + + public class Accessrightdefinition + { + public string name { get; set; } + public string description { get; set; } + } + + public class Roleace + { + public int roleId { get; set; } + public string name { get; set; } + public bool isAdmin { get; set; } + public Accessright[] accessRights { get; set; } + } + + public class Accessright + { + public string name { get; set; } + public bool allowed { get; set; } + } + + public class Build + { + public int buildId { get; set; } + public object[] jobs { get; set; } + public int buildNumber { get; set; } + public string version { get; set; } + public string message { get; set; } + public string messageExtended { get; set; } + public string branch { get; set; } + public bool isTag { get; set; } + public string commitId { get; set; } + public string authorName { get; set; } + public string authorUsername { get; set; } + public string committerName { get; set; } + public string committerUsername { get; set; } + public DateTime committed { get; set; } + public object[] messages { get; set; } + public string status { get; set; } + public DateTime started { get; set; } + public DateTime finished { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + public string pullRequestId { get; set; } + public string pullRequestName { get; set; } + } + } +} \ No newline at end of file diff --git a/src/Ombi.Api.Service/IAppVeyorApi.cs b/src/Ombi.Api.Service/IAppVeyorApi.cs new file mode 100644 index 000000000..c442260d4 --- /dev/null +++ b/src/Ombi.Api.Service/IAppVeyorApi.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace Ombi.Api.Service +{ + public interface IAppVeyorApi + { + Task GetProjectHistory(string branchName, int records = 10); + } +} \ No newline at end of file diff --git a/src/Ombi.Api/IOmbiHttpClient.cs b/src/Ombi.Api/IOmbiHttpClient.cs index 6c2e22f7c..97db4180b 100644 --- a/src/Ombi.Api/IOmbiHttpClient.cs +++ b/src/Ombi.Api/IOmbiHttpClient.cs @@ -1,4 +1,5 @@ -using System.Net.Http; +using System; +using System.Net.Http; using System.Threading.Tasks; namespace Ombi.Api @@ -6,5 +7,6 @@ namespace Ombi.Api public interface IOmbiHttpClient { Task SendAsync(HttpRequestMessage request); + Task GetStringAsync(Uri requestUri); } } \ No newline at end of file diff --git a/src/Ombi.Api/OmbiHttpClient.cs b/src/Ombi.Api/OmbiHttpClient.cs index ba21ddaa3..77ea9d012 100644 --- a/src/Ombi.Api/OmbiHttpClient.cs +++ b/src/Ombi.Api/OmbiHttpClient.cs @@ -56,6 +56,18 @@ namespace Ombi.Api public async Task SendAsync(HttpRequestMessage request) + { + await Setup(); + return await _client.SendAsync(request); + } + + public async Task GetStringAsync(Uri requestUri) + { + await Setup(); + return await _client.GetStringAsync(requestUri); + } + + private async Task Setup() { if (_client == null) { @@ -66,8 +78,6 @@ namespace Ombi.Api } _client = new HttpClient(_handler); } - - return await _client.SendAsync(request); } private async Task GetHandler() diff --git a/src/Ombi.DependencyInjection/IocExtensions.cs b/src/Ombi.DependencyInjection/IocExtensions.cs index 770109854..619d18be1 100644 --- a/src/Ombi.DependencyInjection/IocExtensions.cs +++ b/src/Ombi.DependencyInjection/IocExtensions.cs @@ -50,8 +50,10 @@ using Ombi.Store.Repository.Requests; using Ombi.Updater; using PlexContentCacher = Ombi.Schedule.Jobs.Plex; using Ombi.Api.Telegram; +using Ombi.Core.Processor; using Ombi.Schedule.Jobs.Plex.Interfaces; using Ombi.Schedule.Jobs.SickRage; +using Ombi.Schedule.Processor; namespace Ombi.DependencyInjection { @@ -107,6 +109,7 @@ namespace Ombi.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); } public static void RegisterStore(this IServiceCollection services) { @@ -143,6 +146,7 @@ namespace Ombi.DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); } public static void RegisterJobs(this IServiceCollection services) diff --git a/src/Ombi.Schedule/Jobs/Ombi/OmbiAutomaticUpdater.cs b/src/Ombi.Schedule/Jobs/Ombi/OmbiAutomaticUpdater.cs index bac3ddfca..463f41595 100644 --- a/src/Ombi.Schedule/Jobs/Ombi/OmbiAutomaticUpdater.cs +++ b/src/Ombi.Schedule/Jobs/Ombi/OmbiAutomaticUpdater.cs @@ -17,8 +17,10 @@ using Microsoft.Extensions.Logging; using Ombi.Api.Service; using Ombi.Api.Service.Models; +using Ombi.Core.Processor; using Ombi.Core.Settings; using Ombi.Helpers; +using Ombi.Schedule.Processor; using Ombi.Settings.Settings.Models; using Ombi.Store.Entities; using Ombi.Store.Repository; @@ -30,11 +32,11 @@ namespace Ombi.Schedule.Jobs.Ombi { public class OmbiAutomaticUpdater : IOmbiAutomaticUpdater { - public OmbiAutomaticUpdater(ILogger log, IOmbiService service, + public OmbiAutomaticUpdater(ILogger log, IChangeLogProcessor service, ISettingsService s, IProcessProvider proc, IRepository appConfig) { Logger = log; - OmbiService = service; + Processor = service; Settings = s; _processProvider = proc; _appConfig = appConfig; @@ -42,7 +44,7 @@ namespace Ombi.Schedule.Jobs.Ombi } private ILogger Logger { get; } - private IOmbiService OmbiService { get; } + private IChangeLogProcessor Processor { get; } private ISettingsService Settings { get; } private readonly IProcessProvider _processProvider; private static PerformContext Ctx { get; set; } @@ -57,7 +59,7 @@ namespace Ombi.Schedule.Jobs.Ombi public async Task UpdateAvailable(string branch, string currentVersion) { - var updates = await OmbiService.GetUpdates(branch); + var updates = await Processor.Process(branch); var serverVersion = updates.UpdateVersionString; return !serverVersion.Equals(currentVersion, StringComparison.CurrentCultureIgnoreCase); @@ -95,7 +97,7 @@ namespace Ombi.Schedule.Jobs.Ombi Logger.LogInformation(LoggingEvents.Updater, "Branch {0}", branch); Logger.LogInformation("Looking for updates now"); - var updates = await OmbiService.GetUpdates(branch); + var updates = await Processor.Process(branch); Logger.LogInformation("Updates: {0}", updates); var serverVersion = updates.UpdateVersionString; @@ -110,7 +112,7 @@ namespace Ombi.Schedule.Jobs.Ombi Logger.LogInformation(LoggingEvents.Updater, "OS Information: {0} {1}", desc, proce); Logger.LogInformation("OS Information: {0} {1}", desc, proce); - Download download; + Downloads download; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { Logger.LogInformation(LoggingEvents.Updater, "We are Windows"); diff --git a/src/Ombi.Schedule/Ombi.Schedule.csproj b/src/Ombi.Schedule/Ombi.Schedule.csproj index 02dcc9dec..cb8cef8ab 100644 --- a/src/Ombi.Schedule/Ombi.Schedule.csproj +++ b/src/Ombi.Schedule/Ombi.Schedule.csproj @@ -19,6 +19,9 @@ + + + diff --git a/src/Ombi.Schedule/Processor/AppVeyor.cs b/src/Ombi.Schedule/Processor/AppVeyor.cs new file mode 100644 index 000000000..6bb4e001c --- /dev/null +++ b/src/Ombi.Schedule/Processor/AppVeyor.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using Ombi.Api.Service; +using Ombi.Schedule.Processor; + +namespace Ombi.Core.Processor +{ + public class AppveyorBranchResult + { + public Project1 project { get; set; } + public Build1 build { get; set; } + } + + public class Project1 + { + public int projectId { get; set; } + public int accountId { get; set; } + public string accountName { get; set; } + public object[] builds { get; set; } + public string name { get; set; } + public string slug { get; set; } + public string repositoryType { get; set; } + public string repositoryScm { get; set; } + public string repositoryName { get; set; } + public string repositoryBranch { get; set; } + public bool isPrivate { get; set; } + public bool skipBranchesWithoutAppveyorYml { get; set; } + public bool enableSecureVariablesInPullRequests { get; set; } + public bool enableSecureVariablesInPullRequestsFromSameRepo { get; set; } + public bool enableDeploymentInPullRequests { get; set; } + public bool saveBuildCacheInPullRequests { get; set; } + public bool rollingBuilds { get; set; } + public bool rollingBuildsDoNotCancelRunningBuilds { get; set; } + public bool alwaysBuildClosedPullRequests { get; set; } + public string tags { get; set; } + public Nugetfeed nuGetFeed { get; set; } + public AppVeyorApi.Securitydescriptor securityDescriptor { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + } + + public class Nugetfeed + { + public string id { get; set; } + public string name { get; set; } + public int accountId { get; set; } + public int projectId { get; set; } + public bool publishingEnabled { get; set; } + public DateTime created { get; set; } + } + + + public class Build1 + { + public int buildId { get; set; } + public Job[] jobs { get; set; } + public int buildNumber { get; set; } + public string version { get; set; } + public string message { get; set; } + public string branch { get; set; } + public bool isTag { get; set; } + public string commitId { get; set; } + public string authorName { get; set; } + public string committerName { get; set; } + public DateTime committed { get; set; } + public object[] messages { get; set; } + public string status { get; set; } + public DateTime started { get; set; } + public DateTime finished { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + } + + public class Job + { + public string jobId { get; set; } + public string name { get; set; } + public string osType { get; set; } + public bool allowFailure { get; set; } + public int messagesCount { get; set; } + public int compilationMessagesCount { get; set; } + public int compilationErrorsCount { get; set; } + public int compilationWarningsCount { get; set; } + public int testsCount { get; set; } + public int passedTestsCount { get; set; } + public int failedTestsCount { get; set; } + public int artifactsCount { get; set; } + public string status { get; set; } + public DateTime started { get; set; } + public DateTime finished { get; set; } + public DateTime created { get; set; } + public DateTime updated { get; set; } + } + + public class BuildArtifactsContainer + { + public BuildArtifacts[] artifacts { get; set; } + } + + public class BuildArtifacts + { + public string fileName { get; set; } + public string type { get; set; } + public int size { get; set; } + } + + public class UpdateModel + { + public string UpdateVersionString { get; set; } + public int UpdateVersion { get; set; } + public DateTime UpdateDate { get; set; } + + public List ChangeLogs { get; set; } + public List Downloads { get; set; } + } + + public class ChangeLog + { + public string Type { get; set; } // New, Fixed, Updated + public string Descripion { get; set; } + } + +} \ No newline at end of file diff --git a/src/Ombi.Schedule/Processor/ChangeLogProcessor.cs b/src/Ombi.Schedule/Processor/ChangeLogProcessor.cs new file mode 100644 index 000000000..83d55aec3 --- /dev/null +++ b/src/Ombi.Schedule/Processor/ChangeLogProcessor.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Markdig; +using Octokit; +using Ombi.Api; +using Ombi.Api.Service; +using Ombi.Core.Processor; + +namespace Ombi.Schedule.Processor +{ + public class ChangeLogProcessor : IChangeLogProcessor + { + public ChangeLogProcessor(IApi api, IOmbiHttpClient client) + { + _api = api; + _client = client; + } + + private readonly IApi _api; + private readonly IOmbiHttpClient _client; + private const string _changeLogUrl = "https://raw.githubusercontent.com/tidusjar/Ombi/{0}/CHANGELOG.md"; + private const string AppveyorApiUrl = "https://ci.appveyor.com/api"; + private string ChangeLogUrl(string branch) => string.Format(_changeLogUrl, branch); + + public async Task Process(string branch) + { + var masterBranch = branch.Equals("master", StringComparison.CurrentCultureIgnoreCase); + string githubChangeLog; + + githubChangeLog = await _client.GetStringAsync(new Uri(ChangeLogUrl(branch))); + + + var html = Markdown.ToHtml(githubChangeLog); + + + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + HtmlNode latestRelease; + if (masterBranch) + { + latestRelease = doc.DocumentNode.Descendants("h2") + .FirstOrDefault(x => x.InnerText != "(unreleased)"); + } + else + { + latestRelease = doc.DocumentNode.Descendants("h2") + .FirstOrDefault(x => x.InnerText == "(unreleased)"); + + if (latestRelease == null) + { + latestRelease = doc.DocumentNode.Descendants("h2") + .FirstOrDefault(x => x.InnerText != "(unreleased)"); + } + } + + var newFeatureList = latestRelease.NextSibling.NextSibling.NextSibling.NextSibling; + var featuresString = newFeatureList.ChildNodes.Where(x => x.Name != "#text").Select(x => x.InnerText.Replace("\\n", "")).ToList(); + var fixes = newFeatureList.NextSibling.NextSibling.NextSibling.NextSibling; + var fixesString = fixes.ChildNodes.Where(x => x.Name != "#text").Select(x => x.InnerText.Replace("\\n", "")).ToList(); + + // Cleanup + var featuresList = featuresString.Distinct().ToList(); + var fixesList = fixesString.Distinct().ToList(); + + // Get release + var release = new Release + { + Version = latestRelease.InnerText, + Features = featuresList, + Fixes = fixesList, + Downloads = new List() + }; + + var releaseTag = latestRelease.InnerText.Substring(0, 6); + if (masterBranch) + { + await GetGitubRelease(release, releaseTag); + } + else + { + // Get AppVeyor + await GetAppVeyorRelease(release, branch); + } + + + return TransformUpdate(release); + + } + + private UpdateModel TransformUpdate(Release release) + { + var newUpdate = new UpdateModel + { + UpdateVersionString = release.Version, + UpdateVersion = release.Version == "(unreleased)" ? 0 : int.Parse(release.Version.Substring(1, 5).Replace(".", "")), + UpdateDate = DateTime.Now, + ChangeLogs = new List(), + Downloads = new List() + }; + + foreach (var dl in release.Downloads) + { + newUpdate.Downloads.Add(new Downloads + { + Name = dl.Name, + Url = dl.Url + }); + } + + foreach (var f in release.Features) + { + var change = new ChangeLog + { + Descripion = f, + Type = "New", + }; + + newUpdate.ChangeLogs.Add(change); + } + + foreach (var f in release.Fixes) + { + var change = new ChangeLog + { + Descripion = f, + Type = "Fixed", + }; + + newUpdate.ChangeLogs.Add(change); + } + + return newUpdate; + } + + private async Task GetAppVeyorRelease(Release release, string branch) + { + var request = new Request($"/projects/tidusjar/requestplex/branch/{branch}", AppVeyorApi.AppveyorApiUrl, HttpMethod.Get); + request.ApplicationJsonContentType(); + + var builds = await _api.Request(request); + var jobId = builds.build.jobs.FirstOrDefault()?.jobId ?? string.Empty; + + if (builds.build.finished == DateTime.MinValue) + { + return; + } + release.Version = builds.build.version; + // get the artifacts + request = new Request($"/buildjobs/{jobId}/artifacts", AppVeyorApi.AppveyorApiUrl, HttpMethod.Get); + request.ApplicationJsonContentType(); + + var artifacts = await _api.Request>(request); + + foreach (var item in artifacts) + { + var d = new Downloads + { + Name = item.fileName, + Url = $"{AppveyorApiUrl}/buildjobs/{jobId}/artifacts/{item.fileName}" + }; + release.Downloads.Add(d); + } + } + + private async Task GetGitubRelease(Release release, string releaseTag) + { + var client = new GitHubClient(Octokit.ProductHeaderValue.Parse("OmbiV3")); + + var releases = await client.Repository.Release.GetAll("tidusjar", "ombi"); + var latest = releases.FirstOrDefault(x => x.TagName == releaseTag); + + if (latest == null) + { + latest = releases.OrderBy(x => x.CreatedAt).FirstOrDefault(); + } + foreach (var item in latest.Assets) + { + var d = new Downloads + { + Name = item.Name, + Url = item.BrowserDownloadUrl + }; + release.Downloads.Add(d); + } + } + } + public class Release + { + public string Version { get; set; } + public string CheckinVersion { get; set; } + public List Downloads { get; set; } + public List Features { get; set; } + public List Fixes { get; set; } + } + + public class Downloads + { + public string Name { get; set; } + public string Url { get; set; } + } +} \ No newline at end of file diff --git a/src/Ombi.Schedule/Processor/IChangeLogProcessor.cs b/src/Ombi.Schedule/Processor/IChangeLogProcessor.cs new file mode 100644 index 000000000..97780245f --- /dev/null +++ b/src/Ombi.Schedule/Processor/IChangeLogProcessor.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace Ombi.Core.Processor +{ + public interface IChangeLogProcessor + { + Task Process(string branch); + } +} \ No newline at end of file diff --git a/src/Ombi/Controllers/UpdateController.cs b/src/Ombi/Controllers/UpdateController.cs new file mode 100644 index 000000000..0a6c3b80c --- /dev/null +++ b/src/Ombi/Controllers/UpdateController.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Ombi.Core.Processor; +using Ombi.Helpers; + +namespace Ombi.Controllers +{ + [ApiV1] + [Produces("application/json")] + [AllowAnonymous] + public class UpdateController : Controller + { + public UpdateController(ICacheService cache, IChangeLogProcessor processor) + { + _cache = cache; + _processor = processor; + } + + private readonly ICacheService _cache; + private readonly IChangeLogProcessor _processor; + + [HttpGet("{branch}")] + public async Task UpdateAvailable(string branch) + { + return await _cache.GetOrAdd(branch, async () => await _processor.Process(branch)); + } + } +} \ No newline at end of file