From ebf0174e004cf414b10a3ce27f2a2f1784c91aee Mon Sep 17 00:00:00 2001 From: ta264 Date: Tue, 17 Sep 2019 21:58:19 +0100 Subject: [PATCH] Try to fix browserstack tests --- azure-pipelines.yml | 17 ++- .../AutomationTest.cs | 10 +- .../BrowserStackAutomationTest.cs | 130 ++++++++++++++++-- .../BrowserStackFixture.cs | 22 +-- .../Lidarr.Automation.Test.csproj | 6 +- src/NzbDrone.Automation.Test/MainPagesTest.cs | 11 +- .../PageModel/PageBase.cs | 71 ++++++++-- .../PageModel/PageBaseMobile.cs | 44 ++++++ src/NzbDrone.Test.Common/NzbDroneRunner.cs | 25 +++- 9 files changed, 286 insertions(+), 50 deletions(-) create mode 100644 src/NzbDrone.Automation.Test/PageModel/PageBaseMobile.cs diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d37fdcdc7..5c162254e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -499,6 +499,16 @@ stages: - job: Automation strategy: matrix: + Linux: + osName: 'Linux' + imageName: 'ubuntu-16.04' + pattern: 'Lidarr.**.linux.tar.gz' + failBuild: true + Mac: + osName: 'Mac' + imageName: 'macos-10.13' # Fails due to firefox not being installed on image + pattern: 'Lidarr.**.osx.tar.gz' + failBuild: true Windows: osName: 'Windows' imageName: 'vs2017-win2016' @@ -532,9 +542,6 @@ stages: mkdir -p ./bin/ cp -r -v ${BUILD_ARTIFACTSTAGINGDIRECTORY}/bin/Lidarr/. ./bin/ displayName: Move Package Contents - - script: | - call bin\serviceinstall.exe - displayName: Start Lidarr Service - task: Bash@3 displayName: Run Automation Tests inputs: @@ -544,10 +551,6 @@ stages: env: BROWSERSTACK_USERNAME: $(browserStackUser) BROWSERSTACK_ACCESS_KEY: $(browserStackKey) - - script: | - call sc stop lidarr - call bin\serviceuninstall.exe - displayName: Stop and Remove Lidarr Service - task: PublishTestResults@2 inputs: testResultsFormat: 'NUnit' diff --git a/src/NzbDrone.Automation.Test/AutomationTest.cs b/src/NzbDrone.Automation.Test/AutomationTest.cs index 9210f9940..c53f2cd0a 100644 --- a/src/NzbDrone.Automation.Test/AutomationTest.cs +++ b/src/NzbDrone.Automation.Test/AutomationTest.cs @@ -20,7 +20,7 @@ namespace NzbDrone.Automation.Test [AutomationTest] public abstract class AutomationTest { - private NzbDroneRunner _runner; + protected NzbDroneRunner _runner; protected RemoteWebDriver driver; public AutomationTest() @@ -64,21 +64,21 @@ namespace NzbDrone.Automation.Test protected IEnumerable GetPageErrors() { - return driver.FindElements(By.CssSelector("#errors div")) + return driver?.FindElements(By.CssSelector("#errors div")) .Select(e => e.Text); } [OneTimeTearDown] public virtual void SmokeTestTearDown() { - _runner.KillAll(); - driver.Quit(); + _runner?.KillAll(); + driver?.Quit(); } [TearDown] public void AutomationTearDown() { - GetPageErrors().Should().BeEmpty(); + GetPageErrors().Should().BeNullOrEmpty(); } } } diff --git a/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs b/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs index c5b9c4e6e..1771b3efb 100644 --- a/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs +++ b/src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs @@ -1,11 +1,20 @@ using System; -using System.Collections.Generic; -using BrowserStack; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading; using FluentAssertions; +using Mono.Unix.Native; +using NLog; using NUnit.Framework; using NzbDrone.Automation.Test.PageModel; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Processes; +using NzbDrone.Test.Common; using OpenQA.Selenium.Remote; namespace NzbDrone.Automation.Test @@ -20,15 +29,23 @@ namespace NzbDrone.Automation.Test protected string os; protected string osVersion; protected string device; - private Local browserStackLocal; - public BrowserStackAutomationTest(string device, string os, string osVersion, string browser, string browserVersion) + private readonly Logger _logger; + + private ProcessProvider _processProvider; + private Process _browserStackLocalProcess; + + public BrowserStackAutomationTest(string device, string os, string osVersion, string browser, string browserVersion, int port) { this.device = device; this.browser = browser; this.browserVersion = browserVersion; this.os = os; this.osVersion = osVersion; + + _logger = LogManager.GetCurrentClassLogger(); + _processProvider = new ProcessProvider(_logger); + _runner = new NzbDroneRunner(_logger, port); } [OneTimeSetUp] @@ -42,9 +59,12 @@ namespace NzbDrone.Automation.Test Assert.Ignore("BrowserStack Tests Disabled, No Credentials"); } + _runner.Start(); + string browserstackLocal = "true"; string browserstackLocalIdentifier = string.Format("Lidarr_{0}_{1}", DateTime.UtcNow.Ticks, new Random().Next()); string buildName = BuildInfo.Version.ToString(); + string serverOs = OsInfo.Os.ToString(); DesiredCapabilities capabilities = new DesiredCapabilities(); @@ -56,21 +76,19 @@ namespace NzbDrone.Automation.Test capabilities.SetCapability("browserstack.local", browserstackLocal); capabilities.SetCapability("browserstack.localIdentifier", browserstackLocalIdentifier); capabilities.SetCapability("browserstack.debug", "true"); - capabilities.SetCapability("name", "Function Tests: " + browser); + capabilities.SetCapability("browserstack.console", "verbose"); + capabilities.SetCapability("name", "Functional Tests: " + serverOs + " - " + browser); capabilities.SetCapability("project", "Lidarr"); capabilities.SetCapability("build", buildName); - browserStackLocal = new Local(); - List> bsLocalArgs = new List>(); - bsLocalArgs.Add(new KeyValuePair("key", accessKey)); - bsLocalArgs.Add(new KeyValuePair("localIdentifier", browserstackLocalIdentifier)); - browserStackLocal.start(bsLocalArgs); + var bsLocalArgs = $"--key {accessKey} --local-identifier {browserstackLocalIdentifier} --verbose"; + _browserStackLocalProcess = StartBrowserStackLocal(_runner.AppData, bsLocalArgs); driver = new RemoteWebDriver(new Uri("https://" + username + ":" + accessKey + "@hub.browserstack.com/wd/hub"), capabilities); - driver.Url = "http://localhost:8686"; + driver.Url = $"http://{LocalIPAddress()}:{_runner.Port}"; - var page = new PageBase(driver); + var page = GetPageBase(driver, device); page.WaitForNoSpinner(); driver.ExecuteScript("window.Lidarr.NameViews = true;"); @@ -78,14 +96,96 @@ namespace NzbDrone.Automation.Test GetPageErrors().Should().BeEmpty(); } + private IPAddress LocalIPAddress() + { + IPHostEntry host = Dns.GetHostEntry(Dns.GetHostName()); + + return host.AddressList.FirstOrDefault(ip => ip.AddressFamily == AddressFamily.InterNetwork); + } + + private PageBase GetPageBase(RemoteWebDriver driver, string device) + { + if (device.IsNullOrWhiteSpace()) + { + return new PageBase(driver); + } + else + { + return new PageBaseMobile(driver); + } + } + + [SetUp] + public override void Setup() + { + page = GetPageBase(driver, device); + } + [OneTimeTearDown] public override void SmokeTestTearDown() { - driver.Quit(); - if (browserStackLocal != null) + driver?.Quit(); + _browserStackLocalProcess?.Kill(); + _runner?.Kill(); + } + + private Process StartBrowserStackLocal(string tempDir, string args = null) + { + string url; + string name; + + if (OsInfo.IsWindows) + { + url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-win32.zip"; + name = "BrowserStackLocal.exe"; + } + else if (OsInfo.IsOsx) { - browserStackLocal.stop(); + url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-darwin-x64.zip"; + name = "BrowserStackLocal"; } + else + { + url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-linux-x64.zip"; + name = "BrowserStackLocal"; + } + + var dest = Path.Combine(tempDir, "browserstack.zip"); + TestContext.Progress.WriteLine("Fetching browserstack local"); + + using (var client = new WebClient()) + { + client.DownloadFile(url, dest); + } + ZipFile.ExtractToDirectory(dest, tempDir); + + var browserStack = Path.Combine(tempDir, name); + + if (OsInfo.IsNotWindows) + { + Syscall.chmod(browserStack, FilePermissions.DEFFILEMODE | FilePermissions.S_IRWXU | FilePermissions.S_IXGRP | FilePermissions.S_IXOTH); + } + + TestContext.Progress.WriteLine("Starting browserstack local"); + + var processStarted = new ManualResetEventSlim(); + + var process = _processProvider.Start(browserStack, args, onOutputDataReceived: (string data) => { + TestContext.Progress.WriteLine(data); + if (data.Contains("You can now access your local server")) + { + processStarted.Set(); + } + }); + + if (!processStarted.Wait(5000)) + { + Assert.Fail("Failed to start browserstack within 5 sec"); + } + + TestContext.Progress.WriteLine($"Successfully started browserstacklocal pid {process.Id}"); + + return process; } } } diff --git a/src/NzbDrone.Automation.Test/BrowserStackFixture.cs b/src/NzbDrone.Automation.Test/BrowserStackFixture.cs index 9812b8afb..9df904127 100644 --- a/src/NzbDrone.Automation.Test/BrowserStackFixture.cs +++ b/src/NzbDrone.Automation.Test/BrowserStackFixture.cs @@ -2,15 +2,17 @@ using NUnit.Framework; namespace NzbDrone.Automation.Test { - [TestFixture("","Windows","10","Chrome", "63")] - [TestFixture("", "Windows","10", "Firefox", "67")] - [TestFixture("", "Windows","10", "Edge", "18")] - [TestFixture("iPhone X", "", "11", "iPhone", "")] - [TestFixture("Samsung Galaxy S9 Plus", "", "9.0", "android", "")] - public class BSMainPagesTest : BrowserStackAutomationTest + [TestFixture("", "Windows", "10", "Chrome", "63", 9901)] + [TestFixture("", "Windows", "10", "Firefox", "67", 9902)] + [TestFixture("", "Windows", "10", "Edge", "18", 9903)] + [TestFixture("", "OS X", "Mojave", "Safari", "12.1", 9904)] + // [TestFixture("iPhone X", "", "11", "iPhone", "", 9905)] + // [TestFixture("Samsung Galaxy S9 Plus", "", "9.0", "android", "", 9906)] + public class BrowserStackFixture : BrowserStackAutomationTest { - public BSMainPagesTest(string device, string os, string osVersion, string browser, string browserVersion) : - base(device, os, osVersion, browser, browserVersion) { } - + public BrowserStackFixture(string device, string os, string osVersion, string browser, string browserVersion, int port) : + base(device, os, osVersion, browser, browserVersion, port) + { + } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj b/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj index a4f6dfe06..9a78e25cd 100644 --- a/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj +++ b/src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj @@ -4,11 +4,15 @@ x86 - + + + ..\Libraries\Mono.Posix.dll + + diff --git a/src/NzbDrone.Automation.Test/MainPagesTest.cs b/src/NzbDrone.Automation.Test/MainPagesTest.cs index 95eae8f8d..594dbe784 100644 --- a/src/NzbDrone.Automation.Test/MainPagesTest.cs +++ b/src/NzbDrone.Automation.Test/MainPagesTest.cs @@ -8,10 +8,10 @@ namespace NzbDrone.Automation.Test [TestFixture] public class MainPagesTest : AutomationTest { - private PageBase page; + protected PageBase page; [SetUp] - public void Setup() + public virtual void Setup() { page = new PageBase(driver); } @@ -22,6 +22,8 @@ namespace NzbDrone.Automation.Test page.LibraryNavIcon.Click(); page.WaitForNoSpinner(); page.Find(By.CssSelector("div[class*='ArtistIndex']")).Should().NotBeNull(); + + page.CloseSidebar(); } [Test] @@ -42,6 +44,8 @@ namespace NzbDrone.Automation.Test page.Find(By.LinkText("Queue")).Should().NotBeNull(); page.Find(By.LinkText("History")).Should().NotBeNull(); page.Find(By.LinkText("Blacklist")).Should().NotBeNull(); + + page.CloseSidebar(); } [Test] @@ -52,6 +56,8 @@ namespace NzbDrone.Automation.Test page.Find(By.LinkText("Missing")).Should().NotBeNull(); page.Find(By.LinkText("Cutoff Unmet")).Should().NotBeNull(); + + page.CloseSidebar(); } [Test] @@ -61,6 +67,7 @@ namespace NzbDrone.Automation.Test page.WaitForNoSpinner(); page.Find(By.CssSelector("div[class*='Health']")).Should().NotBeNull(); + page.CloseSidebar(); } [Test] diff --git a/src/NzbDrone.Automation.Test/PageModel/PageBase.cs b/src/NzbDrone.Automation.Test/PageModel/PageBase.cs index acf8cfe73..36e56a00c 100644 --- a/src/NzbDrone.Automation.Test/PageModel/PageBase.cs +++ b/src/NzbDrone.Automation.Test/PageModel/PageBase.cs @@ -8,12 +8,25 @@ namespace NzbDrone.Automation.Test.PageModel { public class PageBase { - private readonly RemoteWebDriver _driver; + protected readonly RemoteWebDriver _driver; public PageBase(RemoteWebDriver driver) { _driver = driver; - driver.Manage().Window.Maximize(); + MaximizeWindow(); + } + + public virtual void MaximizeWindow() + { + _driver.Manage().Window.Maximize(); + } + + public virtual void OpenSidebar() + { + } + + public virtual void CloseSidebar() + { } public IWebElement FindByClass(string className, int timeout = 5) @@ -47,16 +60,58 @@ namespace NzbDrone.Automation.Test.PageModel }); } - public IWebElement LibraryNavIcon => Find(By.LinkText("Library")); + public virtual IWebElement LibraryNavIcon + { + get + { + OpenSidebar(); + return Find(By.LinkText("Library")); + } + } - public IWebElement CalendarNavIcon => Find(By.LinkText("Calendar")); + public virtual IWebElement CalendarNavIcon + { + get + { + OpenSidebar(); + return Find(By.LinkText("Calendar")); + } + } - public IWebElement ActivityNavIcon => Find(By.LinkText("Activity")); + public virtual IWebElement ActivityNavIcon + { + get + { + OpenSidebar(); + return Find(By.LinkText("Activity")); + } + } - public IWebElement WantedNavIcon => Find(By.LinkText("Wanted")); + public virtual IWebElement WantedNavIcon + { + get + { + OpenSidebar(); + return Find(By.LinkText("Wanted")); + } + } - public IWebElement SettingNavIcon => Find(By.LinkText("Settings")); + public virtual IWebElement SettingNavIcon + { + get + { + OpenSidebar(); + return Find(By.LinkText("Setting")); + } + } - public IWebElement SystemNavIcon => Find(By.PartialLinkText("System")); + public virtual IWebElement SystemNavIcon + { + get + { + OpenSidebar(); + return Find(By.PartialLinkText("System")); + } + } } } diff --git a/src/NzbDrone.Automation.Test/PageModel/PageBaseMobile.cs b/src/NzbDrone.Automation.Test/PageModel/PageBaseMobile.cs new file mode 100644 index 000000000..46654315e --- /dev/null +++ b/src/NzbDrone.Automation.Test/PageModel/PageBaseMobile.cs @@ -0,0 +1,44 @@ +using OpenQA.Selenium; +using OpenQA.Selenium.Remote; + +namespace NzbDrone.Automation.Test.PageModel +{ + public class PageBaseMobile : PageBase + { + public PageBaseMobile(RemoteWebDriver driver) + : base(driver) + { + } + + public override void MaximizeWindow() + { + } + + public override void OpenSidebar() + { + // if (!SidebarIsOpen()) + // { + ToggleSidebar(); + // } + } + + public override void CloseSidebar() + { + // if (SidebarIsOpen()) + // { + ToggleSidebar(); + // } + } + + private void ToggleSidebar() + { + Find(By.Id("sidebar-toggle-button")).Click(); + } + + private bool SidebarIsOpen() + { + var sidebar = _driver.FindElement(By.CssSelector("div[class*='PageSidebar-sidebar']")); + return sidebar != null; + } + } +} diff --git a/src/NzbDrone.Test.Common/NzbDroneRunner.cs b/src/NzbDrone.Test.Common/NzbDroneRunner.cs index 284e45dba..2efd134e3 100644 --- a/src/NzbDrone.Test.Common/NzbDroneRunner.cs +++ b/src/NzbDrone.Test.Common/NzbDroneRunner.cs @@ -20,11 +20,14 @@ namespace NzbDrone.Test.Common public string AppData { get; private set; } public string ApiKey { get; private set; } + public int Port { get; private set; } public NzbDroneRunner(Logger logger, int port = 8686) { _processProvider = new ProcessProvider(logger); - _restClient = new RestClient("http://localhost:8686/api/v1"); + _restClient = new RestClient($"http://localhost:{port}/api/v1"); + + Port = port; } public void Start() @@ -74,6 +77,23 @@ namespace NzbDrone.Test.Common } } + public void Kill() + { + try + { + if (_nzbDroneProcess != null) + { + _processProvider.Kill(_nzbDroneProcess.Id); + } + } + catch (InvalidOperationException) + { + // May happen if the process closes while being closed + } + + TestBase.DeleteTempFolder(AppData); + } + public void KillAll() { try @@ -124,7 +144,8 @@ namespace NzbDrone.Test.Common new XDeclaration("1.0", "utf-8", "yes"), new XElement(ConfigFileProvider.CONFIG_ELEMENT_NAME, new XElement(nameof(ConfigFileProvider.ApiKey), apiKey), - new XElement(nameof(ConfigFileProvider.AnalyticsEnabled), false) + new XElement(nameof(ConfigFileProvider.AnalyticsEnabled), false), + new XElement(nameof(ConfigFileProvider.Port), Port) ) );