Try to fix browserstack tests

azure-pipelines.yml

@@ -499,6 +499,16 @@ stages:
     - job: Automation
       strategy:
         matrix:
+          Linux:
+            osName: 'Linux'
+            imageName: 'ubuntu-16.04'
+            pattern: 'Lidarr.**.linux.tar.gz'
+            failBuild: true
+          Mac:
+            osName: 'Mac'
+            imageName: 'macos-10.13' # Fails due to firefox not being installed on image
+            pattern: 'Lidarr.**.osx.tar.gz'
+            failBuild: true
           Windows:
             osName: 'Windows'
             imageName: 'vs2017-win2016'
@@ -532,9 +542,6 @@ stages:
           mkdir -p ./bin/
           cp -r -v ${BUILD_ARTIFACTSTAGINGDIRECTORY}/bin/Lidarr/. ./bin/
         displayName: Move Package Contents
-      - script: |
-          call bin\serviceinstall.exe
-        displayName: Start Lidarr Service
       - task: Bash@3
         displayName: Run Automation Tests
         inputs:
@@ -544,10 +551,6 @@ stages:
         env:
           BROWSERSTACK_USERNAME: $(browserStackUser)
           BROWSERSTACK_ACCESS_KEY: $(browserStackKey)
-      - script: |
-          call sc stop lidarr
-          call bin\serviceuninstall.exe
-        displayName: Stop and Remove Lidarr Service
       - task: PublishTestResults@2
         inputs:
           testResultsFormat: 'NUnit'

src/NzbDrone.Automation.Test/AutomationTest.cs

@@ -20,7 +20,7 @@ namespace NzbDrone.Automation.Test
     [AutomationTest]
     public abstract class AutomationTest
     {
-        private NzbDroneRunner _runner;
+        protected NzbDroneRunner _runner;
         protected RemoteWebDriver driver;
 
         public AutomationTest()
@@ -64,21 +64,21 @@ namespace NzbDrone.Automation.Test
 
         protected IEnumerable<string> GetPageErrors()
         {
-            return driver.FindElements(By.CssSelector("#errors div"))
+            return driver?.FindElements(By.CssSelector("#errors div"))
                 .Select(e => e.Text);
         }
 
         [OneTimeTearDown]
         public virtual void SmokeTestTearDown()
         {
-            _runner.KillAll();
+            _runner?.KillAll();
-            driver.Quit();
+            driver?.Quit();
         }
 
         [TearDown]
         public void AutomationTearDown()
         {
-            GetPageErrors().Should().BeEmpty();
+            GetPageErrors().Should().BeNullOrEmpty();
         }
     }
 }

src/NzbDrone.Automation.Test/BrowserStackAutomationTest.cs

@@ -1,11 +1,20 @@
 using System;
-using System.Collections.Generic;
+using System.Diagnostics;
-using BrowserStack;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Threading;
 using FluentAssertions;
+using Mono.Unix.Native;
+using NLog;
 using NUnit.Framework;
 using NzbDrone.Automation.Test.PageModel;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Processes;
+using NzbDrone.Test.Common;
 using OpenQA.Selenium.Remote;
 
 namespace NzbDrone.Automation.Test
@@ -20,15 +29,23 @@ namespace NzbDrone.Automation.Test
         protected string os;
         protected string osVersion;
         protected string device;
-        private Local browserStackLocal;
 
-        public BrowserStackAutomationTest(string device, string os, string osVersion, string browser, string browserVersion)
+        private readonly Logger _logger;
+
+        private ProcessProvider _processProvider;
+        private Process _browserStackLocalProcess;
+
+        public BrowserStackAutomationTest(string device, string os, string osVersion, string browser, string browserVersion, int port)
         {
             this.device = device;
             this.browser = browser;
             this.browserVersion = browserVersion;
             this.os = os;
             this.osVersion = osVersion;
+
+            _logger = LogManager.GetCurrentClassLogger();
+            _processProvider = new ProcessProvider(_logger);
+            _runner = new NzbDroneRunner(_logger, port);
         }
 
         [OneTimeSetUp]
@@ -42,9 +59,12 @@ namespace NzbDrone.Automation.Test
                 Assert.Ignore("BrowserStack Tests Disabled, No Credentials");
             }
 
+            _runner.Start();
+
             string browserstackLocal = "true";
             string browserstackLocalIdentifier = string.Format("Lidarr_{0}_{1}", DateTime.UtcNow.Ticks, new Random().Next());
             string buildName = BuildInfo.Version.ToString();
+            string serverOs = OsInfo.Os.ToString();
 
             DesiredCapabilities capabilities = new DesiredCapabilities();
 
@@ -56,21 +76,19 @@ namespace NzbDrone.Automation.Test
             capabilities.SetCapability("browserstack.local", browserstackLocal);
             capabilities.SetCapability("browserstack.localIdentifier", browserstackLocalIdentifier);
             capabilities.SetCapability("browserstack.debug", "true");
-            capabilities.SetCapability("name", "Function Tests: " + browser);
+            capabilities.SetCapability("browserstack.console", "verbose");
+            capabilities.SetCapability("name", "Functional Tests: " + serverOs + " - " +  browser);
             capabilities.SetCapability("project", "Lidarr");
             capabilities.SetCapability("build", buildName);
 
-            browserStackLocal = new Local();
+            var bsLocalArgs = $"--key {accessKey} --local-identifier {browserstackLocalIdentifier} --verbose";
-            List<KeyValuePair<string, string>> bsLocalArgs = new List<KeyValuePair<string, string>>();
+            _browserStackLocalProcess = StartBrowserStackLocal(_runner.AppData, bsLocalArgs);
-            bsLocalArgs.Add(new KeyValuePair<string, string>("key", accessKey));
-            bsLocalArgs.Add(new KeyValuePair<string, string>("localIdentifier", browserstackLocalIdentifier));
-            browserStackLocal.start(bsLocalArgs);
 
             driver = new RemoteWebDriver(new Uri("https://" + username + ":" + accessKey + "@hub.browserstack.com/wd/hub"), capabilities);
 
-            driver.Url = "http://localhost:8686";
+            driver.Url = $"http://{LocalIPAddress()}:{_runner.Port}";
 
-            var page = new PageBase(driver);
+            var page = GetPageBase(driver, device);
             page.WaitForNoSpinner();
 
             driver.ExecuteScript("window.Lidarr.NameViews = true;");
@@ -78,14 +96,96 @@ namespace NzbDrone.Automation.Test
             GetPageErrors().Should().BeEmpty();
         }
 
+        private IPAddress LocalIPAddress()
+        {
+            IPHostEntry host = Dns.GetHostEntry(Dns.GetHostName());
+
+            return host.AddressList.FirstOrDefault(ip => ip.AddressFamily == AddressFamily.InterNetwork);
+        }
+
+        private PageBase GetPageBase(RemoteWebDriver driver, string device)
+        {
+            if (device.IsNullOrWhiteSpace())
+            {
+                return new PageBase(driver);
+            }
+            else
+            {
+                return new PageBaseMobile(driver);
+            }
+        }
+
+        [SetUp]
+        public override void Setup()
+        {
+            page = GetPageBase(driver, device);
+        }
+
         [OneTimeTearDown]
         public override void SmokeTestTearDown()
         {
-            driver.Quit();
+            driver?.Quit();
-            if (browserStackLocal != null)
+            _browserStackLocalProcess?.Kill();
+            _runner?.Kill();
+        }
+
+        private Process StartBrowserStackLocal(string tempDir, string args = null)
+        {
+            string url;
+            string name;
+
+            if (OsInfo.IsWindows)
+            {
+                url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-win32.zip";
+                name = "BrowserStackLocal.exe";
+            }
+            else if (OsInfo.IsOsx)
             {
-                browserStackLocal.stop();
+                url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-darwin-x64.zip";
+                name = "BrowserStackLocal";
             }
+            else
+            {
+                url = "https://www.browserstack.com/browserstack-local/BrowserStackLocal-linux-x64.zip";
+                name = "BrowserStackLocal";
+            }
+
+            var dest = Path.Combine(tempDir, "browserstack.zip");
+            TestContext.Progress.WriteLine("Fetching browserstack local");
+
+            using (var client = new WebClient())
+            {
+                client.DownloadFile(url, dest);
+            }
+            ZipFile.ExtractToDirectory(dest, tempDir);
+
+            var browserStack = Path.Combine(tempDir, name);
+
+            if (OsInfo.IsNotWindows)
+            {
+                Syscall.chmod(browserStack, FilePermissions.DEFFILEMODE | FilePermissions.S_IRWXU | FilePermissions.S_IXGRP | FilePermissions.S_IXOTH);
+            }
+
+            TestContext.Progress.WriteLine("Starting browserstack local");
+
+            var processStarted = new ManualResetEventSlim();
+
+            var process = _processProvider.Start(browserStack, args, onOutputDataReceived: (string data) => {
+                    TestContext.Progress.WriteLine(data);
+                    if (data.Contains("You can now access your local server"))
+                    {
+                        processStarted.Set();
+                    }
+                });
+
+            if (!processStarted.Wait(5000))
+            {
+                Assert.Fail("Failed to start browserstack within 5 sec");
+            }
+
+            TestContext.Progress.WriteLine($"Successfully started browserstacklocal pid {process.Id}");
+
+            return process;
         }
     }
 }

src/NzbDrone.Automation.Test/BrowserStackFixture.cs

@@ -2,15 +2,17 @@ using NUnit.Framework;
 
 namespace NzbDrone.Automation.Test
 {
-    [TestFixture("","Windows","10","Chrome", "63")]
+    [TestFixture("", "Windows", "10", "Chrome", "63", 9901)]
-    [TestFixture("", "Windows","10", "Firefox", "67")]
+    [TestFixture("", "Windows", "10", "Firefox", "67", 9902)]
-    [TestFixture("", "Windows","10", "Edge", "18")]
+    [TestFixture("", "Windows", "10", "Edge", "18", 9903)]
-    [TestFixture("iPhone X", "", "11", "iPhone", "")]
+    [TestFixture("", "OS X", "Mojave", "Safari", "12.1", 9904)]
-    [TestFixture("Samsung Galaxy S9 Plus", "", "9.0", "android", "")]
+    // [TestFixture("iPhone X", "", "11", "iPhone", "", 9905)]
-    public class BSMainPagesTest : BrowserStackAutomationTest
+    // [TestFixture("Samsung Galaxy S9 Plus", "", "9.0", "android", "", 9906)]
+    public class BrowserStackFixture : BrowserStackAutomationTest
     {
-        public BSMainPagesTest(string device, string os, string osVersion, string browser, string browserVersion) : 
+        public BrowserStackFixture(string device, string os, string osVersion, string browser, string browserVersion, int port) :
-            base(device, os, osVersion, browser, browserVersion) { }
+        base(device, os, osVersion, browser, browserVersion, port)
-
+        {
+        }
     }
-}
+}

src/NzbDrone.Automation.Test/Lidarr.Automation.Test.csproj

@@ -4,11 +4,15 @@
     <Platforms>x86</Platforms>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="BrowserStackLocal" Version="1.4.0" />
     <PackageReference Include="Selenium.Firefox.WebDriver" Version="0.24.0" />
     <PackageReference Include="Selenium.Support" Version="3.141.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NzbDrone.Test.Common\Lidarr.Test.Common.csproj" />
   </ItemGroup>
+  <ItemGroup>
+    <Reference Include="Mono.Posix">
+      <HintPath>..\Libraries\Mono.Posix.dll</HintPath>
+    </Reference>
+  </ItemGroup>
 </Project>

src/NzbDrone.Automation.Test/MainPagesTest.cs

@@ -8,10 +8,10 @@ namespace NzbDrone.Automation.Test
     [TestFixture]
     public class MainPagesTest : AutomationTest
     {
-        private PageBase page;
+        protected PageBase page;
 
         [SetUp]
-        public void Setup()
+        public virtual void Setup()
         {
             page = new PageBase(driver);
         }
@@ -22,6 +22,8 @@ namespace NzbDrone.Automation.Test
             page.LibraryNavIcon.Click();
             page.WaitForNoSpinner();
             page.Find(By.CssSelector("div[class*='ArtistIndex']")).Should().NotBeNull();
+
+            page.CloseSidebar();
         }
 
         [Test]
@@ -42,6 +44,8 @@ namespace NzbDrone.Automation.Test
             page.Find(By.LinkText("Queue")).Should().NotBeNull();
             page.Find(By.LinkText("History")).Should().NotBeNull();
             page.Find(By.LinkText("Blacklist")).Should().NotBeNull();
+
+            page.CloseSidebar();
         }
 
         [Test]
@@ -52,6 +56,8 @@ namespace NzbDrone.Automation.Test
 
             page.Find(By.LinkText("Missing")).Should().NotBeNull();
             page.Find(By.LinkText("Cutoff Unmet")).Should().NotBeNull();
+
+            page.CloseSidebar();
         }
 
         [Test]
@@ -61,6 +67,7 @@ namespace NzbDrone.Automation.Test
             page.WaitForNoSpinner();
 
             page.Find(By.CssSelector("div[class*='Health']")).Should().NotBeNull();
+            page.CloseSidebar();
         }
 
         [Test]

src/NzbDrone.Automation.Test/PageModel/PageBase.cs

@@ -8,12 +8,25 @@ namespace NzbDrone.Automation.Test.PageModel
 {
     public class PageBase
     {
-        private readonly RemoteWebDriver _driver;
+        protected readonly RemoteWebDriver _driver;
 
         public PageBase(RemoteWebDriver driver)
         {
             _driver = driver;
-            driver.Manage().Window.Maximize();
+            MaximizeWindow();
+        }
+
+        public virtual void MaximizeWindow()
+        {
+            _driver.Manage().Window.Maximize();
+        }
+
+        public virtual void OpenSidebar()
+        {
+        }
+
+        public virtual void CloseSidebar()
+        {
         }
 
         public IWebElement FindByClass(string className, int timeout = 5)
@@ -47,16 +60,58 @@ namespace NzbDrone.Automation.Test.PageModel
             });
         }
 
-        public IWebElement LibraryNavIcon => Find(By.LinkText("Library"));
+        public virtual IWebElement LibraryNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.LinkText("Library"));
+            }
+        }
 
-        public IWebElement CalendarNavIcon => Find(By.LinkText("Calendar"));
+        public virtual IWebElement CalendarNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.LinkText("Calendar"));
+            }
+        }
 
-        public IWebElement ActivityNavIcon => Find(By.LinkText("Activity"));
+        public virtual IWebElement ActivityNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.LinkText("Activity"));
+            }
+        }
 
-        public IWebElement WantedNavIcon => Find(By.LinkText("Wanted"));
+        public virtual IWebElement WantedNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.LinkText("Wanted"));
+            }
+        }
 
-        public IWebElement SettingNavIcon => Find(By.LinkText("Settings"));
+        public virtual IWebElement SettingNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.LinkText("Setting"));
+            }
+        }
 
-        public IWebElement SystemNavIcon => Find(By.PartialLinkText("System"));
+        public virtual IWebElement SystemNavIcon
+        {
+            get
+            {
+                OpenSidebar();
+                return Find(By.PartialLinkText("System"));
+            }
+        }
     }
 }

src/NzbDrone.Automation.Test/PageModel/PageBaseMobile.cs

@@ -0,0 +1,44 @@
+using OpenQA.Selenium;
+using OpenQA.Selenium.Remote;
+
+namespace NzbDrone.Automation.Test.PageModel
+{
+    public class PageBaseMobile : PageBase
+    {
+        public PageBaseMobile(RemoteWebDriver driver)
+        : base(driver)
+        {
+        }
+
+        public override void MaximizeWindow()
+        {
+        }
+
+        public override void OpenSidebar()
+        {
+            // if (!SidebarIsOpen())
+            // {
+                ToggleSidebar();
+            // }
+        }
+
+        public override void CloseSidebar()
+        {
+            // if (SidebarIsOpen())
+            // {
+                ToggleSidebar();
+            // }
+        }
+
+        private void ToggleSidebar()
+        {
+            Find(By.Id("sidebar-toggle-button")).Click();
+        }
+
+        private bool SidebarIsOpen()
+        {
+            var sidebar = _driver.FindElement(By.CssSelector("div[class*='PageSidebar-sidebar']"));
+            return sidebar != null;
+        }
+    }
+}

src/NzbDrone.Test.Common/NzbDroneRunner.cs

@@ -20,11 +20,14 @@ namespace NzbDrone.Test.Common
 
         public string AppData { get; private set; }
         public string ApiKey { get; private set; }
+        public int Port { get; private set; }
 
         public NzbDroneRunner(Logger logger, int port = 8686)
         {
             _processProvider = new ProcessProvider(logger);
-            _restClient = new RestClient("http://localhost:8686/api/v1");
+            _restClient = new RestClient($"http://localhost:{port}/api/v1");
+
+            Port = port;
         }
 
         public void Start()
@@ -74,6 +77,23 @@ namespace NzbDrone.Test.Common
             }
         }
 
+        public void Kill()
+        {
+            try
+            {
+                if (_nzbDroneProcess != null)
+                {
+                    _processProvider.Kill(_nzbDroneProcess.Id);
+                }
+            }
+            catch (InvalidOperationException)
+            {
+                // May happen if the process closes while being closed
+            }
+
+            TestBase.DeleteTempFolder(AppData);
+        }
+
         public void KillAll()
         {
             try
@@ -124,7 +144,8 @@ namespace NzbDrone.Test.Common
                 new XDeclaration("1.0", "utf-8", "yes"), 
                 new XElement(ConfigFileProvider.CONFIG_ELEMENT_NAME,
                              new XElement(nameof(ConfigFileProvider.ApiKey), apiKey),
-                             new XElement(nameof(ConfigFileProvider.AnalyticsEnabled), false)
+                             new XElement(nameof(ConfigFileProvider.AnalyticsEnabled), false),
+                             new XElement(nameof(ConfigFileProvider.Port), Port)
                     )
                 );