Attempt to read the logs from the browser (#26289)

pranavkm · web-flow · commit 0e5d7ef1d4b9 · 2020-09-28T17:07:22.000-07:00
* Attempt to read the logs from the browser Seleniunm currently does not return log messages (See SeleniumHQ/selenium#8229). This making figuring out blazor WASM test failures a lot harder. This change adds some JS to the test apps to read the console output. * WIP
diff --git a/src/Components/WebAssembly/testassets/HostedInAspNet.Client/wwwroot/index.html b/src/Components/WebAssembly/testassets/HostedInAspNet.Client/wwwroot/index.html
@@ -6,6 +6,7 @@
 </head>
 <body>
     <app>Loading...</app>
+    <script src="seleniumworkaround.js"></script>
     <script src="customJsFileForTests.js"></script>
     <script src="_framework/blazor.webassembly.js" autostart="false"></script>
 
diff --git a/src/Components/WebAssembly/testassets/HostedInAspNet.Client/wwwroot/seleniumworkaround.js b/src/Components/WebAssembly/testassets/HostedInAspNet.Client/wwwroot/seleniumworkaround.js
@@ -0,0 +1,26 @@
+(function () {
+  // Note: there are multiple copies of this file throughout the repo. If you're editing it, please look for
+  // other seleniumworkaround.js files and keep them all in sync.
+  const logs = [];
+
+  const defaultLog = console.log;
+  console.log = function () {
+    defaultLog.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultError = console.error;
+  console.error = function () {
+    defaultError.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultWarn = console.warn;
+  console.warn = function () {
+    defaultWarn.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  window.getBrowserLogs = () => logs;
+})();
+
diff --git a/src/Components/WebAssembly/testassets/Wasm.Authentication.Client/wwwroot/index.html b/src/Components/WebAssembly/testassets/Wasm.Authentication.Client/wwwroot/index.html
@@ -17,6 +17,8 @@
         <a href="" class="reload">Reload</a>
         <a class="dismiss">🗙</a>
     </div>
+
+    <script src="seleniumworkaround.js"></script>
     <script src="_content/Microsoft.AspNetCore.Components.WebAssembly.Authentication/AuthenticationService.js"></script>
     <script src="_framework/blazor.webassembly.js"></script>
 </body>
diff --git a/src/Components/WebAssembly/testassets/Wasm.Authentication.Client/wwwroot/seleniumworkaround.js b/src/Components/WebAssembly/testassets/Wasm.Authentication.Client/wwwroot/seleniumworkaround.js
@@ -0,0 +1,26 @@
+(function () {
+  // Note: there are multiple copies of this file throughout the repo. If you're editing it, please look for
+  // other seleniumworkaround.js files and keep them all in sync.
+  const logs = [];
+
+  const defaultLog = console.log;
+  console.log = function () {
+    defaultLog.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultError = console.error;
+  console.error = function () {
+    defaultError.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultWarn = console.warn;
+  console.warn = function () {
+    defaultWarn.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  window.getBrowserLogs = () => logs;
+})();
+
diff --git a/src/Components/test/E2ETest/ServerExecutionTests/ServerAuthTest.cs b/src/Components/test/E2ETest/ServerExecutionTests/ServerAuthTest.cs
@@ -62,8 +62,8 @@ private void PerformReconnection()
             ((IJavaScriptExecutor)Browser).ExecuteScript("Blazor._internal.forceCloseConnection()");
 
             // Wait until the reconnection dialog has been shown but is now hidden
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(10))
-                .Until(driver => driver.FindElement(By.Id("components-reconnect-modal"))?.GetCssValue("display") == "none");
+            var reconnectModel = Browser.Exists(By.Id("components-reconnect-modal"));
+            Browser.Equal("none", () => reconnectModel.GetCssValue("display"));
         }
     }
 }
diff --git a/src/Components/test/E2ETest/ServerExecutionTests/ServerInteropTestDefaultExceptionsBehavior.cs b/src/Components/test/E2ETest/ServerExecutionTests/ServerInteropTestDefaultExceptionsBehavior.cs
@@ -48,8 +48,7 @@ public void DotNetExceptionDetailsAreNotLoggedByDefault()
             var interopButton = Browser.FindElement(By.Id("btn-interop"));
             interopButton.Click();
 
-            var wait = new WebDriverWait(Browser, TimeSpan.FromSeconds(10))
-                .Until(d => d.FindElement(By.Id("done-with-interop")));
+            Browser.Exists(By.Id("done-with-interop"));
 
             foreach (var expectedValue in expectedValues)
             {
diff --git a/src/Components/test/E2ETest/Tests/BinaryHttpClientTest.cs b/src/Components/test/E2ETest/Tests/BinaryHttpClientTest.cs
@@ -2,13 +2,11 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using BasicTestApp;
 using BasicTestApp.HttpClientTest;
 using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures;
 using Microsoft.AspNetCore.E2ETesting;
 using Microsoft.AspNetCore.Testing;
 using OpenQA.Selenium;
-using OpenQA.Selenium.Support.UI;
 using TestServer;
 using Xunit;
 using Xunit.Abstractions;
@@ -61,9 +59,7 @@ private void IssueRequest(string relativeUri)
 
             _appElement.FindElement(By.Id("send-request")).Click();
 
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.Id("response-status")) != null);
-            _responseStatus = _appElement.FindElement(By.Id("response-status"));
+            _responseStatus = Browser.Exists(By.Id("response-status"));
             _responseStatusText = _appElement.FindElement(By.Id("response-status-text"));
             _testOutcome = _appElement.FindElement(By.Id("test-outcome"));
         }
diff --git a/src/Components/test/E2ETest/Tests/BootResourceCachingTest.cs b/src/Components/test/E2ETest/Tests/BootResourceCachingTest.cs
@@ -148,8 +148,8 @@ private IReadOnlyCollection<string> GetAndClearRequestedPaths()
 
         private void WaitUntilLoaded()
         {
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.TagName("h1")).Text == "Hello, world!");
+            var element = Browser.Exists(By.TagName("h1"));
+            Browser.Equal("Hello, world!", () => element.Text);
         }
     }
 }
diff --git a/src/Components/test/E2ETest/Tests/ClientSideHostingTest.cs b/src/Components/test/E2ETest/Tests/ClientSideHostingTest.cs
@@ -61,11 +61,10 @@ public void MapFallbackToClientSideBlazor_AssemblyPath_Pattern_FilePath()
             Assert.NotNull(Browser.FindElement(By.Id("test-selector")));
         }
 
-
         private void WaitUntilLoaded()
         {
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.TagName("app")).Text != "Loading...");
+            var app = Browser.Exists(By.TagName("app"));
+            Browser.NotEqual("Loading...", () => app.Text);
         }
     }
 }
diff --git a/src/Components/test/E2ETest/Tests/HostedInAspNetTest.cs b/src/Components/test/E2ETest/Tests/HostedInAspNetTest.cs
@@ -48,8 +48,8 @@ public void ServesStaticAssetsFromClientAppWebRoot()
 
         private void WaitUntilLoaded()
         {
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.TagName("app")).Text != "Loading...");
+            var app = Browser.Exists(By.TagName("app"));
+            Browser.NotEqual("Loading...", () => app.Text);
         }
     }
 }
diff --git a/src/Components/test/E2ETest/Tests/HttpClientTest.cs b/src/Components/test/E2ETest/Tests/HttpClientTest.cs
@@ -134,9 +134,7 @@ public void CanSendAndReceiveCookies()
 
             string WaitAndGetResponseText()
             {
-                new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                    driver => driver.FindElement(By.Id("response-text")) != null);
-                return app.FindElement(By.Id("response-text")).Text;
+                return Browser.Exists(By.Id("response-text")).Text;
             }
         }
 
@@ -150,9 +148,7 @@ private void IssueRequest(string requestMethod, string relativeUri, string reque
 
             _appElement.FindElement(By.Id("send-request")).Click();
 
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.Id("response-status")) != null);
-            _responseStatus = _appElement.FindElement(By.Id("response-status"));
+            _responseStatus = Browser.Exists(By.Id("response-status"));
             _responseBody = _appElement.FindElement(By.Id("response-body"));
             _responseHeaders = _appElement.FindElement(By.Id("response-headers"));
         }
diff --git a/src/Components/test/E2ETest/Tests/InteropTest.cs b/src/Components/test/E2ETest/Tests/InteropTest.cs
@@ -135,8 +135,7 @@ public void CanInvokeDotNetMethods()
             var interopButton = Browser.FindElement(By.Id("btn-interop"));
             interopButton.Click();
 
-            var wait = new WebDriverWait(Browser, TimeSpan.FromSeconds(10))
-                .Until(d => d.FindElement(By.Id("done-with-interop")));
+            Browser.Exists(By.Id("done-with-interop"));
 
             foreach (var expectedValue in expectedValues)
             {
diff --git a/src/Components/test/E2ETest/Tests/RoutingTest.cs b/src/Components/test/E2ETest/Tests/RoutingTest.cs
@@ -546,10 +546,7 @@ public void OnNavigate_CanRenderLoadingFragment()
 
             SetUrlViaPushState("/LongPage1");
 
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(2)).Until(
-                driver => driver.FindElement(By.Id("loading-banner")) != null);
-
-            Assert.True(app.FindElement(By.Id("loading-banner")) != null);
+            Browser.Exists(By.Id("loading-banner"));
         }
 
         [Fact]
diff --git a/src/Components/test/E2ETest/Tests/StandaloneAppTest.cs b/src/Components/test/E2ETest/Tests/StandaloneAppTest.cs
@@ -96,8 +96,7 @@ public void HasFetchDataPage()
 
             // Wait until loaded
             var tableSelector = By.CssSelector("table.table");
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(10)).Until(
-                driver => driver.FindElement(tableSelector) != null);
+            Browser.Exists(tableSelector);
 
             // Check the table is displayed correctly
             var rows = Browser.FindElements(By.CssSelector("table.table tbody tr"));
@@ -111,8 +110,8 @@ public void HasFetchDataPage()
 
         private void WaitUntilLoaded()
         {
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(
-                driver => driver.FindElement(By.TagName("app")).Text != "Loading...");
+            var app = Browser.Exists(By.TagName("app"));
+            Browser.NotEqual("Loading...", () => app.Text);
         }
 
         public void Dispose()
diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyLazyLoadTest.cs
@@ -66,13 +66,10 @@ public void CanLazyLoadOnFirstVisit()
         {
             // Navigate to a page with lazy loaded assemblies for the first time
             SetUrlViaPushState("/WithLazyAssembly");
-            var app = Browser.MountTestComponent<TestRouterWithLazyAssembly>();
+            Browser.MountTestComponent<TestRouterWithLazyAssembly>();
 
             // Wait for the page to finish loading
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(2)).Until(
-                driver => driver.FindElement(By.Id("use-package-button")) != null);
-
-            var button = app.FindElement(By.Id("use-package-button"));
+            var button = Browser.Exists(By.Id("use-package-button"));
 
             // We should have requested the DLL
             Assert.True(HasLoadedAssembly("Newtonsoft.Json.dll"));
@@ -98,8 +95,7 @@ public void CanLazyLoadAssemblyWithRoutes()
             SetUrlViaPushState("/WithLazyLoadedRoutes");
 
             // Wait for the page to finish loading
-            new WebDriverWait(Browser, TimeSpan.FromSeconds(2)).Until(
-                 driver => driver.FindElement(By.Id("lazy-load-msg")) != null);
+            Browser.Exists(By.Id("lazy-load-msg"));
 
             // Now the assembly has been loaded
             Assert.True(HasLoadedAssembly("LazyTestContentPackage.dll"));
diff --git a/src/Components/test/testassets/BasicTestApp/wwwroot/index.html b/src/Components/test/testassets/BasicTestApp/wwwroot/index.html
@@ -26,6 +26,7 @@
     </div>
 
     <!-- Used for specific test cases -->
+    <script src="js/seleniumworkaround.js"></script>
     <script src="js/jsinteroptests.js"></script>
     <script src="js/renderattributestest.js"></script>
     <script src="js/webComponentPerformingJsInterop.js"></script>
diff --git a/src/Components/test/testassets/BasicTestApp/wwwroot/js/seleniumworkaround.js b/src/Components/test/testassets/BasicTestApp/wwwroot/js/seleniumworkaround.js
@@ -0,0 +1,26 @@
+(function () {
+  // Note: there are multiple copies of this file throughout the repo. If you're editing it, please look for
+  // other seleniumworkaround.js files and keep them all in sync.
+  const logs = [];
+
+  const defaultLog = console.log;
+  console.log = function () {
+    defaultLog.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultError = console.error;
+  console.error = function () {
+    defaultError.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  const defaultWarn = console.warn;
+  console.warn = function () {
+    defaultWarn.apply(console, arguments);
+    logs.push(Array.from(arguments).join(' '));
+  }
+
+  window.getBrowserLogs = () => logs;
+})();
+
diff --git a/src/Shared/E2ETesting/BrowserAssertFailedException.cs b/src/Shared/E2ETesting/BrowserAssertFailedException.cs
@@ -14,12 +14,12 @@ namespace OpenQA.Selenium
     // case.
     public class BrowserAssertFailedException : XunitException
     {
-        public BrowserAssertFailedException(IReadOnlyList<LogEntry> logs, Exception innerException, string screenShotPath, string innerHTML)
+        public BrowserAssertFailedException(IReadOnlyCollection<string> logs, Exception innerException, string screenShotPath, string innerHTML)
             : base(BuildMessage(innerException, logs, screenShotPath, innerHTML), innerException)
         {
         }
 
-        private static string BuildMessage(Exception exception, IReadOnlyList<LogEntry> logs, string screenShotPath, string innerHTML)
+        private static string BuildMessage(Exception exception, IReadOnlyCollection<string> logs, string screenShotPath, string innerHTML)
         {
             var builder = new StringBuilder();
             builder.AppendLine(exception.ToString());
diff --git a/src/Shared/E2ETesting/WaitAssert.cs b/src/Shared/E2ETesting/WaitAssert.cs
@@ -5,8 +5,11 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.ExceptionServices;
 using OpenQA.Selenium;
+using OpenQA.Selenium.DevTools.Page;
+using OpenQA.Selenium.Interactions;
 using OpenQA.Selenium.Support.UI;
 using Xunit;
 
@@ -117,7 +120,17 @@ private static TResult WaitAssertCore<TResult>(IWebDriver driver, Func<TResult>
 
                 var fileId = $"{Guid.NewGuid():N}.png";
                 var screenShotPath = Path.Combine(Path.GetFullPath(E2ETestOptions.Instance.ScreenShotsPath), fileId);
-                var errors = driver.GetBrowserLogs(LogLevel.All);
+                var errors = driver.GetBrowserLogs(LogLevel.All).Select(c => c.ToString()).ToList();
+                if (errors.Count == 0)
+                {
+                    // Workaround for selenium bug https://github.com/SeleniumHQ/selenium/issues/8229. Getting log does
+                    // not work. However some of our test apps provide a mechnanism to read the logs. Try that.
+
+                    var logs = (IReadOnlyCollection<object>)((IJavaScriptExecutor)driver).ExecuteScript(
+                        "return window.getBrowserLogs && window.getBrowserLogs() || []");
+
+                    errors = logs.Select(l => l.ToString()).ToList();
+                }
 
                 TakeScreenShot(driver, screenShotPath);
                 var exceptionInfo = lastException != null ? ExceptionDispatchInfo.Capture(lastException) :

Original file line number	Diff line number	Diff line change
`@@ -62,8 +62,8 @@ private void PerformReconnection()`
`62`	`62`	`((IJavaScriptExecutor)Browser).ExecuteScript("Blazor._internal.forceCloseConnection()");`
`63`	`63`
`64`	`64`	`// Wait until the reconnection dialog has been shown but is now hidden`
`65`		`- new WebDriverWait(Browser, TimeSpan.FromSeconds(10))`
`66`		`- .Until(driver => driver.FindElement(By.Id("components-reconnect-modal"))?.GetCssValue("display") == "none");`
	`65`	`+ var reconnectModel = Browser.Exists(By.Id("components-reconnect-modal"));`
	`66`	`+ Browser.Equal("none", () => reconnectModel.GetCssValue("display"));`
`67`	`67`	`}`
`68`	`68`	`}`
`69`	`69`	`}`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,7 @@ public void DotNetExceptionDetailsAreNotLoggedByDefault()`
`48`	`48`	`var interopButton = Browser.FindElement(By.Id("btn-interop"));`
`49`	`49`	`interopButton.Click();`
`50`	`50`
`51`		`- var wait = new WebDriverWait(Browser, TimeSpan.FromSeconds(10))`
`52`		`- .Until(d => d.FindElement(By.Id("done-with-interop")));`
	`51`	`+ Browser.Exists(By.Id("done-with-interop"));`
`53`	`52`
`54`	`53`	`foreach (var expectedValue in expectedValues)`
`55`	`54`	`{`
Original file line number	Diff line number	Diff line change
`@@ -148,8 +148,8 @@ private IReadOnlyCollection<string> GetAndClearRequestedPaths()`
`148`	`148`
`149`	`149`	`private void WaitUntilLoaded()`
`150`	`150`	`{`
`151`		`- new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(`
`152`		`- driver => driver.FindElement(By.TagName("h1")).Text == "Hello, world!");`
	`151`	`+ var element = Browser.Exists(By.TagName("h1"));`
	`152`	`+ Browser.Equal("Hello, world!", () => element.Text);`
`153`	`153`	`}`
`154`	`154`	`}`
`155`	`155`	`}`
Original file line number	Diff line number	Diff line change
`@@ -61,11 +61,10 @@ public void MapFallbackToClientSideBlazor_AssemblyPath_Pattern_FilePath()`
`61`	`61`	`Assert.NotNull(Browser.FindElement(By.Id("test-selector")));`
`62`	`62`	`}`
`63`	`63`
`64`		`-`
`65`	`64`	`private void WaitUntilLoaded()`
`66`	`65`	`{`
`67`		`- new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(`
`68`		`- driver => driver.FindElement(By.TagName("app")).Text != "Loading...");`
	`66`	`+ var app = Browser.Exists(By.TagName("app"));`
	`67`	`+ Browser.NotEqual("Loading...", () => app.Text);`
`69`	`68`	`}`
`70`	`69`	`}`
`71`	`70`	`}`
Original file line number	Diff line number	Diff line change
`@@ -48,8 +48,8 @@ public void ServesStaticAssetsFromClientAppWebRoot()`
`48`	`48`
`49`	`49`	`private void WaitUntilLoaded()`
`50`	`50`	`{`
`51`		`- new WebDriverWait(Browser, TimeSpan.FromSeconds(30)).Until(`
`52`		`- driver => driver.FindElement(By.TagName("app")).Text != "Loading...");`
	`51`	`+ var app = Browser.Exists(By.TagName("app"));`
	`52`	`+ Browser.NotEqual("Loading...", () => app.Text);`
`53`	`53`	`}`
`54`	`54`	`}`
`55`	`55`	`}`