From 8ade00f2327a0cb5d8c3b3fad09ad86edaad0fb2 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Wed, 16 Oct 2024 10:47:06 +0200 Subject: [PATCH 01/17] added LighthouseUtils --- pom.xml | 2 +- .../neodymium/util/LighthouseUtils.java | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java diff --git a/pom.xml b/pom.xml index 3eaae160..03cc2ade 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.xceptance neodymium-library - 5.0.2 + 5.0.2-SNAPSHOT neodymium-library https://github.com/Xceptance/neodymium-library diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java new file mode 100644 index 00000000..abf5c95e --- /dev/null +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -0,0 +1,55 @@ +package com.xceptance.neodymium.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WindowType; + +public class LighthouseUtils +{ + public static void createLightHouseReport(WebDriver driver, String URL, String reportName) throws InterruptedException, IOException + { + // close window to avoid conflict with lighthouse + String newWindow = windowOperations(driver); + + // start lighthouse report + lighthouseAudit(URL, reportName); + + // TODO - add lighthouse report as attachment to allure report + + // switch back to saved URL + driver.switchTo().window(newWindow); + driver.get(URL); + } + + private static String windowOperations(WebDriver driver) throws InterruptedException + { + String originalWindow = driver.getWindowHandle(); + driver.switchTo().newWindow(WindowType.TAB); + String newWindow = driver.getWindowHandle(); + driver.switchTo().window(originalWindow); + driver.close(); + return newWindow; + } + + private static void lighthouseAudit(String URL, String ReportName) throws IOException + { + ProcessBuilder builder = new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=html", "--output-path=target/" + + ReportName + + ".html"); + builder.redirectErrorStream(true); + Process p = builder.start(); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line; + while (true) + { + line = r.readLine(); + if (line == null) + { + break; + } + } + } +} From be17fa3fdd1f848927ad2d8f7938c9be4592ed62 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Tue, 12 Nov 2024 10:15:07 +0100 Subject: [PATCH 02/17] added lighthouse report handling --- config/neodymium.properties | 6 ++ .../recording/TakeScreenshotsThread.java | 17 ++++- .../neodymium/util/LighthouseUtils.java | 73 +++++++++++++++---- .../util/NeodymiumConfiguration.java | 16 ++++ 4 files changed, 94 insertions(+), 18 deletions(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index 02d0c6eb..3188ed9e 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -226,6 +226,12 @@ neodymium.webDriver.keepBrowserOpenOnFailure = false # If false: all tests of a test class are executed neodymium.workInProgress = false +# lighthouse report thresholds +neodymium.lighthouse.performance = 0.89 +neodymium.lighthouse.accessiblity = 0.89 +neodymium.lighthouse.bestPractices = 0.89 +neodymium.lighthouse.seo = 0.89 + ############################# # # Proxy configuration properties diff --git a/src/main/java/com/xceptance/neodymium/common/recording/TakeScreenshotsThread.java b/src/main/java/com/xceptance/neodymium/common/recording/TakeScreenshotsThread.java index 9e308970..929dd7d9 100644 --- a/src/main/java/com/xceptance/neodymium/common/recording/TakeScreenshotsThread.java +++ b/src/main/java/com/xceptance/neodymium/common/recording/TakeScreenshotsThread.java @@ -6,6 +6,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.Date; +import org.openqa.selenium.NoSuchWindowException; import org.openqa.selenium.OutputType; import org.openqa.selenium.TakesScreenshot; import org.openqa.selenium.WebDriver; @@ -84,10 +85,18 @@ public synchronized void run() { long start = new Date().getTime(); - File file = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); - writer.compressImageIfNeeded(file, recordingConfigurations.imageScaleFactor(), recordingConfigurations.imageQuality()); - writer.write(file); - + try + { + File file = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); + + writer.compressImageIfNeeded(file, recordingConfigurations.imageScaleFactor(), recordingConfigurations.imageQuality()); + writer.write(file); + } + catch (NoSuchWindowException e) + { + // catching the exception prevents the video from failing + } + long duration = new Date().getTime() - start; millis += duration; turns++; diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index abf5c95e..0b4cfa32 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -1,24 +1,66 @@ package com.xceptance.neodymium.util; import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WindowType; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.qameta.allure.Allure; + public class LighthouseUtils { - public static void createLightHouseReport(WebDriver driver, String URL, String reportName) throws InterruptedException, IOException + public static void createLightHouseReport(WebDriver driver, String URL, String reportName) throws Exception { + // validate that lighthouse is installed + try + { + new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--version").start(); + } + catch (Exception e) + { + throw new Exception("lighthouse binary not found, please install lighthouse and add it to the PATH"); + } + + // validate chrome browser (lighthouse only works for chrome) + Assert.assertTrue("the current browser is " + Neodymium.getBrowserName() + ", but lighthouse only works in combination with chrome", Neodymium.getBrowserName().contains("chrome")); + // close window to avoid conflict with lighthouse String newWindow = windowOperations(driver); // start lighthouse report lighthouseAudit(URL, reportName); - // TODO - add lighthouse report as attachment to allure report - + // get report json + FileReader reader = new FileReader("target/" + reportName + ".report.json"); + JsonObject json = JsonParser.parseReader(reader).getAsJsonObject(); + + // get report json scores + JsonObject categories = json.getAsJsonObject("categories"); + double performanceScore = categories.getAsJsonObject("performance").get("score").getAsDouble(); + double accessibilityScore = categories.getAsJsonObject("accessibility").get("score").getAsDouble(); + double bestPracticesScore = categories.getAsJsonObject("best-practices").get("score").getAsDouble(); + double seoScore = categories.getAsJsonObject("seo").get("score").getAsDouble(); + + // validate if values in report json are greater than defined threshold in config + SelenideAddons.wrapAssertionError(() -> { + Assert.assertTrue("the performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthousePerformance() <= performanceScore); + Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAccessibility() <= accessibilityScore); + Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseBestPractices() <= bestPracticesScore); + Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseSeo() <= seoScore); + }); + + // add report html to allure + Allure.addAttachment(reportName, "text/html", FileUtils.openInputStream(new File("target/" + reportName + ".report.html")), "html"); + // switch back to saved URL driver.switchTo().window(newWindow); driver.get(URL); @@ -34,22 +76,25 @@ private static String windowOperations(WebDriver driver) throws InterruptedExcep return newWindow; } - private static void lighthouseAudit(String URL, String ReportName) throws IOException + private static void lighthouseAudit(String URL, String reportName) throws IOException { - ProcessBuilder builder = new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=html", "--output-path=target/" - + ReportName - + ".html"); + ProcessBuilder builder = new ProcessBuilder(); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + builder = new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux")) + { + // TODO (with luck linux and macOS are the same, ask Olha for help) + } + builder.redirectErrorStream(true); Process p = builder.start(); BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line; - while (true) + while (r.readLine() != null) { - line = r.readLine(); - if (line == null) - { - break; - } + continue; } } } diff --git a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java index a6d4a7be..dbccd698 100644 --- a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java +++ b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java @@ -266,5 +266,21 @@ public interface NeodymiumConfiguration extends Mutable @Key("neodymium.logNeoVersion") @DefaultValue("true") + public boolean logNeoVersion(); + @Key("neodymium.lighthouse.performance") + @DefaultValue("0.50") + public double lighthousePerformance(); + + @Key("neodymium.lighthouse.accessibilty") + @DefaultValue("0.50") + public double lighthouseAccessibility(); + + @Key("neodymium.lighthouse.bestPractices") + @DefaultValue("0.50") + public double lighthouseBestPractices(); + + @Key("neodymium.lighthouse.seo") + @DefaultValue("0.50") + public double lighthouseSeo(); } From c7f6f4cc28c8cfab3071ee4dfcb9783eeb185724 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Wed, 13 Nov 2024 09:38:37 +0100 Subject: [PATCH 03/17] added lighthouse binary validation and report creation for linux --- .../xceptance/neodymium/util/LighthouseUtils.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 0b4cfa32..d6018ed3 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -23,7 +23,14 @@ public static void createLightHouseReport(WebDriver driver, String URL, String r // validate that lighthouse is installed try { - new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--version").start(); + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--version").start(); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux")) + { + new ProcessBuilder("lighthouse", "--version"); + } } catch (Exception e) { @@ -31,7 +38,9 @@ public static void createLightHouseReport(WebDriver driver, String URL, String r } // validate chrome browser (lighthouse only works for chrome) - Assert.assertTrue("the current browser is " + Neodymium.getBrowserName() + ", but lighthouse only works in combination with chrome", Neodymium.getBrowserName().contains("chrome")); + SelenideAddons.wrapAssertionError(() -> { + Assert.assertTrue("the current browser is " + Neodymium.getBrowserName() + ", but lighthouse only works in combination with chrome", Neodymium.getBrowserName().contains("chrome")); + }); // close window to avoid conflict with lighthouse String newWindow = windowOperations(driver); @@ -86,7 +95,7 @@ private static void lighthouseAudit(String URL, String reportName) throws IOExce } else if (System.getProperty("os.name").toLowerCase().contains("linux")) { - // TODO (with luck linux and macOS are the same, ask Olha for help) + builder = new ProcessBuilder("lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } builder.redirectErrorStream(true); From 35b2d6ad647b8ab52c248ab955d081e50a69662b Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Wed, 13 Nov 2024 09:48:49 +0100 Subject: [PATCH 04/17] added lighthouse binary validation and report creation for mac --- .../java/com/xceptance/neodymium/util/LighthouseUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index d6018ed3..6b9ad79e 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -27,7 +27,7 @@ public static void createLightHouseReport(WebDriver driver, String URL, String r { new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--version").start(); } - else if (System.getProperty("os.name").toLowerCase().contains("linux")) + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { new ProcessBuilder("lighthouse", "--version"); } @@ -93,7 +93,7 @@ private static void lighthouseAudit(String URL, String reportName) throws IOExce { builder = new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } - else if (System.getProperty("os.name").toLowerCase().contains("linux")) + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { builder = new ProcessBuilder("lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } From aa4460ae42af28d4ce05bd45daade3e58f137079 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Thu, 14 Nov 2024 11:19:52 +0100 Subject: [PATCH 05/17] #302 added javadoc --- config/neodymium.properties | 15 ++++-- .../neodymium/util/LighthouseUtils.java | 50 ++++++++++++++++--- .../util/NeodymiumConfiguration.java | 5 ++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index 3188ed9e..342fecdb 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -226,11 +226,16 @@ neodymium.webDriver.keepBrowserOpenOnFailure = false # If false: all tests of a test class are executed neodymium.workInProgress = false -# lighthouse report thresholds -neodymium.lighthouse.performance = 0.89 -neodymium.lighthouse.accessiblity = 0.89 -neodymium.lighthouse.bestPractices = 0.89 -neodymium.lighthouse.seo = 0.89 +############################# +# +# Lighthouse +# +############################# +neodymium.lighthouse.binaryPath = lighthouse +neodymium.lighthouse.performance = 0.50 +neodymium.lighthouse.accessiblity = 0.50 +neodymium.lighthouse.bestPractices = 0.50 +neodymium.lighthouse.seo = 0.50 ############################# # diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 6b9ad79e..2ab72760 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -18,23 +18,33 @@ public class LighthouseUtils { - public static void createLightHouseReport(WebDriver driver, String URL, String reportName) throws Exception + /** + * Creates a Lighthouse report + * (Copyright Google) of the current URL and adds it to the Allure report. + * + * @param driver + * The current webdriver + * @param reportName + * The name of the Lighthouse report attachment in the Allure report + * @throws Exception + */ + public static void createLightHouseReport(WebDriver driver, String reportName) throws Exception { // validate that lighthouse is installed try { if (System.getProperty("os.name").toLowerCase().contains("win")) { - new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--version").start(); + new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - new ProcessBuilder("lighthouse", "--version"); + new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--version"); } } catch (Exception e) { - throw new Exception("lighthouse binary not found, please install lighthouse and add it to the PATH"); + throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location. " + e); } // validate chrome browser (lighthouse only works for chrome) @@ -42,6 +52,9 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System Assert.assertTrue("the current browser is " + Neodymium.getBrowserName() + ", but lighthouse only works in combination with chrome", Neodymium.getBrowserName().contains("chrome")); }); + // get the current URL + String URL = driver.getCurrentUrl(); + // close window to avoid conflict with lighthouse String newWindow = windowOperations(driver); @@ -75,7 +88,18 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System driver.get(URL); } - private static String windowOperations(WebDriver driver) throws InterruptedException + /** + *

+ * Opens a new tab apart from the first tab with the test automation, adds a handle and closes the first tab. + * If the first tab with the test automation is not closed, the Lighthouse report will not have proper values, + * because it will interfere with the Lighthouse report generation. + *

+ * + * @param driver + * The current webdriver + * @return A new empty tab with a window handle + */ + private static String windowOperations(WebDriver driver) { String originalWindow = driver.getWindowHandle(); driver.switchTo().newWindow(WindowType.TAB); @@ -85,17 +109,29 @@ private static String windowOperations(WebDriver driver) throws InterruptedExcep return newWindow; } + /** + *

+ * Uses Lighthouse (Copyright Google) + * to create a Lighthouse report of the current URL. + *

+ * + * @param URL + * The current URL the Lighthouse report should be generated on + * @param reportName + * The name of the Lighthouse report attachment in the Allure report + * @throws IOException + */ private static void lighthouseAudit(String URL, String reportName) throws IOException { ProcessBuilder builder = new ProcessBuilder(); if (System.getProperty("os.name").toLowerCase().contains("win")) { - builder = new ProcessBuilder("cmd.exe", "/c", "lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - builder = new ProcessBuilder("lighthouse", "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + builder = new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } builder.redirectErrorStream(true); diff --git a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java index dbccd698..37ed66fd 100644 --- a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java +++ b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java @@ -268,6 +268,11 @@ public interface NeodymiumConfiguration extends Mutable @DefaultValue("true") public boolean logNeoVersion(); + + @Key("neodymium.lighthouse.binaryPath") + @DefaultValue("lighthouse") + public String lighthouseBinaryPath(); + @Key("neodymium.lighthouse.performance") @DefaultValue("0.50") public double lighthousePerformance(); From 9765b9fa6b6175f3d2ff89577f7d67816e9c6992 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Thu, 14 Nov 2024 13:53:37 +0100 Subject: [PATCH 06/17] #302 added usage of free port for chrome --- .../common/browser/BrowserRunnerHelper.java | 7 +++++ .../neodymium/util/LighthouseUtils.java | 6 ++--- .../xceptance/neodymium/util/Neodymium.java | 27 ++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/xceptance/neodymium/common/browser/BrowserRunnerHelper.java b/src/main/java/com/xceptance/neodymium/common/browser/BrowserRunnerHelper.java index 8c7f884c..2a1b5318 100644 --- a/src/main/java/com/xceptance/neodymium/common/browser/BrowserRunnerHelper.java +++ b/src/main/java/com/xceptance/neodymium/common/browser/BrowserRunnerHelper.java @@ -33,6 +33,7 @@ import org.openqa.selenium.ie.InternetExplorerDriver; import org.openqa.selenium.ie.InternetExplorerDriverService; import org.openqa.selenium.ie.InternetExplorerOptions; +import org.openqa.selenium.net.PortProber; import org.openqa.selenium.os.ExecutableFinder; import org.openqa.selenium.remote.CapabilityType; import org.openqa.selenium.remote.HttpCommandExecutor; @@ -228,6 +229,12 @@ else if (Neodymium.configuration().useProxy()) { options.addArguments("--headless"); } + + // find a free port for each chrome session (important for lighthouse) + var remoteDebuggingPort = PortProber.findFreePort(); + Neodymium.setRemoteDebuggingPort(remoteDebuggingPort); + options.addArguments("--remote-debugging-port=" + remoteDebuggingPort); + if (config.getArguments() != null && config.getArguments().size() > 0) { options.addArguments(config.getArguments()); diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 2ab72760..88232292 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -127,13 +127,13 @@ private static void lighthouseAudit(String URL, String reportName) throws IOExce if (System.getProperty("os.name").toLowerCase().contains("win")) { - builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - builder = new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=9999", "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + builder = new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } - + builder.redirectErrorStream(true); Process p = builder.start(); BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); diff --git a/src/main/java/com/xceptance/neodymium/util/Neodymium.java b/src/main/java/com/xceptance/neodymium/util/Neodymium.java index 3e123e3d..8741d180 100644 --- a/src/main/java/com/xceptance/neodymium/util/Neodymium.java +++ b/src/main/java/com/xceptance/neodymium/util/Neodymium.java @@ -33,9 +33,12 @@ public class Neodymium // keep our current browser profile name private String browserProfileName; - + // keep our current browser name private String browserName; + + // keep our current remote debugging port + private int remoteDebuggingPort; // our global configuration private final NeodymiumConfiguration configuration; @@ -255,6 +258,28 @@ public static void setBrowserName(String browserName) { getContext().browserName = browserName; } + + /** + * Remote debugging port of the current bowser + * + * @return remote debugging port + */ + public static int getRemoteDebuggingPort() + { + return getContext().remoteDebuggingPort; + } + + /** + * Set the remote debugging port of the current browser.
+ * Attention: This function is mainly used to set information within the context internally. + * + * @param remoteDebuggingPort + * the current browser port + */ + public static void setRemoteDebuggingPort(int remoteDebuggingPort) + { + getContext().remoteDebuggingPort = remoteDebuggingPort; + } /** * Current window width and height From 5d0daac0d975f1d34ba021f1f406d6710925af06 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Mon, 18 Nov 2024 08:57:45 +0100 Subject: [PATCH 07/17] added unit tests --- config/neodymium.properties | 8 +- .../neodymium/util/LighthouseUtils.java | 33 ++- .../util/NeodymiumConfiguration.java | 10 +- .../neodymium/util/LighthouseUtilsTest.java | 212 ++++++++++++++++++ 4 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java diff --git a/config/neodymium.properties b/config/neodymium.properties index 342fecdb..a5dacbbb 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -232,10 +232,10 @@ neodymium.workInProgress = false # ############################# neodymium.lighthouse.binaryPath = lighthouse -neodymium.lighthouse.performance = 0.50 -neodymium.lighthouse.accessiblity = 0.50 -neodymium.lighthouse.bestPractices = 0.50 -neodymium.lighthouse.seo = 0.50 +neodymium.lighthouse.performance = 0.5 +neodymium.lighthouse.accessibility = 0.5 +neodymium.lighthouse.bestPractices = 0.5 +neodymium.lighthouse.seo = 0.5 ############################# # diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 88232292..18387fa8 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -16,6 +16,9 @@ import io.qameta.allure.Allure; +/** + * Powered by Lighthouse (Copyright Google) + */ public class LighthouseUtils { /** @@ -35,11 +38,23 @@ public static void createLightHouseReport(WebDriver driver, String reportName) t { if (System.getProperty("os.name").toLowerCase().contains("win")) { - new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); + ProcessBuilder builder = new ProcessBuilder(); + builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + Process p = builder.start(); + int errorCode = p.waitFor(); + + if (errorCode != 0) + { + throw new IOException(); + } } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--version"); + new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); } } catch (Exception e) @@ -75,9 +90,9 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System // validate if values in report json are greater than defined threshold in config SelenideAddons.wrapAssertionError(() -> { Assert.assertTrue("the performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthousePerformance() <= performanceScore); - Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAccessibility() <= accessibilityScore); - Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseBestPractices() <= bestPracticesScore); - Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseSeo() <= seoScore); + Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAccessibility() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAccessibility() <= accessibilityScore); + Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseBestPractices() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseBestPractices() <= bestPracticesScore); + Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseSeo() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseSeo() <= seoScore); }); // add report html to allure @@ -119,9 +134,9 @@ private static String windowOperations(WebDriver driver) * The current URL the Lighthouse report should be generated on * @param reportName * The name of the Lighthouse report attachment in the Allure report - * @throws IOException + * @throws Exception */ - private static void lighthouseAudit(String URL, String reportName) throws IOException + private static void lighthouseAudit(String URL, String reportName) throws Exception { ProcessBuilder builder = new ProcessBuilder(); @@ -133,6 +148,10 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System { builder = new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } builder.redirectErrorStream(true); Process p = builder.start(); diff --git a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java index 37ed66fd..b5d95a32 100644 --- a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java +++ b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java @@ -274,18 +274,18 @@ public interface NeodymiumConfiguration extends Mutable public String lighthouseBinaryPath(); @Key("neodymium.lighthouse.performance") - @DefaultValue("0.50") + @DefaultValue("0.5") public double lighthousePerformance(); - @Key("neodymium.lighthouse.accessibilty") - @DefaultValue("0.50") + @Key("neodymium.lighthouse.accessibility") + @DefaultValue("0.5") public double lighthouseAccessibility(); @Key("neodymium.lighthouse.bestPractices") - @DefaultValue("0.50") + @DefaultValue("0.5") public double lighthouseBestPractices(); @Key("neodymium.lighthouse.seo") - @DefaultValue("0.50") + @DefaultValue("0.5") public double lighthouseSeo(); } diff --git a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java new file mode 100644 index 00000000..a2fd41e6 --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java @@ -0,0 +1,212 @@ +package com.xceptance.neodymium.util; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +import org.junit.Assert; + +import com.codeborne.selenide.Selenide; +import com.xceptance.neodymium.common.browser.Browser; +import com.xceptance.neodymium.junit5.NeodymiumTest; + +@Browser("Chrome_headless") +public class LighthouseUtilsTest +{ + @NeodymiumTest + public void testLighthouseUtilsHappyPath() throws Exception + { + Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.accessiblity", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + + ProcessBuilder builder = new ProcessBuilder(); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); + builder = new ProcessBuilder("cmd.exe", "/c", "fabricatedHtml > target/lighthouseUtilsReport.report.html"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + builder = new ProcessBuilder("fabricatedHtml > target/lighthouseUtilsReport.report.html"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + builder.redirectErrorStream(true); + Process p = builder.start(); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while (r.readLine() != null) + { + continue; + } + + Selenide.open("https://blog.xceptance.com/"); + + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + + @NeodymiumTest + public void testLighthouseUtilsPerfromanceException() throws Exception + { + Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + Selenide.open("https://blog.xceptance.com/"); + + try + { + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + catch (AssertionError e) + { + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + } + } + + @NeodymiumTest + public void testLighthouseUtilsAccessibilityException() throws Exception + { + Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + Selenide.open("https://blog.xceptance.com/"); + + try + { + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + catch (AssertionError e) + { + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + } + } + + @NeodymiumTest + public void testLighthouseUtilsBestPracticesException() throws Exception + { + Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + Selenide.open("https://blog.xceptance.com/"); + + try + { + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + catch (AssertionError e) + { + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + } + } + + @NeodymiumTest + public void testLighthouseUtilsSeoException() throws Exception + { + Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.51"); + + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + Selenide.open("https://blog.xceptance.com/"); + + try + { + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + catch (AssertionError e) + { + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + } + } + + @NeodymiumTest + public void testLighthouseUtilsBinNotFound() throws Exception + { + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "programmWhichIsDefinitelyNotInstalled"); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "programmWhichIsDefinitelyNotInstalled"); + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } + + Selenide.open("https://blog.xceptance.com/"); + + try + { + LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + } + catch (Exception e) + { + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("lighthouse binary not found at")); + } + } +} \ No newline at end of file From def4f49a3481f8a222ce0e20af2c77ddf7b8d4e0 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Mon, 18 Nov 2024 08:59:45 +0100 Subject: [PATCH 08/17] corrected grammar mistake --- .../java/com/xceptance/neodymium/util/LighthouseUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java index a2fd41e6..a8162243 100644 --- a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java +++ b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java @@ -51,7 +51,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } @NeodymiumTest - public void testLighthouseUtilsPerfromanceException() throws Exception + public void testLighthouseUtilsPerformanceException() throws Exception { Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.51"); Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); From 85358f2390b1c59ac6abe84eecc8ef820dc0bfaa Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Mon, 18 Nov 2024 09:19:40 +0100 Subject: [PATCH 09/17] changed back version in pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 03cc2ade..3eaae160 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.xceptance neodymium-library - 5.0.2-SNAPSHOT + 5.0.2 neodymium-library https://github.com/Xceptance/neodymium-library From 4a816337143699dd24d86c43724601db6bf6b077 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Wed, 20 Nov 2024 14:53:07 +0100 Subject: [PATCH 10/17] #302 - added lighthouse audits validation --- config/neodymium.properties | 9 ++-- .../neodymium/util/LighthouseUtils.java | 51 +++++++++++++++++-- .../util/NeodymiumConfiguration.java | 19 ++++--- 3 files changed, 63 insertions(+), 16 deletions(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index a5dacbbb..c481b2d9 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -232,10 +232,11 @@ neodymium.workInProgress = false # ############################# neodymium.lighthouse.binaryPath = lighthouse -neodymium.lighthouse.performance = 0.5 -neodymium.lighthouse.accessibility = 0.5 -neodymium.lighthouse.bestPractices = 0.5 -neodymium.lighthouse.seo = 0.5 +neodymium.lighthouse.assert.thresholdScore.performance = 0.5 +neodymium.lighthouse.assert.thresholdScore.accessibility = 0.5 +neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5 +neodymium.lighthouse.assert.thresholdScore.seo = 0.5 +#neodymium.lighthouse.assert.audits = ############################# # diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 18387fa8..b34aa7da 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -5,6 +5,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.io.FileUtils; import org.junit.Assert; @@ -13,6 +15,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; import io.qameta.allure.Allure; @@ -77,6 +81,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System lighthouseAudit(URL, reportName); // get report json + File jsonFile = new File("target/" + reportName + ".report.json"); FileReader reader = new FileReader("target/" + reportName + ".report.json"); JsonObject json = JsonParser.parseReader(reader).getAsJsonObject(); @@ -89,12 +94,15 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System // validate if values in report json are greater than defined threshold in config SelenideAddons.wrapAssertionError(() -> { - Assert.assertTrue("the performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthousePerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthousePerformance() <= performanceScore); - Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAccessibility() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAccessibility() <= accessibilityScore); - Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseBestPractices() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseBestPractices() <= bestPracticesScore); - Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseSeo() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseSeo() <= seoScore); + Assert.assertTrue("the performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertPerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertPerformance() <= performanceScore); + Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertAccessibility() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertAccessibility() <= accessibilityScore); + Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertBestPractices() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertBestPractices() <= bestPracticesScore); + Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertSeo() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertSeo() <= seoScore); }); + // validate jsonpaths in neodymium properties + validateAudits(jsonFile); + // add report html to allure Allure.addAttachment(reportName, "text/html", FileUtils.openInputStream(new File("target/" + reportName + ".report.html")), "html"); @@ -161,4 +169,39 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System continue; } } + + // javadoc + // test for jsonobject as well + private static void validateAudits(File json) throws Exception + { + String assertAuditsString = Neodymium.configuration().lighthouseAssertAudits(); + List errorAudits = new ArrayList<>(); + + if (!assertAuditsString.isEmpty()) + { + for (String audit : assertAuditsString.split(" ")) + { + String jsonPath = ("$.audits." + audit.trim() + ".details.items.length()"); + + try + { + int value = JsonPath.read(json, jsonPath); + + if (value > 0) + { + errorAudits.add(audit.trim()); + } + } + catch (PathNotFoundException e) + { + continue; + } + + } + if (errorAudits.size() > 0) + { + throw new Exception("the following audits " + errorAudits + " contain errors that need to be fixed, please look into the corresponding html for further information"); + } + } + } } diff --git a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java index b5d95a32..ee067b12 100644 --- a/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java +++ b/src/main/java/com/xceptance/neodymium/util/NeodymiumConfiguration.java @@ -273,19 +273,22 @@ public interface NeodymiumConfiguration extends Mutable @DefaultValue("lighthouse") public String lighthouseBinaryPath(); - @Key("neodymium.lighthouse.performance") + @Key("neodymium.lighthouse.assert.thresholdScore.performance") @DefaultValue("0.5") - public double lighthousePerformance(); + public double lighthouseAssertPerformance(); - @Key("neodymium.lighthouse.accessibility") + @Key("neodymium.lighthouse.assert.thresholdScore.accessibility") @DefaultValue("0.5") - public double lighthouseAccessibility(); + public double lighthouseAssertAccessibility(); - @Key("neodymium.lighthouse.bestPractices") + @Key("neodymium.lighthouse.assert.thresholdScore.bestPractices") @DefaultValue("0.5") - public double lighthouseBestPractices(); + public double lighthouseAssertBestPractices(); - @Key("neodymium.lighthouse.seo") + @Key("neodymium.lighthouse.assert.thresholdScore.seo") @DefaultValue("0.5") - public double lighthouseSeo(); + public double lighthouseAssertSeo(); + + @Key("neodymium.lighthouse.assert.audits") + public String lighthouseAssertAudits(); } From 88024183a904c724cac93b91d77bf0f6710cd307 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Wed, 20 Nov 2024 14:59:08 +0100 Subject: [PATCH 11/17] #302 - added javadoc for validate audits method --- .../xceptance/neodymium/util/LighthouseUtils.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index b34aa7da..fcf335ec 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -170,8 +170,18 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } } - // javadoc - // test for jsonobject as well + /** + *

+ * Validates Lighthouse (Copyright Google) + * Audits specified in the Neodymium configuration. + *

+ * + * @param json + * The json file of the Lighthouse + * (Copyright Google) report + * + * @throws Exception + */ private static void validateAudits(File json) throws Exception { String assertAuditsString = Neodymium.configuration().lighthouseAssertAudits(); From 549219d6ec2b5cd209d2b097b65b31db95bf2a3b Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Thu, 21 Nov 2024 13:25:02 +0100 Subject: [PATCH 12/17] #302 - updated audits condition --- src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index fcf335ec..552720cb 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -187,7 +187,7 @@ private static void validateAudits(File json) throws Exception String assertAuditsString = Neodymium.configuration().lighthouseAssertAudits(); List errorAudits = new ArrayList<>(); - if (!assertAuditsString.isEmpty()) + if (!(assertAuditsString == null)) { for (String audit : assertAuditsString.split(" ")) { From 1d0f013c5e26cc2f146cc156afe72a3160ad2b14 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Fri, 22 Nov 2024 10:54:47 +0100 Subject: [PATCH 13/17] #302 - code review changes --- config/neodymium.properties | 27 ++++++++++++ .../neodymium/util/LighthouseUtils.java | 43 +++++++++++-------- .../neodymium/util/LighthouseUtilsTest.java | 16 +++---- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index 469ca895..2201ba4a 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -253,11 +253,38 @@ neodymium.report.enableTestDataInReport = true # Lighthouse # ############################# + +# Specifies the path to the Lighthouse executable +# If Lighthouse is globally installed and available in PATH, use only the name of the Lighthouse binary +# If Lighthouse is not globally installed and available in PATH, use the absolute/relative path to the Lighthouse binary neodymium.lighthouse.binaryPath = lighthouse + +# Specifies the minimum acceptable score for the performance category in Lighthouse reports +# If the Lighthouse performance score falls below this threshold, the test will fail +# Range: 0.0 - 1.0 (representing 0% to 100%) +# The actual value for the performance score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.performance = 0.5 + +# Specifies the minimum acceptable score for the accessibility category in Lighthouse reports +# If the Lighthouse accessibility score falls below this threshold, the test will fail +# Range: 0.0 - 1.0 (representing 0% to 100%) +# The actual value for the accessibility score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.accessibility = 0.5 + +# Specifies the minimum acceptable score for the best practices category in Lighthouse reports +# If the Lighthouse best practices score falls below this threshold, the test will fail +# Range: 0.0 - 1.0 (representing 0% to 100%) +# The actual value for the best practices score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5 + +# Specifies the minimum acceptable score for the seo category in Lighthouse reports +# If the Lighthouse seo score falls below this threshold, the test will fail +# Range: 0.0 - 1.0 (representing 0% to 100%) +# The actual value for the seo score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.seo = 0.5 + +# To be able to validate Lighthouse report audits, wo use internal json id's from the report itself +# A full list of all id's can be find here: TODO LINK TO DOCUMENTATION #neodymium.lighthouse.assert.audits = ############################# diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 552720cb..4ce51657 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -9,6 +9,7 @@ import java.util.List; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WindowType; @@ -22,6 +23,10 @@ /** * Powered by Lighthouse (Copyright Google) + *

+ * This class is used to create Lighthouse (Copyright Google) + * reports and validate them using defined values ​​in the Neodymium configuration + *

*/ public class LighthouseUtils { @@ -29,13 +34,11 @@ public class LighthouseUtils * Creates a Lighthouse report * (Copyright Google) of the current URL and adds it to the Allure report. * - * @param driver - * The current webdriver * @param reportName * The name of the Lighthouse report attachment in the Allure report * @throws Exception */ - public static void createLightHouseReport(WebDriver driver, String reportName) throws Exception + public static void createLightHouseReport(String reportName) throws Exception { // validate that lighthouse is installed try @@ -54,7 +57,7 @@ public static void createLightHouseReport(WebDriver driver, String reportName) t } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); + new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); } else { @@ -71,6 +74,9 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System Assert.assertTrue("the current browser is " + Neodymium.getBrowserName() + ", but lighthouse only works in combination with chrome", Neodymium.getBrowserName().contains("chrome")); }); + // get the current webdriver + WebDriver driver = Neodymium.getDriver(); + // get the current URL String URL = driver.getCurrentUrl(); @@ -80,6 +86,9 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System // start lighthouse report lighthouseAudit(URL, reportName); + // add report html to allure + Allure.addAttachment(reportName, "text/html", FileUtils.openInputStream(new File("target/" + reportName + ".report.html")), "html"); + // get report json File jsonFile = new File("target/" + reportName + ".report.json"); FileReader reader = new FileReader("target/" + reportName + ".report.json"); @@ -94,17 +103,15 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System // validate if values in report json are greater than defined threshold in config SelenideAddons.wrapAssertionError(() -> { - Assert.assertTrue("the performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertPerformance() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertPerformance() <= performanceScore); - Assert.assertTrue("the accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertAccessibility() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertAccessibility() <= accessibilityScore); - Assert.assertTrue("the best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertBestPractices() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertBestPractices() <= bestPracticesScore); - Assert.assertTrue("the seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertSeo() + ", please improve the score to match expectations", Neodymium.configuration().lighthouseAssertSeo() <= seoScore); + Assert.assertTrue("The Lighthouse performance score " + performanceScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertPerformance() + ", please improve the score to match expectations and look into the corresponding Lighthouse report named \"" + reportName + "\"", Neodymium.configuration().lighthouseAssertPerformance() <= performanceScore); + Assert.assertTrue("The Lighthouse accessibility score " + accessibilityScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertAccessibility() + ", please improve the score to match expectations and look into the corresponding Lighthouse report named \"" + reportName + "\"", Neodymium.configuration().lighthouseAssertAccessibility() <= accessibilityScore); + Assert.assertTrue("The Lighthouse best practices score " + bestPracticesScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertBestPractices() + ", please improve the score to match expectations and look into the corresponding Lighthouse report named \"" + reportName + "\"", Neodymium.configuration().lighthouseAssertBestPractices() <= bestPracticesScore); + Assert.assertTrue("The Lighthouse seo score " + seoScore + " doesn't exceed nor match the required threshold of " + Neodymium.configuration().lighthouseAssertSeo() + ", please improve the score to match expectations and look into the corresponding Lighthouse report named \"" + reportName + "\"", Neodymium.configuration().lighthouseAssertSeo() <= seoScore); }); // validate jsonpaths in neodymium properties - validateAudits(jsonFile); + validateAudits(jsonFile, reportName); - // add report html to allure - Allure.addAttachment(reportName, "text/html", FileUtils.openInputStream(new File("target/" + reportName + ".report.html")), "html"); // switch back to saved URL driver.switchTo().window(newWindow); @@ -154,7 +161,7 @@ private static void lighthouseAudit(String URL, String reportName) throws Except } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - builder = new ProcessBuilder(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + builder = new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); } else { @@ -182,12 +189,12 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System * * @throws Exception */ - private static void validateAudits(File json) throws Exception + private static void validateAudits(File json, String reportName) throws Exception { String assertAuditsString = Neodymium.configuration().lighthouseAssertAudits(); List errorAudits = new ArrayList<>(); - if (!(assertAuditsString == null)) + if (StringUtils.isNotBlank(assertAuditsString)) { for (String audit : assertAuditsString.split(" ")) { @@ -208,10 +215,10 @@ private static void validateAudits(File json) throws Exception } } - if (errorAudits.size() > 0) - { - throw new Exception("the following audits " + errorAudits + " contain errors that need to be fixed, please look into the corresponding html for further information"); - } + + SelenideAddons.wrapAssertionError(() -> { + Assert.assertTrue("the following Lighthouse audits " + errorAudits + " contain errors that need to be fixed, please look into the Lighthouse report named \"" + reportName + "\" for further information. ", errorAudits.size() > 0); + }); } } } diff --git a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java index a8162243..4e2faaea 100644 --- a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java +++ b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java @@ -25,12 +25,12 @@ public void testLighthouseUtilsHappyPath() throws Exception if (System.getProperty("os.name").toLowerCase().contains("win")) { Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); - builder = new ProcessBuilder("cmd.exe", "/c", "fabricatedHtml > target/lighthouseUtilsReport.report.html"); + builder = new ProcessBuilder("cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); - builder = new ProcessBuilder("fabricatedHtml > target/lighthouseUtilsReport.report.html"); + builder = new ProcessBuilder("sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); } else { @@ -47,7 +47,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System Selenide.open("https://blog.xceptance.com/"); - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } @NeodymiumTest @@ -75,7 +75,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } catch (AssertionError e) { @@ -108,7 +108,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } catch (AssertionError e) { @@ -141,7 +141,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } catch (AssertionError e) { @@ -174,7 +174,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } catch (AssertionError e) { @@ -202,7 +202,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { - LighthouseUtils.createLightHouseReport(Neodymium.getDriver(), "lighthouseUtilsReport"); + LighthouseUtils.createLightHouseReport("lighthouseUtilsReport"); } catch (Exception e) { From 30e10d1de6ced2cb618939f0883d7ddbb82724fd Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Fri, 22 Nov 2024 13:41:34 +0100 Subject: [PATCH 14/17] #302 - added missing parameter to javadoc --- src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 4ce51657..6be80a82 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -186,6 +186,8 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System * @param json * The json file of the Lighthouse * (Copyright Google) report + * @param reportName + * The name of the Lighthouse report attachment in the Allure report * * @throws Exception */ From ef9429bfd8cb2eb8d5fe50ce8e4fbb7332ccb3e5 Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Fri, 22 Nov 2024 14:40:15 +0100 Subject: [PATCH 15/17] #302 - added link to docu in new audit property --- config/neodymium.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index 2201ba4a..6b67cadb 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -284,7 +284,7 @@ neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5 neodymium.lighthouse.assert.thresholdScore.seo = 0.5 # To be able to validate Lighthouse report audits, wo use internal json id's from the report itself -# A full list of all id's can be find here: TODO LINK TO DOCUMENTATION +# A full list of all audit id's and their corresponding titles can be found here: https://github.com/Xceptance/neodymium/wiki/Reports#lighthouse-audit-validation #neodymium.lighthouse.assert.audits = ############################# From fe414b8cf8dcc8d02724b3d3acccb69fb2f9ffda Mon Sep 17 00:00:00 2001 From: Jonas Hoyer Date: Mon, 2 Dec 2024 12:40:02 +0100 Subject: [PATCH 16/17] #302 - added requested changes in review --- config/neodymium.properties | 2 +- .../neodymium/util/LighthouseUtils.java | 128 ++++++++++++++---- .../neodymium/util/LighthouseUtilsTest.java | 69 ++++++---- 3 files changed, 141 insertions(+), 58 deletions(-) diff --git a/config/neodymium.properties b/config/neodymium.properties index 6b67cadb..3469a844 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -283,7 +283,7 @@ neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5 # The actual value for the seo score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.seo = 0.5 -# To be able to validate Lighthouse report audits, wo use internal json id's from the report itself +# To be able to validate Lighthouse report audits, we use internal json id's from the report itself # A full list of all audit id's and their corresponding titles can be found here: https://github.com/Xceptance/neodymium/wiki/Reports#lighthouse-audit-validation #neodymium.lighthouse.assert.audits = diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 6be80a82..0f47a004 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -45,19 +45,21 @@ public static void createLightHouseReport(String reportName) throws Exception { if (System.getProperty("os.name").toLowerCase().contains("win")) { - ProcessBuilder builder = new ProcessBuilder(); - builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); - Process p = builder.start(); - int errorCode = p.waitFor(); - - if (errorCode != 0) - { - throw new IOException(); - } + Process p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); + try + { + Process p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); + } + catch (Exception e) + { + Process p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); + } } else { @@ -66,7 +68,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (Exception e) { - throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location. " + e); + throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location.", e); } // validate chrome browser (lighthouse only works for chrome) @@ -153,28 +155,64 @@ private static String windowOperations(WebDriver driver) */ private static void lighthouseAudit(String URL, String reportName) throws Exception { - ProcessBuilder builder = new ProcessBuilder(); + Process p = null; + String readerOutput = ""; + String readerLine= ""; - if (System.getProperty("os.name").toLowerCase().contains("win")) - { - builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); - } - else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + try { - builder = new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while ((readerLine = r.readLine()) != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + try + { + p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while (r.readLine() != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + catch (Exception e) + { + p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while (r.readLine() != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } } - else + catch (IOException e) { - throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + throw new Exception("failed to run the process\n\nProcess Log: " + readerOutput, e); } - builder.redirectErrorStream(true); - Process p = builder.start(); - BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); - while (r.readLine() != null) - { - continue; - } } /** @@ -223,4 +261,40 @@ private static void validateAudits(File json, String reportName) throws Exceptio }); } } + + /** + * Runs a process with the in params specified command + * + * @param + * params the command with all required parameters + * @return + * the process + * @throws IOException + */ + private static Process runProcess(String... params) throws IOException + { + ProcessBuilder builder = new ProcessBuilder(); + builder = new ProcessBuilder(params); + builder.redirectErrorStream(true); + + return builder.start(); + } + + /** + * Checks the error code for the specified process + * + * @param p + * the process of which the error code needs to be checked + * @throws InterruptedException + * @throws IOException + */ + private static void checkErrorCodeProcess(Process p) throws InterruptedException, IOException + { + int errorCode = p.waitFor(); + + if (errorCode != 0) + { + throw new IOException("process error code differs from 0"); + } + } } diff --git a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java index 4e2faaea..001c0294 100644 --- a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java +++ b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java @@ -2,6 +2,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import java.lang.reflect.Method; import org.junit.Assert; @@ -15,30 +16,38 @@ public class LighthouseUtilsTest @NeodymiumTest public void testLighthouseUtilsHappyPath() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessiblity", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessiblity", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); - ProcessBuilder builder = new ProcessBuilder(); + Class clazz = LighthouseUtils.class; + Method runProcess = clazz.getDeclaredMethod("runProcess", String[].class); + runProcess.setAccessible(true); + Process p = null; if (System.getProperty("os.name").toLowerCase().contains("win")) { Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); - builder = new ProcessBuilder("cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); - builder = new ProcessBuilder("sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); + try + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); + } + catch (Exception e) + { + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); + } } else { throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); } - builder.redirectErrorStream(true); - Process p = builder.start(); BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); while (r.readLine() != null) { @@ -53,10 +62,10 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System @NeodymiumTest public void testLighthouseUtilsPerformanceException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -79,17 +88,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsAccessibilityException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -112,17 +121,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsBestPracticesException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -145,17 +154,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsSeoException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.51"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -178,7 +187,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } From 5f5228ecb85b70d7dc0e35911241e2bd3e424de1 Mon Sep 17 00:00:00 2001 From: Bernd Weigel Date: Tue, 3 Dec 2024 06:25:36 +0100 Subject: [PATCH 17/17] [#302] some refactoring + improved error message --- .../xceptance/neodymium/util/LighthouseUtils.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 0f47a004..da34df5c 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -41,6 +41,7 @@ public class LighthouseUtils public static void createLightHouseReport(String reportName) throws Exception { // validate that lighthouse is installed + String readerOutput = ""; try { if (System.getProperty("os.name").toLowerCase().contains("win")) @@ -53,6 +54,14 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System try { Process p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + String readerLine; + while ((readerLine = r.readLine()) != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + checkErrorCodeProcess(p); } catch (Exception e) @@ -68,7 +77,9 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (Exception e) { - throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location.", e); + throw new Exception("lighthouse binary can't be run (" + Neodymium.configuration().lighthouseBinaryPath() + + "), please install lighthouse and add it to the PATH or enter the correct lighthouse binary location.\n\nProcess Log: " + + readerOutput, e); } // validate chrome browser (lighthouse only works for chrome) @@ -210,7 +221,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (IOException e) { - throw new Exception("failed to run the process\n\nProcess Log: " + readerOutput, e); + throw new Exception("failed to run the lighthouse process. \n\nProcess Log: " + readerOutput, e); } }