diff --git a/config/neodymium.properties b/config/neodymium.properties index 6b67cadb..3469a844 100644 --- a/config/neodymium.properties +++ b/config/neodymium.properties @@ -283,7 +283,7 @@ neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5 # The actual value for the seo score varies alot, so consider using a lower threshold to avoid a lot of false alerts neodymium.lighthouse.assert.thresholdScore.seo = 0.5 -# To be able to validate Lighthouse report audits, wo use internal json id's from the report itself +# To be able to validate Lighthouse report audits, we use internal json id's from the report itself # A full list of all audit id's and their corresponding titles can be found here: https://github.com/Xceptance/neodymium/wiki/Reports#lighthouse-audit-validation #neodymium.lighthouse.assert.audits = diff --git a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java index 6be80a82..0f47a004 100644 --- a/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java +++ b/src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java @@ -45,19 +45,21 @@ public static void createLightHouseReport(String reportName) throws Exception { if (System.getProperty("os.name").toLowerCase().contains("win")) { - ProcessBuilder builder = new ProcessBuilder(); - builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); - Process p = builder.start(); - int errorCode = p.waitFor(); - - if (errorCode != 0) - { - throw new IOException(); - } + Process p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start(); + try + { + Process p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); + } + catch (Exception e) + { + Process p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--version"); + checkErrorCodeProcess(p); + } } else { @@ -66,7 +68,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (Exception e) { - throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location. " + e); + throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location.", e); } // validate chrome browser (lighthouse only works for chrome) @@ -153,28 +155,64 @@ private static String windowOperations(WebDriver driver) */ private static void lighthouseAudit(String URL, String reportName) throws Exception { - ProcessBuilder builder = new ProcessBuilder(); + Process p = null; + String readerOutput = ""; + String readerLine= ""; - if (System.getProperty("os.name").toLowerCase().contains("win")) - { - builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); - } - else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + try { - builder = new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + if (System.getProperty("os.name").toLowerCase().contains("win")) + { + p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while ((readerLine = r.readLine()) != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) + { + try + { + p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while (r.readLine() != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + catch (Exception e) + { + p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json"); + + BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); + while (r.readLine() != null) + { + readerOutput = readerOutput + readerLine; + continue; + } + + checkErrorCodeProcess(p); + } + } + else + { + throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + } } - else + catch (IOException e) { - throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); + throw new Exception("failed to run the process\n\nProcess Log: " + readerOutput, e); } - builder.redirectErrorStream(true); - Process p = builder.start(); - BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); - while (r.readLine() != null) - { - continue; - } } /** @@ -223,4 +261,40 @@ private static void validateAudits(File json, String reportName) throws Exceptio }); } } + + /** + * Runs a process with the in params specified command + * + * @param + * params the command with all required parameters + * @return + * the process + * @throws IOException + */ + private static Process runProcess(String... params) throws IOException + { + ProcessBuilder builder = new ProcessBuilder(); + builder = new ProcessBuilder(params); + builder.redirectErrorStream(true); + + return builder.start(); + } + + /** + * Checks the error code for the specified process + * + * @param p + * the process of which the error code needs to be checked + * @throws InterruptedException + * @throws IOException + */ + private static void checkErrorCodeProcess(Process p) throws InterruptedException, IOException + { + int errorCode = p.waitFor(); + + if (errorCode != 0) + { + throw new IOException("process error code differs from 0"); + } + } } diff --git a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java index 4e2faaea..001c0294 100644 --- a/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java +++ b/src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java @@ -2,6 +2,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import java.lang.reflect.Method; import org.junit.Assert; @@ -15,30 +16,38 @@ public class LighthouseUtilsTest @NeodymiumTest public void testLighthouseUtilsHappyPath() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessiblity", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessiblity", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); - ProcessBuilder builder = new ProcessBuilder(); + Class clazz = LighthouseUtils.class; + Method runProcess = clazz.getDeclaredMethod("runProcess", String[].class); + runProcess.setAccessible(true); + Process p = null; if (System.getProperty("os.name").toLowerCase().contains("win")) { Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #"); - builder = new ProcessBuilder("cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); } else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac")) { - Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); - builder = new ProcessBuilder("sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"); + try + { + Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json"); + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); + } + catch (Exception e) + { + p = (Process) runProcess.invoke(null, new Object[]{new String[]{"echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}}); + } } else { throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS"); } - builder.redirectErrorStream(true); - Process p = builder.start(); BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream())); while (r.readLine() != null) { @@ -53,10 +62,10 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System @NeodymiumTest public void testLighthouseUtilsPerformanceException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -79,17 +88,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsAccessibilityException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -112,17 +121,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsBestPracticesException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.51"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -145,17 +154,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } } @NeodymiumTest public void testLighthouseUtilsSeoException() throws Exception { - Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5"); - Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.51"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5"); + Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.51"); if (System.getProperty("os.name").toLowerCase().contains("win")) { @@ -178,7 +187,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System } catch (AssertionError e) { - Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); + Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations")); } }