Skip to content

Commit

Permalink
#302 - added requested changes in review
Browse files Browse the repository at this point in the history
  • Loading branch information
Jonas Hoyer committed Dec 2, 2024
1 parent ef9429b commit fe414b8
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 58 deletions.
2 changes: 1 addition & 1 deletion config/neodymium.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ neodymium.lighthouse.assert.thresholdScore.bestPractices = 0.5
# The actual value for the seo score varies alot, so consider using a lower threshold to avoid a lot of false alerts
neodymium.lighthouse.assert.thresholdScore.seo = 0.5

# To be able to validate Lighthouse report audits, wo use internal json id's from the report itself
# To be able to validate Lighthouse report audits, we use internal json id's from the report itself
# A full list of all audit id's and their corresponding titles can be found here: https://github.com/Xceptance/neodymium/wiki/Reports#lighthouse-audit-validation
#neodymium.lighthouse.assert.audits =

Expand Down
128 changes: 101 additions & 27 deletions src/main/java/com/xceptance/neodymium/util/LighthouseUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,21 @@ public static void createLightHouseReport(String reportName) throws Exception
{
if (System.getProperty("os.name").toLowerCase().contains("win"))
{
ProcessBuilder builder = new ProcessBuilder();
builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version");
Process p = builder.start();
int errorCode = p.waitFor();

if (errorCode != 0)
{
throw new IOException();
}
Process p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--version");
checkErrorCodeProcess(p);
}
else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac"))
{
new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version").start();
try
{
Process p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--version");
checkErrorCodeProcess(p);
}
catch (Exception e)
{
Process p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--version");
checkErrorCodeProcess(p);
}
}
else
{
Expand All @@ -66,7 +68,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
}
catch (Exception e)
{
throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location. " + e);
throw new Exception("lighthouse binary not found at " + Neodymium.configuration().lighthouseBinaryPath() + ", please install lighthouse and add it to the PATH or enter the correct lighthouse binary location.", e);
}

// validate chrome browser (lighthouse only works for chrome)
Expand Down Expand Up @@ -153,28 +155,64 @@ private static String windowOperations(WebDriver driver)
*/
private static void lighthouseAudit(String URL, String reportName) throws Exception
{
ProcessBuilder builder = new ProcessBuilder();
Process p = null;
String readerOutput = "";
String readerLine= "";

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
builder = new ProcessBuilder("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json");
}
else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac"))
try
{
builder = new ProcessBuilder("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json");
if (System.getProperty("os.name").toLowerCase().contains("win"))
{
p = runProcess("cmd.exe", "/c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json");

BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((readerLine = r.readLine()) != null)
{
readerOutput = readerOutput + readerLine;
continue;
}

checkErrorCodeProcess(p);
}
else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac"))
{
try
{
p = runProcess("sh", "-c", Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json");

BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
while (r.readLine() != null)
{
readerOutput = readerOutput + readerLine;
continue;
}

checkErrorCodeProcess(p);
}
catch (Exception e)
{
p = runProcess(Neodymium.configuration().lighthouseBinaryPath(), "--chrome-flags=\"--ignore-certificate-errors\"", URL, "--port=" + Neodymium.getRemoteDebuggingPort(), "--preset=desktop", "--output=json", "--output=html", "--output-path=target/" + reportName + ".json");

BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
while (r.readLine() != null)
{
readerOutput = readerOutput + readerLine;
continue;
}

checkErrorCodeProcess(p);
}
}
else
{
throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS");
}
}
else
catch (IOException e)
{
throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS");
throw new Exception("failed to run the process\n\nProcess Log: " + readerOutput, e);
}

builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
while (r.readLine() != null)
{
continue;
}
}

/**
Expand Down Expand Up @@ -223,4 +261,40 @@ private static void validateAudits(File json, String reportName) throws Exceptio
});
}
}

/**
* Runs a process with the in params specified command
*
* @param
* params the command with all required parameters
* @return
* the process
* @throws IOException
*/
private static Process runProcess(String... params) throws IOException
{
ProcessBuilder builder = new ProcessBuilder();
builder = new ProcessBuilder(params);
builder.redirectErrorStream(true);

return builder.start();
}

/**
* Checks the error code for the specified process
*
* @param p
* the process of which the error code needs to be checked
* @throws InterruptedException
* @throws IOException
*/
private static void checkErrorCodeProcess(Process p) throws InterruptedException, IOException
{
int errorCode = p.waitFor();

if (errorCode != 0)
{
throw new IOException("process error code differs from 0");
}
}
}
69 changes: 39 additions & 30 deletions src/test/java/com/xceptance/neodymium/util/LighthouseUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.reflect.Method;

import org.junit.Assert;

Expand All @@ -15,30 +16,38 @@ public class LighthouseUtilsTest
@NeodymiumTest
public void testLighthouseUtilsHappyPath() throws Exception
{
Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.accessiblity", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessiblity", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5");

ProcessBuilder builder = new ProcessBuilder();
Class<LighthouseUtils> clazz = LighthouseUtils.class;
Method runProcess = clazz.getDeclaredMethod("runProcess", String[].class);
runProcess.setAccessible(true);
Process p = null;

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json | echo makeCommentWork #");
builder = new ProcessBuilder("cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html");
p = (Process) runProcess.invoke(null, new Object[]{new String[]{"cmd.exe", "/c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}});
}
else if (System.getProperty("os.name").toLowerCase().contains("linux") || System.getProperty("os.name").toLowerCase().contains("mac"))
{
Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json");
builder = new ProcessBuilder("sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html");
try
{
Neodymium.configuration().setProperty("neodymium.lighthouse.binaryPath", "echo {\"categories\": {\"performance\": {\"score\": 0.5}, \"accessibility\": {\"score\": 0.5}, \"best-practices\": {\"score\": 0.5}, \"seo\": {\"score\": 0.5}}} > target/lighthouseUtilsReport.report.json");
p = (Process) runProcess.invoke(null, new Object[]{new String[]{"sh", "-c", "echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}});
}
catch (Exception e)
{
p = (Process) runProcess.invoke(null, new Object[]{new String[]{"echo fabricatedHtml > target/lighthouseUtilsReport.report.html"}});
}
}
else
{
throw new Exception("your current operation system is not supported, please use Windows, Linux or MacOS");
}

builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
while (r.readLine() != null)
{
Expand All @@ -53,10 +62,10 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
@NeodymiumTest
public void testLighthouseUtilsPerformanceException() throws Exception
{
Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5");

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
Expand All @@ -79,17 +88,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
}
catch (AssertionError e)
{
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse performance score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
}
}

@NeodymiumTest
public void testLighthouseUtilsAccessibilityException() throws Exception
{
Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5");

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
Expand All @@ -112,17 +121,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
}
catch (AssertionError e)
{
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse accessibility score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
}
}

@NeodymiumTest
public void testLighthouseUtilsBestPracticesException() throws Exception
{
Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.5");

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
Expand All @@ -145,17 +154,17 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
}
catch (AssertionError e)
{
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse best practices score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
}
}

@NeodymiumTest
public void testLighthouseUtilsSeoException() throws Exception
{
Neodymium.configuration().setProperty("neodymium.lighthouse.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.seo", "0.51");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.performance", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.accessibility", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.bestPractices", "0.5");
Neodymium.configuration().setProperty("neodymium.lighthouse.assert.thresholdScore.seo", "0.51");

if (System.getProperty("os.name").toLowerCase().contains("win"))
{
Expand All @@ -178,7 +187,7 @@ else if (System.getProperty("os.name").toLowerCase().contains("linux") || System
}
catch (AssertionError e)
{
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("the seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
Assert.assertTrue("the compared error messages doesn't match", e.getMessage().contains("The Lighthouse seo score 0.5 doesn't exceed nor match the required threshold of 0.51, please improve the score to match expectations"));
}
}

Expand Down

0 comments on commit fe414b8

Please sign in to comment.