Gauge Taiko-Accessibility plugin issue

[harimadusumilli]

Taiko-Accessibility plugin appears to have discrepancies in the results that it provides when compared to selenium-accessibility using Axe-Core library by Deque labs

Here is what we did:
Installed the following softwares:
Gauge version: 1.0.7
Commit Hash: ed7b4fd

Plugins

dotnet (0.1.3)
html-report (4.0.9)
js (2.3.7)
json-report (0.3.3)
screenshot (0.0.1)
xml-report (0.2.3)

Installed taiko-accessibility plugin
Installed the axe-core plugin version - version 4.0.2
Added the following piece of code in index.js present in the taiko-accessbility\node_modules\axe-core. The code that has been modified in order to get results from runAudit method of taiko-accessbility plugin is as follows:

const runAudit = async () => {
    try{
    await injectAxe();
    const response = await _runtime.evaluate({expression: '{testResult: axe.run({resultTypes: ["violations"]})}', awaitPromise: true, returnByValue: true});
  //console.log(response);
    const result = response.result.value;
    
    return {
        score: calculateScore(result.passes, result.violations),
        violations: result.violations, incomplete: result.incomplete, passes: result.passes, inapplicable: result.inapplicable, url: result.url, timestamp: result.timestamp, testEngine: result.testEngine, testEnvironment: result.testEnvironment
    };
}
catch(error){
    console.log("ErrorAudit" + error);
}
};

Now, after adding this piece of code we created the following spec file and its implementation as follows:

Spec file :

# Testing Axe Taiko plugin

## Scenario of Taiko - Axe Plugin
* Launch Application url: "http://puppies.herokuapp.com/"

Implementation file Code (js) file is as given below:

"use strict";
var assert = require("assert");
const fs = require('fs');
var axe = require('axe-core');
let _runtime;

const {accessibility, openBrowser, goto, currentURL,write, click, link, into, textBox, button,closeBrowser, waitFor} = require('taiko');

step("Launch Application url: ", async function(url) {
    await openBrowser({headless:false,args:['--start-maximized']});
    await goto(url);
    waitFor(20000);
    await accessibilityReport();
      gauge.screenshot();
    assert.ok(await currentURL(),url);
});
const accessibilityReport = async() => {
    const audit = await accessibility.runAudit();
    console.log("Accessiblity Check");
    console.log("Context:");
    console.log("Url : " + audit.url);
    console.log("Orientation : " + audit.testEnvironment.orientationType);
    console.log("Size : " + audit.testEnvironment.windowWidth + " X " + audit.testEnvironment.windowHeight);
    console.log("Time : " + audit.timestamp);
    console.log("User agent : " + audit.testEnvironment.userAgent);
    console.log("Using : " + audit.testEngine.name + " " + audit.testEngine.version);
    console.log("Counts:");
    console.log("score : " + audit.score);
    console.log("violations : " + audit.violations.length);
    console.log("incomplete : " + audit.incomplete.length);
    console.log("passes : " + audit.passes.length);
    console.log("inapplicable : " + audit.inapplicable.length);
    /console.log("Violations : ");
    for(let val of audit.violations) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Impact : " + val.impact);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("incomplete : ");
    for(let val of audit.incomplete) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Impact : " + val.impact);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("passes : ");
    for(let val of audit.passes) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Tags : " + val.tags);
        console.log("Nodes : " + val.nodes.length);
        for(let a of val.nodes) {
            console.log("Html : " + a.html);
            console.log("Selector(s): " + a.target[0]);
        }
    }
    console.log("inapplicable : ");
    for(let val of audit.inapplicable) {
        console.log("Description : " + val.description);
        console.log("Help : " + val.help);
        console.log("HelpUrl : " + val.helpUrl);
        console.log("Tags : " + val.tags);
    }/
};

Output of the run is as given below:
Accessiblity Check with Taiko-Accessibility-Plugin
Context:
Url : http://puppies.herokuapp.com/
Orientation : landscape-primary
Size : 1920 X 877
Time : 2020-12-18T16:33:18.341Z
User agent : Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60
Using : axe-core 4.0.2
Counts:
score : 75
violations : 6
incomplete : 1
passes : 18
inapplicable : 53

When the same test is run using Selenium axe plugin we get the following results:
Environment: Gauge + Selenium + Axe-Core + Dotnet Core Context: Url: http://puppies.herokuapp.com/ Orientation: landscape-primary Size: 1280 x 565 Time: 12/7/2020 2:37:11 PM +00:00 User agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Using: axe-core (4.0.2)
Counts: Violation: 11
Incomplete: 14
Pass: 191
Inapplicable: 53

Notice that except for the inapplicable all the other metrics such as Violations, Incomplete and passes do not match there is a huge discrepancy.

[BugDiver]

The taiko-accessibility plugin is developed and maintained by @andreas-ku. Please try to raise an issue there

[harimadusumilli]

I have raised an issue in the taiko-accessibility plugin forum. There has been no response. So, I had raised it in the spectrum/chat and Zabil had requested that I post it and since that forum is closed. i have raised it here and also in the taiko forum, hoping that I could find some help.

[andreas-ku]

@harimadusumilli just answered :) andreas-ku/taiko-accessibility#15 (comment)

[harimadusumilli]

Thank you Andreas