feat: new parsing algorithm

forsti0506 · Jan 10, 2021 · d2c9817 · d2c9817
1 parent b503a7b
commit d2c9817
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 31 deletions.
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/bin/a11y-sitechecker.ts b/bin/a11y-sitechecker.ts
@@ -15,6 +15,7 @@ import {
 import { analyzeSite } from '../lib/a11y-sitechecker';
 import { Result, Spec } from 'axe-core';
 import * as puppeteer from 'puppeteer';
+import { Page } from 'puppeteer';
 
 const config: Config = { json: true, resultsPath: 'results', axeConfig: {} };
 
@@ -142,6 +143,9 @@ function mergeResults(report: A11ySitecheckerResult): A11ySitecheckerResult {
         if (configFile.axeConfig.localePath && typeof configFile.axeConfig.localePath === 'string') {
             config.axeConfig.localePath = configFile.axeConfig.localePath;
         }
+        if (configFile.login) {
+            config.login = configFile.login;
+        }
     }
     const axeConfig: Spec = {};
     if (config.axeConfig?.locale) {
@@ -171,7 +175,10 @@ async function next(axeSpecs: Spec): Promise<void> {
             height: 1080,
         });
 
-        let report = await analyzeSite(commander.args[0], axeSpecs);
+        await executeLogin(commander.args[0], page);
+
+        let report = await analyzeSite(commander.args[0], axeSpecs, page, config);
+        await browser.close();
         report.url = commander.args[0];
         report = mergeResults(report);
         if (config.json) {
@@ -196,3 +203,27 @@ async function next(axeSpecs: Spec): Promise<void> {
         process.exit(1);
     }
 }
+
+async function executeLogin(url: string, page: Page): Promise<void> {
+    if (!config.login) {
+        return;
+    }
+    await page.goto(url, { waitUntil: 'networkidle2' });
+    // await page.waitForNavigation({ waitUntil: 'networkidle2' });
+    await page.screenshot({ path: 'images/yeah.png' });
+    for (const step of config.login) {
+        for (const input of step.input) {
+            await page.type(input.selector, input.value);
+            await page.screenshot({ path: 'images/yeah.png' });
+        }
+        await page.click(step.submit);
+    }
+    await page.screenshot({ path: 'images/login.png' });
+    try {
+        await page.waitForNavigation({ waitUntil: 'networkidle2' });
+        await page.screenshot({ path: 'images/lopin2.png' });
+    } catch (e) {
+        // eslint-disable-next-line prettier/prettier
+        console.log(chalk.red('No Navigation after Login. Please check if it\'s working as expected!'));
+    }
+}
diff --git a/lib/a11y-sitechecker.ts b/lib/a11y-sitechecker.ts
@@ -1,16 +1,19 @@
 import * as fs from 'fs';
 import { AxePuppeteer } from '@axe-core/puppeteer';
 import * as chalk from 'chalk';
-import * as puppeteer from 'puppeteer';
 import * as JSDOM from 'jsdom';
 import { Spec } from 'axe-core';
 import { A11ySitecheckerResult } from './models/a11y-sitechecker-result';
+import { Page } from 'puppeteer';
+import { Config } from './models/config';
 
 const alreadyVisited = [];
 const alreadyParsed = [];
 const notCheckedLinks = [];
 
 const log = console.log;
+const debug = console.debug;
+let rootDomain;
 
 let results: A11ySitecheckerResult = {
     testEngine: undefined,
@@ -31,49 +34,50 @@ function getEscaped(link: string): string {
     return link.replace(/[`~!@#$%^&*()_|+\-=?;:'",.<>{}\\[\]/]/gi, '_');
 }
 
-export async function analyzeSite(url: string, axeSpecs: Spec): Promise<A11ySitecheckerResult> {
+export async function analyzeSite(
+    url: string,
+    axeSpecs: Spec,
+    page: Page,
+    config: Config,
+): Promise<A11ySitecheckerResult> {
     if (!url.startsWith('https://')) {
         url = 'https://' + url;
     }
     if (!url.endsWith('/')) {
         url = url + '/';
     }
 
-    const browser = await puppeteer.launch();
-    const page = await browser.newPage();
-    await page.setViewport({
-        width: 1920,
-        height: 1080,
-    });
+    log(chalk.blue('Start analyze of ' + url));
+
+    await analyzeUrl(page, url, axeSpecs, null, config);
 
-    await analyzeUrl(page, url, axeSpecs);
     const html = await page.content();
     const links = getLinks(html, url);
 
     for (const link of links) {
-        if (!alreadyVisited.includes(link)) {
-            await analyzeUrl(page, link, axeSpecs, url);
-        }
+        await analyzeUrl(page, link, axeSpecs, url, config);
     }
-    await browser.close();
-    let count = 0;
     for (const link of links) {
-        if (count >= 1) break;
         if (!alreadyParsed.includes(link)) {
-            const res: A11ySitecheckerResult = await analyzeSite(link, axeSpecs);
-            alreadyParsed.push(link);
+            const res: A11ySitecheckerResult = await analyzeSite(link, axeSpecs, page, config);
+            debug(chalk.yellow('Finished analyze of: ' + link));
             results = { ...results, ...res };
         }
-        count++;
     }
     results.analyzedUrls = alreadyParsed;
     return results;
 }
 
 function getLinks(html: string, url: string): string[] {
+    if (alreadyParsed.includes(url)) {
+        return [];
+    }
     const dom = new JSDOM.JSDOM(html, { contentType: 'text/html' });
     const links: string[] = [];
-    const rootDomain = new URL(url).hostname;
+    if (!rootDomain) {
+        debug(chalk.green('RootDomain was set to: ' + url));
+        rootDomain = url;
+    }
     dom.window.document.querySelectorAll('a').forEach((element: HTMLAnchorElement) => {
         let link = element.href;
         if (isAbsoluteUrl(link) && link.includes(rootDomain)) {
@@ -83,36 +87,55 @@ function getLinks(html: string, url: string): string[] {
             if (!link.endsWith('/') && !link.includes('#') && !isFile(link)) {
                 link = link + '/';
             }
-            if (!links.includes(link) && !isFile(link)) {
+            if (!links.includes(link) && !alreadyVisited.includes(link)) {
                 links.push(link);
             }
         } else if (!isAbsoluteUrl(link) && !link.includes('#')) {
-            let absoluteUrl = url + '/' + link;
+            let absoluteUrl = new URL(link, url).href;
             if (!absoluteUrl.endsWith('/') && !isFile(absoluteUrl)) {
                 absoluteUrl = absoluteUrl + '/';
             }
-            if (!links.includes(absoluteUrl) && !isFile(absoluteUrl)) {
+            if (
+                !links.includes(absoluteUrl) &&
+                !alreadyVisited.includes(absoluteUrl) &&
+                absoluteUrl.includes(rootDomain)
+            ) {
                 links.push(absoluteUrl);
             }
         } else if (!notCheckedLinks.includes(link)) {
             notCheckedLinks.push(link);
         }
     });
+    alreadyParsed.push(url);
     return links;
 }
 
 function isAbsoluteUrl(url): boolean {
     return /^(?:[a-z]+:)?\/\//i.test(url);
 }
 
-async function analyzeUrl(page, url: string, axeSpecs: Spec, backUrl?: string): Promise<void> {
-    log(chalk.blue('Currently analyzing ' + url));
+async function analyzeUrl(page, url: string, axeSpecs: Spec, backUrl: string, config: Config): Promise<void> {
     await page.goto(url, { waitUntil: 'networkidle2' });
+    if (alreadyVisited.includes(url)) {
+        debug(chalk.yellow('Already visited: ' + url));
+        return;
+    }
+    log(chalk.blue('Currently analyzing ' + url));
     if (!fs.existsSync('images')) {
         fs.mkdirSync('images');
     }
-    await page.screenshot({ path: 'images/' + getEscaped(url) + '.png' });
-    alreadyParsed.push(url);
+
+    if (config.saveImages) {
+        try {
+            const imagePath = 'images/' + getEscaped(url) + '.png';
+            await page.screenshot({ path: imagePath });
+            debug(chalk.yellow('Saved Image ' + imagePath));
+        } catch (error) {
+            log(chalk.red(error + '. Image not saved. Analyze not stopped!'));
+        }
+    }
+
+    // alreadyParsed.push(url);
     try {
         const axe = await new AxePuppeteer(page);
         axe.configure(axeSpecs);
@@ -131,7 +154,7 @@ async function analyzeUrl(page, url: string, axeSpecs: Spec, backUrl?: string):
             toolOptions: axeResults.toolOptions,
         });
         alreadyVisited.push(url);
-        log(chalk.green('Finished analyze of url.'));
+        log(chalk.green('Finished analyze of url: ' + url));
     } catch (error) {
         log(chalk.red(error + '. Analyze not stopped.'));
     }

diff --git a/lib/models/config.ts b/lib/models/config.ts
@@ -5,4 +5,17 @@ export interface Config {
         locale?: string;
         localePath?: string;
     };
+    login?: LoginStep[];
+    saveImages?: boolean;
+    imagesPath?: string;
+}
+
+interface LoginStep {
+    input: Input[];
+    submit: string;
+}
+
+interface Input {
+    selector: string;
+    value: string;
 }