fix: work with extremely large results (dequelabs#667)

* fix: work with extremely large partial results * fix tests * try again
mrtnvh · Feb 13, 2023 · 395d5fc · 395d5fc
1 parent 31bb4c5
commit 395d5fc
Show file tree

Hide file tree

Showing 17 changed files with 232 additions and 71 deletions.
diff --git a/packages/playwright/package-lock.json b/packages/playwright/package-lock.json
diff --git a/packages/playwright/src/browser.ts b/packages/playwright/src/browser.ts
@@ -14,6 +14,7 @@ import axeCore from 'axe-core';
 declare global {
   interface Window {
     axe: typeof axeCore;
+    partialResults: string;
   }
 }
 export const axeGetFrameContexts = ({
@@ -36,8 +37,12 @@ export const axeRunPartial = ({
 };
 
 export const axeFinishRun = ({
-  partialResults,
   options
 }: FinishRunParams): Promise<AxeResults> => {
-  return window.axe.finishRun(partialResults, options);
+  return window.axe.finishRun(JSON.parse(window.partialResults), options);
 };
+
+export function chunkResultString(chunk: string) {
+  window.partialResults ??= '';
+  window.partialResults += chunk;
+}
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
@@ -17,7 +17,8 @@ import {
   axeFinishRun,
   axeGetFrameContexts,
   axeRunPartial,
-  axeShadowSelect
+  axeShadowSelect,
+  chunkResultString
 } from './browser';
 import AxePartialRunner from './AxePartialRunner';
 
@@ -283,9 +284,24 @@ export default class AxeBuilder {
 
     blankPage.evaluate(this.script());
     blankPage.evaluate(await this.axeConfigure());
+
+    // evaluate has a size limit on the number of characters so we'll need
+    // to split partialResults into chunks if it exceeds that limit.
+    const sizeLimit = 60_000_000;
+    const partialString = JSON.stringify(partialResults);
+
+    async function chunkResults(result: string): Promise<void> {
+      const chunk = result.substring(0, sizeLimit);
+      await blankPage.evaluate(chunkResultString, chunk);
+
+      if (result.length > sizeLimit) {
+        return await chunkResults(result.substr(sizeLimit));
+      }
+    }
+
+    await chunkResults(partialString);
     return await blankPage
       .evaluate(axeFinishRun, {
-        partialResults,
         options
       })
       .finally(async () => {

diff --git a/packages/playwright/src/types.ts b/packages/playwright/src/types.ts
@@ -39,6 +39,5 @@ export interface RunPartialParams {
 }
 
 export interface FinishRunParams {
-  partialResults: PartialResults[];
   options: RunOptions;
 }
diff --git a/packages/playwright/tests/axe-playwright.spec.ts b/packages/playwright/tests/axe-playwright.spec.ts
@@ -35,6 +35,10 @@ describe('@axe-core/playwright', () => {
     path.join(externalPath, 'axe-force-legacy.js'),
     'utf8'
   );
+  const axeLargePartial = fs.readFileSync(
+    path.join(externalPath, 'axe-large-partial.js'),
+    'utf8'
+  );
 
   before(async () => {
     const app = express();
@@ -118,6 +122,22 @@ describe('@axe-core/playwright', () => {
       assert.isUndefined(err);
     });
 
+    it('handles large results', async function () {
+      /* this test handles a large amount of partial results a timeout may be required */
+      this.timeout(100_000);
+      const res = await await page.goto(`${addr}/external/index.html`);
+
+      assert.equal(res?.status(), 200);
+
+      const results = await new AxeBuilder({
+        page,
+        axeSource: axeSource + axeLargePartial
+      }).analyze();
+
+      assert.lengthOf(results.passes, 1);
+      assert.equal(results.passes[0].id, 'duplicate-id');
+    });
+
     it('reports frame-tested', async () => {
       const res = await page.goto(`${addr}/external/crash-parent.html`);
       const results = await new AxeBuilder({

diff --git a/packages/puppeteer/package-lock.json b/packages/puppeteer/package-lock.json
diff --git a/packages/puppeteer/src/axePuppeteer.ts b/packages/puppeteer/src/axePuppeteer.ts
@@ -14,7 +14,8 @@ import {
   axeFinishRun,
   axeConfigure,
   axeRunLegacy,
-  axeRunPartialSupport
+  axeRunPartialSupport,
+  chunkResultString
 } from './browser';
 import { AnalyzeCB, PartialResults } from './types';
 import { iframeSelector, injectJS } from './legacy';
@@ -280,8 +281,24 @@ export class AxePuppeteer {
     );
 
     await frameSourceInject(blankPage.mainFrame(), axeSource, config);
+
+    // evaluate has a size limit on the number of characters so we'll need
+    // to split partialResults into chunks if it exceeds that limit.
+    const sizeLimit = 60_000_000;
+    const partialString = JSON.stringify(partialResults);
+
+    async function chunkResults(result: string): Promise<void> {
+      const chunk = result.substring(0, sizeLimit);
+      await blankPage.evaluate(chunkResultString, chunk);
+
+      if (result.length > sizeLimit) {
+        return await chunkResults(result.substr(sizeLimit));
+      }
+    }
+
+    await chunkResults(partialString);
     return await blankPage
-      .evaluate(axeFinishRun, partialResults, axeOptions)
+      .evaluate(axeFinishRun, axeOptions)
       .finally(async () => {
         await blankPage.close();
       });

diff --git a/packages/puppeteer/src/browser.ts b/packages/puppeteer/src/browser.ts
@@ -9,6 +9,7 @@ declare global {
   // tslint:disable-next-line:interface-name
   interface Window {
     axe: typeof Axe;
+    partialResults: string;
   }
 }
 
@@ -48,11 +49,8 @@ export function axeRunPartial(
   return window.axe.runPartial(context, options);
 }
 
-export function axeFinishRun(
-  partials: PartialResults,
-  options: Axe.RunOptions
-): Promise<Axe.AxeResults> {
-  return window.axe.finishRun(partials, options);
+export function axeFinishRun(options: Axe.RunOptions): Promise<Axe.AxeResults> {
+  return window.axe.finishRun(JSON.parse(window.partialResults), options);
 }
 
 // Defined at top-level to clarify that it can't capture variables from outer scope.
@@ -62,3 +60,8 @@ export function axeRunLegacy(
 ): Promise<Axe.AxeResults> {
   return window.axe.run(context || document, options || {});
 }
+
+export function chunkResultString(chunk: string) {
+  window.partialResults ??= '';
+  window.partialResults += chunk;
+}
diff --git a/packages/puppeteer/test/axePuppeteer.test.ts b/packages/puppeteer/test/axePuppeteer.test.ts
@@ -33,6 +33,7 @@ describe('AxePuppeteer', function () {
   let axeSource: string;
   let axeCrasherSource: string;
   let axeForceLegacy: string;
+  let axeLargePartial: string;
 
   before(async () => {
     const axePath = require.resolve('axe-core');
@@ -46,6 +47,10 @@ describe('AxePuppeteer', function () {
       path.join(externalPath, 'axe-force-legacy.js'),
       'utf8'
     );
+    axeLargePartial = fs.readFileSync(
+      path.join(externalPath, 'axe-large-partial.js'),
+      'utf8'
+    );
   });
 
   before(async () => {
@@ -171,6 +176,22 @@ describe('AxePuppeteer', function () {
       assert.isUndefined(err);
     });
 
+    it('handles large results', async function () {
+      /* this test handles a large amount of partial results a timeout may be required */
+      this.timeout(50_000);
+      const res = await await page.goto(`${addr}/external/index.html`);
+
+      assert.equal(res?.status(), 200);
+
+      const results = await new AxePuppeteer(
+        page,
+        axeSource + axeLargePartial
+      ).analyze();
+
+      assert.lengthOf(results.passes, 1);
+      assert.equal(results.passes[0].id, 'duplicate-id');
+    });
+
     it('returns the same results from runPartial as from legacy mode', async () => {
       const res = await page.goto(`${addr}/external/nested-iframes.html`);
       const legacyResults = await new AxePuppeteer(

diff --git a/packages/webdriverio/package-lock.json b/packages/webdriverio/package-lock.json
diff --git a/packages/webdriverio/src/test.ts b/packages/webdriverio/src/test.ts
@@ -85,6 +85,10 @@ describe('@axe-core/webdriverio', () => {
         path.join(axeTestFixtures, 'axe-force-legacy.js'),
         'utf8'
       );
+      const axeLargePartial = fs.readFileSync(
+        path.join(axeTestFixtures, 'axe-large-partial.js'),
+        'utf8'
+      );
 
       beforeEach(async () => {
         const app = express();
@@ -482,6 +486,20 @@ describe('@axe-core/webdriverio', () => {
             assert.isNull(error);
           });
 
+          it('handles large results', async function () {
+            /* this test handles a large amount of partial results a timeout may be required */
+            this.timeout(100_000);
+            await client.url(`${addr}/external/index.html`);
+
+            const results = await new AxeBuilder({
+              client,
+              axeSource: axeSource + axeLargePartial
+            }).analyze();
+
+            assert.lengthOf(results.passes, 1);
+            assert.equal(results.passes[0].id, 'duplicate-id');
+          });
+
           it('returns correct results metadata', async () => {
             await client.url(`${addr}/index.html`);
             const title = await client.getTitle();

diff --git a/packages/webdriverio/src/utils.ts b/packages/webdriverio/src/utils.ts
@@ -165,23 +165,48 @@ export const axeFinishRun = (
   partialResults: PartialResults,
   options: RunOptions
 ): Promise<AxeResults> => {
-  return promisify(
-    client
-      .executeAsync<string, never>(
-        `var callback = arguments[arguments.length - 1];
+  // executeScript has a size limit of ~32 million characters so we'll need
+  // to split partialResults into chunks if it exceeds that limit.
+  // since we need to stringify twice we need to leave room for the double escaped quotes
+  const sizeLimit = 15_000_000;
+  const partialString = JSON.stringify(
+    partialResults.map(res => JSON.stringify(res))
+  );
+  function chunkResults(result: string): Promise<void> {
+    const chunk = JSON.stringify(result.substring(0, sizeLimit));
+    return promisify(
+      client.execute(
+        `
+        window.partialResults ??= '';
+        window.partialResults += ${chunk};
+        `
+      )
+    ).then(() => {
+      if (result.length > sizeLimit) {
+        return chunkResults(result.substr(sizeLimit));
+      }
+    });
+  }
+
+  return chunkResults(partialString)
+    .then(() => {
+      return promisify(
+        client.executeAsync<string, never>(
+          `var callback = arguments[arguments.length - 1];
       ${axeSource};
       window.axe.configure({
         branding: { application: 'webdriverio' }
       });
 
-      var partialResults = ${JSON.stringify(partialResults)};
+      var partialResults = JSON.parse(window.partialResults).map(res => JSON.parse(res));
       var options = ${JSON.stringify(options || {})};
       window.axe.finishRun(partialResults, options).then(function (axeResults) {
         callback(JSON.stringify(axeResults))
       });`
-      )
-      .then((r: string) => deserialize<AxeResults>(r))
-  );
+        )
+      );
+    })
+    .then((r: string) => deserialize<AxeResults>(r));
 };
 
 export const configureAllowedOrigins = (

diff --git a/packages/webdriverjs/package-lock.json b/packages/webdriverjs/package-lock.json