Merge pull request #267 from gemini-testing/feat/FEI-12127

feat: save & show error details
gemini-testing · Oct 3, 2019 · 9b47661 · 9b47661
2 parents f2b981d + ef1bac9
commit 9b47661
Show file tree

Hide file tree

Showing 20 changed files with 511 additions and 70 deletions.
diff --git a/README.md b/README.md
@@ -20,6 +20,16 @@ Plugin has following configuration:
 * **path** (optional) `String` - path to directory for saving html report file; by
 default html report will be saved into `gemini-report/index.html` inside current work
 directory.
+* **saveErrorDetails** (optional) `Boolean` – save/don't save error details to json-files (to error-details folder); `false` by default.
+
+  Any plugin of hermione can add error details when throwing an error. Details can help a user to debug a problem in a test. Html-reporter saves these details to a file with name `<hash of suite path>-<browser>_<retry number>_<timestamp>.json` in the error-details folder. Below a stacktrace html-reporter adds the section `Error details` with the link `title` pointing to the json-file. A user can open it in a browser or any IDE.
+
+  How to add error details when throwing an error from a plugin:
+```
+   const err = new Error('some error');
+   err.details = {title: 'description, will be used as url title', data: {} | [] | 'some additional info'};
+   throw err;
+```
 * **defaultView** (optional) `String` - default view mode. Available values are:
   * `all` - show all tests. Default value.
   * `failed` - show only failed tests.

diff --git a/hermione.js b/hermione.js
@@ -32,6 +32,10 @@ async function prepare(hermione, reportBuilder, pluginConfig) {
         const formattedResult = reportBuilder.format(testResult);
         const actions = [formattedResult.saveTestImages(reportPath, workers)];
 
+        if (formattedResult.errorDetails) {
+            actions.push(formattedResult.saveErrorDetails(reportPath));
+        }
+
         if (formattedResult.screenshot) {
             actions.push(formattedResult.saveBase64Screenshot(reportPath));
         }

diff --git a/lib/config.js b/lib/config.js
@@ -76,6 +76,12 @@ const getParser = () => {
             defaultValue: 'html-report',
             validate: assertString('path')
         }),
+        saveErrorDetails: option({
+            defaultValue: false,
+            parseEnv: JSON.parse,
+            parseCli: JSON.parse,
+            validate: assertBoolean('saveErrorDetails')
+        }),
         defaultView: option({
             defaultValue: configDefaults.defaultView,
             validate: assertString('defaultView')

diff --git a/lib/constants/defaults.js b/lib/constants/defaults.js
@@ -3,6 +3,7 @@
 module.exports = {
     CIRCLE_RADIUS: 150,
     config: {
+        saveErrorDetails: false,
         defaultView: 'all',
         baseHost: '',
         scaleImages: false,

diff --git a/lib/constants/paths.js b/lib/constants/paths.js
@@ -0,0 +1,6 @@
+'use strict';
+
+module.exports = {
+    IMAGES_PATH: 'images',
+    ERROR_DETAILS_PATH: 'error-details'
+};
diff --git a/lib/gui/server.js b/lib/gui/server.js
@@ -7,6 +7,7 @@ const Promise = require('bluebird');
 const bodyParser = require('body-parser');
 const App = require('./app');
 const {MAX_REQUEST_SIZE, KEEP_ALIVE_TIMEOUT, HEADERS_TIMEOUT} = require('./constants/server');
+const {IMAGES_PATH, ERROR_DETAILS_PATH} = require('../constants/paths');
 const {logger} = require('../server-utils');
 
 exports.start = ({paths, tool, guiApi, configs}) => {
@@ -20,7 +21,8 @@ exports.start = ({paths, tool, guiApi, configs}) => {
 
     server.use(express.static(path.join(__dirname, '../static'), {index: 'gui.html'}));
     server.use(express.static(process.cwd()));
-    server.use('/images', express.static(path.join(process.cwd(), pluginConfig.path, 'images')));
+    server.use(`/${IMAGES_PATH}`, express.static(path.join(process.cwd(), pluginConfig.path, IMAGES_PATH)));
+    server.use(`/${ERROR_DETAILS_PATH}`, express.static(path.join(process.cwd(), pluginConfig.path, ERROR_DETAILS_PATH)));
 
     server.get('/', (req, res) => res.sendFile(path.join(__dirname, '../static', 'gui.html')));
 

diff --git a/lib/gui/tool-runner-factory/hermione/report-subscriber.js b/lib/gui/tool-runner-factory/hermione/report-subscriber.js
@@ -16,6 +16,10 @@ module.exports = (hermione, reportBuilder, client, reportPath) => {
             actions.push(formattedResult.saveBase64Screenshot(reportPath));
         }
 
+        if (formattedResult.errorDetails) {
+            actions.push(formattedResult.saveErrorDetails(reportPath));
+        }
+
         return Promise.all(actions);
     }
 

diff --git a/lib/report-builder-factory/report-builder.js b/lib/report-builder-factory/report-builder.js
@@ -115,15 +115,24 @@ module.exports = class ReportBuilder {
     }
 
     _createTestResult(result, props) {
-        const {browserId, suite, sessionId, description, imagesInfo, screenshot, multipleTabs} = result;
-        const {baseHost} = this._pluginConfig;
+        const {
+            browserId, suite, sessionId, description, imagesInfo, screenshot, multipleTabs, errorDetails
+        } = result;
+
+        const {baseHost, saveErrorDetails} = this._pluginConfig;
         const suiteUrl = suite.getUrl({browserId, baseHost});
         const metaInfo = _.merge(_.cloneDeep(result.meta), {url: suite.fullUrl, file: suite.file, sessionId});
 
-        return Object.assign({
+        const testResult = Object.assign({
             suiteUrl, name: browserId, metaInfo, description, imagesInfo,
             screenshot: Boolean(screenshot), multipleTabs
         }, props);
+
+        if (saveErrorDetails && errorDetails) {
+            testResult.errorDetails = _.pick(errorDetails, ['title', 'filePath']);
+        }
+
+        return testResult;
     }
 
     _getResultNode(formattedResult) {

diff --git a/lib/server-utils.js b/lib/server-utils.js
@@ -7,6 +7,7 @@ const _ = require('lodash');
 const fs = require('fs-extra');
 
 const {ERROR, UPDATED, IDLE} = require('./constants/test-statuses');
+const {IMAGES_PATH} = require('./constants/paths');
 
 const getReferencePath = (testResult, stateName) => createPath('ref', testResult, stateName);
 const getCurrentPath = (testResult, stateName) => createPath('current', testResult, stateName);
@@ -38,7 +39,7 @@ const getDiffAbsolutePath = (testResult, reportDir, stateName) => {
  */
 function createPath(kind, result, stateName) {
     const attempt = result.attempt || 0;
-    const imageDir = _.compact(['images', result.imageDir, stateName]);
+    const imageDir = _.compact([IMAGES_PATH, result.imageDir, stateName]);
     const components = imageDir.concat(`${result.browserId}~${kind}_${attempt}.png`);
 
     return path.join.apply(null, components);
@@ -89,7 +90,13 @@ function shouldUpdateAttempt(status) {
     return ![UPDATED, IDLE].includes(status);
 }
 
+function getDetailsFileName(testId, browserId, attempt) {
+    return `${testId}-${browserId}_${Number(attempt) + 1}_${Date.now()}.json`;
+}
+
 module.exports = {
+    getDetailsFileName,
+
     getReferencePath,
     getCurrentPath,
     getDiffPath,

diff --git a/lib/static/components/section/body/index.js b/lib/static/components/section/body/index.js
@@ -118,33 +118,43 @@ class Body extends Component {
     }
 
     _getActiveResult = () => {
+        if (this._activeResult) {
+            return this._activeResult;
+        }
+
         const {result, retries} = this.props;
 
-        return retries.concat(result)[this.state.retry];
+        this._activeResult = retries.concat(result)[this.state.retry];
+
+        return this._activeResult;
     }
 
     _getTabs() {
         const activeResult = this._getActiveResult();
+        const {errorDetails} = activeResult;
         const retryIndex = this.state.retry;
 
         if (isEmpty(activeResult.imagesInfo)) {
-            return isSuccessStatus(activeResult.status) ? null : this._drawTab(activeResult);
+            return isSuccessStatus(activeResult.status) ? null : this._drawTab(activeResult, {errorDetails});
         }
 
         const tabs = activeResult.imagesInfo.map((imageInfo, idx) => {
             const {stateName} = imageInfo;
             const error = imageInfo.error || activeResult.error;
 
-            return this._drawTab(imageInfo, `${stateName || idx}_${retryIndex}`, {image: true, error});
+            const options = imageInfo.stateName ? {} : {errorDetails};
+
+            return this._drawTab(imageInfo, {...options, image: true, error}, `${stateName || idx}_${retryIndex}`);
         });
 
         return this._shouldAddErrorTab(activeResult)
-            ? tabs.concat(this._drawTab(activeResult))
+            ? tabs.concat(this._drawTab(activeResult, {errorDetails}))
             : tabs;
     }
 
-    _drawTab(state, key = '', opts = {}) {
+    _drawTab(state, opts = {}, key = '') {
         const {result: {name: browserId}, suite: {suitePath}} = this.props;
+
         opts = defaults({error: state.error}, opts);
 
         return (

diff --git a/lib/static/components/state/error-details.js b/lib/static/components/state/error-details.js
@@ -0,0 +1,18 @@
+'use strict';
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import ToggleOpen from '../toggle-open';
+
+export default class ErrorDetails extends Component {
+    static propTypes = {
+        errorDetails: PropTypes.object.isRequired
+    };
+
+    render() {
+        const {title, filePath} = this.props.errorDetails;
+        const content = <div className="toggle-open__item"><a href={filePath} target="_blank">{title}</a></div>;
+
+        return <ToggleOpen title='Error details' content={content} extendClassNames="toggle-open_type_text"/>;
+    }
+}
diff --git a/lib/static/components/state/index.js b/lib/static/components/state/index.js
@@ -119,7 +119,7 @@ class State extends Component {
 
     render() {
         const {status, expectedImg, actualImg, diffImg, stateName, diffClusters} = this.props.state;
-        const {image, error} = this.props;
+        const {image, error, errorDetails} = this.props;
         let elem = null;
 
         if (!this.state.opened) {
@@ -132,12 +132,12 @@ class State extends Component {
         }
 
         if (isErroredStatus(status)) {
-            elem = <StateError image={Boolean(image)} actualImg={actualImg} error={error}/>;
+            elem = <StateError image={Boolean(image)} actualImg={actualImg} error={error} errorDetails={errorDetails}/>;
         } else if (isSuccessStatus(status) || isUpdatedStatus(status) || (isIdleStatus(status) && get(expectedImg, 'path'))) {
             elem = <StateSuccess status={status} expectedImg={expectedImg} />;
         } else if (isFailStatus(status)) {
             elem = error
-                ? <StateError image={Boolean(image)} actualImg={actualImg} error={error}/>
+                ? <StateError image={Boolean(image)} actualImg={actualImg} error={error} errorDetails={errorDetails}/>
                 : <StateFail expectedImg={expectedImg} actualImg={actualImg} diffImg={diffImg} diffClusters={diffClusters}/>;
         }
 

diff --git a/lib/static/components/state/state-error.js b/lib/static/components/state/state-error.js
@@ -6,6 +6,7 @@ import {map} from 'lodash';
 import Screenshot from './screenshot';
 import ToggleOpen from '../toggle-open';
 import {isNoRefImageError} from '../../modules/utils';
+import ErrorDetails from './error-details';
 
 export default class StateError extends Component {
     static propTypes = {
@@ -15,11 +16,12 @@ export default class StateError extends Component {
     };
 
     render() {
-        const {image, error, actualImg} = this.props;
+        const {image, error, errorDetails, actualImg} = this.props;
 
         return (
             <div className="image-box__image image-box__image_single">
                 <div className="error">{errorToElements(error)}</div>
+                {errorDetails && <ErrorDetails errorDetails={errorDetails}/>}
                 {this._drawImage(image, actualImg)}
             </div>
         );

diff --git a/lib/test-adapter/hermione-test-adapter.js b/lib/test-adapter/hermione-test-adapter.js
@@ -9,6 +9,7 @@ const TestAdapter = require('./test-adapter');
 const HermioneSuiteAdapter = require('../suite-adapter/hermione-suite-adapter.js');
 const {getSuitePath} = require('../plugin-utils').getHermioneUtils();
 const {SUCCESS, FAIL, ERROR} = require('../constants/test-statuses');
+const {ERROR_DETAILS_PATH} = require('../constants/paths');
 const utils = require('../server-utils');
 const fs = require('fs-extra');
 const crypto = require('crypto');
@@ -32,6 +33,8 @@ module.exports = class HermioneTestResultAdapter extends TestAdapter {
         this._suite = HermioneSuiteAdapter.create(this._testResult);
         this._imagesSaver = this._tool.htmlReporter.imagesSaver;
         this._testId = `${this._testResult.fullTitle()}.${this._testResult.browserId}`;
+        this._errorDetails = undefined;
+
         if (utils.shouldUpdateAttempt(status)) {
             testsAttempts.set(this._testId, _.isUndefined(testsAttempts.get(this._testId)) ? 0 : testsAttempts.get(this._testId) + 1);
         }
@@ -141,6 +144,28 @@ module.exports = class HermioneTestResultAdapter extends TestAdapter {
         return this._testResult.meta;
     }
 
+    get errorDetails() {
+        if (!_.isUndefined(this._errorDetails)) {
+            return this._errorDetails;
+        }
+
+        const details = _.get(this._testResult, 'err.details', null);
+
+        if (details) {
+            this._errorDetails = {
+                title: details.title || 'error details',
+                data: details.data,
+                filePath: `${ERROR_DETAILS_PATH}/${utils.getDetailsFileName(
+                    this._testResult.id, this._testResult.browserId, this.attempt
+                )}`
+            };
+        } else {
+            this._errorDetails = null;
+        }
+
+        return this._errorDetails;
+    }
+
     getRefImg(stateName) {
         return _.get(_.find(this.assertViewResults, {stateName}), 'refImg', {});
     }
@@ -164,6 +189,20 @@ module.exports = class HermioneTestResultAdapter extends TestAdapter {
         return true;
     }
 
+    async saveErrorDetails(reportPath) {
+        if (!this.errorDetails) {
+            return;
+        }
+
+        const detailsFilePath = path.resolve(reportPath, this.errorDetails.filePath);
+        const detailsData = _.isObject(this.errorDetails.data)
+            ? JSON.stringify(this.errorDetails.data, null, 2)
+            : this.errorDetails.data;
+
+        await utils.makeDirFor(detailsFilePath);
+        await fs.writeFile(detailsFilePath, detailsData);
+    }
+
     saveTestImages(reportPath, workers) {
         return Promise.map(this.assertViewResults, async (assertResult) => {
             const {stateName} = assertResult;

diff --git a/test/unit/lib/config.js b/test/unit/lib/config.js
@@ -68,6 +68,30 @@ describe('config', () => {
         });
     });
 
+    describe('"saveErrorDetails" option', () => {
+        it('should be false by default', () => {
+            assert.isFalse(parseConfig({}).saveErrorDetails);
+        });
+
+        it('should set from configuration file', () => {
+            const config = parseConfig({saveErrorDetails: true});
+
+            assert.isTrue(config.saveErrorDetails);
+        });
+
+        it('should set from environment variable', () => {
+            process.env['html_reporter_save_error_details'] = 'true';
+
+            assert.isTrue(parseConfig({}).saveErrorDetails);
+        });
+
+        it('should set from cli', () => {
+            process.argv = process.argv.concat('--html-reporter-save-error-details', 'true');
+
+            assert.isTrue(parseConfig({}).saveErrorDetails);
+        });
+    });
+
     describe('"defaultView" option', () => {
         it('should show all suites by default', () => {
             assert.equal(parseConfig({}).defaultView, 'all');