diff --git a/lib/report-builder-factory/report-builder.js b/lib/report-builder-factory/report-builder.js index 794860478..2c15490f4 100644 --- a/lib/report-builder-factory/report-builder.js +++ b/lib/report-builder-factory/report-builder.js @@ -6,7 +6,7 @@ const _ = require('lodash'); const fs = require('fs-extra'); const {IDLE, RUNNING, SUCCESS, FAIL, ERROR, SKIPPED, UPDATED} = require('../constants/test-statuses'); const {logger, getPathsFor, hasImage} = require('../server-utils'); -const {setStatusForBranch, hasFails} = require('../static/modules/utils'); +const {setStatusForBranch, hasFails, hasNoRefImageErrors} = require('../static/modules/utils'); const NO_STATE = 'NO_STATE'; @@ -118,7 +118,6 @@ module.exports = class ReportBuilder { _addTestResult(formattedResult, props) { const testResult = this._createTestResult(formattedResult, _.extend(props, {attempt: 0})); const {suite, browserId} = formattedResult; - const suitePath = suite.path.concat(formattedResult.state ? formattedResult.state.name : NO_STATE); const node = findOrCreate(this._tree, suitePath, testResult.status); node.browsers = Array.isArray(node.browsers) ? node.browsers : []; @@ -128,6 +127,11 @@ module.exports = class ReportBuilder { formattedResult.attempt = testResult.attempt; formattedResult.image = hasImage(formattedResult); extendTestWithImagePaths(testResult, formattedResult); + + if (hasNoRefImageErrors(formattedResult)) { + testResult.status = FAIL; + } + node.browsers.push({name: browserId, result: testResult, retries: []}); setStatusForBranch(this._tree, node.suitePath, testResult.status); @@ -157,6 +161,8 @@ module.exports = class ReportBuilder { if (!hasFails(stateInBrowser)) { stateInBrowser.result.status = SUCCESS; + } else if (hasNoRefImageErrors(stateInBrowser.result)) { + stateInBrowser.result.status = FAIL; } setStatusForBranch(this._tree, node.suitePath, testResult.status); diff --git a/lib/static/components/section/body/index.js b/lib/static/components/section/body/index.js index 04d6f07a4..e96c4ca39 100644 --- a/lib/static/components/section/body/index.js +++ b/lib/static/components/section/body/index.js @@ -92,9 +92,7 @@ class Body extends Component { return this._drawTab(state, stateName || idx); }); - const {multipleTabs, status, screenshot} = activeResult; - - return multipleTabs && isErroredStatus(status) && !screenshot + return this._shouldAddErrorTab(activeResult) ? tabs.concat(this._drawTab(activeResult)) : tabs; } @@ -109,6 +107,10 @@ class Body extends Component { ); } + _shouldAddErrorTab({multipleTabs, status, screenshot}) { + return multipleTabs && isErroredStatus(status) && !screenshot; + } + render() { const {retries} = this.props; const activeResult = this._getActiveResult(); diff --git a/lib/static/modules/utils.js b/lib/static/modules/utils.js index d179289a2..fd7d88080 100644 --- a/lib/static/modules/utils.js +++ b/lib/static/modules/utils.js @@ -13,6 +13,10 @@ function hasFailedImages(result) { || isErroredStatus(status) || isFailStatus(status); } +function hasNoRefImageErrors({imagesInfo = []}) { + return Boolean(imagesInfo.filter((v) => get(v, 'reason.stack', '').startsWith(NO_REF_IMAGE_ERROR)).length); +} + function hasFails(node) { const {result} = node; const isFailed = result && hasFailedImages(result); @@ -103,6 +107,7 @@ function setStatusForBranch(nodes, suitePath, status) { } module.exports = { + hasNoRefImageErrors, hasFails, isSuiteFailed, isAcceptable, diff --git a/test/lib/report-builder-factory/report-builder.js b/test/lib/report-builder-factory/report-builder.js index 15e1169c5..a923ff4eb 100644 --- a/test/lib/report-builder-factory/report-builder.js +++ b/test/lib/report-builder-factory/report-builder.js @@ -5,6 +5,8 @@ const _ = require('lodash'); const {logger} = require('../../../lib/server-utils'); const proxyquire = require('proxyquire'); const {SUCCESS, FAIL, ERROR, SKIPPED, IDLE, UPDATED} = require('../../../lib/constants/test-statuses'); +const {getCommonErrors} = require('../../../lib/constants/errors'); +const {NO_REF_IMAGE_ERROR} = getCommonErrors(); describe('ReportBuilder', () => { const sandbox = sinon.sandbox.create(); @@ -261,6 +263,43 @@ describe('ReportBuilder', () => { }); }); + describe('should rewrite suite status to "fail" if test failed with no reference image error', () => { + it('and test does not exist in tests tree', () => { + const reportBuilder = mkReportBuilder_(); + const test = stubTest_({ + status: ERROR, + imagesInfo: [{ + stateName: 'plain', status: ERROR, + reason: {stack: `${NO_REF_IMAGE_ERROR}: ...`} + }] + }); + + reportBuilder.addError(test); + + const suiteResult = getSuiteResult_(reportBuilder); + + assert.equal(suiteResult.status, FAIL); + }); + + it('and test exists in tests tree', () => { + const reportBuilder = mkReportBuilder_(); + const test = stubTest_({ + status: ERROR, + imagesInfo: [{ + stateName: 'plain', status: ERROR, + reason: {stack: `${NO_REF_IMAGE_ERROR}: ...`} + }] + }); + + reportBuilder.addIdle(test); + reportBuilder.addError(test); + + const suiteResult = getSuiteResult_(reportBuilder); + + assert.equal(suiteResult.status, FAIL); + }); + }); + describe('should not rewrite suite status to "success" if image comparison is successful, but test', () => { [ {status: FAIL, methodName: 'addFail'},