Merge pull request #136 from gemini-testing/dd.fix_no_ref_img

fix: ability to accept image on no reference image error
gemini-testing · Aug 1, 2018 · 4c409f1 · 4c409f1
2 parents eeddd07 + f4c5890
commit 4c409f1
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 5 deletions.
diff --git a/lib/report-builder-factory/report-builder.js b/lib/report-builder-factory/report-builder.js
@@ -6,7 +6,7 @@ const _ = require('lodash');
 const fs = require('fs-extra');
 const {IDLE, RUNNING, SUCCESS, FAIL, ERROR, SKIPPED, UPDATED} = require('../constants/test-statuses');
 const {logger, getPathsFor, hasImage} = require('../server-utils');
-const {setStatusForBranch, hasFails} = require('../static/modules/utils');
+const {setStatusForBranch, hasFails, hasNoRefImageErrors} = require('../static/modules/utils');
 
 const NO_STATE = 'NO_STATE';
 
@@ -118,7 +118,6 @@ module.exports = class ReportBuilder {
     _addTestResult(formattedResult, props) {
         const testResult = this._createTestResult(formattedResult, _.extend(props, {attempt: 0}));
         const {suite, browserId} = formattedResult;
-
         const suitePath = suite.path.concat(formattedResult.state ? formattedResult.state.name : NO_STATE);
         const node = findOrCreate(this._tree, suitePath, testResult.status);
         node.browsers = Array.isArray(node.browsers) ? node.browsers : [];
@@ -128,6 +127,11 @@ module.exports = class ReportBuilder {
             formattedResult.attempt = testResult.attempt;
             formattedResult.image = hasImage(formattedResult);
             extendTestWithImagePaths(testResult, formattedResult);
+
+            if (hasNoRefImageErrors(formattedResult)) {
+                testResult.status = FAIL;
+            }
+
             node.browsers.push({name: browserId, result: testResult, retries: []});
             setStatusForBranch(this._tree, node.suitePath, testResult.status);
 
@@ -157,6 +161,8 @@ module.exports = class ReportBuilder {
 
         if (!hasFails(stateInBrowser)) {
             stateInBrowser.result.status = SUCCESS;
+        } else if (hasNoRefImageErrors(stateInBrowser.result)) {
+            stateInBrowser.result.status = FAIL;
         }
 
         setStatusForBranch(this._tree, node.suitePath, testResult.status);

diff --git a/lib/static/components/section/body/index.js b/lib/static/components/section/body/index.js
@@ -92,9 +92,7 @@ class Body extends Component {
             return this._drawTab(state, stateName || idx);
         });
 
-        const {multipleTabs, status, screenshot} = activeResult;
-
-        return multipleTabs && isErroredStatus(status) && !screenshot
+        return this._shouldAddErrorTab(activeResult)
             ? tabs.concat(this._drawTab(activeResult))
             : tabs;
     }
@@ -109,6 +107,10 @@ class Body extends Component {
         );
     }
 
+    _shouldAddErrorTab({multipleTabs, status, screenshot}) {
+        return multipleTabs && isErroredStatus(status) && !screenshot;
+    }
+
     render() {
         const {retries} = this.props;
         const activeResult = this._getActiveResult();

diff --git a/lib/static/modules/utils.js b/lib/static/modules/utils.js
@@ -13,6 +13,10 @@ function hasFailedImages(result) {
         || isErroredStatus(status) || isFailStatus(status);
 }
 
+function hasNoRefImageErrors({imagesInfo = []}) {
+    return Boolean(imagesInfo.filter((v) => get(v, 'reason.stack', '').startsWith(NO_REF_IMAGE_ERROR)).length);
+}
+
 function hasFails(node) {
     const {result} = node;
     const isFailed = result && hasFailedImages(result);
@@ -103,6 +107,7 @@ function setStatusForBranch(nodes, suitePath, status) {
 }
 
 module.exports = {
+    hasNoRefImageErrors,
     hasFails,
     isSuiteFailed,
     isAcceptable,

diff --git a/test/lib/report-builder-factory/report-builder.js b/test/lib/report-builder-factory/report-builder.js
@@ -5,6 +5,8 @@ const _ = require('lodash');
 const {logger} = require('../../../lib/server-utils');
 const proxyquire = require('proxyquire');
 const {SUCCESS, FAIL, ERROR, SKIPPED, IDLE, UPDATED} = require('../../../lib/constants/test-statuses');
+const {getCommonErrors} = require('../../../lib/constants/errors');
+const {NO_REF_IMAGE_ERROR} = getCommonErrors();
 
 describe('ReportBuilder', () => {
     const sandbox = sinon.sandbox.create();
@@ -261,6 +263,43 @@ describe('ReportBuilder', () => {
             });
         });
 
+        describe('should rewrite suite status to "fail" if test failed with no reference image error', () => {
+            it('and test does not exist in tests tree', () => {
+                const reportBuilder = mkReportBuilder_();
+                const test = stubTest_({
+                    status: ERROR,
+                    imagesInfo: [{
+                        stateName: 'plain', status: ERROR,
+                        reason: {stack: `${NO_REF_IMAGE_ERROR}: ...`}
+                    }]
+                });
+
+                reportBuilder.addError(test);
+
+                const suiteResult = getSuiteResult_(reportBuilder);
+
+                assert.equal(suiteResult.status, FAIL);
+            });
+
+            it('and test exists in tests tree', () => {
+                const reportBuilder = mkReportBuilder_();
+                const test = stubTest_({
+                    status: ERROR,
+                    imagesInfo: [{
+                        stateName: 'plain', status: ERROR,
+                        reason: {stack: `${NO_REF_IMAGE_ERROR}: ...`}
+                    }]
+                });
+
+                reportBuilder.addIdle(test);
+                reportBuilder.addError(test);
+
+                const suiteResult = getSuiteResult_(reportBuilder);
+
+                assert.equal(suiteResult.status, FAIL);
+            });
+        });
+
         describe('should not rewrite suite status to "success" if image comparison is successful, but test', () => {
             [
                 {status: FAIL, methodName: 'addFail'},