From 51f7f67b58faf31186d15a7b6c07d302b392436d Mon Sep 17 00:00:00 2001 From: AnanyaJha Date: Mon, 26 Apr 2021 17:20:13 -0700 Subject: [PATCH] fix: set correct exit codes on failure (#185) forcedotcom/salesforcedx-vscode#3163 @W-9177437@ --- packages/apex-node/src/index.ts | 2 +- .../src/commands/force/apex/test/report.ts | 16 +++-- .../src/commands/force/apex/test/run.ts | 18 ++++- packages/plugin-apex/src/utils.ts | 2 + .../commands/force/apex/test/report.test.ts | 70 ++++++++++++++++++- .../test/commands/force/apex/test/run.test.ts | 52 ++++++++++++-- 6 files changed, 146 insertions(+), 14 deletions(-) diff --git a/packages/apex-node/src/index.ts b/packages/apex-node/src/index.ts index 4f03c27c..8662e998 100644 --- a/packages/apex-node/src/index.ts +++ b/packages/apex-node/src/index.ts @@ -15,9 +15,9 @@ export { LogService, ApexLogGetOptions, LogRecord, LogResult } from './logs'; export { JUnitReporter, TapReporter, HumanReporter } from './reporters'; export { ApexTestProgressValue, - ApexTestRunResultStatus, ApexTestResultData, ApexTestResultOutcome, + ApexTestRunResultStatus, AsyncTestArrayConfiguration, AsyncTestConfiguration, CodeCoverageResult, diff --git a/packages/plugin-apex/src/commands/force/apex/test/report.ts b/packages/plugin-apex/src/commands/force/apex/test/report.ts index b809e5b4..f3452c4a 100644 --- a/packages/plugin-apex/src/commands/force/apex/test/report.ts +++ b/packages/plugin-apex/src/commands/force/apex/test/report.ts @@ -9,7 +9,8 @@ import { HumanReporter, TapReporter, TestService, - TestResult + TestResult, + ApexTestRunResultStatus } from '@salesforce/apex-node'; import { flags, SfdxCommand } from '@salesforce/command'; import { Messages, Org } from '@salesforce/core'; @@ -19,7 +20,12 @@ import { CliJsonFormat, buildOutputDirConfig } from '../../../../reporters'; -import { buildDescription, logLevels, resultFormat } from '../../../../utils'; +import { + buildDescription, + logLevels, + resultFormat, + FAILURE_EXIT_CODE +} from '../../../../utils'; Messages.importMessagesDirectory(__dirname); const messages = Messages.loadMessages('@salesforce/plugin-apex', 'report'); @@ -109,6 +115,9 @@ export default class Report extends SfdxCommand { } try { + if (result.summary.outcome === ApexTestRunResultStatus.Failed) { + process.exitCode = FAILURE_EXIT_CODE; + } switch (this.flags.resultformat) { case 'tap': this.logTap(result); @@ -119,7 +128,7 @@ export default class Report extends SfdxCommand { case 'json': // when --json flag is specified, we should log CLI json format if (!this.flags.json) { - this.ux.logJson(jsonOutput); + this.ux.logJson({ status: process.exitCode, result: jsonOutput }); } break; default: @@ -130,7 +139,6 @@ export default class Report extends SfdxCommand { const msg = messages.getMessage('testResultProcessErr', [e]); this.ux.error(msg); } - return jsonOutput as AnyJson; } diff --git a/packages/plugin-apex/src/commands/force/apex/test/run.ts b/packages/plugin-apex/src/commands/force/apex/test/run.ts index a8db9f61..60761534 100644 --- a/packages/plugin-apex/src/commands/force/apex/test/run.ts +++ b/packages/plugin-apex/src/commands/force/apex/test/run.ts @@ -11,7 +11,8 @@ import { JUnitReporter, HumanReporter, TestResult, - TestLevel + TestLevel, + ApexTestRunResultStatus } from '@salesforce/apex-node'; import { flags, SfdxCommand } from '@salesforce/command'; import { Messages, Org, SfdxError } from '@salesforce/core'; @@ -21,7 +22,12 @@ import { CliJsonFormat, JsonReporter } from '../../../../reporters'; -import { buildDescription, logLevels, resultFormat } from '../../../../utils'; +import { + buildDescription, + logLevels, + resultFormat, + FAILURE_EXIT_CODE +} from '../../../../utils'; Messages.importMessagesDirectory(__dirname); const messages = Messages.loadMessages('@salesforce/plugin-apex', 'run'); @@ -193,6 +199,9 @@ export default class Run extends SfdxCommand { } try { + if (result.summary.outcome === ApexTestRunResultStatus.Failed) { + process.exitCode = FAILURE_EXIT_CODE; + } switch (this.flags.resultformat) { case 'human': this.logHuman( @@ -210,7 +219,10 @@ export default class Run extends SfdxCommand { case 'json': // when --json flag is specified, we should log CLI json format if (!this.flags.json) { - this.ux.logJson(this.formatResultInJson(result)); + this.ux.logJson({ + status: process.exitCode, + result: this.formatResultInJson(result) + }); } break; default: diff --git a/packages/plugin-apex/src/utils.ts b/packages/plugin-apex/src/utils.ts index bc61f819..a21feea1 100644 --- a/packages/plugin-apex/src/utils.ts +++ b/packages/plugin-apex/src/utils.ts @@ -7,6 +7,8 @@ import * as chalk from 'chalk'; +export const FAILURE_EXIT_CODE = 100; + export const colorSuccess = chalk.bold.green; export const colorError = chalk.bold.red; diff --git a/packages/plugin-apex/test/commands/force/apex/test/report.test.ts b/packages/plugin-apex/test/commands/force/apex/test/report.test.ts index 81f059ca..0d8b9752 100644 --- a/packages/plugin-apex/test/commands/force/apex/test/report.test.ts +++ b/packages/plugin-apex/test/commands/force/apex/test/report.test.ts @@ -23,7 +23,8 @@ import { cliWithCoverage, runWithCoverage, jsonWithCoverage, - jsonResult + jsonResult, + runWithFailures } from './testData'; Messages.importMessagesDirectory(__dirname); @@ -285,6 +286,31 @@ describe('force:apex:test:report', () => { } ); + test + .withOrg({ username: TEST_USERNAME }, true) + .loadConfig({ + root: __dirname + }) + .stub(process, 'cwd', () => projectPath) + .stub(TestService.prototype, 'reportAsyncResults', () => testRunSimple) + .stdout() + .command([ + 'force:apex:test:report', + '-i', + '01pxx00000NWwb3', + '--resultformat', + 'json' + ]) + .it( + 'should return a CLI json result with json result flag are specified', + ctx => { + const result = ctx.stdout; + expect(result).to.not.be.empty; + const resultJSON = JSON.parse(result); + expect(resultJSON).to.deep.equal(cliJsonResult); + } + ); + test .withOrg({ username: TEST_USERNAME }, true) .loadConfig({ @@ -309,6 +335,48 @@ describe('force:apex:test:report', () => { expect(resultJSON).to.deep.equal(cliWithCoverage); }); + test + .withOrg({ username: TEST_USERNAME }, true) + .loadConfig({ + root: __dirname + }) + .stub(process, 'cwd', () => projectPath) + .stub(TestService.prototype, 'reportAsyncResults', () => runWithFailures) + .stdout() + .command([ + 'force:apex:test:report', + '-i', + '01pxx00000NWwb3', + '--json', + '--resultformat', + 'junit', + '-c' + ]) + .it('should set exit code as 100 for run with failures', () => { + expect(process.exitCode).to.eql(100); + }); + + test + .withOrg({ username: TEST_USERNAME }, true) + .loadConfig({ + root: __dirname + }) + .stub(process, 'cwd', () => projectPath) + .stub(TestService.prototype, 'reportAsyncResults', () => testRunSimple) + .stdout() + .command([ + 'force:apex:test:report', + '-i', + '01pxx00000NWwb3', + '--json', + '--resultformat', + 'junit', + '-c' + ]) + .it('should set exit code as 0 for passing run', () => { + expect(process.exitCode).to.eql(0); + }); + test .withOrg({ username: TEST_USERNAME }, true) .loadConfig({ diff --git a/packages/plugin-apex/test/commands/force/apex/test/run.test.ts b/packages/plugin-apex/test/commands/force/apex/test/run.test.ts index 371adef2..147706f4 100644 --- a/packages/plugin-apex/test/commands/force/apex/test/run.test.ts +++ b/packages/plugin-apex/test/commands/force/apex/test/run.test.ts @@ -23,10 +23,10 @@ import { runWithCoverage, cliJsonResult, cliWithCoverage, - jsonResult, jsonWithCoverage, jsonSyncResult, - rawSyncResult + rawSyncResult, + runWithFailures } from './testData'; Messages.importMessagesDirectory(__dirname); @@ -126,7 +126,7 @@ describe('force:apex:test:run', () => { expect(ctx.stdout).to.contain('{\n "tests": []\n}\n'); expect(ctx.stderr).to.contain( messages.getMessage('testResultProcessErr', [ - "TypeError: Cannot read property 'testRunId' of undefined" + "TypeError: Cannot read property 'outcome' of undefined" ]) ); }); @@ -177,7 +177,7 @@ describe('force:apex:test:run', () => { expect(ctx.stdout).to.contain('{\n "tests": []\n}\n'); expect(ctx.stderr).to.contain( messages.getMessage('testResultProcessErr', [ - "TypeError: Cannot read property 'testStartTime' of undefined" + "TypeError: Cannot read property 'outcome' of undefined" ]) ); }); @@ -279,7 +279,7 @@ describe('force:apex:test:run', () => { const result = ctx.stdout; expect(result).to.not.be.empty; const resultJSON = JSON.parse(result); - expect(resultJSON).to.deep.equal(jsonResult); + expect(resultJSON).to.deep.equal(cliJsonResult); } ); @@ -1166,4 +1166,46 @@ describe('force:apex:test:run', () => { expect(ctx.stderr).to.include(messages.getMessage('warningMessage')); } ); + + test + .withOrg({ username: TEST_USERNAME }, true) + .loadConfig({ + root: __dirname + }) + .stub(process, 'cwd', () => projectPath) + .stub(TestService.prototype, 'runTestAsynchronous', () => runWithFailures) + .stdout() + .command([ + 'force:apex:test:run', + '--tests', + 'MyApexClass.testInsertTrigger', + '--outputdir', + 'my/path/to/dir', + '-r', + 'human' + ]) + .it('should set exit code as 100 for run with failures', () => { + expect(process.exitCode).to.eql(100); + }); + + test + .withOrg({ username: TEST_USERNAME }, true) + .loadConfig({ + root: __dirname + }) + .stub(process, 'cwd', () => projectPath) + .stub(TestService.prototype, 'runTestAsynchronous', () => testRunSimple) + .stdout() + .command([ + 'force:apex:test:run', + '--tests', + 'MyApexClass.testInsertTrigger', + '--outputdir', + 'my/path/to/dir', + '-r', + 'human' + ]) + .it('should set exit code as 0 for passing run', () => { + expect(process.exitCode).to.eql(0); + }); });