fix: set correct exit codes on failure (#185)

forcedotcom/salesforcedx-vscode#3163 @W-9177437@
forcedotcom · May 4, 2021 · 51f7f67 · 51f7f67
1 parent 902ea7c
commit 51f7f67
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 14 deletions.
diff --git a/packages/apex-node/src/index.ts b/packages/apex-node/src/index.ts
@@ -15,9 +15,9 @@ export { LogService, ApexLogGetOptions, LogRecord, LogResult } from './logs';
 export { JUnitReporter, TapReporter, HumanReporter } from './reporters';
 export {
   ApexTestProgressValue,
-  ApexTestRunResultStatus,
   ApexTestResultData,
   ApexTestResultOutcome,
+  ApexTestRunResultStatus,
   AsyncTestArrayConfiguration,
   AsyncTestConfiguration,
   CodeCoverageResult,

diff --git a/packages/plugin-apex/src/commands/force/apex/test/report.ts b/packages/plugin-apex/src/commands/force/apex/test/report.ts
@@ -9,7 +9,8 @@ import {
   HumanReporter,
   TapReporter,
   TestService,
-  TestResult
+  TestResult,
+  ApexTestRunResultStatus
 } from '@salesforce/apex-node';
 import { flags, SfdxCommand } from '@salesforce/command';
 import { Messages, Org } from '@salesforce/core';
@@ -19,7 +20,12 @@ import {
   CliJsonFormat,
   buildOutputDirConfig
 } from '../../../../reporters';
-import { buildDescription, logLevels, resultFormat } from '../../../../utils';
+import {
+  buildDescription,
+  logLevels,
+  resultFormat,
+  FAILURE_EXIT_CODE
+} from '../../../../utils';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/plugin-apex', 'report');
@@ -109,6 +115,9 @@ export default class Report extends SfdxCommand {
     }
 
     try {
+      if (result.summary.outcome === ApexTestRunResultStatus.Failed) {
+        process.exitCode = FAILURE_EXIT_CODE;
+      }
       switch (this.flags.resultformat) {
         case 'tap':
           this.logTap(result);
@@ -119,7 +128,7 @@ export default class Report extends SfdxCommand {
         case 'json':
           // when --json flag is specified, we should log CLI json format
           if (!this.flags.json) {
-            this.ux.logJson(jsonOutput);
+            this.ux.logJson({ status: process.exitCode, result: jsonOutput });
           }
           break;
         default:
@@ -130,7 +139,6 @@ export default class Report extends SfdxCommand {
       const msg = messages.getMessage('testResultProcessErr', [e]);
       this.ux.error(msg);
     }
-
     return jsonOutput as AnyJson;
   }
 

diff --git a/packages/plugin-apex/src/commands/force/apex/test/run.ts b/packages/plugin-apex/src/commands/force/apex/test/run.ts
@@ -11,7 +11,8 @@ import {
   JUnitReporter,
   HumanReporter,
   TestResult,
-  TestLevel
+  TestLevel,
+  ApexTestRunResultStatus
 } from '@salesforce/apex-node';
 import { flags, SfdxCommand } from '@salesforce/command';
 import { Messages, Org, SfdxError } from '@salesforce/core';
@@ -21,7 +22,12 @@ import {
   CliJsonFormat,
   JsonReporter
 } from '../../../../reporters';
-import { buildDescription, logLevels, resultFormat } from '../../../../utils';
+import {
+  buildDescription,
+  logLevels,
+  resultFormat,
+  FAILURE_EXIT_CODE
+} from '../../../../utils';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/plugin-apex', 'run');
@@ -193,6 +199,9 @@ export default class Run extends SfdxCommand {
     }
 
     try {
+      if (result.summary.outcome === ApexTestRunResultStatus.Failed) {
+        process.exitCode = FAILURE_EXIT_CODE;
+      }
       switch (this.flags.resultformat) {
         case 'human':
           this.logHuman(
@@ -210,7 +219,10 @@ export default class Run extends SfdxCommand {
         case 'json':
           // when --json flag is specified, we should log CLI json format
           if (!this.flags.json) {
-            this.ux.logJson(this.formatResultInJson(result));
+            this.ux.logJson({
+              status: process.exitCode,
+              result: this.formatResultInJson(result)
+            });
           }
           break;
         default:

diff --git a/packages/plugin-apex/src/utils.ts b/packages/plugin-apex/src/utils.ts
@@ -7,6 +7,8 @@
 
 import * as chalk from 'chalk';
 
+export const FAILURE_EXIT_CODE = 100;
+
 export const colorSuccess = chalk.bold.green;
 export const colorError = chalk.bold.red;
 

diff --git a/packages/plugin-apex/test/commands/force/apex/test/report.test.ts b/packages/plugin-apex/test/commands/force/apex/test/report.test.ts
@@ -23,7 +23,8 @@ import {
   cliWithCoverage,
   runWithCoverage,
   jsonWithCoverage,
-  jsonResult
+  jsonResult,
+  runWithFailures
 } from './testData';
 
 Messages.importMessagesDirectory(__dirname);
@@ -285,6 +286,31 @@ describe('force:apex:test:report', () => {
       }
     );
 
+  test
+    .withOrg({ username: TEST_USERNAME }, true)
+    .loadConfig({
+      root: __dirname
+    })
+    .stub(process, 'cwd', () => projectPath)
+    .stub(TestService.prototype, 'reportAsyncResults', () => testRunSimple)
+    .stdout()
+    .command([
+      'force:apex:test:report',
+      '-i',
+      '01pxx00000NWwb3',
+      '--resultformat',
+      'json'
+    ])
+    .it(
+      'should return a CLI json result with json result flag are specified',
+      ctx => {
+        const result = ctx.stdout;
+        expect(result).to.not.be.empty;
+        const resultJSON = JSON.parse(result);
+        expect(resultJSON).to.deep.equal(cliJsonResult);
+      }
+    );
+
   test
     .withOrg({ username: TEST_USERNAME }, true)
     .loadConfig({
@@ -309,6 +335,48 @@ describe('force:apex:test:report', () => {
       expect(resultJSON).to.deep.equal(cliWithCoverage);
     });
 
+  test
+    .withOrg({ username: TEST_USERNAME }, true)
+    .loadConfig({
+      root: __dirname
+    })
+    .stub(process, 'cwd', () => projectPath)
+    .stub(TestService.prototype, 'reportAsyncResults', () => runWithFailures)
+    .stdout()
+    .command([
+      'force:apex:test:report',
+      '-i',
+      '01pxx00000NWwb3',
+      '--json',
+      '--resultformat',
+      'junit',
+      '-c'
+    ])
+    .it('should set exit code as 100 for run with failures', () => {
+      expect(process.exitCode).to.eql(100);
+    });
+
+  test
+    .withOrg({ username: TEST_USERNAME }, true)
+    .loadConfig({
+      root: __dirname
+    })
+    .stub(process, 'cwd', () => projectPath)
+    .stub(TestService.prototype, 'reportAsyncResults', () => testRunSimple)
+    .stdout()
+    .command([
+      'force:apex:test:report',
+      '-i',
+      '01pxx00000NWwb3',
+      '--json',
+      '--resultformat',
+      'junit',
+      '-c'
+    ])
+    .it('should set exit code as 0 for passing run', () => {
+      expect(process.exitCode).to.eql(0);
+    });
+
   test
     .withOrg({ username: TEST_USERNAME }, true)
     .loadConfig({

diff --git a/packages/plugin-apex/test/commands/force/apex/test/run.test.ts b/packages/plugin-apex/test/commands/force/apex/test/run.test.ts
@@ -23,10 +23,10 @@ import {
   runWithCoverage,
   cliJsonResult,
   cliWithCoverage,
-  jsonResult,
   jsonWithCoverage,
   jsonSyncResult,
-  rawSyncResult
+  rawSyncResult,
+  runWithFailures
 } from './testData';
 
 Messages.importMessagesDirectory(__dirname);
@@ -126,7 +126,7 @@ describe('force:apex:test:run', () => {
       expect(ctx.stdout).to.contain('{\n  "tests": []\n}\n');
       expect(ctx.stderr).to.contain(
         messages.getMessage('testResultProcessErr', [
-          "TypeError: Cannot read property 'testRunId' of undefined"
+          "TypeError: Cannot read property 'outcome' of undefined"
         ])
       );
     });
@@ -177,7 +177,7 @@ describe('force:apex:test:run', () => {
       expect(ctx.stdout).to.contain('{\n  "tests": []\n}\n');
       expect(ctx.stderr).to.contain(
         messages.getMessage('testResultProcessErr', [
-          "TypeError: Cannot read property 'testStartTime' of undefined"
+          "TypeError: Cannot read property 'outcome' of undefined"
         ])
       );
     });
@@ -279,7 +279,7 @@ describe('force:apex:test:run', () => {
         const result = ctx.stdout;
         expect(result).to.not.be.empty;
         const resultJSON = JSON.parse(result);
-        expect(resultJSON).to.deep.equal(jsonResult);
+        expect(resultJSON).to.deep.equal(cliJsonResult);
       }
     );
 
@@ -1166,4 +1166,46 @@ describe('force:apex:test:run', () => {
         expect(ctx.stderr).to.include(messages.getMessage('warningMessage'));
       }
     );
+
+  test
+    .withOrg({ username: TEST_USERNAME }, true)
+    .loadConfig({
+      root: __dirname
+    })
+    .stub(process, 'cwd', () => projectPath)
+    .stub(TestService.prototype, 'runTestAsynchronous', () => runWithFailures)
+    .stdout()
+    .command([
+      'force:apex:test:run',
+      '--tests',
+      'MyApexClass.testInsertTrigger',
+      '--outputdir',
+      'my/path/to/dir',
+      '-r',
+      'human'
+    ])
+    .it('should set exit code as 100 for run with failures', () => {
+      expect(process.exitCode).to.eql(100);
+    });
+
+  test
+    .withOrg({ username: TEST_USERNAME }, true)
+    .loadConfig({
+      root: __dirname
+    })
+    .stub(process, 'cwd', () => projectPath)
+    .stub(TestService.prototype, 'runTestAsynchronous', () => testRunSimple)
+    .stdout()
+    .command([
+      'force:apex:test:run',
+      '--tests',
+      'MyApexClass.testInsertTrigger',
+      '--outputdir',
+      'my/path/to/dir',
+      '-r',
+      'human'
+    ])
+    .it('should set exit code as 0 for passing run', () => {
+      expect(process.exitCode).to.eql(0);
+    });
 });