Merge pull request #723 from aws-samples/test-output-env

Pass test output to test hooks

test/util/README.md

@@ -176,3 +176,76 @@ Here is a complete list of the available annotations:
 | hookTimeout | Time limit in seconds for the hooks to complete before the script block will be marked as failed                                                                                      | 300     |
 | expectError | Ignore any errors that occur when the script block is executed                                                                                                                        | false   |
 | raw         | By default a script block will be smartly interpreted to extract commands and distinguish this from sample output. Enabling this flag will executed the entire script block, assuming there is no output, and will not look for `$` as a prefix for commands | false   |
+
+## Hooks
+
+Test hooks are designed to be able to wrap the script included in the content with instructions that can be run before or after. This can be especially helpful to perform assertions/validation.
+
+Hooks are expressed using an annotation like so:
+
+````
+```bash hook=my-hook
+$ echo "Showing hooks"
+```
+````
+
+This will check for a shell script at `tests/hook-my-hook.sh` relative to the location of the Markdown file. For example if the Markdown above was in `introduction.md` the file structure would look something like this.
+
+```
+├── index.md
+└── chapter1
+    ├── tests
+    │   └── hook-my-hook.sh
+    └── introduction.md
+```
+
+The shell script will be invoked with a single argument with a value of either `before` or `after` indicating whether its being run as a before or after hook.
+
+A good template to use for a hook file is:
+
+```bash
+set -Eeuo pipefail
+
+before() {
+  echo "This hook executes before the test"
+}
+
+after() {
+  echo "This hook executes after the test"
+}
+
+"$@"
+```
+
+If the hook file fails then the associated test will fail:
+
+```bash
+set -Eeuo pipefail
+
+before() {
+  echo "noop"
+}
+
+after() {
+  echo "This will fail the test"
+  exit 1
+}
+
+"$@"
+```
+
+When invoked as an "after" hook the stdout/stderr of the test script will be provided using the environment variable `TEST_OUTPUT`:
+
+```bash
+set -Eeuo pipefail
+
+before() {
+  echo "noop"
+}
+
+after() {
+  echo "The actual test output: ${TEST_OUTPUT}"
+}
+
+"$@"
+```

test/util/src/lib/markdownsh.ts

@@ -3,7 +3,7 @@ import Mocha, { Suite, Test } from 'mocha'
 import path from 'path'
 import GlobToRegExp from 'glob-to-regexp'
 import { assert } from 'chai'
-import { DefaultShell, Shell, ShellError, ShellTimeout } from "./shell/shell.js";
+import { DefaultShell, ExecutionResult, Shell, ShellError, ShellTimeout } from "./shell/shell.js";
 import fs from 'fs'
 
 export class MarkdownSh {
@@ -138,7 +138,7 @@ export class MarkdownSh {
         this.debugMessage(`Calling suite ${hook} hook at ${hookPath}`)
 
         if(!dryRun) {
-          let response = await shell.exec(`bash ${hookPath} ${hook}`, hookTimeout, false)
+          let response = await shell.exec(`bash ${hookPath} ${hook}`, hookTimeout, false, {})
 
           this.debugMessage(response.output)
         }
@@ -187,18 +187,22 @@ class CustomTest extends Test {
               hookTimeout = testCase.hookTimeout
             }
 
-            await this.hook(testCase, category, 'before', hookTimeout)
+            await this.hook(testCase, category, 'before', hookTimeout, {})
+
+            let result: ExecutionResult | undefined;
 
             try {
-              await this.executeShell(testCase.command, testCase.timeout, testCase.expectError)
+              result = await this.executeShell(testCase.command, testCase.timeout, testCase.expectError, {})
             }
             catch(e: any) {
               e.message = `Error running test case command at line ${testCase.lineNumber} - ${e.message}`
 
               throw e
             }
 
-            await this.hook(testCase, category, 'after', hookTimeout)
+            await this.hook(testCase, category, 'after', hookTimeout, {
+              'TEST_OUTPUT': result?.output
+            });
 
             if(testCase.wait > 0) {
               await this.sleep(testCase.wait * 1000)
@@ -233,14 +237,14 @@ class CustomTest extends Test {
     });
   }
 
-  async hook(testCase: Script, category: Category, hook: string, timeout: number) {
+  async hook(testCase: Script, category: Category, hook: string, timeout: number, env: { [key: string]: string | undefined }) {
     if(testCase.hook) {
       this.debugMessage(`Calling ${hook} hook ${testCase.hook}`)
 
       const hookPath = `${category.path}/tests/hook-${testCase.hook}.sh`;
 
       try {
-        const response = await this.executeShell(`bash ${hookPath} ${hook}`, timeout, false)
+        const response = await this.executeShell(`bash ${hookPath} ${hook}`, timeout, false, env)
 
         this.debugMessage(`Completed ${hook} hook ${testCase.hook}`)
       }
@@ -252,18 +256,22 @@ class CustomTest extends Test {
     }
   }
 
-  async executeShell(command: string, timeout: number, expectError: boolean) {
+  async executeShell(command: string, timeout: number, expectError: boolean, env: { [key: string]: string | undefined }): Promise<ExecutionResult | undefined> {
 
     this.debugMessage(`Executing shell:
   Command ${command}
   Timeout ${timeout}
   `)
 
     if (!this.dryRun) {
-      let response = await this.shell.exec(command, timeout, expectError)
+      let response = await this.shell.exec(command, timeout, expectError, env)
 
       this.debugMessage(response.output)
+
+      return response;
     }
+
+    return undefined;
   }
 
   async sleep(ms: number): Promise<void> {

test/util/src/lib/shell/shell.ts

@@ -3,7 +3,7 @@ import * as child from 'child_process';
 import * as os from 'os';
 
 export interface Shell {
-  exec:(command: string, timeout: number, expect: boolean) => Promise<ExecutionResult>
+  exec:(command: string, timeout: number, expect: boolean, additionalEnv: { [key: string]: string | undefined }) => Promise<ExecutionResult>
 }
 
 export class ExecutionResult {
@@ -19,7 +19,7 @@ export class DefaultShell implements Shell {
 
   constructor(private beforeEach: string) {}
 
-  exec(command: string, timeout: number = 300, expect: boolean = false) : Promise<ExecutionResult> {
+  exec(command: string, timeout: number = 300, expect: boolean = false, additionalEnv: { [key: string]: string | undefined }) : Promise<ExecutionResult> {
     if(!command) {
       throw new Error("Command should not be empty")
     }
@@ -32,7 +32,10 @@ export class DefaultShell implements Shell {
         killSignal: 'SIGKILL',
         stdio: ['inherit', 'pipe', 'pipe'],
         shell: '/bin/bash',
-        env: this.environment
+        env: {
+          ...this.environment,
+          ...additionalEnv
+        }
       });
 
       const output = String(buffer);
@@ -61,7 +64,7 @@ export class DefaultShell implements Shell {
             if(key.startsWith("BASH_FUNC")) {
               processingFunction = true;
             }
-            else {
+            else if(!(key in additionalEnv)) {
               env[key] = val;
             }
           }
@@ -91,7 +94,11 @@ export class DefaultShell implements Shell {
         throw new ShellTimeout(`Timed out after ${timeout} seconds`, e.stdout, e.stderr, timeout)
       }
 
-      throw new ShellError(e.status, e.message, e.stdout, e.stderr)
+      if(!expect) {
+        throw new ShellError(e.status, e.message, e.stdout, e.stderr)
+      }
+
+      return Promise.resolve(new ExecutionResult(e.stderr));
     }
   }
 }