From c627afc9d5d83c1bdb54320485ce6b2c081c9bd6 Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Mon, 7 Sep 2020 17:23:11 +0800
Subject: [PATCH 1/6] Allow users to set pre-command in remote mode

---
 src/nni_manager/rest_server/restValidationSchemas.ts   |  3 ++-
 .../remote_machine/extends/linuxCommands.ts            |  8 ++++++++
 .../remote_machine/extends/windowsCommands.ts          | 10 +++++++++-
 .../training_service/remote_machine/osCommands.ts      |  1 +
 .../remote_machine/remoteMachineData.ts                |  1 +
 .../training_service/remote_machine/shellExecutor.ts   |  5 +++++
 tools/nni_cmd/config_schema.py                         |  6 ++++--
 7 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts
index cb1a1282e7..337bdcf0c8 100644
--- a/src/nni_manager/rest_server/restValidationSchemas.ts
+++ b/src/nni_manager/rest_server/restValidationSchemas.ts
@@ -17,7 +17,8 @@ export namespace ValidationSchemas {
                 passphrase: joi.string(),
                 gpuIndices: joi.string(),
                 maxTrialNumPerGpu: joi.number(),
-                useActiveGpu: joi.boolean()
+                useActiveGpu: joi.boolean(),
+                preCommand: joi.string()
             })),
             local_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase
                 gpuIndices: joi.string(),
diff --git a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
index 76d4ac19fc..fa966ca68c 100644
--- a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
+++ b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
@@ -128,6 +128,14 @@ class LinuxCommands extends OsCommands {
         }
         return command;
     }
+
+    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+            return command;
+        } else {
+            return `${preCommand} && ${command}`;
+        }
+    }
 }
 
 export { LinuxCommands };
diff --git a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts
index fd2fd2118b..2a81157e29 100644
--- a/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts
+++ b/src/nni_manager/training_service/remote_machine/extends/windowsCommands.ts
@@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands {
     }
 
     public generateGpuStatsScript(scriptFolder: string): string {
-        return `powershell -command $env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
+        return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`;
     }
 
     public createFolder(folderName: string, sharedFolder: boolean = false): string {
@@ -122,6 +122,14 @@ class WindowsCommands extends OsCommands {
         const command = `${script}`;
         return command;
     }
+
+    public addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined{
+        if (command === undefined || command === '' || preCommand === undefined || preCommand === ''){
+            return command;
+        } else {
+            return `${preCommand} && set prePath=%path% && ${command}`;
+        }
+    }
 }
 
 export { WindowsCommands };
diff --git a/src/nni_manager/training_service/remote_machine/osCommands.ts b/src/nni_manager/training_service/remote_machine/osCommands.ts
index 3c4bcb0f1c..cb110c6694 100644
--- a/src/nni_manager/training_service/remote_machine/osCommands.ts
+++ b/src/nni_manager/training_service/remote_machine/osCommands.ts
@@ -28,6 +28,7 @@ abstract class OsCommands {
     public abstract killChildProcesses(pidFileName: string, killSelf: boolean): string;
     public abstract extractFile(tarFileName: string, targetFolder: string): string;
     public abstract executeScript(script: string, isFile: boolean): string;
+    public abstract addPreCommand(preCommand: string | undefined, command: string | undefined): string | undefined;
 
     public joinPath(...paths: string[]): string {
         let dir: string = paths.filter((path: any) => path !== '').join(this.pathSpliter);
diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts
index 61024c1fdd..e48d1fbf57 100644
--- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts
+++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts
@@ -23,6 +23,7 @@ export class RemoteMachineMeta {
     //TODO: initialize varialbe in constructor
     public occupiedGpuIndexMap?: Map<number, number>;
     public readonly useActiveGpu?: boolean = false;
+    public readonly preCommand?: string;
 }
 
 /**
diff --git a/src/nni_manager/training_service/remote_machine/shellExecutor.ts b/src/nni_manager/training_service/remote_machine/shellExecutor.ts
index 5f3a0bc78b..b093dfcd0e 100644
--- a/src/nni_manager/training_service/remote_machine/shellExecutor.ts
+++ b/src/nni_manager/training_service/remote_machine/shellExecutor.ts
@@ -32,6 +32,7 @@ class ShellExecutor {
     private tempPath: string = "";
     private isWindows: boolean = false;
     private channelDefaultOutputs: string[] = [];
+    private preCommand: string | undefined;
 
     constructor() {
         this.log = getLogger();
@@ -47,6 +48,7 @@ class ShellExecutor {
             username: rmMeta.username,
             tryKeyboard: true,
         };
+        this.preCommand = rmMeta.preCommand;
         this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`;
         if (rmMeta.passwd !== undefined) {
             connectConfig.password = rmMeta.passwd;
@@ -349,6 +351,9 @@ class ShellExecutor {
         let exitCode: number;
 
         const commandIndex = randomInt(10000);
+        if(this.osCommands !== undefined){
+            command = this.osCommands.addPreCommand(this.preCommand, command);
+        }
         this.log.debug(`remoteExeCommand(${commandIndex}): [${command}]`);
 
         // Windows always uses shell, and it needs to disable to get it works.
diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py
index 81dc1c77a5..2a7d929779 100644
--- a/tools/nni_cmd/config_schema.py
+++ b/tools/nni_cmd/config_schema.py
@@ -382,7 +382,8 @@ def validate(self, data):
             Optional('passphrase'): setType('passphrase', str),
             Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
             Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
-            Optional('useActiveGpu'): setType('useActiveGpu', bool)
+            Optional('useActiveGpu'): setType('useActiveGpu', bool),
+            Optional('preCommand'): setType('preCommand', str)
         },
         {
             'ip': setType('ip', str),
@@ -391,7 +392,8 @@ def validate(self, data):
             'passwd': setType('passwd', str),
             Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
             Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
-            Optional('useActiveGpu'): setType('useActiveGpu', bool)
+            Optional('useActiveGpu'): setType('useActiveGpu', bool),
+            Optional('preCommand'): setType('preCommand', str)
         })]
 }
 

From 1fd74c50aa205e538e9d1b27c9db8b25ba8bc714 Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Wed, 16 Sep 2020 11:16:47 +0800
Subject: [PATCH 2/6] add doc and example for remote-machineList-preCommand

---
 .../TrainingService/RemoteMachineMode.md      | 76 +++++++++++++++++++
 docs/en_US/Tutorial/ExperimentConfig.md       | 13 ++++
 examples/trials/mnist-tfv2/config_remote.yml  | 32 ++++++++
 3 files changed, 121 insertions(+)
 create mode 100644 examples/trials/mnist-tfv2/config_remote.yml

diff --git a/docs/en_US/TrainingService/RemoteMachineMode.md b/docs/en_US/TrainingService/RemoteMachineMode.md
index bb8c9d5d67..78e3a196a5 100644
--- a/docs/en_US/TrainingService/RemoteMachineMode.md
+++ b/docs/en_US/TrainingService/RemoteMachineMode.md
@@ -107,3 +107,79 @@ Files in `codeDir` will be uploaded to remote machines automatically. You can ru
 ```bash
 nnictl create --config examples/trials/mnist-annotation/config_remote.yml
 ```
+
+### Configure python environment
+
+By default, commands and scripts will be executed in the default environment in remote machine. If there are multiple python virtual environments in your remote machine, and you want to run experiments in a specific environment, then use `preCommand` to specify a python environment on your remote machine. 
+
+Use `examples/trials/mnist-tfv2` as the example. Below is content of `examples/trials/mnist-tfv2/config_remote.yml`:
+
+```yaml
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: remote
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 mnist.py
+  codeDir: .
+  gpuNum: 0
+#machineList can be empty if the platform is local
+machineList:
+  - ip: ${replace_to_your_remote_machine_ip}
+    username: ${replace_to_your_remote_machine_username}
+    sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath}
+    # Pre-command will be executed before the remote machine executes other commands.
+    # Below is an example of specifying python environment.
+    # If you want to execute multiple commands, please use "&&" to connect them.
+    # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
+    # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
+    preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
+```
+
+The `preCommand` will be executed before the remote machine executes other commands. So you can configure python environment path like this:
+
+```yaml
+# Linux remote machine
+preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
+# Windows remote machine
+preCommand: set path=${replace_to_python_environment_path_in_your_remote_machine};%path%
+```
+
+Or if you want to activate the `virtualenv` environment:
+
+```yaml
+# Linux remote machine
+preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
+# Windows remote machine
+preCommand: ${replace_to_absolute_path_recommended_here}\\scripts\\activate
+```
+
+Or if you want to activate the `conda` environment:
+
+```yaml
+# Linux remote machine
+preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
+# Windows remote machine
+preCommand: call activate ${replace_to_conda_env_name}
+```
+
+If there are multiple commands want to execute, you can use `&&` to connect these commands:
+
+```yaml
+preCommand: command1 && command2 && command3
+```
+
+
diff --git a/docs/en_US/Tutorial/ExperimentConfig.md b/docs/en_US/Tutorial/ExperimentConfig.md
index 687e9980f6..881dcb4e56 100644
--- a/docs/en_US/Tutorial/ExperimentConfig.md
+++ b/docs/en_US/Tutorial/ExperimentConfig.md
@@ -58,6 +58,7 @@ This document describes the rules to write the config file, and provides some ex
       - [gpuIndices](#gpuindices-3)
       - [maxTrialNumPerGpu](#maxtrialnumpergpu-1)
       - [useActiveGpu](#useactivegpu-1)
+      - [preCommand](#preCommand)
     + [kubeflowConfig](#kubeflowconfig)
       - [operator](#operator)
       - [storage](#storage)
@@ -583,6 +584,12 @@ Optional. Bool. Default: false.
 
 Used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. If __useActiveGpu__ is set to true, NNI will use the GPU regardless of another processes. This field is not applicable for NNI on Windows.
 
+#### preCommand
+
+Optional. String.
+
+Specifies the pre-command that will be executed before the remote machine executes other commands. Users can configure the experimental environment on remote machine by setting __preCommand__. If there are multiple commands need to execute, use `&&` to connect them, such as `preCommand: command1 && command2 && ...`.
+
 ### kubeflowConfig
 
 #### operator
@@ -795,6 +802,12 @@ If run trial jobs in remote machine, users could specify the remote machine info
       username: test
       sshKeyPath: /nni/sshkey
       passphrase: qwert
+      # Pre-command will be executed before the remote machine executes other commands.
+      # Below is an example of specifying python environment.
+      # If you want to execute multiple commands, please use "&&" to connect them.
+      # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
+      # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
+      preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
   ```
 
 ### PAI mode
diff --git a/examples/trials/mnist-tfv2/config_remote.yml b/examples/trials/mnist-tfv2/config_remote.yml
new file mode 100644
index 0000000000..0c39914dbc
--- /dev/null
+++ b/examples/trials/mnist-tfv2/config_remote.yml
@@ -0,0 +1,32 @@
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+#choice: local, remote, pai
+trainingServicePlatform: remote
+searchSpacePath: search_space.json
+#choice: true, false
+useAnnotation: false
+tuner:
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
+  #SMAC (SMAC should be installed through nnictl)
+  builtinTunerName: TPE
+  classArgs:
+    #choice: maximize, minimize
+    optimize_mode: maximize
+trial:
+  command: python3 mnist.py
+  codeDir: .
+  gpuNum: 0
+#machineList can be empty if the platform is local
+machineList:
+  - ip: ${replace_to_your_remote_machine_ip}
+    username: ${replace_to_your_remote_machine_username}
+    sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath}
+    # Pre-command will be executed before the remote machine executes other commands.
+    # Below is an example of specifying python environment.
+    # If you want to execute multiple commands, please use "&&" to connect them.
+    # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate
+    # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name}
+    preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH

From c8a3d805fdc0b0c509f8b13e196bf4ea51a8c6b7 Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Wed, 16 Sep 2020 16:01:38 +0800
Subject: [PATCH 3/6] add note in doc about using preCommand

---
 docs/en_US/TrainingService/RemoteMachineMode.md | 6 +++---
 docs/en_US/Tutorial/ExperimentConfig.md         | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/en_US/TrainingService/RemoteMachineMode.md b/docs/en_US/TrainingService/RemoteMachineMode.md
index 78e3a196a5..f6e540e0db 100644
--- a/docs/en_US/TrainingService/RemoteMachineMode.md
+++ b/docs/en_US/TrainingService/RemoteMachineMode.md
@@ -110,7 +110,7 @@ nnictl create --config examples/trials/mnist-annotation/config_remote.yml
 
 ### Configure python environment
 
-By default, commands and scripts will be executed in the default environment in remote machine. If there are multiple python virtual environments in your remote machine, and you want to run experiments in a specific environment, then use `preCommand` to specify a python environment on your remote machine. 
+By default, commands and scripts will be executed in the default environment in remote machine. If there are multiple python virtual environments in your remote machine, and you want to run experiments in a specific environment, then use __preCommand__ to specify a python environment on your remote machine. 
 
 Use `examples/trials/mnist-tfv2` as the example. Below is content of `examples/trials/mnist-tfv2/config_remote.yml`:
 
@@ -149,7 +149,7 @@ machineList:
     preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH
 ```
 
-The `preCommand` will be executed before the remote machine executes other commands. So you can configure python environment path like this:
+The __preCommand__ will be executed before the remote machine executes other commands. So you can configure python environment path like this:
 
 ```yaml
 # Linux remote machine
@@ -182,4 +182,4 @@ If there are multiple commands want to execute, you can use `&&` to connect thes
 preCommand: command1 && command2 && command3
 ```
 
-
+__Note__: Because __preCommand__ will execute before other commands each time, it is strongly not recommended to set __preCommand__ that will make changes to system, i.e. `mkdir` or `touch`.
diff --git a/docs/en_US/Tutorial/ExperimentConfig.md b/docs/en_US/Tutorial/ExperimentConfig.md
index 881dcb4e56..0321154e58 100644
--- a/docs/en_US/Tutorial/ExperimentConfig.md
+++ b/docs/en_US/Tutorial/ExperimentConfig.md
@@ -590,6 +590,8 @@ Optional. String.
 
 Specifies the pre-command that will be executed before the remote machine executes other commands. Users can configure the experimental environment on remote machine by setting __preCommand__. If there are multiple commands need to execute, use `&&` to connect them, such as `preCommand: command1 && command2 && ...`.
 
+__Note__: Because __preCommand__ will execute before other commands each time, it is strongly not recommended to set __preCommand__ that will make changes to system, i.e. `mkdir` or `touch`.
+
 ### kubeflowConfig
 
 #### operator

From 1c84f6abc542996354d305b4e943a7107647094d Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Wed, 16 Sep 2020 19:07:47 +0800
Subject: [PATCH 4/6] add preCommand UT

---
 .../remote_machine/test/shellExecutor.test.ts | 29 +++++++++++++++++++
 .../test/remoteMachineTrainingService.test.ts |  4 +--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
index 4e9d9ffb68..a263e70ceb 100644
--- a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
+++ b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
@@ -36,6 +36,7 @@ async function getRemoteFileContentLoop(executor: ShellExecutor): Promise<void>
 
 describe('ShellExecutor test', () => {
     let skip: boolean = false;
+    let isWindows: boolean;
     let rmMeta: any;
     try {
         rmMeta = JSON.parse(fs.readFileSync('../../.vscode/rminfo.json', 'utf8'));
@@ -86,4 +87,32 @@ describe('ShellExecutor test', () => {
         await getRemoteFileContentLoop(executor);
         await executor.close();
     });
+
+    it('Test preCommand-1', async () => {
+        if (skip) {
+            return;
+        }
+        const executor: ShellExecutor = new ShellExecutor();
+        await executor.initialize(rmMeta);
+        const result = await executor.executeScript("ver", false, false);
+        isWindows = result.exitCode == 0 && result.stdout.search("Windows") > -1;
+        await executor.close();
+    });
+
+    it('Test preCommand-2', async () => {
+        if (skip) {
+            return;
+        }
+        const executor: ShellExecutor = new ShellExecutor();
+        rmMeta.preCommand = isWindows ? "set TEST_PRE_COMMAND=test_pre_command" : "export TEST_PRE_COMMAND=test_pre_command";
+        await executor.initialize(rmMeta);
+        const command = isWindows ? "echo %TEST_PRE_COMMAND%" : "echo $TEST_PRE_COMMAND";
+        const result = await executor.executeScript(command, false, false);
+        if (isWindows) {
+            chai.expect(result.stdout).eq("%TEST_PRE_COMMAND%\r\n");
+        } else {
+            chai.expect(result.stdout).eq("test_pre_command\n");
+        }
+        await executor.close();
+    });
 });
diff --git a/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts b/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts
index d2446460ee..551d281f76 100644
--- a/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts
+++ b/src/nni_manager/training_service/test/remoteMachineTrainingService.test.ts
@@ -25,8 +25,8 @@ describe('Unit Test for RemoteMachineTrainingService', () => {
     Default/.vscode/rminfo.json,  whose content looks like:
     {
         "ip": "10.172.121.40",
-        "user": "user1",
-        "password": "mypassword"
+        "username": "user1",
+        "passwd": "mypassword"
     }
     */
     let skip: boolean = false;

From 22afaa54641ab18d1b7dd17859ed0b66e660a4c1 Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Thu, 17 Sep 2020 14:17:25 +0800
Subject: [PATCH 5/6] make preCommand UT more accurate and fix bug in
 linuxCommands

---
 .../remote_machine/extends/linuxCommands.ts            |  2 +-
 .../remote_machine/test/shellExecutor.test.ts          | 10 +++-------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
index fa966ca68c..0dab8a63ea 100644
--- a/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
+++ b/src/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
@@ -123,7 +123,7 @@ class LinuxCommands extends OsCommands {
         if (isFile) {
             command = `bash '${script}'`;
         } else {
-            script = script.replace('"', '\\"');
+            script = script.replace(/"/g, '\\"');
             command = `bash -c "${script}"`;
         }
         return command;
diff --git a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
index a263e70ceb..3aee8ba020 100644
--- a/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
+++ b/src/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts
@@ -106,13 +106,9 @@ describe('ShellExecutor test', () => {
         const executor: ShellExecutor = new ShellExecutor();
         rmMeta.preCommand = isWindows ? "set TEST_PRE_COMMAND=test_pre_command" : "export TEST_PRE_COMMAND=test_pre_command";
         await executor.initialize(rmMeta);
-        const command = isWindows ? "echo %TEST_PRE_COMMAND%" : "echo $TEST_PRE_COMMAND";
-        const result = await executor.executeScript(command, false, false);
-        if (isWindows) {
-            chai.expect(result.stdout).eq("%TEST_PRE_COMMAND%\r\n");
-        } else {
-            chai.expect(result.stdout).eq("test_pre_command\n");
-        }
+        const command = isWindows ? "python -c \"import os; print(os.environ.get(\'TEST_PRE_COMMAND\'))\"" : "python3 -c \"import os; print(os.environ.get(\'TEST_PRE_COMMAND\'))\"";
+        const result = (await executor.executeScript(command, false, false)).stdout.replace(/[\ +\r\n]/g, "");
+        chai.expect(result).eq("test_pre_command");
         await executor.close();
     });
 });

From 01020d890af26632dc12b2c7fd606065aedb7f16 Mon Sep 17 00:00:00 2001
From: Ning Shang <nishang@microsoft.com>
Date: Mon, 21 Sep 2020 12:11:42 +0800
Subject: [PATCH 6/6] update preCommand doc

---
 docs/en_US/TrainingService/RemoteMachineMode.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en_US/TrainingService/RemoteMachineMode.md b/docs/en_US/TrainingService/RemoteMachineMode.md
index f6e540e0db..f25aa364bc 100644
--- a/docs/en_US/TrainingService/RemoteMachineMode.md
+++ b/docs/en_US/TrainingService/RemoteMachineMode.md
@@ -176,7 +176,7 @@ preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_
 preCommand: call activate ${replace_to_conda_env_name}
 ```
 
-If there are multiple commands want to execute, you can use `&&` to connect these commands:
+If you want multiple commands to be executed, you can use `&&` to connect these commands:
 
 ```yaml
 preCommand: command1 && command2 && command3