Merge pull request #245 from microsoft/master

merge master

Makefile

@@ -173,12 +173,12 @@ install-python-modules:
 dev-install-python-modules:
 	#$(_INFO) Installing Python SDK $(_END)
 	mkdir -p build
-	ln -sf ../src/sdk/pynni/nni build
-	ln -sf ../src/sdk/pycli/nnicli build
-	ln -sf ../tools/nni_annotation build
-	ln -sf ../tools/nni_cmd build
-	ln -sf ../tools/nni_trial_tool build
-	ln -sf ../tools/nni_gpu_tool build
+	ln -sfT ../src/sdk/pynni/nni build/nni
+	ln -sfT ../src/sdk/pycli/nnicli build/nnicli
+	ln -sfT ../tools/nni_annotation build/nni_annotation
+	ln -sfT ../tools/nni_cmd build/nni_cmd
+	ln -sfT ../tools/nni_trial_tool build/nni_trial_tool
+	ln -sfT ../tools/nni_gpu_tool build/nni_gpu_tool
 	cp setup.py build/
 	cp README.md build/
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' build/setup.py
@@ -205,16 +205,14 @@ install-node-modules:
 .PHONY: dev-install-node-modules
 dev-install-node-modules:
 	#$(_INFO) Installing NNI Package $(_END)
-	rm -rf $(NNI_PKG_FOLDER)
-	ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
+	ln -sfT ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
 	cp src/nni_manager/package.json $(NNI_PKG_FOLDER)
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
-	ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)
-	ln -sf ${PWD}/src/webui/build -t $(NNI_PKG_FOLDER)
-	mv $(NNI_PKG_FOLDER)/build $(NNI_PKG_FOLDER)/static
+	ln -sfT ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
+	ln -sfT ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static
 	mkdir -p $(NASUI_PKG_FOLDER)
-	ln -sf ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)
-	ln -sf ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)
+	ln -sfT ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)/build
+	ln -sfT ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)/server.js
 
 .PHONY: install-scripts
 install-scripts:

README.md

@@ -342,7 +342,7 @@ With authors' permission, we listed a set of NNI usage examples and relevant art
 Join IM discussion groups:
 |Gitter||WeChat|
 |----|----|----|
-|![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://user-images.githubusercontent.com/39592018/80665762-f06f2a00-8acc-11ea-8d22-e461e68e2d9b.png)|
+|![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://github.com/JSong-Jia/NNI-user-group/blob/master/NNI%20user%20group_3.png)|
 
 
 ## Related Projects

docs/en_US/TrainingService/PaiMode.md

@@ -92,8 +92,18 @@ Compared with [LocalMode](LocalMode.md) and [RemoteMachineMode](RemoteMachineMod
     * Required key. Set the mount path in your container used in PAI.
 * paiStoragePlugin
     * Optional key. Set the storage plugin name used in PAI. If it is not set in trial configuration, it should be set in the config file specified in `paiConfigPath` field.
+* command  
+    * Optional key. Set the commands used in PAI container.
 * paiConfigPath
     * Optional key. Set the file path of pai job configuration, the file is in yaml format.
+    If users set `paiConfigPath` in NNI's configuration file, no need to specify the fields `command`, `paiStoragePlugin`, `virtualCluster`, `image`, `memoryMB`, `cpuNum`, `gpuNum` in `trial` configuration. These fields will use the values from the config file specified by  `paiConfigPath`. 
+    ```
+    Note:
+      1. The job name in PAI's configuration file will be replaced by a new job name, the new job name is created by NNI, the name format is nni_exp_${this.experimentId}_trial_${trialJobId}.
+
+      2.  If users set multiple taskRoles in PAI's configuration file, NNI will wrap all of these taksRoles and start multiple tasks in one trial job, users should ensure that only one taskRole report metric to NNI, otherwise there might be some conflict error. 
+
+    ```  
 
 
 Once complete to fill NNI experiment config file and save (for example, save as exp_pai.yml), then run the following command

src/nni_manager/.eslintrc

@@ -23,6 +23,13 @@
         "@typescript-eslint/consistent-type-assertions": 0,
         "@typescript-eslint/no-inferrable-types": 0,
         "no-inner-declarations": 0,
+        "@typescript-eslint/explicit-function-return-type": "error",
+        "@typescript-eslint/no-unused-vars": [
+            "error",
+            {
+                "argsIgnorePattern": "^_"
+            }
+        ],
         "@typescript-eslint/no-var-requires": 0
     },
     "ignorePatterns": [

src/nni_manager/core/sqlDatabase.ts

@@ -98,7 +98,7 @@ class SqlDB implements Database {
                 this.resolve(this.initTask, err);
             } else {
                 if (createNew) {
-                    this.db.exec(createTables, (error: Error | null) => {
+                    this.db.exec(createTables, (_error: Error | null) => {
                         this.resolve(this.initTask, err);
                     });
                 } else {

src/nni_manager/package.json

@@ -14,7 +14,6 @@
     "azure-storage": "^2.10.2",
     "chai-as-promised": "^7.1.1",
     "child-process-promise": "^2.2.1",
-    "deepmerge": "^4.2.2",
     "express": "^4.16.3",
     "express-joi-validator": "^2.0.0",
     "js-base64": "^2.4.9",

src/nni_manager/rest_server/restHandler.ts

@@ -60,7 +60,7 @@ class NNIRestHandler {
         this.exportData(router);
 
         // Express-joi-validator configuration
-        router.use((err: any, req: Request, res: Response, next: any) => {
+        router.use((err: any, _req: Request, res: Response, _next: any) => {
             if (err.isBoom) {
                 this.log.error(err.output.payload);
 

src/nni_manager/training_service/dlts/dltsTrainingService.ts

@@ -131,7 +131,7 @@ class DLTSTrainingService implements TrainingService {
     private async statusCheckingLoop(): Promise<void> {
         while (!this.stopping) {
             const updateDLTSTrialJobs: Promise<void>[] = [];
-            for (const [trialJobId, dltsTrialJob] of this.trialJobsMap) {
+            for (const dltsTrialJob of this.trialJobsMap.values()) {
                 updateDLTSTrialJobs.push(this.getDLTSTrialJobInfo(dltsTrialJob));
             }
 
@@ -405,7 +405,7 @@ class DLTSTrainingService implements TrainingService {
         }
     }
 
-    public async getClusterMetadata(key: string): Promise<string> {
+    public async getClusterMetadata(_key: string): Promise<string> {
         return '';
     }
 
@@ -545,7 +545,7 @@ class DLTSTrainingService implements TrainingService {
             body: parameterFileMeta
         };
         await new Promise((resolve, reject) => {
-            request(req, (err: Error, res: request.Response) => {
+            request(req, (err: Error, _res: request.Response) => {
                 if (err) {
                     reject(err);
                 } else {

src/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts

@@ -20,7 +20,7 @@ export namespace AzureStorageClientUtility {
      */
     export async function createShare(fileServerClient: any, azureShare: any): Promise<boolean> {
         const deferred: Deferred<boolean> = new Deferred<boolean>();
-        fileServerClient.createShareIfNotExists(azureShare, (error: any, result: any, response: any) => {
+        fileServerClient.createShareIfNotExists(azureShare, (error: any, _result: any, _response: any) => {
             if (error) {
                 getLogger()
                   .error(`Create share failed:, ${error}`);
@@ -41,7 +41,7 @@ export namespace AzureStorageClientUtility {
      */
     export async function createDirectory(fileServerClient: azureStorage.FileService, azureFoler: any, azureShare: any): Promise<boolean> {
         const deferred: Deferred<boolean> = new Deferred<boolean>();
-        fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, result: any, response: any) => {
+        fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, _result: any, _response: any) => {
             if (error) {
                 getLogger()
                   .error(`Create directory failed:, ${error}`);
@@ -89,7 +89,7 @@ export namespace AzureStorageClientUtility {
                                      localFilePath: string): Promise<boolean> {
         const deferred: Deferred<boolean> = new Deferred<boolean>();
         await fileServerClient.createFileFromLocalFile(azureShare, azureDirectory, azureFileName, localFilePath,
-                                                       (error: any, result: any, response: any) => {
+                                                       (error: any, _result: any, _response: any) => {
             if (error) {
                 getLogger()
                   .error(`Upload file failed:, ${error}`);
@@ -114,7 +114,7 @@ export namespace AzureStorageClientUtility {
                                 localFilePath: string): Promise<boolean> {
         const deferred: Deferred<boolean> = new Deferred<boolean>();
         await fileServerClient.getFileToStream(azureShare, azureDirectory, azureFileName, fs.createWriteStream(localFilePath),
-                                               (error: any, result: any, response: any) => {
+                                               (error: any, _result: any, _response: any) => {
             if (error) {
                 getLogger()
                   .error(`Download file failed:, ${error}`);
@@ -183,7 +183,7 @@ export namespace AzureStorageClientUtility {
         const deferred: Deferred<void> = new Deferred<void>();
         await mkDirP(localDirectory);
         fileServerClient.listFilesAndDirectoriesSegmented(azureShare, azureDirectory, 'null',
-                                                          async (error: any, result: any, response: any) => {
+                                                          async (_error: any, result: any, _response: any) => {
             if (('entries' in result) === false) {
                 getLogger()
                   .error(`list files failed, can't get entries in result`);

src/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts

@@ -40,8 +40,8 @@ export class KubernetesJobInfoCollector {
         await Promise.all(updateKubernetesTrialJobs);
     }
 
-    protected async retrieveSingleTrialJobInfo(kubernetesCRDClient: KubernetesCRDClient | undefined,
-                                               kubernetesTrialJob: KubernetesTrialJobDetail): Promise<void> {
+    protected async retrieveSingleTrialJobInfo(_kubernetesCRDClient: KubernetesCRDClient | undefined,
+                                               _kubernetesTrialJob: KubernetesTrialJobDetail): Promise<void> {
             throw new MethodNotImplementedError();
     }
 }

src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts

@@ -77,7 +77,7 @@ abstract class KubernetesTrainingService {
     public async listTrialJobs(): Promise<TrialJobDetail[]> {
         const jobs: TrialJobDetail[] = [];
 
-        for (const [key, value] of this.trialJobsMap) {
+        for (const key of this.trialJobsMap.keys()) {
             jobs.push(await this.getTrialJob(key));
         }
 
@@ -107,7 +107,7 @@ abstract class KubernetesTrainingService {
         return false;
     }
 
-    public getClusterMetadata(key: string): Promise<string> {
+    public getClusterMetadata(_key: string): Promise<string> {
         return Promise.resolve('');
     }
 

src/nni_manager/training_service/pai/paiConfig.ts

@@ -3,7 +3,6 @@
 
 'use strict';
 
-import {TrialConfig} from '../common/trialConfig';
 import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
 
 export class PAIClusterConfig {

src/nni_manager/training_service/pai/paiJobInfoCollector.ts

@@ -62,7 +62,7 @@ export class PAIJobInfoCollector {
         };
 
         //TODO : pass in request timeout param?
-        request(getJobInfoRequest, (error: Error, response: request.Response, body: any) => {
+        request(getJobInfoRequest, (error: Error, response: request.Response, _body: any) => {
             if ((error !== undefined && error !== null) || response.statusCode >= 500) {
                 this.log.error(`PAI Training service: get job info for trial ${paiTrialJob.id} from PAI Cluster failed!`);
                 // Queried PAI job info failed, set job status to UNKNOWN

src/nni_manager/training_service/pai/paiJobRestServer.ts

@@ -4,8 +4,6 @@
 'use strict';
 
 import { Request, Response, Router } from 'express';
-import { Inject } from 'typescript-ioc';
-import * as component from '../../common/component';
 import { ClusterJobRestServer } from '../common/clusterJobRestServer';
 import { PAITrainingService } from './paiTrainingService';