Show trial environment information in reuse mode (#3389)

microsoft · Feb 24, 2021 · bd2543e · bd2543e
1 parent 9caf521
commit bd2543e
Show file tree

Hide file tree

Showing 9 changed files with 10 additions and 29 deletions.
diff --git a/ts/nni_manager/common/datastore.ts b/ts/nni_manager/common/datastore.ts
@@ -23,6 +23,7 @@ interface TrialJobEventRecord {
     readonly data?: string;
     readonly logPath?: string;
     readonly sequenceId?: number;
+    readonly message?: string;
 }
 
 interface MetricData {

diff --git a/ts/nni_manager/common/manager.ts b/ts/nni_manager/common/manager.ts
@@ -105,7 +105,6 @@ abstract class Manager {
     public abstract getTrialLog(trialJobId: string, logType: LogType): Promise<string>;
 
     public abstract getTrialJobStatistics(): Promise<TrialJobStatistics[]>;
-    public abstract getTrialJobMessage(trialJobId: string): string | undefined;
     public abstract getStatus(): NNIManagerStatus;
 }
 

diff --git a/ts/nni_manager/core/nniDataStore.ts b/ts/nni_manager/core/nniDataStore.ts
@@ -87,7 +87,6 @@ class NNIDataStore implements DataStore {
         if (timestamp === undefined) {
             timestamp = Date.now();
         }
-
         return this.db.storeTrialJobEvent(event, trialJobId, timestamp, hyperParameter, jobDetail).catch(
                 (err: Error) => {
                     throw NNIError.FromError(err, 'Datastore error: ');
@@ -381,6 +380,7 @@ class NNIDataStore implements DataStore {
             if (record.sequenceId !== undefined && jobInfo.sequenceId === undefined) {
                 jobInfo.sequenceId = record.sequenceId;
             }
+            jobInfo.message = record.message;
             map.set(record.trialJobId, jobInfo);
         }
 

diff --git a/ts/nni_manager/core/nnimanager.ts b/ts/nni_manager/core/nnimanager.ts
@@ -353,14 +353,6 @@ class NNIManager implements Manager {
         return this.status;
     }
 
-    public getTrialJobMessage(trialJobId: string): string | undefined {
-        const trialJob = this.trialJobs.get(trialJobId);
-        if (trialJob !== undefined){
-            return trialJob.message
-        }
-        return undefined
-    }
-
     public async listTrialJobs(status?: TrialJobStatus): Promise<TrialJobInfo[]> {
         return this.dataStore.listTrialJobs(status);
     }

diff --git a/ts/nni_manager/core/sqlDatabase.ts b/ts/nni_manager/core/sqlDatabase.ts
@@ -22,7 +22,7 @@ import { TrialJobDetail } from '../common/trainingService';
 
 
 const createTables: string = `
-create table TrialJobEvent (timestamp integer, trialJobId text, event text, data text, logPath text, sequenceId integer);
+create table TrialJobEvent (timestamp integer, trialJobId text, event text, data text, logPath text, sequenceId integer, message text);
 create index TrialJobEvent_trialJobId on TrialJobEvent(trialJobId);
 create index TrialJobEvent_event on TrialJobEvent(event);
 
@@ -62,7 +62,8 @@ function loadTrialJobEvent(row: any): TrialJobEventRecord {
         event: row.event,
         data: row.data === null ? undefined : row.data,
         logPath: row.logPath === null ? undefined : row.logPath,
-        sequenceId: row.sequenceId === null ? undefined : row.sequenceId
+        sequenceId: row.sequenceId === null ? undefined : row.sequenceId,
+        message: row.message === null ? undefined: row.message
     };
 }
 
@@ -163,10 +164,11 @@ class SqlDB implements Database {
 
     public storeTrialJobEvent(
         event: TrialJobEvent, trialJobId: string, timestamp: number, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void> {
-        const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?)';
+        const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?,?)';
         const logPath: string | undefined = jobDetail === undefined ? undefined : jobDetail.url;
         const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.form.sequenceId;
-        const args: any[] = [timestamp, trialJobId, event, hyperParameter, logPath, sequenceId];
+        const message: string | undefined = jobDetail === undefined ? undefined : jobDetail.message;
+        const args: any[] = [timestamp, trialJobId, event, hyperParameter, logPath, sequenceId, message];
 
         this.log.trace(`storeTrialJobEvent: SQL: ${sql}, args: ${JSON.stringify(args)}`);
         const deferred: Deferred<void> = new Deferred<void>();

diff --git a/ts/nni_manager/rest_server/restHandler.ts b/ts/nni_manager/rest_server/restHandler.ts
@@ -213,7 +213,6 @@ class NNIRestHandler {
             this.nniManager.listTrialJobs(req.query.status).then((jobInfos: TrialJobInfo[]) => {
                 jobInfos.forEach((trialJob: TrialJobInfo) => {
                     this.setErrorPathForFailedJob(trialJob);
-                    this.setMessageforJob(trialJob);
                 });
                 res.send(jobInfos);
             }).catch((err: Error) => {
@@ -226,7 +225,6 @@ class NNIRestHandler {
         router.get('/trial-jobs/:id', (req: Request, res: Response) => {
             this.nniManager.getTrialJob(req.params.id).then((jobDetail: TrialJobInfo) => {
                 const jobInfo: TrialJobInfo = this.setErrorPathForFailedJob(jobDetail);
-                this.setMessageforJob(jobInfo);
                 res.send(jobInfo);
             }).catch((err: Error) => {
                 this.handleError(err, res);
@@ -327,14 +325,6 @@ class NNIRestHandler {
 
         return jobInfo;
     }
-
-    private setMessageforJob(jobInfo: TrialJobInfo): TrialJobInfo {
-        if (jobInfo === undefined){
-            return jobInfo
-        }
-        jobInfo.message = this.nniManager.getTrialJobMessage(jobInfo.trialJobId);
-        return jobInfo
-    }
 }
 
 export function createRestHandler(rs: NNIRestServer): Router {

diff --git a/ts/nni_manager/rest_server/test/mockedNNIManager.ts b/ts/nni_manager/rest_server/test/mockedNNIManager.ts
@@ -111,10 +111,6 @@ export class MockedNNIManager extends Manager {
         return deferred.promise;
     }
 
-    public getTrialJobMessage(trialJobId: string): string | undefined {
-        return "TEST-MESSAGE"
-    }
-
     public stopExperiment(): Promise<void> {
         throw new MethodNotImplementedError();
     }

diff --git a/ts/nni_manager/training_service/reusable/trial.ts b/ts/nni_manager/training_service/reusable/trial.ts
@@ -19,6 +19,7 @@ export class TrialDetail implements TrialJobDetail {
     public form: TrialJobApplicationForm;
     public isEarlyStopped?: boolean;
     public environment?: EnvironmentInformation;
+    public message?: string;
 
     // init settings of trial
     public settings = {};

diff --git a/ts/nni_manager/training_service/reusable/trialDispatcher.ts b/ts/nni_manager/training_service/reusable/trialDispatcher.ts
@@ -435,7 +435,6 @@ class TrialDispatcher implements TrainingService {
                             if (environment.environmentService === undefined) {
                                 throw new Error(`${environment.id} does not has environment service!`);
                             }
-
                             trial.url = environment.trackingUrl;
                             const environmentStatus = environment.status;
 
@@ -704,6 +703,7 @@ class TrialDispatcher implements TrainingService {
         if (environment.environmentService === undefined) {
             throw new Error(`${environment.id} environmentService not initialized!`);
         }
+        trial.message = `Platform: ${environment.environmentService.getName}, environment: ${environment.id}`;
         if (environment.environmentService.hasStorageService) {	
             const storageService = component.get<StorageService>(StorageService);	
             trial.workingDirectory = storageService.joinPath('trials', trial.id);