change SIGKILL to SIGTERM in local mode cancel trial job (#3173)

microsoft · Dec 15, 2020 · e9f832d · e9f832d
1 parent 12a5e3d
commit e9f832d
Showing 1 changed file with 14 additions and 1 deletion.
diff --git a/ts/nni_manager/training_service/local/localTrainingService.ts b/ts/nni_manager/training_service/local/localTrainingService.ts
@@ -253,7 +253,20 @@ class LocalTrainingService implements TrainingService {
 
             return Promise.resolve();
         }
-        tkill(trialJob.pid, 'SIGKILL');
+        tkill(trialJob.pid, 'SIGTERM');
+        const startTime = Date.now();
+        while(await isAlive(trialJob.pid)) {    
+            if (Date.now() - startTime > 4999) {
+                tkill(trialJob.pid, 'SIGKILL', (err) => {
+                    if (err) {
+                        this.log.error(`kill trial job error: ${err}`);
+                    }
+                });
+                break;
+            }
+            await delay(500);
+        }
+
         this.setTrialJobStatus(trialJob, getJobCancelStatus(isEarlyStopped));
 
         return Promise.resolve();