fix multi-thread event handling issue

nni/retiarii/experiment/pytorch.py

@@ -286,6 +286,11 @@ def stop(self) -> None:
         """
         _logger.info('Stopping experiment, please wait...')
         atexit.unregister(self.stop)
+
+        # stop strategy first
+        if self._dispatcher_thread is not None:
+            self._dispatcher.stopping = True
+            self._dispatcher_thread.join(timeout=1)
 
         if self.id is not None:
             nni.runtime.log.stop_experiment_log(self.id)
@@ -302,9 +307,6 @@ def stop(self) -> None:
 
         if self._pipe is not None:
             self._pipe.close()
-        if self._dispatcher_thread is not None:
-            self._dispatcher.stopping = True
-            self._dispatcher_thread.join(timeout=1)
 
         self.id = None
         self.port = None

ts/nni_manager/core/nnimanager.ts

@@ -620,8 +620,6 @@ class NNIManager implements Manager {
                 this.trialConcurrencyChange = requestTrialNum;
             }
 
-            this.requestTrialJobs(requestTrialNum);
-
             // check maxtrialnum and maxduration here
             // NO_MORE_TRIAL is more like a subset of RUNNING, because during RUNNING tuner
             // might tell nnimanager that this is no more trials. In NO_MORE_TRIAL state, the experiment is viewed
@@ -646,6 +644,8 @@ class NNIManager implements Manager {
                     }
                 }
             } else {
+                this.requestTrialJobs(requestTrialNum);
+
                 if (this.status.status === 'DONE') {
                     delete this.experimentProfile.endTime;
                     await this.storeExperimentProfile();