diff --git a/nni/runtime/msg_dispatcher.py b/nni/runtime/msg_dispatcher.py index 4f24294fc7..20b9597f9e 100644 --- a/nni/runtime/msg_dispatcher.py +++ b/nni/runtime/msg_dispatcher.py @@ -39,7 +39,7 @@ def _sort_history(history): # Tuner global variables _next_parameter_id = 0 _trial_params = {} -'''key: trial job ID; value: parameters''' +'''key: parameter ID; value: parameters''' _customized_parameter_ids = set() @@ -114,7 +114,7 @@ def handle_import_data(self, data): data: a list of dictionaries, each of which has at least two keys, 'parameter' and 'value' """ for entry in data: - entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) + entry['value'] = entry['value'] if type(entry['value']) is str else json_tricks.dumps(entry['value']) entry['value'] = json_tricks.loads(entry['value']) self.tuner.import_data(data) @@ -182,8 +182,11 @@ def _handle_final_metric_data(self, data): customized = True else: customized = False + if id_ in _trial_params: self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, trial_job_id=data.get('trial_job_id')) + else: + _logger.warning('Find unknown job parameter id %s, maybe something goes wrong.', _trial_params[id_]) def _handle_intermediate_metric_data(self, data): """Call assessor to process intermediate results diff --git a/ts/nni_manager/core/nnimanager.ts b/ts/nni_manager/core/nnimanager.ts index 6fa1271523..29705f3ec4 100644 --- a/ts/nni_manager/core/nnimanager.ts +++ b/ts/nni_manager/core/nnimanager.ts @@ -678,11 +678,15 @@ class NNIManager implements Manager { private async onTrialJobMetrics(metric: TrialJobMetric): Promise { this.log.debug(`NNIManager received trial job metrics: ${metric}`); - await this.dataStore.storeMetricData(metric.id, metric.data); - if (this.dispatcher === undefined) { - throw new Error('Error: tuner has not been setup'); + if (this.trialJobs.has(metric.id)){ + await this.dataStore.storeMetricData(metric.id, metric.data); + if (this.dispatcher === undefined) { + throw new Error('Error: tuner has not been setup'); + } + this.dispatcher.sendCommand(REPORT_METRIC_DATA, metric.data); + } else { + this.log.warning(`NNIManager received non-existent trial job metrics: ${metric}`); } - this.dispatcher.sendCommand(REPORT_METRIC_DATA, metric.data); } private requestTrialJobs(jobNum: number): void {