From 5ae751f288de85aefe875a2202f46122b3c3c58f Mon Sep 17 00:00:00 2001 From: Jariwala Date: Mon, 30 Apr 2018 08:26:04 -0700 Subject: [PATCH 1/2] store metrics to log file for benchmarkAI --- benchmark/scripts/logging_metrics.py | 72 +++++++++++++++++++++++++ benchmark/scripts/models/timehistory.py | 64 ++++++++++++++++++++-- 2 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 benchmark/scripts/logging_metrics.py diff --git a/benchmark/scripts/logging_metrics.py b/benchmark/scripts/logging_metrics.py new file mode 100644 index 00000000000..714950e5724 --- /dev/null +++ b/benchmark/scripts/logging_metrics.py @@ -0,0 +1,72 @@ +import keras + + +class LoggingMetrics: + """Callback that save metrics to a logfile. + + # Arguments + history_callback: instance of `keras.callbacks.History`. + Training parameters + (eg. batch size, number of epochs, loss, acc). + time_callback: instance of `keras.callbacks.Callback`. + Training parameters + (eg. time, time-step, speed). + + # Raises + TypeError: In case of invalid object instance. + """ + + def __init__(self, history_callback, time_callback): + self.num_iteration = None + self.parameter_list = [] + self.pattern_list = [] + self.retrieve_metrics(history_callback, time_callback) + + def retrieve_metrics(self, history_callback, time_callback): + if not isinstance(history_callback, keras.callbacks.History): + raise TypeError('`history_callback` should be an instance of ' + '`keras.callbacks.History`') + if not isinstance(time_callback, keras.callbacks.Callback): + raise TypeError('`time_callback` should be an instance of ' + '`keras.callbacks.Callback`') + + if hasattr(history_callback, 'epoch'): + self.parameter_list.append(history_callback.epoch) + self.pattern_list.append('[Epoch %d]\t') + + if hasattr(time_callback, 'times'): + self.parameter_list.append(time_callback.get_time()) + self.parameter_list.append(time_callback.get_time_step()) + self.parameter_list.append(time_callback.get_speed()) + self.pattern_list.append('time: %s\t') + self.pattern_list.append('time_step: %s\t') + self.pattern_list.append('speed: %s\t') + + if 'loss' in history_callback.history: + self.parameter_list.append(history_callback.history['loss']) + self.pattern_list.append('train_loss: %.4f\t') + + if 'acc' in history_callback.history: + self.parameter_list.append(history_callback.history['acc']) + self.pattern_list.append('train_acc: %.4f\t') + + if 'val_loss' in history_callback.history: + self.parameter_list.append(history_callback.history['val_loss']) + self.pattern_list.append('val_loss: %.4f\t') + + if 'val_acc' in history_callback.history: + self.parameter_list.append(history_callback.history['val_acc']) + self.pattern_list.append('val_acc: %.4f\t') + + self.num_iteration = history_callback.params['epochs'] + + def get_metrics_index(self, idx): + idx_param_list = [] + for param in self.parameter_list: + idx_param_list.append(param[idx]) + return tuple(idx_param_list) + + def save_metrics_to_log(self, logging): + pattern_str = ''.join(self.pattern_list) + for i in range(self.num_iteration): + logging.info(pattern_str % self.get_metrics_index(i)) diff --git a/benchmark/scripts/models/timehistory.py b/benchmark/scripts/models/timehistory.py index 32caccca8b8..f6d5fdcfcf5 100644 --- a/benchmark/scripts/models/timehistory.py +++ b/benchmark/scripts/models/timehistory.py @@ -8,11 +8,69 @@ class TimeHistory(keras.callbacks.Callback): - def on_train_begin(self, logs={}): + """Callback that extract execution time of every epoch, time-step, + and speed in terms of sample per sec + """ + + def __init__(self): + super(TimeHistory, self).__init__() + self.times = [] + + def on_train_begin(self, logs=None): self.times = [] - def on_epoch_begin(self, batch, logs={}): + def on_epoch_begin(self, batch, logs=None): self.epoch_time_start = time.time() - def on_epoch_end(self, batch, logs={}): + def on_epoch_end(self, batch, logs=None): self.times.append(time.time() - self.epoch_time_start) + + def get_num_samples(self): + if 'samples' in self.params: + return self.params['samples'] + elif 'steps' in self.params: + return self.params['steps'] + else: + raise ValueError('Incorrect metric parameter') + + def reformat(self, var): + if var >= 1: + var = '%.2f ' % var + time_format = 'sec' + elif var >= 1e-3: + var = '%.2f ' % (var * 1e3) + time_format = 'msec' + else: + var = '%.2f ' % (var * 1e6) + time_format = 'usec' + return var, time_format + + def get_time_step(self): + time_list = [] + num_samples = self.get_num_samples() + for t in self.times: + speed = t / num_samples + speed, time_format = self.reformat(speed) + time_list.append(speed + time_format + '/step') + return time_list + + def get_total_time(self): + total_time = sum(self.times) + total_time, time_format = self.reformat(total_time) + return total_time + time_format + + def get_time(self): + time_list = [] + for t in self.times: + time, time_format = self.reformat(t) + time_list.append(time + time_format) + return time_list + + def get_speed(self): + samples_list = [] + num_samples = self.get_num_samples() + for t in self.times: + sample_sec = num_samples / t + sample_sec, time_format = self.reformat(sample_sec) + samples_list.append(sample_sec + 'samples/' + time_format) + return samples_list From 38c788822e5eb92268cfc2324671b5f637c66e68 Mon Sep 17 00:00:00 2001 From: Jariwala Date: Mon, 30 Apr 2018 08:31:24 -0700 Subject: [PATCH 2/2] modified naming conventions --- benchmark/scripts/logging_metrics.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmark/scripts/logging_metrics.py b/benchmark/scripts/logging_metrics.py index 714950e5724..8102180fa3d 100644 --- a/benchmark/scripts/logging_metrics.py +++ b/benchmark/scripts/logging_metrics.py @@ -18,7 +18,7 @@ class LoggingMetrics: def __init__(self, history_callback, time_callback): self.num_iteration = None - self.parameter_list = [] + self.metric_list = [] self.pattern_list = [] self.retrieve_metrics(history_callback, time_callback) @@ -31,40 +31,40 @@ def retrieve_metrics(self, history_callback, time_callback): '`keras.callbacks.Callback`') if hasattr(history_callback, 'epoch'): - self.parameter_list.append(history_callback.epoch) + self.metric_list.append(history_callback.epoch) self.pattern_list.append('[Epoch %d]\t') if hasattr(time_callback, 'times'): - self.parameter_list.append(time_callback.get_time()) - self.parameter_list.append(time_callback.get_time_step()) - self.parameter_list.append(time_callback.get_speed()) + self.metric_list.append(time_callback.get_time()) + self.metric_list.append(time_callback.get_time_step()) + self.metric_list.append(time_callback.get_speed()) self.pattern_list.append('time: %s\t') self.pattern_list.append('time_step: %s\t') self.pattern_list.append('speed: %s\t') if 'loss' in history_callback.history: - self.parameter_list.append(history_callback.history['loss']) + self.metric_list.append(history_callback.history['loss']) self.pattern_list.append('train_loss: %.4f\t') if 'acc' in history_callback.history: - self.parameter_list.append(history_callback.history['acc']) + self.metric_list.append(history_callback.history['acc']) self.pattern_list.append('train_acc: %.4f\t') if 'val_loss' in history_callback.history: - self.parameter_list.append(history_callback.history['val_loss']) + self.metric_list.append(history_callback.history['val_loss']) self.pattern_list.append('val_loss: %.4f\t') if 'val_acc' in history_callback.history: - self.parameter_list.append(history_callback.history['val_acc']) + self.metric_list.append(history_callback.history['val_acc']) self.pattern_list.append('val_acc: %.4f\t') self.num_iteration = history_callback.params['epochs'] def get_metrics_index(self, idx): - idx_param_list = [] - for param in self.parameter_list: - idx_param_list.append(param[idx]) - return tuple(idx_param_list) + idx_metric_list = [] + for metric in self.metric_list: + idx_metric_list.append(metric[idx]) + return tuple(idx_metric_list) def save_metrics_to_log(self, logging): pattern_str = ''.join(self.pattern_list)