diff --git a/mlperf_logging/benchmark_meta.py b/mlperf_logging/benchmark_meta.py index 0d2cd18..aa57eff 100644 --- a/mlperf_logging/benchmark_meta.py +++ b/mlperf_logging/benchmark_meta.py @@ -119,7 +119,7 @@ 'ssd', 'unet3d', 'stable_diffusion', - 'llama2_70b_lora' + 'llama2_70b_lora', 'rnnt', 'unet3d', 'stable_diffusion', diff --git a/mlperf_logging/rcp_checker/rcp_checker.py b/mlperf_logging/rcp_checker/rcp_checker.py index 53d6d8f..45f07b4 100644 --- a/mlperf_logging/rcp_checker/rcp_checker.py +++ b/mlperf_logging/rcp_checker/rcp_checker.py @@ -79,6 +79,10 @@ def read_submission_file(result_file, ruleset, use_train_samples): eval_metric = json.loads(eval_accuracy_str)["metadata"]["metric"] eval_score = json.loads(eval_accuracy_str)["value"] stable_diffusion_eval_results[eval_step][eval_metric] = eval_score + elif benchmark == "llama2_70b_lora" and ("eval_error" in str or "eval_accuracy" in str): + eval_accuracy_str = str + conv_epoch = json.loads(eval_accuracy_str)["metadata"]["samples_count"] + eval_score = json.loads(eval_accuracy_str)["value"] elif not use_train_samples and ("eval_error" in str or "eval_accuracy" in str): eval_accuracy_str = str conv_epoch = json.loads(eval_accuracy_str)["metadata"]["epoch_num"] @@ -202,10 +206,11 @@ def _process_raw_rcp_data(self, raw_rcp_data): ''' processed_rcps = {} for record, record_contents in raw_rcp_data.items(): + conv_unit = "samples to converge" if record_contents['Benchmark']=='llama2_70b_lora' else "Epochs to converge" processed_record = {'Benchmark': record_contents['Benchmark'], 'BS': record_contents['BS'], 'Hyperparams': record_contents['Hyperparams'], - 'Epochs to converge': record_contents['Epochs to converge'], + 'Epochs to converge': record_contents[conv_unit], 'RCP Mean': 0.0, 'RCP Stdev': 0.0, 'Max Speedup': 0.0}