diff --git a/examples/4_train_policy_with_script.md b/examples/4_train_policy_with_script.md index d8ffa09f2..454ea4263 100644 --- a/examples/4_train_policy_with_script.md +++ b/examples/4_train_policy_with_script.md @@ -177,7 +177,7 @@ When you start the training process, you will first see your full configuration After that, you will see training log like this one: ``` -INFO 2024-08-14 13:35:12 smpl:2K ep:3 epch:0.06 loss:3.706 grdn:94.749 lr:1.0e-05 pu_mx_av:1.472|1.159 dl_mx_av:0.022|0.010 +INFO 2024-08-14 13:35:12 smpl:2K ep:3 epch:0.06 loss:3.706 grdn:94.749 lr:1.0e-05 updt_max|avg:1472|1159 data_max|avg:22|10 ``` or evaluation log like: diff --git a/lerobot/scripts/train.py b/lerobot/scripts/train.py index 6f1a2c000..98782f693 100644 --- a/lerobot/scripts/train.py +++ b/lerobot/scripts/train.py @@ -181,7 +181,7 @@ def log_train_info(logger: Logger, info, step, cfg, dataset, is_online): f"smpl:{format_big_number(num_samples)}", # number of episodes seen during training f"ep:{format_big_number(num_episodes)}", - # number of time all unique samples are seen + # number of passes through all of the training samples since the start of training f"epch:{num_epochs:.2f}", # loss in the past step f"loss:{loss:.3f}", @@ -189,10 +189,10 @@ def log_train_info(logger: Logger, info, step, cfg, dataset, is_online): f"grdn:{grad_norm:.3f}", # learning rate at the end of the past step f"lr:{lr:0.1e}", - # aggregated policy update time(s) in format max_update_time|average_update_time since the last log - f"pu_mx_av:{max_policy_updating_s:.3f}|{avg_policy_updating_s:.3f}", - # data loading time(s) in format max_loading_time|average_loading_time since the last log - f"dl_mx_av:{max_data_loading_s:.3f}|{avg_data_loading_s:.3f}", # if not ~0, you are bottlenecked by cpu or io + # time taken for a policy update (forward + backward + optimizer step) in milliseconds. Includes the maximum and average over all training steps since the last log. + f"updt_max|avg:{round(max_policy_updating_s * 1000)}|{round(avg_policy_updating_s * 1000)}", + # data loading time in milliseconds. Includes the maximum and average over all training steps since the last log. + f"data_max|avg:{round(max_data_loading_s *1000)}|{round(avg_data_loading_s*1000)}", # if not ~0, you are bottlenecked by cpu or io ] logging.info(" ".join(log_items))