Support d*lr for ProdigyPlus optimizer

rockerBOO · Nov 20, 2024 · ef9a9c1 · ef9a9c1
1 parent 0dbb0d9
commit ef9a9c1
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/train_network.py b/train_network.py
@@ -53,7 +53,7 @@ def __init__(self):
 
     # TODO 他のスクリプトと共通化する
     def generate_step_logs(
-        self, args: argparse.Namespace, current_loss, avr_loss, lr_scheduler, keys_scaled=None, mean_norm=None, maximum_norm=None
+        self, args: argparse.Namespace, current_loss, avr_loss, lr_scheduler, optimizer, keys_scaled=None, mean_norm=None, maximum_norm=None
     ):
         logs = {"loss/current": current_loss, "loss/average": avr_loss}
 
@@ -79,6 +79,12 @@ def generate_step_logs(
                 logs["lr/d*lr"] = (
                     lr_scheduler.optimizers[-1].param_groups[0]["d"] * lr_scheduler.optimizers[-1].param_groups[0]["lr"]
                 )
+            if (
+                args.optimizer_type.lower().endswith("ProdigyPlusScheduleFree".lower())
+            ):  # tracking d*lr value of unet.
+                logs["lr/d*lr"] = (
+                    optimizer.param_groups[0]["d"] * optimizer.param_groups[0]["lr"]
+                )
         else:
             idx = 0
             if not args.network_train_unet_only:
@@ -91,6 +97,12 @@ def generate_step_logs(
                     logs[f"lr/d*lr/group{i}"] = (
                         lr_scheduler.optimizers[-1].param_groups[i]["d"] * lr_scheduler.optimizers[-1].param_groups[i]["lr"]
                     )
+                if (
+                    args.optimizer_type.lower().endswith("ProdigyPlusScheduleFree".lower())
+                ):  
+                    logs[f"lr/d*lr/group{i}"] = (
+                        optimizer.param_groups[0]["d"] * optimizer.param_groups[0]["lr"]
+                    )
 
         return logs
 
@@ -965,7 +977,7 @@ def remove_model(old_ckpt_name):
                     progress_bar.set_postfix(**{**max_mean_logs, **logs})
 
                 if args.logging_dir is not None:
-                    logs = self.generate_step_logs(args, current_loss, avr_loss, lr_scheduler, keys_scaled, mean_norm, maximum_norm)
+                    logs = self.generate_step_logs(args, current_loss, avr_loss, lr_scheduler, optimizer, keys_scaled, mean_norm, maximum_norm)
                     accelerator.log(logs, step=global_step)
 
                 if global_step >= args.max_train_steps: