add a basic ds zero3 config

better defaults for ds
axolotl-ai-cloud · Aug 6, 2023 · cc3b6c7 · cc3b6c7
1 parent 10405b9
commit cc3b6c7
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/ds_config.json → deepspeed/zero3.json b/ds_config.json → deepspeed/zero3.json
@@ -37,18 +37,18 @@
       "lr": "auto",
       "betas": [
         0.9,
-        0.999
+        0.95
       ],
       "eps": 1e-8,
       "weight_decay": "auto"
     }
   },
   "scheduler": {
-    "type": "OneCycle",
+    "type": "WarmupLR",
     "params": {
-      "cycle_min_lr": 0.00001,
-      "cycle_max_lr": 0.00003,
-      "cycle_first_step_size": 120
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
     }
   },
   "train_batch_size": "auto",