Update README.md

wuhouming · Jan 13, 2025 · 301a393 · 301a393
1 parent f7b9582
commit 301a393
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -161,4 +161,19 @@ torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \
 
 -->
 
+## Activations recompute-enhanced
+<pre>
+
+# The default and recommended 
+SELECTIVE_RECOMPUTE_ARGS="
+    --recompute-granularity selective
+"
 
+# For cases where memory is very tight
+FULL_RECOMPUTE_ARGS="
+    --recompute-granularity full \
+    --recompute-method uniform \
+    --recompute-num-layers 1
+"
+
+</pre>