feat: add HuggingFace - AutoTrain

gist-rs · Oct 1, 2023 · 26f6b3e · 26f6b3e
1 parent 07eaebe
commit 26f6b3e
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 18 deletions.
diff --git a/src/SUMMARY.md b/src/SUMMARY.md
@@ -64,4 +64,5 @@
   - [Large Language Model](ml/large-language-model.md)
   - [Stable Diffusion](ml/stable-diffusion.md)
   - [Hugging Face - Candle](ml/huggingface-candle.md)
+  - [Hugging Face - AutoTrain](ml/huggingface-autotrain.md)
 - [Bye](bye.md)
diff --git a/src/ml/huggingface-autotrain.md b/src/ml/huggingface-autotrain.md
@@ -1,13 +1,13 @@
-## HuggingFace Auto train
+## 🤗 HuggingFace - AutoTrain
 
-### Knowledge Base
+### 📚 Knowledge Base
 
-- `PEFT` Parameter-Efficient Fine-Tuning: https://github.com/huggingface/peft
-- `TRL` Transformer Reinforcement Learning: https://github.com/huggingface/trl
+- `PEFT` [Parameter-Efficient Fine-Tuning](https://github.com/huggingface/peft)
+- `TRL` [Transformer Reinforcement Learning](https://github.com/huggingface/trl)
 
 > ref: https://github.com/huggingface/trl/blob/main/examples/README.md
 
-### Setup
+### ⚙️ Setup
 
 ```bash
 # Install Python3 pip
@@ -30,7 +30,7 @@ pip3 install deepspeed
 pip install accelerate
 ```
 
-### Config
+### 📝 Config
 
 ```bash
 accelerate config
@@ -99,7 +99,9 @@ bf16
 accelerate configuration saved at /home/katopz/.cache/huggingface/accelerate/default_config.yaml
 ```
 
-### Fine-tuned
+### 🪛 Fine tune
+
+![](/assets/kat.png) <span class="speech-bubble">Let's fine-tune [Mistral](https://mistral.ai/) locally.</span>
 
 ```bash
 # Setup
@@ -110,17 +112,17 @@ autotrain llm --help
 # Fine tune
 autotrain llm \
   --train \
-  --model mistralai/Mistral-7B-Instruct-v0.1  \
-  --data-path $DATA_PATH  \
-  --use-peft  \
-  --use-int4 \
-  --lr 20-4 \
-  --batch-size 4  \
-  --epochs 1  \
+  --model mistralai/Mistral-7B-Instruct-v0.1 \
+  --data-path $DATA_PATH \
+  --use-peft\
+  --use-int4\
+  --lr 2e-4 \
+  --batch-size 4 \
+  --epochs 1 \
   --trainer sft \
-  --username $USERNAME \
+  --username $USER_NAME \
   --token $HF_TOKEN \
-  --project-name foo-bar  \
-  --target-modules q_proj,v_proj  \
+  --project-name $PROJECT_NAME \
+  --target-modules q_proj,v_proj \
   --push-to-hub
 ```
diff --git a/src/ml/huggingface-candle.md b/src/ml/huggingface-candle.md
@@ -1,4 +1,4 @@
-# Hugging Face - Candle
+# 🤗 HuggingFace - Candle
 
 > [https://github.com/huggingface/candle]()