-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathtrain_script_clutrr.sh
130 lines (130 loc) · 4.43 KB
/
train_script_clutrr.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/bin/bash
## Uncomment the following to run experiments for: llama2-llama2_13B-base
# model_name="llama2-llama2_13B-base";
# task="clutrr";
# seed=42;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component A \
# --use_topk_heads 160 \
# --lr 1e-3 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 1e-3 \
# --eval_batch 32 \
# --train_size 300 \
# --seed $seed;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component v \
# --use_topk_heads 48 \
# --lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
# --lr 1e-2 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 0 \
# --eval_batch 32 \
# --train_size 300 \
# --seed $seed;
## Uncomment the following to run experiments for: llama2_7B
model_name="llama2_7B";
task="clutrr";
seed=42;
CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
--task $task \
--base_model_name $model_name \
--apply_chat_template False \
--ft_method lofit \
--lofit_component A \
--use_topk_heads 160 \
--lr 5e-4 \
--train_batch 8 \
--num_epoch 5 \
--output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
--run_mode train \
--output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
--applied_module attention \
--save_strategy no \
--l1_lambda 5e-3 \
--eval_batch 32 \
--train_size 300 \
--seed $seed;
CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
--task $task \
--base_model_name $model_name \
--apply_chat_template False \
--ft_method lofit \
--lofit_component v \
--use_topk_heads 32 \
--lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
--lr 1e-2 \
--train_batch 8 \
--num_epoch 5 \
--output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
--run_mode train \
--output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
--applied_module attention \
--save_strategy no \
--l1_lambda 0 \
--eval_batch 32 \
--train_size 300 \
--seed $seed;
## Uncomment the following to run experiments for: gemma_7b
# model_name="gemma_7b";
# task="clutrr";
# seed=42;
# CUDA_VISIBLE_DEVICES=0 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component A \
# --use_topk_heads 160 \
# --lr 5e-4 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_Aonly_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 5e-3 \
# --eval_batch 32 \
# --train_size 300 \
# --seed $seed;
# CUDA_VISIBLE_DEVICES=5 python lofit_trainer.py \
# --task $task \
# --base_model_name $model_name \
# --apply_chat_template False \
# --ft_method lofit \
# --lofit_component v \
# --use_topk_heads 16 \
# --lofit_heads "./top_heads/${model_name}_${task}_Aonly_top160heads_${seed}.npy"\
# --lr 1e-2 \
# --train_batch 8 \
# --num_epoch 5 \
# --output_dir "./finetuned_checkpoints/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --run_mode train \
# --output_file_name "./finetuned_outputs/${task}/${model_name}_${task}_lofit_seed${seed}"\
# --applied_module attention \
# --save_strategy no \
# --l1_lambda 0 \
# --eval_batch 32 \
# --train_size 300 \
# --seed $seed;