-
Notifications
You must be signed in to change notification settings - Fork 0
/
iclr_reb_trunction_pre_all_blank.sh
111 lines (105 loc) · 4.46 KB
/
iclr_reb_trunction_pre_all_blank.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
for i in 0 # 1 1.5 2 2.5 3 3.5 4 4.5 5 #3 4 5 6 7 8 9 2 all #0 # 5 15 30 # 10 20 25 # # 40 45 #
do
base_noise="shuffle"
cuda=3
if [ ! -d "log/out/"$preix ]; then
mkdir "log/out/"$preix
echo "crete folder log/out/"$preix
fi
preix=$base_noise"_iclr_reb_trunc_pre"
LR=1e-06
sym="ce_trunc$i""_lr_"$LR
loss="cross_entropy_truncation" # cross_entropy
model="bart_base" # "bart_base_iter"
tf_ratio=1.0
noise="$base_noise$i"
data="multi30k-bin/$noise"
max_epoch=20
name=${loss:0:2}"_"${noise}"_"${model:0:2}"_"$tf_ratio"_"$sym
TOTAL_NUM_UPDATES=10000
WARMUP_UPDATES=500
MAX_TOKENS=6000
UPDATE_FREQ=1
BART_PATH="/home/lptang/fairseq/checkpoints/denoising_bart/$base_noise/cr_"$noise"_ba_1.0_cetf_"$i"/checkpoint_best.pt"
BATCH_SIZE=128
if [ ! -d "log/out/"$preix ]; then
mkdir "log/out/"$preix
echo "crete folder log/out/"$preix
fi
CUDA_VISIBLE_DEVICES=$cuda fairseq-train multi30k-bin/$noise \
--restore-file $BART_PATH --reset-optimizer --reset-dataloader --reset-meters \
--max-tokens $MAX_TOKENS \
--task translation \
--source-lang de --target-lang en \
--truncate-source \
--layernorm-embedding \
--share-all-embeddings \
--share-decoder-input-output-embed \
--required-batch-size-multiple 1 \
--arch $model \
--criterion $loss \
--dropout 0.1 --attention-dropout 0.1 \
--weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.999)" --adam-eps 1e-08 \
--clip-norm 0.1 \
--lr-scheduler polynomial_decay --lr $LR --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES\
--fp16 --update-freq $UPDATE_FREQ \
--skip-invalid-size-inputs-valid-test \
--find-unused-parameters\
--batch-size $BATCH_SIZE\
--eval-bleu --eval-bleu-args '{"beam": 5, "max_len_a": 1.2, "max_len_b": 10}' \
--eval-bleu-detok moses --eval-bleu-remove-bpe --best-checkpoint-metric bleu \
--maximize-best-checkpoint-metric\
--save-dir checkpoints/denoising_bart/$preix/$name \
--tf-ratio $tf_ratio --tensorboard-logdir log/tf/denoising_bart/$preix/$name \
--max-epoch $max_epoch --no-epoch-checkpoints --patience 5
# preix=$base_noise
# sym="ce_trunc$i""_lr_"$LR
# loss="cross_entropy_truncation" # cross_entropy
# model="bart_base" # "bart_base_iter"
# tf_ratio=1.0 #'1.0'
# noise="$preix$i"
# data="multi30k-bin/$noise"
# max_epoch=20
# name=${loss:0:2}"_"${noise}"_"${model:0:2}"_"$tf_ratio"_"$sym
# TOTAL_NUM_UPDATES=10000
# WARMUP_UPDATES=500
# LR=3e-05
# MAX_TOKENS=6000
# UPDATE_FREQ=4
# # BART_PATH=/home/lptang/fairseq/checkpoints/510_bart/cr_shuffle5_ba_1.0_s5_ce/checkpoint_best.pt
# BART_PATH=/home/lptang/fairseq/examples/bart/pretrained/bart.base/model.pt
# # BART_PATH="/home/lptang/fairseq/checkpoints/denoising_bart/$base_noise/cr_"$noise"_ba_1.0_cetf_"$i"/checkpoint_best.pt"
# BATCH_SIZE=128
# if [ ! -d "log/out/"$preix ]; then
# mkdir "log/out/"$preix
# echo "crete folder log/out/"$preix
# fi
# mkdir log/tf/denoising_bart/$preix
# CUDA_VISIBLE_DEVICES=$cuda fairseq-train multi30k-bin/$noise \
# --restore-file $BART_PATH \
# --max-tokens $MAX_TOKENS \
# --task translation \
# --source-lang de --target-lang en \
# --truncate-source \
# --layernorm-embedding \
# --share-all-embeddings \
# --share-decoder-input-output-embed \
# --reset-optimizer --reset-dataloader --reset-meters \
# --required-batch-size-multiple 1 \
# --arch $model \
# --criterion $loss \
# --dropout 0.1 --attention-dropout 0.1 \
# --weight-decay 0.01 --optimizer adam --adam-betas "(0.9, 0.999)" --adam-eps 1e-08 \
# --clip-norm 0.1 \
# --lr-scheduler polynomial_decay --lr $LR --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES\
# --fp16 --update-freq $UPDATE_FREQ \
# --skip-invalid-size-inputs-valid-test \
# --find-unused-parameters\
# --batch-size $BATCH_SIZE\
# --eval-bleu --eval-bleu-args '{"beam": 5, "max_len_a": 1.2, "max_len_b": 10}' \
# --eval-bleu-detok moses --eval-bleu-remove-bpe --best-checkpoint-metric bleu \
# --maximize-best-checkpoint-metric\
# --save-dir checkpoints/denoising_bart/$preix/$name \
# --tf-ratio $tf_ratio --tensorboard-logdir log/tf/denoising_bart/$preix/$name \
# --max-epoch $max_epoch --no-epoch-checkpoints --patience 3 #--eval-bleu-print-samples --log-interval 5
done