-
Notifications
You must be signed in to change notification settings - Fork 0
/
common-allennlp-eval-v2
145 lines (127 loc) · 5.58 KB
/
common-allennlp-eval-v2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/bin/bash
readonly archive_dir="$MTL_ALLENNLP_OUTPUTS"
readonly data_dir="$MTL_DATA"
ARGS=( "$@" )
COMMANDS="${ARGS[@]}"
for i in $COMMANDS; do
echo $i
if [[ $i == ms* ]]; then
# ms*
# e.g., msner_mtlalt_ug_tva_best
lang=`echo $i | cut -d"_" -f3`
base=`echo $i | cut -d"_" -f2`
task=`echo $i | cut -d"_" -f1`
else
# mtl, mtlud, mtlner, mtlpos, mtlhom, mtlalthom, mtlalt
# e.g., mtl_ug_tva_best
lang=`echo $i | cut -d"_" -f2`
task=`echo $i | cut -d"_" -f1`
fi
for fold in $(seq 0 4); do
archive="${archive_dir}/${i}.${fold}/model.tar.gz"
if [[ $task == "mtlner" || $task == "msner" ]]; then
test_file="${data_dir}/${lang}/panx/test"
allennlp evaluate $archive $test_file \
--data-key ner \
--output-file ${archive_dir}/${i}.${fold}/metrics_ner.json \
--cuda-device 0 \
--include-package modules-v2
fi
if [[ $task == "mtlpos" || $task == "mspos" ]]; then
data_key=`[[ $task == "mspos" && ($base == "mtlalt" || $base == "mtlalthom") ]] \
&& echo "ud" || echo "pos"`
test_file="${data_dir}/${lang}/ud/test.conllu"
allennlp evaluate $archive $test_file \
--data-key ${data_key} \
--output-file ${archive_dir}/${i}.${fold}/metrics_pos.json \
--cuda-device 0 \
--extend-namespace head_tags \
--extend-namespace head_indices \
--include-package modules-v2
fi
if [[ $task == "mtlud" || $task == "msud" ]]; then
test_file="${data_dir}/${lang}/ud/test.conllu"
allennlp evaluate $archive $test_file \
--data-key ud \
--output-file ${archive_dir}/${i}.${fold}/metrics_ud.json \
--cuda-device 0 \
--include-package modules-v2 \
--extend-namespace head_tags \
--extend-namespace head_indices
allennlp predict $archive $test_file \
--use-dataset-reader \
--include-package modules-v2 \
--predictor dependency-parser \
--multitask-head ud \
--extend-namespace head_tags \
--extend-namespace head_indices \
--silent \
--cuda-device 0 \
--output-file ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu
(python scripts/eval/conll18_ud_eval.py \
$test_file ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu \
--skip-checks) \
> ${archive_dir}/${i}.${fold}/metrics_ud.txt
fi
if [[ $task == "mtl" || $task == "mtlhom" ]]; then
panx_test="${data_dir}/${lang}/panx/test"
ud_test="${data_dir}/${lang}/ud/test.conllu"
if [[ $lang == "wo" ]]; then
json_input="{\"ud\":\"${ud_test}\",\"pos\":\"${ud_test}\"}"
else
json_input="{\"ud\":\"${ud_test}\",\"ner\":\"${panx_test}\",\"pos\":\"${ud_test}\"}"
fi
allennlp evaluate $archive $json_input \
--truly-multitask \
--output-file ${archive_dir}/${i}.${fold}/metrics_mtl.json \
--cuda-device 0 \
--extend-namespace head_tags \
--extend-namespace head_indices \
--include-package modules-v2
allennlp predict $archive $ud_test \
--use-dataset-reader \
--include-package modules-v2 \
--predictor dependency-parser \
--multitask-head ud \
--extend-namespace head_tags \
--extend-namespace head_indices \
--silent \
--cuda-device 0 \
--output-file ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu
(python scripts/eval/conll18_ud_eval.py \
$ud_test ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu \
--skip-checks) \
> ${archive_dir}/${i}.${fold}/metrics_ud.txt
fi
if [[ $task == "mtlalt" || $task == "mtlalthom" ]]; then
panx_test="${data_dir}/${lang}/panx/test"
ud_test="${data_dir}/${lang}/ud/test.conllu"
if [[ $lang == "wo" ]]; then
json_input="{\"ud\":\"${ud_test}\"}"
else
json_input="{\"ud\":\"${ud_test}\",\"ner\":\"${panx_test}\"}"
fi
allennlp evaluate $archive $json_input \
--truly-multitask \
--output-file ${archive_dir}/${i}.${fold}/metrics_mtl.json \
--cuda-device 0 \
--extend-namespace head_tags \
--extend-namespace head_indices \
--include-package modules-v2
allennlp predict $archive $ud_test \
--use-dataset-reader \
--include-package modules-v2 \
--predictor dependency-parser \
--multitask-head ud \
--extend-namespace head_tags \
--extend-namespace head_indices \
--silent \
--cuda-device 0 \
--output-file ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu
(python scripts/eval/conll18_ud_eval.py \
$ud_test ${archive_dir}/${i}.${fold}/test_predictions_ud.conllu \
--skip-checks) \
> ${archive_dir}/${i}.${fold}/metrics_ud.txt
fi
done
done