From 6311a419d6d1d7a1f4d23a8d35d8ec7101e1b845 Mon Sep 17 00:00:00 2001 From: Desh Raj Date: Thu, 5 Dec 2019 15:08:20 -0500 Subject: [PATCH 1/3] changed scoring tool for diarization --- egs/chime6/s5_track2/local/diarize.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/egs/chime6/s5_track2/local/diarize.sh b/egs/chime6/s5_track2/local/diarize.sh index 2ca95dc0fbc..95370389018 100755 --- a/egs/chime6/s5_track2/local/diarize.sh +++ b/egs/chime6/s5_track2/local/diarize.sh @@ -86,10 +86,17 @@ fi if [ $stage -le 5 ]; then if [ -f $ref_rttm ]; then echo "$0: computing diariztion error rate (DER) using reference ${ref_rttm}" - mkdir -p $out_dir/tuning/ - md-eval.pl -c 0.25 -1 -r $ref_rttm -s $out_dir/rttm 2> $out_dir/log/der.log > $out_dir/der - der=$(grep -oP 'DIARIZATION\ ERROR\ =\ \K[0-9]+([.][0-9]+)?' ${out_dir}/der) - echo "DER: $der%" + ref_rttm_path=$((readlink -f $ref_rttm)) + out_rttm_path=$((readling -f $out_dir/rttm)) + if ! [ -d dscore ]; then + git clone https://github.com/nryant/dscore.git || exit 1; + cd dscore + python -m pip install --user -r requirements.txt + cd .. + fi + cd dscore + python score.py -r $ref_rttm_path -s $out_rttm_path + cd .. fi fi From 76c5a41b1f97ae5cf172cab660e4d990c19399aa Mon Sep 17 00:00:00 2001 From: Desh Raj Date: Thu, 5 Dec 2019 15:45:50 -0500 Subject: [PATCH 2/3] added comment for scoring --- egs/chime6/s5_track2/local/diarize.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/egs/chime6/s5_track2/local/diarize.sh b/egs/chime6/s5_track2/local/diarize.sh index 95370389018..2990709b173 100755 --- a/egs/chime6/s5_track2/local/diarize.sh +++ b/egs/chime6/s5_track2/local/diarize.sh @@ -83,11 +83,14 @@ if [ $stage -le 4 ]; then echo "$0: wrote RTTM to output directory ${out_dir}" fi +# For scoring the diarization system, we use the same tool that was +# used in the DIHARD II challenge. This is available at: +# https://github.com/nryant/dscore if [ $stage -le 5 ]; then if [ -f $ref_rttm ]; then echo "$0: computing diariztion error rate (DER) using reference ${ref_rttm}" ref_rttm_path=$((readlink -f $ref_rttm)) - out_rttm_path=$((readling -f $out_dir/rttm)) + out_rttm_path=$((readlink -f $out_dir/rttm)) if ! [ -d dscore ]; then git clone https://github.com/nryant/dscore.git || exit 1; cd dscore From 3b754a0fa2406d2b25c2abc6f4ed5a417dd8b133 Mon Sep 17 00:00:00 2001 From: Desh Raj Date: Thu, 5 Dec 2019 16:21:10 -0500 Subject: [PATCH 3/3] updated RESULTS for new diarization scoring --- egs/chime6/s5_track2/RESULTS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/egs/chime6/s5_track2/RESULTS b/egs/chime6/s5_track2/RESULTS index eacee196584..6ba5deb9c5d 100644 --- a/egs/chime6/s5_track2/RESULTS +++ b/egs/chime6/s5_track2/RESULTS @@ -10,8 +10,8 @@ Eval 5.6 5.9 11.5 # Diarization DER JER -Dev 31.37 20.45 -Eval 30.67 18.97 +Dev 57.15 83.96 +Eval 54.12 80.33 # ASR nnet3 tdnn+chain Dev: U06 58881 48061 81.62