Merge pull request kaldi-asr#41 from hhadian/attention

Add test template for AttentionForward/Backward
hhadian · Jul 3, 2017 · f29f8a6 · f29f8a6
2 parents ecffd43 + 0c75eaa
commit f29f8a6
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 3 deletions.
diff --git a/src/nnet3/attention-test.cc b/src/nnet3/attention-test.cc
@@ -1,6 +1,6 @@
 // nnet3/attention-test.cc
 
-// Copyright 2017    Johns Hopkins University (author:  Hossein Hadian)
+// Copyright      2017 Hossein Hadian
 
 // See ../../COPYING for clarification regarding multiple authors
 //
@@ -116,8 +116,35 @@ void UnitTestAttentionDotProductAndAddScales() {
   AssertEqual(B, B2);
 }
 
+void TestAttentionForwardBackward() {
+  BaseFloat key_scale = 0.5 * RandInt(1, 3);
+  int32 output_num_rows = RandInt(1, 50),
+      value_dim = RandInt(1, 10), key_dim = RandInt(1, 10),
+      row_shift = RandInt(1, 5), context_dim = RandInt(2, 5),
+      num_extra_rows = (context_dim - 1) * row_shift,
+      input_num_rows = output_num_rows + num_extra_rows,
+      query_dim = key_dim + context_dim;
+  CuMatrix<BaseFloat> keys(input_num_rows, key_dim),
+      queries(output_num_rows, query_dim),
+      values(input_num_rows, value_dim),
+      C(output_num_rows, context_dim),
+      output(output_num_rows, value_dim + context_dim);
+
+  AttentionForward(key_scale, keys, queries, values, &C, &output);
+
+
+  CuMatrix<BaseFloat> keys_deriv(input_num_rows, key_dim),
+      queries_deriv(output_num_rows, query_dim),
+      values_deriv(input_num_rows, value_dim),
+      output_deriv(output_num_rows, value_dim + context_dim);
+
+  AttentionBackward(key_scale, keys, queries, values, C,
+                    output_deriv, &keys_deriv, &queries_deriv, &values_deriv);
+}
+
 void UnitTestAttention() {
   UnitTestAttentionDotProductAndAddScales();
+  TestAttentionForwardBackward();
 }
 
 

diff --git a/src/nnet3/attention.cc b/src/nnet3/attention.cc
@@ -1,7 +1,7 @@
 // nnet3/attention.cc
 
 // Copyright      2017  Johns Hopkins University (author: Daniel Povey)
-//                      Johns Hopkins University (author: Hossein Hadian)
+//                      Hossein Hadian
 
 // See ../../COPYING for clarification regarding multiple authors
 //

diff --git a/src/nnet3/attention.h b/src/nnet3/attention.h
@@ -1,7 +1,7 @@
 // nnet3/attention.h
 
 // Copyright      2017  Johns Hopkins University (author: Daniel Povey)
-//                      Johns Hopkins University (author: Hossein Hadian)
+//                      Hossein Hadian
 
 // See ../../COPYING for clarification regarding multiple authors
 //