Merge pull request #5626 from RasaHQ/bug-activation

Move activation to transformer block
RasaHQ · Apr 15, 2020 · dc58581 · dc58581
2 parents 356ce59 + 37bdde7
commit dc58581
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/changelog/5626.msic.rst b/changelog/5626.msic.rst
@@ -0,0 +1 @@
+Apply ``tfa.activations.gelu(x)`` only if min 1 transformer block is used in ``DIETClassifier``.
diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py
@@ -1249,7 +1249,10 @@ def _create_sequence(
         outputs = self._tf_layers[f"{name}_transformer"](
             inputs, 1 - mask, self._training
         )
-        outputs = tfa.activations.gelu(outputs)
+
+        if self.config[NUM_TRANSFORMER_LAYERS] > 0:
+            # apply activation
+            outputs = tfa.activations.gelu(outputs)
 
         return outputs, inputs, seq_ids, lm_mask_bool
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Apply ``tfa.activations.gelu(x)`` only if min 1 transformer block is used in ``DIETClassifier``.