RasaHQ · tmbo · Dec 16, 2020 · Sep 16, 2020 · Sep 16, 2020 · Sep 16, 2020
diff --git a/changelog/6410.removal.md b/changelog/6410.removal.md
@@ -0,0 +1,5 @@
+`Domain.random_template_for` is deprecated and will be removed in Rasa Open Source 
+3.0.0. You can alternatively use the `TemplatedNaturalLanguageGenerator`.
+
+`Domain.action_names` is deprecated and will be removed in Rasa Open Source 
+3.0.0. Please use `Domain.action_names_or_texts` instead.
diff --git a/changelog/7436.improvement.md b/changelog/7436.improvement.md
@@ -0,0 +1,4 @@
+Make `rasa data validate stories` work for end-to-end.
+
+The `rasa data validate stories` function now considers the tokenized user text instead of the plain text that is part of a state. 
+This is closer to what Rasa Core actually uses to distinguish states and thus captures more story structure problems.
diff --git a/changelog/7496.feature.md b/changelog/7496.feature.md
@@ -0,0 +1,36 @@
+Make [TED Policy](./policies.mdx#ted-policy) an end-to-end policy. Namely, make it possible to train TED on stories that contain
+intent and entities or user text and bot actions or bot text.
+If you don't have text in your stories, TED will behave the same way as before.
+Add possibility to predict entities using TED.
+
+Here's an example of a dialogue in the Rasa story format:
+
+```rasa-yaml
+stories:
+- story: collect restaurant booking info  # name of the story - just for debugging
+  steps:
+  - intent: greet                          # user message with no entities
+  - action: utter_ask_howcanhelp           # action that the bot should execute
+  - intent: inform                         # user message with entities
+    entities:
+    - location: "rome"
+    - price: "cheap"
+  - bot: On it                             # actual text that bot can output
+  - action: utter_ask_cuisine
+  - user: I would like [spanish](cuisine). # actual text that user input
+  - action: utter_ask_num_people
+```
+
+Some model options for `TEDPolicy` got renamed.
+Please update your configuration files using the following mapping:
+
+|      Old model option       |                  New model option                      |
+|-----------------------------|--------------------------------------------------------|
+|transformer_size             |dictionary “transformer_size” with keys                 |
+|                             |“text”, “action_text”, “label_action_text”, “dialogue”  |
+|number_of_transformer_layers |dictionary “number_of_transformer_layers” with keys     |
+|                             |“text”, “action_text”, “label_action_text”, “dialogue”  |
+|dense_dimension              |dictionary “dense_dimension” with keys                  |
+|                             |“text”, “action_text”, “label_action_text”, “intent”,   |
+|                             |“action_name”, “label_action_name”, “entities”, “slots”,|
+|                             |“active_loop”                                           |
diff --git a/data/test_dialogues/default.json b/data/test_dialogues/default.json
@@ -45,7 +45,8 @@
         "text": "Hi my name is Peter"
       },
       "text": "Hi my name is Peter",
-      "timestamp": 1551953035.076376
+      "timestamp": 1551953035.076376,
+      "use_text_for_featurization": false
     },
     {
       "py/object": "rasa.shared.core.events.SlotSet",

diff --git a/data/test_dialogues/formbot.json b/data/test_dialogues/formbot.json
@@ -5,6 +5,7 @@
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"action_listen",
+      "action_text": null,
       "confidence":null,
       "policy":null,
       "timestamp":1551884035.892855,
@@ -32,11 +33,13 @@
         "text":"Hi I'm desperate to talk to you"
       },
       "text":"Hi I'm desperate to talk to you",
-      "timestamp":1551884050.259948
+      "timestamp":1551884050.259948,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"utter_greet",
+      "action_text": null,
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884060.466681,
@@ -55,6 +58,7 @@
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"action_listen",
+      "action_text": null,
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884061.9350882,
@@ -104,11 +108,13 @@
         "text":"I'm looking for an indian restaurant...in Bombay"
       },
       "text":"I'm looking for an indian restaurant...in Bombay",
-      "timestamp":1551884090.9653602
+      "timestamp":1551884090.9653602,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"restaurant_form",
+      "action_text": null,
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884095.542748,
@@ -117,6 +123,7 @@
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"utter_slots_values",
+      "action_text": null,
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884097.570883,
@@ -135,6 +142,7 @@
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"action_listen",
+      "action_text": null,
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884098.8006358,
@@ -162,11 +170,13 @@
         "text":"Let's just pretend everything went correctly"
       },
       "text":"Let's just pretend everything went correctly",
-      "timestamp":1551884208.092693
+      "timestamp":1551884208.092693,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"action_deactivate_loop",
+      "action_text": null,
       "confidence":null,
       "policy":null,
       "timestamp":1551884214.951055,
@@ -186,6 +196,7 @@
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
       "action_name":"action_listen",
+      "action_text": null,
       "confidence":0.7680902069097734,
       "policy":"policy_0_TEDPolicy",
       "timestamp":1551884216.705635,

diff --git a/data/test_dialogues/moodbot.json b/data/test_dialogues/moodbot.json
@@ -59,7 +59,8 @@
         "text":"Hi talk to me"
       },
       "text":"Hi talk to me",
-      "timestamp":1551883971.410778
+      "timestamp":1551883971.410778,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
@@ -146,7 +147,8 @@
         "text":"Super sad"
       },
       "text":"Super sad",
-      "timestamp":1551883982.540276
+      "timestamp":1551883982.540276,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
@@ -243,7 +245,8 @@
         "text":"No"
       },
       "text":"No",
-      "timestamp":1551883989.0720608
+      "timestamp":1551883989.0720608,
+      "use_text_for_featurization": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",

diff --git a/data/test_trackers/tracker_moodbot.json b/data/test_trackers/tracker_moodbot.json
@@ -37,6 +37,7 @@
       "timestamp": 1517821726.211031,
       "event": "action",
       "name": "action_listen",
+      "action_text": null,
       "policy": null,
       "confidence": null
     },
@@ -75,13 +76,15 @@
       "timestamp": 1517821726.200373,
       "event": "action",
       "name": "utter_greet",
+      "action_text": null,
       "policy": null,
       "confidence": null
     },
     {
       "timestamp": 1517821726.211038,
       "event": "action",
       "name": "action_listen",
+      "action_text": null,
       "policy": null,
       "confidence": null
     },
@@ -120,13 +123,15 @@
       "timestamp": 1517821726.209908,
       "event": "action",
       "name": "utter_happy",
+      "action_text": null,
       "policy": "policy_1_TEDPolicy",
       "confidence": 0.8
     },
     {
       "timestamp": 1517821726.211042,
       "event": "action",
       "name": "action_listen",
+      "action_text": null,
       "policy": "policy_2_MemoizationPolicy",
       "confidence": 1.0
     }

diff --git a/data/test_yaml_stories/stories_e2e.yml b/data/test_yaml_stories/stories_e2e.yml
@@ -0,0 +1,30 @@
+version: "2.0"
+
+stories:
+- story: happy path (intent to action)
+  steps:
+  - intent: greet
+  - action: utter_greet
+  - intent: mood_great
+  - action: utter_happy
+
+- story: sad path (text to text)
+  steps:
+  - user: "[Hello](bla)"
+  - bot: "Welcome to moodbot. How are you feeling today?"
+  - user: "Horrible"
+  - bot: "Oh no! Here is a kitten photo. Did it help?"
+  - user: "Yes"
+  - bot: "Perfect"
+
+- story: sad path 2 (mixed)
+  steps:
+  - intent: greet
+  - action: utter_greet
+  - user: "I'm not happy"
+  - action: utter_cheer_up
+  - action: utter_did_that_help
+  - user: "Not at all"
+  - bot: "Goodbye!"
+
+
diff --git a/data/test_yaml_stories/stories_hybrid_e2e.yml b/data/test_yaml_stories/stories_hybrid_e2e.yml
@@ -0,0 +1,15 @@
+stories:
+- story: My hybrid End-to-End story
+  steps:
+  # Regular story with labels
+  - intent: simple
+  - action: utter_greet
+  # Actual messages are given instead of labels
+  - user: "I am looking for a [Kenyan](cuisine) restaurant"
+  - bot: "good for you"
+  # Regular labeled events
+  - intent: goodbye
+  - action: utter_goodbye
+  # Actual messages are given instead of labels
+  - user: One more thing
+  - bot: What?
diff --git a/data/test_yaml_stories/stories_simple.yml b/data/test_yaml_stories/stories_simple.yml
@@ -0,0 +1,8 @@
+- story: simple_story
+  steps:
+  - intent: greet
+  - action: utter_greet
+  - intent: default
+  - action: utter_default
+  - intent: goodbye
+  - action: utter_goodbye
diff --git a/docs/docs/command-line-interface.mdx b/docs/docs/command-line-interface.mdx
@@ -391,6 +391,10 @@ rasa data validate stories
 :::note
 Running `rasa data validate` does **not** test if your [rules](./rules.mdx) are consistent with your stories. 
 However, during training, the `RulePolicy` checks for conflicts between rules and stories. Any such conflict will abort training.
+
+Also, if you use end-to-end stories, then this might not capture all conflicts. Specifically, if two user inputs
+result in different tokens yet exactly the same featurization, then conflicting actions after these inputs
+may exist but will not be reported by the tool. 
 :::
 
 To interrupt validation even for minor issues such as unused intents or responses, use the `--fail-on-warnings` flag.

diff --git a/docs/docs/components.mdx b/docs/docs/components.mdx
@@ -1485,7 +1485,7 @@ However, additional parameters exist that can be adapted.
 +=================================+==================+==============================================================+
 | hidden_layers_sizes             | text: []         | Hidden layer sizes for layers before the embedding layers    |
 |                                 | label: []        | for user messages and labels. The number of hidden layers is |
-|                                 |                  | equal to the length of the corresponding.                    |
+|                                 |                  | equal to the length of the corresponding list.                    |
 +---------------------------------+------------------+--------------------------------------------------------------+
 | share_hidden_layers             | False            | Whether to share the hidden layer weights between user       |
 |                                 |                  | messages and labels.                                         |
@@ -1519,8 +1519,8 @@ However, additional parameters exist that can be adapted.
 +---------------------------------+------------------+--------------------------------------------------------------+
 | embedding_dimension             | 20               | Dimension size of embedding vectors.                         |
 +---------------------------------+------------------+--------------------------------------------------------------+
-| dense_dimension                 | text: 128        | Dense dimension for sparse features to use if no dense       |
-|                                 | label: 20        | features are present.                                        |
+| dense_dimension                 | text: 128        | Dense dimension for sparse features to use.                  |
+|                                 | label: 20        |                                                              |
 +---------------------------------+------------------+--------------------------------------------------------------+
 | concat_dimension                | text: 128        | Concat dimension for sequence and sentence features.         |
 |                                 | label: 20        |                                                              |

diff --git a/docs/docs/migration-guide.mdx b/docs/docs/migration-guide.mdx
@@ -12,8 +12,54 @@ how you can migrate from one version to another.
 
 ## Rasa 2.1 to Rasa 2.2
 
+### General
+
+`TEDPolicy`'s  `transformer_size`, `number_of_transformer_layers`,
+and `dense_dimensions` parameters have been renamed.
+Please update your configuration files using the following mapping:
+
+|      Old Model Parameter    |                 New Model Parameter                    |
+|-----------------------------|--------------------------------------------------------|
+|`transformer_size`           |dictionary `transformer_size` with keys                 |
+|                             |`text`, `action_text`, `label_action_text`, `dialogue`  |
+|`number_of_transformer_layers`|dictionary `number_of_transformer_layers` with keys    |
+|                             |`text`, `action_text`, `label_action_text`, `dialogue`  |
+|`dense_dimension`            |dictionary `dense_dimension` with keys                  |
+|                             |`text`, `action_text`, `label_action_text`, `intent`,   |
+|                             |`action_name`, `label_action_name`, `entities`, `slots`,|
+|                             |`active_loop`                                           |
+
+For example:
+
+```yaml-rasa title="config.yml"
+policies:
+  - name: TEDPolicy
+    transformer_size:
+      text: 128
+      action_text: 128
+      label_action_text: 128
+      dialogue: 128
+    number_of_transformer_layers:
+      text: 1
+      action_text: 1
+      label_action_text: 1
+      dialogue: 1
+    dense_dimension:
+      text: 128
+      action_text: 128
+      label_action_text: 128
+      intent: 20
+      action_name: 20
+      label_action_name: 20
+      entities: 20
+      slots: 20
+      active_loop: 20
+```
+
+
 ### Deprecations
 
+#### Markdown Data
 Training and test data in Markdown format is now deprecated. This includes:
 - reading and writing of story files in Markdown format
 - reading and writing of NLU data in Markdown format
@@ -24,13 +70,23 @@ Support for Markdown data will be removed entirely in Rasa Open Source 3.0.0.
 Please convert your existing Markdown data by using the commands
 described [here](./migration-guide.mdx#training-data-files).
 
+
 ### Policies
 
 [Policies](./policies.mdx) now require a `**kwargs` argument in their constructor and `load` method.
 Policies without `**kwargs` will be supported until Rasa version `3.0.0`.
 However when using [incremental training](./command-line-interface.mdx#incremental-training)
 `**kwargs` **must** be included.
 
+
+#### Other
+
+* `Domain.random_template_for` is deprecated and will be removed in Rasa Open Source
+   3.0.0. You can alternatively use the `TemplatedNaturalLanguageGenerator`.
+* `Domain.action_names` is deprecated and will be removed in Rasa Open Source
+   3.0.0. Please use `Domain.action_names_or_texts` instead.
+
+
 ## Rasa 2.0 to Rasa 2.1
 
 ### Deprecations