Skip to content

Commit

Permalink
fix; intent_responder_and_tests (#226)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dilyararimovna
dilyararimovna authored Nov 24, 2022
1 parent 74db932 commit fb8441e
Show file tree
Hide file tree
Showing 11 changed files with 63 additions and 54 deletions.
2 changes: 1 addition & 1 deletion skills/dff_intent_responder_skill/scenario/data/intent_response_phrases.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"Okay, let's chat. What would you like to talk about?"
],
"who_made_you" : [
"I was built by the team of Moscow Institute of Physics and Technology."
"I was built by the DeepPavlov team."
],
"what_is_your_name": [
"My name is DREAM Socialbot."
Expand Down
20 changes: 10 additions & 10 deletions skills/dff_intent_responder_skill/tests/intent_get_dialog_id_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@
"modified_sents": [
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?"
]
},
Expand Down Expand Up @@ -378,9 +378,9 @@
"sentrewrite": {
"clusters": [],
"modified_sents": [
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?"
]
},
Expand Down Expand Up @@ -432,12 +432,12 @@
],
"bot_utterances": [
{
"text": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"text": "I was built by the DeepPavlov team. #+#who_made_you",
"annotations": {
"sentseg": {
"punct_sent": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"punct_sent": "I was built by the DeepPavlov team. #+#who_made_you",
"segments": [
"I was built by the team of Moscow Institute of Physics and Technology.",
"I was built by the DeepPavlov team.",
"#+#who_made_you"
]
},
Expand All @@ -457,9 +457,9 @@
"clusters": [],
"modified_sents": [
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"I was built by the DeepPavlov team. #+#who_made_you"
]
}
},
Expand Down Expand Up @@ -514,8 +514,8 @@
"4": "who made you?"
},
"responses": {
"3": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"3": "I was built by the DeepPavlov team. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you"
},
"validation": false
},
Expand Down
2 changes: 1 addition & 1 deletion skills/dff_intent_responder_skill/tests/intent_get_dialog_id_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"5": "Where are you from?"
},
"responses": {
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you",
"5": "As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from"
},
"misc": {},
Expand Down
2 changes: 1 addition & 1 deletion skills/dff_intent_responder_skill/tests/intent_what_is_your_job_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@
"sentrewrite": {
"clusters": [],
"modified_sents": [
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?",
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"introduce yourself."
Expand Down
10 changes: 5 additions & 5 deletions skills/dff_intent_responder_skill/tests/intent_what_is_your_name_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,9 @@
"sentrewrite": {
"clusters": [],
"modified_sents": [
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?"
]
},
Expand Down Expand Up @@ -378,7 +378,7 @@
"sentrewrite": {
"clusters": [],
"modified_sents": [
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?",
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"introduce yourself."
Expand Down Expand Up @@ -472,7 +472,7 @@
"clusters": [],
"modified_sents": [
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?",
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from"
]
Expand Down Expand Up @@ -529,7 +529,7 @@
"5": "Where are you from?"
},
"responses": {
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you",
"5": "As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from"
},
"validation": false
Expand Down
20 changes: 10 additions & 10 deletions skills/dff_intent_responder_skill/tests/intent_where_are_you_from_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@
"modified_sents": [
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?"
]
},
Expand Down Expand Up @@ -378,9 +378,9 @@
"sentrewrite": {
"clusters": [],
"modified_sents": [
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"Where are you from?"
]
},
Expand Down Expand Up @@ -432,12 +432,12 @@
],
"bot_utterances": [
{
"text": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"text": "I was built by the DeepPavlov team. #+#who_made_you",
"annotations": {
"sentseg": {
"punct_sent": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"punct_sent": "I was built by the DeepPavlov team. #+#who_made_you",
"segments": [
"I was built by the team of Moscow Institute of Physics and Technology.",
"I was built by the DeepPavlov team.",
"#+#who_made_you"
]
},
Expand All @@ -457,9 +457,9 @@
"clusters": [],
"modified_sents": [
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"I was built by the DeepPavlov team. #+#who_made_you"
]
}
},
Expand Down Expand Up @@ -514,8 +514,8 @@
"4": "who made you?"
},
"responses": {
"3": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"3": "I was built by the DeepPavlov team. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you"
},
"validation": false
},
Expand Down
2 changes: 1 addition & 1 deletion skills/dff_intent_responder_skill/tests/intent_where_are_you_from_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
"5": "Where are you from?"
},
"responses": {
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you",
"5": "As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from"
},
"misc": {},
Expand Down
12 changes: 6 additions & 6 deletions skills/dff_intent_responder_skill/tests/intent_who_made_you_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@
"modified_sents": [
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
"who made you?"
]
},
Expand Down Expand Up @@ -443,12 +443,12 @@
],
"bot_utterances": [
{
"text": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"text": "I was built by the DeepPavlov team. #+#who_made_you",
"annotations": {
"sentseg": {
"punct_sent": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"punct_sent": "I was built by the DeepPavlov team. #+#who_made_you",
"segments": [
"I was built by the team of Moscow Institute of Physics and Technology.",
"I was built by the DeepPavlov team.",
"#+#who_made_you"
]
},
Expand All @@ -470,7 +470,7 @@
"Where are you from?",
"As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"who made you?",
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"I was built by the DeepPavlov team. #+#who_made_you"
]
}
},
Expand Down Expand Up @@ -526,7 +526,7 @@
},
"responses": {
"2": "As a socialbot, I live in a cloud. If you want to know me better, let's spend more time talking to each other. Where are you from? #+#where_are_you_from",
"3": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"3": "I was built by the DeepPavlov team. #+#who_made_you"
},
"validation": false
},
Expand Down
6 changes: 3 additions & 3 deletions skills/dff_intent_responder_skill/tests/intent_who_made_you_out.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[
[
"I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"I was built by the DeepPavlov team. #+#who_made_you",
1.0,
{
"dff_intent_responder_skill_state": {
Expand Down Expand Up @@ -46,8 +46,8 @@
"4": "who made you?"
},
"responses": {
"3": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you",
"4": "I was built by the team of Moscow Institute of Physics and Technology. #+#who_made_you"
"3": "I was built by the DeepPavlov team. #+#who_made_you",
"4": "I was built by the DeepPavlov team. #+#who_made_you"
},
"misc": {},
"validation": false,
Expand Down
29 changes: 19 additions & 10 deletions tests/dream/assert_test_dialogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

SPECIAL_SKILLS = {
"RANDOM_SKILLS": [
"program_y",
"dff_program_y_skill",
"dummy_skill",
"movie_tfidf_retrieval",
"entertainment_tfidf_retrieval",
Expand All @@ -42,6 +42,7 @@
"dff_science_skill",
"dff_travel_skill",
"dff_wiki_skill",
"dff_short_story_skill",
"game_cooperative_skill",
"dialogpt",
"dialogpt_persona_based",
Expand Down Expand Up @@ -72,6 +73,9 @@ def main():
assert statistics.mean(proc_times) <= args.time_limit, print(
f"Mean proc time: {mean_proc_time} > {args.time_limit}"
)

error_reports = []

for pred_r, true_r, skill in zip(pred_data, true_data, active_skills):
true_sents = set([sent.lower().replace("\n", " ").replace(" ", " ") for sent in true_r[2:]])
acceptable_skill_names = true_r[0]
Expand Down Expand Up @@ -112,18 +116,23 @@ def main():
checked = True
if not checked:
passed_gold_phrases = False
print("FOUND POSSIBLE ERROR: {} not in {}".format(pred_r[-1], true_sents))
print(f"FOUND POSSIBLE ERROR: {pred_r[-1]} by skill {skill} not in {true_sents}")

if len(acceptable_skill_names) > 0 or len(true_sents) > 0:
assert (len(acceptable_skill_names) > 0 and passed_acceptable_skills) or (
if (len(acceptable_skill_names) > 0 and passed_acceptable_skills) or (
len(true_sents) > 0 and passed_gold_phrases
), (
f"\nERROR!!!\nAcceptable skill names: `{acceptable_skill_names}`.\n"
f"Passed acceptable skill names: `{passed_acceptable_skills}`.\n"
f"True sentences: `{true_sents}`.\n"
f"Passed true sentences: `{passed_gold_phrases}`.\n"
f"Skill: {skill}\nSkill output: {pred_r[-1]}"
)
):
continue
else:
error_reports += [
f"\nERROR!!!\nAcceptable skill names: `{acceptable_skill_names}`.\n"
f"Passed acceptable skill names: `{passed_acceptable_skills}`.\n"
f"True sentences: `{true_sents}`.\n"
f"Passed true sentences: `{passed_gold_phrases}`.\n"
f"Skill: {skill}\nSkill output: {pred_r[-1]}"
]
print("\n\nASSERTION RESULTS:\n")
assert len(error_reports) == 0, print("\n\n".join(error_reports))


if __name__ == "__main__":
Expand Down
Loading

0 comments on commit fb8441e

Please sign in to comment.