-
Notifications
You must be signed in to change notification settings - Fork 1
/
salsa.yml
280 lines (274 loc) · 10.9 KB
/
salsa.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
template_name: salsa
template_label: 💃 SALSA
template_description: Success and FAilure Linguistic Simplification Annotation
instructions: |
## SALSA 💃 Annotation Instructions
Please make sure you select all the edits, some edits **may be easily missed** when the sentence is too long.
If you encounter any bug or have any suggestion on this tool, please write it at this [google doc link](https://docs.google.com/document/d/1oQpOMZij_keFGonBr9ZvXUn4cmgTgfpfueME_bVU-FI/edit).
If you have any question, please don't hesitate to ask us over **slack**.
Have fun!!!
interface_text:
typology:
source_label: "Original Sentence"
target_label: "Simplified Sentence"
edits:
- name: deletion
label: "Deletion"
type: primitive
color: red
icon: fa-delete-left
enable_input: true
annotation:
- name: deletion_type
question: "Select the type of this deletion edit."
options:
- name: good_deletion
label: "Delete Insignificant Information"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: trivial_deletion
label: "Trivial Deletion"
- name: bad_deletion
label: "Delete Significant Information"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: coreference
question: "Does this deletion result in coreference error?"
options: binary
- name: grammar_error
question: "Does this deletion edit introduce any fluency / grammar error?"
options: binary
- name: insertion
label: "Insertion"
type: primitive
color: green
icon: fa-circle-plus
enable_output: true
annotation:
- name: insertion_type
question: "Select the type of this insertion edit."
options:
- name: elaboration
label: "Elaboration"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: trivial_insertion
label: "Trivial Insertion"
question: "Does it help you to read and understand the sentence?"
options:
- name: yes
label: "Yes"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: no
label: "No"
- name: repetition
label: "Repetition"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: contradiction
label: "Contradiction"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: facutal_error
label: "Factual Error"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: irrelevant
label: "Irrelevant"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: grammar_error
question: "Does this insertion edit introduce any fluency / grammar error?"
options: binary
- name: substitution
label: "Substitution"
type: primitive
color: blue
icon: fa-sort
enable_input: true
enable_output: true
multi_span: true
annotation:
- name: substitution_info_change
question: "Compared to the source phrase, the new phrase expresses:"
options:
- name: same
label: "The same meaning"
question: "Select the impact of this paraphrase edit on the sentence simplicity."
options:
- name: bad
label: "Negative impact"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: trivial
label: "No impact"
- name: good
label: "Positive impact"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: less
label: "Less information"
question: "Is the deleted information significant or not?"
options:
- name: good_deletion
label: "Delete Insignificant Information"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: trivial_deletion
label: "Trivial Deletion"
- name: bad_deletion
label: "Delete Significant Information"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: more
label: "More information"
question: "Select the type."
options:
- name: elaboration
label: "Elaboration"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: trivial_insertion
label: "Trivial Insertion"
question: "Does it help you to read and understand the sentence?"
options:
- name: yes
label: "Yes"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: no
label: "No"
- name: repetition
label: "Repetition"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: contradiction
label: "Contradiction"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: facutal_error
label: "Factual Error"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: irrelevant
label: "Irrelevant"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: different
label: "Different information"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: grammar_error
question: "Does this substitution edit introduce any fluency / grammar error?"
options: binary
- name: reorder
label: "Reorder"
type: primitive
color: teal
icon: fa-arrows-left-right
enable_input: true
enable_output: true
annotation:
- name: reorder_level
question: "Does this reordering happen on a word-level or component-level?"
options:
- name: word_level
label: "Word-level"
question: "Select the impact of this reordering edit on the sentence simplicity."
options:
- name: bad
label: "Negative impact"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: trivial
label: "No impact"
- name: good
label: "Positive impact"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: component_level
label: "Component-level"
question: "Select the impact of this reordering edit on the sentence simplicity."
options:
- name: bad
label: "Negative impact"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: trivial
label: "No impact"
- name: good
label: "Positive impact"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: grammar_error
question: "Does this reorder edit introduce any fluency / grammar error?"
options: binary
- name: split
label: "Split"
color: yellow
icon: fa-grip-lines-vertical
type: composite
annotation:
- name: impact
question: "Select the impact of this structure edit on the sentence simplicity."
options:
- name: bad
label: "Negative impact"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: trivial
label: "No impact"
- name: good
label: "Positive impact"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: grammar_error
question: "Does this split edit introduce any fluency / grammar error?"
options: binary
- name: structure
label: "Structure"
color: orange
icon: fa-tree
type: composite
annotation:
- name: structure_type
question: "Select the type of this structure change edit."
options:
- name: voice
label: "Voice Change"
- name: pos
label: "POS Change"
- name: tense
label: "Tense Change"
- name: grammar_number
label: "Grammatical Number"
- name: clausal
label: "Clausal Change"
- name: transition
label: "Transition Change"
- name: impact
question: "Select the impact of this structure edit on the sentence simplicity."
options:
- name: bad
label: "Negative impact"
question: "Rate the severity (how much it makes the sentence harder to read and understand?)."
options: likert-3
- name: trivial
label: "No impact"
- name: good
label: "Positive impact"
question: "Rate the efficacy (how much it helps you to read and understand the sentence?)"
options: likert-3
- name: grammar_error
question: "Does this structure edit introduce any fluency / grammar error?"
options: binary
default_data_link: https://thresh.tools/data/salsa.json
paper_link: https://arxiv.org/abs/2305.14458
citation: |
@article{heineman2023dancing,
title={Dancing Between Success and Failure: Edit-level Simplification Evaluation using SALSA},
author={Heineman, David and Dou, Yao and Maddela, Mounica and Xu, Wei},
journal={arXiv preprint arXiv:2305.14458},
year={2023}
}