Merge pull request #62 from kasnerz/crowdsourcing-instructions

Configure crowdsourcing instructions through UI + add them to config

factgenie/config.yml

@@ -1,5 +1,5 @@
 ---
-debug: false
+debug: true
 host_prefix: ""
 logging_level: INFO
 login:

factgenie/config/crowdsourcing/example.yaml

@@ -8,10 +8,32 @@ annotation_span_categories:
   - name: "Other"
     color: "#bbbbbb"
 examples_per_batch: 10
-# options:
-# - `dataset-level`: shuffles all outputs in the dataset
-# - `example-level`: shuffles outputs for each example keeping the order of examples
 sort_order: dataset-level
-# time after which an unfinished example can be re-assigned to a new annotator (minutes)
 idle_time: 120
-completion_code: "ABCDE"
+has_display_overlay: true
+annotator_instructions: |
+  In this task, you will annotate outputs of an automatic text generation system. For each example, you will see **data** on the left side and the corresponding generated **text** on the right side. Your task is to **annotate errors** in the text with respect to the data.
+
+  There are five types of errors that you can mark in the generated text:
+
+  1.  <span style="background-color: #FFBCBC"><b>Incorrect fact</b></span>: The fact in the text contradicts the data.
+  2.  <span style="background-color: #e9d2ff"><b>Not checkable</b></span> : The fact in the text cannot be checked given the data.
+  3.  <span style="background-color: #FFF79F"><b>Misleading</b></span>: The fact in the text is misleading in the given context.
+  4.  <span style="background-color: #bbbbbb"><b>Other</b></span> : The text is problematic for another reason, e.g. grammatically or stylistically incorrect, irrelevant, or repetitive.
+
+  You can annotate the errors by dragging your mouse over the text, highlighting the error span.
+  
+  Once you think you have marked all the errors present in the text, click the **✅ Mark example as complete** button (you can still update the annotation later). 
+  
+  You will be able to submit the annotations once they are all are marked as complete.
+annotator_prompt: "Please annotate all the errors in the text:"
+final_message: |
+  Your annotations have been submitted.
+
+  Your Prolific completion code is **CK388WFU**.
+
+  **[Return to Prolific](https://app.prolific.co/submissions/complete?cc=CK388WFU)**
+flags:
+  - The text 100% accurate and does not contain any errors.
+  - The text is missing or incomplete.
+  - The text is severely off-topic (seems completely unrelated to the data).

factgenie/main.py

@@ -138,14 +138,13 @@ def annotate():
     utils.generate_campaign_index(app)
     campaign_id = request.args.get("campaign")
     campaign = app.db["campaign_index"]["crowdsourcing"][campaign_id]
-    compl_code = campaign.metadata["config"]["completion_code"]
-    prolific_pid = request.args.get("PROLIFIC_PID", "test")
+    annotator_id = request.args.get("PROLIFIC_PID", "test")
     session_id = request.args.get("SESSION_ID", "test")
     study_id = request.args.get("STUDY_ID", "test")
 
     db = campaign.db
     metadata = campaign.metadata
-    annotation_set = utils.get_annotator_batch(app, campaign, db, prolific_pid, session_id, study_id)
+    annotation_set = utils.get_annotator_batch(app, campaign, db, annotator_id, session_id, study_id)
 
     if not annotation_set:
         # no more available examples
@@ -159,8 +158,7 @@ def annotate():
         f"campaigns/{campaign.campaign_id}/annotate.html",
         host_prefix=app.config["host_prefix"],
         annotation_set=annotation_set,
-        annotator_id=prolific_pid,
-        compl_code=compl_code,
+        annotator_id=annotator_id,
         metadata=metadata,
     )
 
@@ -290,13 +288,8 @@ def crowdsourcing_create():
             indent=4,
         )
 
-    # copy templates/campaigns/annotate_default.html into templates/campaigns/{campaign_id} as "annotate.html"
-    os.makedirs(os.path.join(TEMPLATES_DIR, "campaigns", campaign_id), exist_ok=True)
-
-    shutil.copy(
-        os.path.join(TEMPLATES_DIR, "campaigns", "annotate_default.html"),
-        os.path.join(TEMPLATES_DIR, "campaigns", campaign_id, "annotate.html"),
-    )
+    # prepare the crowdsourcing HTML page
+    utils.create_crowdsourcing_page(campaign_id, config)
 
     # create the campaign object
     campaign = HumanCampaign(campaign_id=campaign_id)

factgenie/static/css/custom.css

@@ -828,4 +828,9 @@ a:hover {
     border-style: solid;
     border-radius: 10px;
     padding: 30px;
+}
+
+.CodeMirror,
+.CodeMirror-scroll {
+    min-height: 150px !important;
 }

factgenie/static/js/factgenie.js

@@ -180,7 +180,6 @@ function loadAnnotations() {
 }
 
 function submitAnnotations(campaign_id) {
-    // console.log(annotation_set);
     $.post({
         url: `${url_prefix}/submit_annotations`,
         contentType: 'application/json', // Specify JSON content type
@@ -197,51 +196,34 @@ function submitAnnotations(campaign_id) {
     });
 }
 
+function collectFlags() {
+    // collect values of all checkboxes within divs of class `flag-checkbox`, save values sequentially (for each id)
+    const flags = [];
+    $(".flag-checkbox").each(function () {
+        const value = $(this).find("input[type='checkbox']").prop("checked");
+        flags.push(value);
+    });
+    console.log("collected flags", flags);
+    return flags;
+}
+
 function saveCurrentOnly() {
     var collection = YPet[`p${example_idx}`].currentView.collection.parentDocument.get('annotations').toJSON();
-
-    const checkbox_correct = $("#checkbox-correct").is(":checked");
-    const checkbox_missing = $("#checkbox-missing").is(":checked");
-    const checkbox_off_topic = $("#checkbox-off-topic").is(":checked");
-
     annotation_set[example_idx]["annotations"] = collection;
-    annotation_set[example_idx]["flags"] = {
-        "is_fully_correct": checkbox_correct,
-        "is_missing": checkbox_missing,
-        "is_off_topic": checkbox_off_topic,
-    };
 }
 
 
 function markAnnotationAsComplete() {
     var collection = YPet[`p${example_idx}`].currentView.collection.parentDocument.get('annotations').toJSON();
-
-    const checkbox_correct = $("#checkbox-correct").is(":checked");
-    const checkbox_missing = $("#checkbox-missing").is(":checked");
-    const checkbox_off_topic = $("#checkbox-off-topic").is(":checked");
-
-
-    // if the collection is empty but the `checkbox-correct` is not checked, display an alert
-    if (collection.length == 0 && !(checkbox_correct || checkbox_missing)) {
-        alert("Are you *really* sure that the example does not contain any errors? If so, please check the last box to mark the example as complete.");
-        return;
-    }
-
     annotation_set[example_idx]["annotations"] = collection;
-    annotation_set[example_idx]["flags"] = {
-        "is_fully_correct": checkbox_correct,
-        "is_missing": checkbox_missing,
-        "is_off_topic": checkbox_off_topic,
-    };
+    console.log(example_idx);
+    annotation_set[example_idx]["flags"] = collectFlags();
 
     $('#page-link-' + example_idx).removeClass("bg-incomplete");
     $('#page-link-' + example_idx).addClass("bg-complete");
 
     // uncheck all checkboxes
-    $("#checkbox-correct").prop("checked", false);
-    $("#checkbox-missing").prop("checked", false);
-    $("#checkbox-off-topic").prop("checked", false);
-
+    $(".flag-checkbox input[type='checkbox']").prop("checked", false);
 
     // if all the examples are annotated, post the annotations
     if ($(".bg-incomplete").length == 0) {
@@ -277,13 +259,13 @@ function showAnnotation() {
     const flags = annotation_set[example_idx].flags;
 
     if (flags !== undefined) {
-        $("#checkbox-correct").prop("checked", flags.is_fully_correct);
-        $("#checkbox-missing").prop("checked", flags.is_missing);
-        $("#checkbox-off-topic").prop("checked", flags.is_off_topic);
+        // flags are an array
+        $(".flag-checkbox").each(function (i) {
+            $(this).find("input[type='checkbox']").prop("checked", flags[i]);
+        });
     } else {
-        $("#checkbox-correct").prop("checked", false);
-        $("#checkbox-missing").prop("checked", false);
-        $("#checkbox-off-topic").prop("checked", false);
+        // uncheck all checkboxes
+        $(".flag-checkbox input[type='checkbox']").prop("checked", false);
     }
     $("#examplearea").html(data.html);
     // $(".text-type").html(`${type}`);
@@ -595,11 +577,15 @@ function gatherConfig() {
     var config = {};
 
     if (window.mode == "crowdsourcing") {
+        config.annotatorInstructions = annotatorInstructionsMDE.value();
+        config.annotatorPrompt = $("#annotatorPrompt").val();
+        config.finalMessage = finalMessageMDE.value();
+        config.hasDisplayOverlay = $("#displayOverlay").is(":checked");
         config.examplesPerBatch = $("#examplesPerBatch").val();
         config.idleTime = $("#idleTime").val();
-        config.completionCode = $("#completionCode").val();
         config.sortOrder = $("#sortOrder").val();
         config.annotationSpanCategories = getAnnotationSpanCategories();
+        config.flags = getKeys($("#flags"));
     } else if (window.mode == "llm_eval") {
         config.metricType = $("#metric-type").val();
         config.modelName = $("#model-name").val();
@@ -670,6 +656,15 @@ function getKeysAndValues(div) {
     return args;
 }
 
+function getKeys(div) {
+    var keys = [];
+    div.children().each(function () {
+        const key = $(this).find("input[name='argName']").val();
+        keys.push(key);
+    });
+    return keys;
+}
+
 function createHumanCampaign() {
     const campaignId = $('#campaignId').val();
     const config = gatherConfig();
@@ -838,10 +833,33 @@ function addExtraArgument() {
     modelArguments.append(newArg);
 }
 
+function addFlag() {
+    const flags = $("#flags");
+    const newFlag = createFlagElem("");
+    flags.append(newFlag);
+}
+
+
 function deleteRow(button) {
     $(button).parent().parent().remove();
 }
 
+function createFlagElem(key) {
+    // text area and selectbox for the flag ("checked" or "unchecked" based on the value)
+    const newFlag = $(`
+        <div class="row mt-1">
+        <div class="col-11">
+        <input type="text" class="form-control" name="argName" value="${key}" placeholder="Question or statement">
+        </div>
+        <div class="col-1">
+        <button type="button" class="btn btn-danger" onclick="deleteRow(this);">x</button>
+        </div>
+        </div>
+    `);
+    return newFlag;
+}
+
+
 function createArgElem(key, value) {
     const newArg = $(`
         <div class="row mt-1">
@@ -973,30 +991,44 @@ function updateCrowdsourcingConfig() {
     const crowdsourcingConfig = $('#crowdsourcingConfig').val();
 
     if (crowdsourcingConfig === "[None]") {
+        annotatorInstructionsMDE.value("");
+        $("#annotatorPrompt").val("");
+        finalMessageMDE.value("");
         $("#examplesPerBatch").val("");
         $("#idleTime").val("");
-        $("#completionCode").val("");
         $("#annotation-span-categories").empty();
+        $("#flags").empty();
         return;
     }
     const cfg = window.configs[crowdsourcingConfig];
 
+    const annotatorInstructions = cfg.annotator_instructions;
+    const annotatorPrompt = cfg.annotator_prompt;
+    const finalMessage = cfg.final_message;
     const examplesPerBatch = cfg.examples_per_batch;
     const idleTime = cfg.idle_time;
-    const completionCode = cfg.completion_code;
     const sortOrder = cfg.sort_order;
     const annotationSpanCategories = cfg.annotation_span_categories;
+    const flags = cfg.flags;
 
+    annotatorInstructionsMDE.value(annotatorInstructions);
+    $("#annotatorPrompt").val(annotatorPrompt);
+    finalMessageMDE.value(finalMessage);
     $("#examplesPerBatch").val(examplesPerBatch);
     $("#idleTime").val(idleTime);
-    $("#completionCode").val(completionCode);
     $("#sortOrder").val(sortOrder);
     $("#annotation-span-categories").empty();
 
     annotationSpanCategories.forEach((annotationSpanCategory) => {
         const newCategory = createAnnotationSpanCategoryElem(annotationSpanCategory.name, annotationSpanCategory.color);
         $("#annotation-span-categories").append(newCategory);
     });
+    $("#flags").empty();
+
+    flags.forEach((flag) => {
+        const newFlag = createFlagElem(flag);
+        $("#flags").append(newFlag);
+    });
 }