From dd9a815c5d572c33fb15f38c052194b9ff3bc67d Mon Sep 17 00:00:00 2001
From: Richard Lobb <trampgeek@gmail.com>
Date: Wed, 20 Mar 2024 20:16:47 +1300
Subject: [PATCH 1/6] Add behat test for combinator template grader.

---
 tests/behat/test_combinator_grader.feature | 92 ++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 tests/behat/test_combinator_grader.feature

diff --git a/tests/behat/test_combinator_grader.feature b/tests/behat/test_combinator_grader.feature
new file mode 100644
index 00000000..5c6245a9
--- /dev/null
+++ b/tests/behat/test_combinator_grader.feature
@@ -0,0 +1,92 @@
+@qtype @qtype_coderunner @javascript @prototypetests
+Feature: test_combinator_grader
+  For the ultimate in grading flexibility
+  As a teacher
+  I must be able to create and use combinator template graders
+
+  Background:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
+      | username | firstname | lastname | email            |
+      | teacher1 | Teacher   | 1        | teacher1@asd.com |
+    And the following "courses" exist:
+      | fullname | shortname | category |
+      | Course 1 | C1        | 0        |
+    And the following "course enrolments" exist:
+      | user     | course | role           |
+      | teacher1 | C1     | editingteacher |
+    And the following "question categories" exist:
+      | contextlevel | reference | questioncategory | name          |
+      | Course       | C1        | Top              | Behat Testing |
+    And I am on the "Course 1" "core_question > course question bank" page logged in as teacher1
+    And I press "Create a new question ..."
+    And I click on "input#item_qtype_coderunner" "css_element"
+    And I press "submitbutton"
+    And I set the field "id_coderunnertype" to "python3"
+    And I set the field "name" to "PROTOTYPE_test_combinator_grader_prototype"
+    And I set the field "id_questiontext" to "Dummy question text"
+    And I set the field "id_customise" to "1"
+    And I set the field "id_useace" to "0"
+    And I set the field "id_uiplugin" to "None"
+    And I set the field "id_template" to:
+      """
+      import subprocess, json, sys
+      {{ STUDENT_ANSWER | e('py') }}
+      tests = json.loads('''{{ TESTCASES | json_encode | e('py') }}''')
+      test_results = [['Test', 'Expected', 'Got', 'iscorrect']]
+      num_right = 0
+      for test in tests:
+              code = test['testcode']
+              expected = int(test['expected'])
+              got = eval(code)
+              test_results.append([code, expected, got, expected == got])
+              if expected == got:
+                      num_right += 1
+      print(json.dumps({
+          'testresults': test_results,
+          'fraction': num_right / len(tests),
+          'prologuehtml': '<h1>I am a prologue</h1>',
+          'epiloguehtml': '<h1>I am an epilogue</h1>'
+      }))
+      """
+    And I set the field "id_iscombinatortemplate" to "1"
+    And I set the field "id_grader" to "TemplateGrader"
+    And I click on "a[aria-controls='id_advancedcustomisationheadercontainer']" "css_element"
+    And I set the field "prototypetype" to "Yes (user defined)"
+    And I set the field "typename" to "python3_test_combinator_grader_prototype"
+    And I press "id_submitbutton"
+
+    # Now try to add a new question of type python3_test_combinator_prototype
+    And I add a "CodeRunner" question filling the form with:
+      | id_coderunnertype | python3_test_combinator_grader_prototype |
+      | name              | Combinator prototype grader tester       |
+      | id_questiontext   | Write the inevitable sqr function        |
+      | id_testcode_0     | sqr(-11)                                 |
+      | id_expected_0     | 121                                      |
+      | id_ordering_0     | 10                                       |
+      | id_testcode_1     | sqr(9)                                   |
+      | id_expected_1     | 80                                       |
+      | id_ordering_1     | 20                                       |
+      | id_answer         | def sqr(n): return n * n                 |
+
+    Then I should see "Failed testing"
+    And I should see "Click on the << button to replace the expected output of this testcase with actual output."
+
+    When I press "<<"
+    And I press "id_submitbutton"
+    Then I should not see "Save changes"
+    And I should not see "Write a sqr function"
+    And I should see "Combinator prototype grader tester"
+
+  Scenario: As a teacher, I get marked right (using combinator template) if I submit a correct answer to a CodeRunner question
+    When I choose "Preview" action for "Combinator prototype grader tester" in the question bank
+    And I click on "a[aria-controls='id_attemptoptionsheadercontainer']" "css_element"
+    And I set the field "id_behaviour" to "Adaptive mode"
+    And I press "id_saverestart"
+    And I set the field with xpath "//textarea[contains(@name, 'answer')]" to "def sqr(n): return n * n"
+    And I press "Check"
+    Then I should see "Passed all tests!"
+    And I should not see "Show differences"
+    And I should see "Marks for this submission: 1.00/1.00"
+    And I should see "I am a prologue"
+    And I should see "I am an epilogue"

From b1f2d7ca715911b726d5b8a6cd3bc04f8f62a950 Mon Sep 17 00:00:00 2001
From: Anupama Sarjoshi <81178902+AnupamaSarjoshi@users.noreply.github.com>
Date: Wed, 20 Mar 2024 21:08:31 +0000
Subject: [PATCH 2/6] Fix warning - undefined property behat_prefix (#208)

---
 classes/jobesandbox.php  |  3 ++-
 classes/sandbox.php      | 22 +++++++++++++++++++++-
 edit_coderunner_form.php |  2 +-
 renderer.php             |  3 ++-
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/classes/jobesandbox.php b/classes/jobesandbox.php
index 725e87fb..848bcb28 100644
--- a/classes/jobesandbox.php
+++ b/classes/jobesandbox.php
@@ -71,7 +71,8 @@ public function __construct() {
         global $CFG;
         qtype_coderunner_sandbox::__construct();
         $this->jobeserver = get_config('qtype_coderunner', 'jobe_host');
-        if ($this->jobeserver === 'jobe2.cosc.canterbury.ac.nz' && $CFG->prefix === $CFG->behat_prefix) {
+        if (qtype_coderunner_sandbox::is_canterbury_server($this->jobeserver)
+                && qtype_coderunner_sandbox::is_using_test_sandbox()) {
             throw new Exception("Please don't use the Canterbury jobe server for test runs");
         }
         $this->apikey = get_config('qtype_coderunner', 'jobe_apikey');
diff --git a/classes/sandbox.php b/classes/sandbox.php
index 922649fd..825518dc 100644
--- a/classes/sandbox.php
+++ b/classes/sandbox.php
@@ -34,9 +34,11 @@
 
 defined('MOODLE_INTERNAL') || die();
 
+use qtype_coderunner\constants;
+
 global $CFG;
 
-if ($CFG->prefix == $CFG->behat_prefix) {
+if (qtype_coderunner_sandbox::is_using_test_sandbox()) {
     require_once($CFG->dirroot .'/question/type/coderunner/tests/fixtures/test-sandbox-config.php');
 }
 
@@ -209,6 +211,24 @@ public static function enabled_sandboxes() {
         return $enabled;
     }
 
+    /**
+     * Returns true if sandbox is being used for tests.
+     * @return bool
+     */
+    public static function is_using_test_sandbox(): bool {
+        global $CFG;
+        return !empty($CFG->behat_prefix) && $CFG->prefix === $CFG->behat_prefix;
+    }
+
+    /**
+     * Returns true if canterbury jobe server is being used.
+     * @param string jobeserver being used.
+     * @return bool
+     */
+    public static function is_canterbury_server(string $jobeserver): bool {
+        return $jobeserver === constants::JOBE_HOST_DEFAULT;
+    }
+
     /**
      * Get the filename containing the given external sandbox name.
      * @param string $externalsandboxname
diff --git a/edit_coderunner_form.php b/edit_coderunner_form.php
index e9aba7d0..7f6f1978 100644
--- a/edit_coderunner_form.php
+++ b/edit_coderunner_form.php
@@ -250,7 +250,7 @@ protected function add_sample_answer_field($mform) {
         $mform->addHelpButton('sampleanswerattachments', 'sampleanswerattachments', 'qtype_coderunner');
         // Unless behat is running, hide the attachments file picker.
         // behat barfs if it's hidden.
-        if ($CFG->prefix !== $CFG->behat_prefix) {
+        if (!qtype_coderunner_sandbox::is_using_test_sandbox()) {
             $method = method_exists($mform, 'hideIf') ? 'hideIf' : 'disabledIf';
             $mform->$method('sampleanswerattachments', 'attachments', 'eq', 0);
         }
diff --git a/renderer.php b/renderer.php
index e81d2663..890c2436 100644
--- a/renderer.php
+++ b/renderer.php
@@ -411,7 +411,8 @@ protected function build_results_table($outcome, qtype_coderunner_question $ques
         if (isset($sandboxinfo['jobeserver'])) {
             $jobeserver = $sandboxinfo['jobeserver'];
             $apikey = $sandboxinfo['jobeapikey'];
-            if ($jobeserver == constants::JOBE_HOST_DEFAULT && $CFG->prefix !== $CFG->behat_prefix) {
+            if (qtype_coderunner_sandbox::is_canterbury_server($jobeserver)
+                    && (!qtype_coderunner_sandbox::is_using_test_sandbox())) {
                 if ($apikey == constants::JOBE_HOST_DEFAULT_API_KEY) {
                     $fb .= get_string('jobe_warning_html', 'qtype_coderunner');
                 } else {

From 95a0fb673b9fa014407322fab19cd6ff23a67a21 Mon Sep 17 00:00:00 2001
From: Richard Lobb <trampgeek@gmail.com>
Date: Thu, 21 Mar 2024 10:30:35 +1300
Subject: [PATCH 3/6] Remove hack to sandbox.php to force loading of test
 config. Instead, do it in each behat feature.

---
 classes/sandbox.php                           |  4 ---
 .../ace_scratchpad_compatibility.feature      |  6 ++--
 tests/behat/attachmentimportexport.feature    |  5 +--
 tests/behat/attachments.feature               |  3 +-
 tests/behat/backup_and_restore.feature        |  1 +
 tests/behat/behat_coderunner.php              | 31 ++++++++++++++++---
 .../behat/check_graph_question_types.feature  |  3 +-
 .../check_python_template_params.feature      |  3 +-
 tests/behat/check_stepinfo.feature            |  3 +-
 .../behat/check_twig_student_variable.feature |  3 +-
 .../behat/create_python3_sqr_function.feature |  5 ++-
 tests/behat/duplicate_prototype.feature       |  3 +-
 tests/behat/edit.feature                      |  3 +-
 tests/behat/edit_question_precheck.feature    |  3 +-
 tests/behat/edit_table.feature                |  3 +-
 tests/behat/export.feature                    |  3 +-
 tests/behat/gapfiller_ui.feature              |  3 +-
 tests/behat/grading_scenarios.feature         |  3 +-
 tests/behat/html_ui.feature                   |  3 +-
 tests/behat/import.feature                    |  3 +-
 tests/behat/make_combinator_prototype.feature | 16 ++++++++--
 tests/behat/make_prototype.feature            |  3 +-
 tests/behat/missing_prototype.feature         |  3 +-
 tests/behat/reset_button.feature              |  3 +-
 tests/behat/run_python3_sqr_function.feature  |  3 +-
 ...run_python3_sqr_function_templated.feature |  3 +-
 tests/behat/sandbox_webservice.feature        |  3 +-
 tests/behat/scratchpad_ui.feature             |  5 +--
 tests/behat/scratchpad_ui_params.feature      |  5 +--
 tests/behat/set_uiplugin.feature              |  3 +-
 tests/behat/showdifferences_button.feature    |  3 +-
 tests/behat/template_params_error.feature     |  3 +-
 tests/behat/twigprefix.feature                |  3 +-
 33 files changed, 106 insertions(+), 44 deletions(-)

diff --git a/classes/sandbox.php b/classes/sandbox.php
index 825518dc..34b9ad56 100644
--- a/classes/sandbox.php
+++ b/classes/sandbox.php
@@ -38,10 +38,6 @@
 
 global $CFG;
 
-if (qtype_coderunner_sandbox::is_using_test_sandbox()) {
-    require_once($CFG->dirroot .'/question/type/coderunner/tests/fixtures/test-sandbox-config.php');
-}
-
 abstract class qtype_coderunner_sandbox {
     protected $user;     // Username supplied when constructing.
     protected $password; // Password supplied when constructing.
diff --git a/tests/behat/ace_scratchpad_compatibility.feature b/tests/behat/ace_scratchpad_compatibility.feature
index 86a3bda2..0a36b15d 100644
--- a/tests/behat/ace_scratchpad_compatibility.feature
+++ b/tests/behat/ace_scratchpad_compatibility.feature
@@ -5,7 +5,9 @@ Feature: Ace UI convert to Scratchpad UI questions with one click
   I should be able to change a question from using Ace to Scratchpad in one click
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the CodeRunner webservice is enabled
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
@@ -20,7 +22,7 @@ Feature: Ace UI convert to Scratchpad UI questions with one click
     And the following "questions" exist:
       | questioncategory | qtype      | name            |
       | Test questions   | coderunner | Square function |
-    And the CodeRunner webservice is enabled
+
 
     When I am on the "Square function" "core_question > edit" page logged in as teacher1
     And I set the following fields to these values:
diff --git a/tests/behat/attachmentimportexport.feature b/tests/behat/attachmentimportexport.feature
index 00e6ae70..43eb7308 100644
--- a/tests/behat/attachmentimportexport.feature
+++ b/tests/behat/attachmentimportexport.feature
@@ -5,7 +5,9 @@ Feature: Test importing and exporting of question with attachments
   I need to be able to import and export them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the CodeRunner webservice is enabled
+    And the following "users" exist:
       | username |
       | teacher  |
     And the following "courses" exist:
@@ -20,7 +22,6 @@ Feature: Test importing and exporting of question with attachments
     And the following "questions" exist:
       | questioncategory | qtype      | name            |
       | Test questions   | coderunner | Square function |
-    And the CodeRunner webservice is enabled
     And I am on the "Square function" "core_question > edit" page logged in as teacher
     And I click on "a[aria-controls='id_attachmentoptionscontainer']" "css_element"
     And I set the field "Answer" to "from sqrmodule import sqr"
diff --git a/tests/behat/attachments.feature b/tests/behat/attachments.feature
index 2403b336..95d52505 100644
--- a/tests/behat/attachments.feature
+++ b/tests/behat/attachments.feature
@@ -5,7 +5,8 @@ Feature: Test editing and using attachments to a CodeRunner question
   I need to enable and configure them, then preview them.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/backup_and_restore.feature b/tests/behat/backup_and_restore.feature
index 87065833..f364425b 100644
--- a/tests/behat/backup_and_restore.feature
+++ b/tests/behat/backup_and_restore.feature
@@ -5,6 +5,7 @@ Feature: Duplicate a course containing a CodeRunner question
   I need to be able to back them up and restore them
 
   Background:
+    Given the CodeRunner test configuration file is loaded
     And the following "courses" exist:
       | fullname | shortname | category |
       | Course 1 | C1        | 0        |
diff --git a/tests/behat/behat_coderunner.php b/tests/behat/behat_coderunner.php
index 82c6fa09..cfed2249 100644
--- a/tests/behat/behat_coderunner.php
+++ b/tests/behat/behat_coderunner.php
@@ -27,15 +27,36 @@
 
 
 class behat_coderunner extends behat_base {
-     /**
-      * Sets the webserver sandbox to enabled for testing purposes.
-      *
-      * @Given /^the CodeRunner webservice is enabled/
-      */
+
+    /**
+     * Loads the default coderunner settings file for testing.
+     * It seems silly that I have to do that. Why is there not
+     * a global behat configuration option apply to all features?
+     * @Given /^the CodeRunner test configuration file is loaded/
+     */
+    public function the_coderunner_test_configuration_file_is_loaded() {
+        global $CFG;
+        require($CFG->dirroot .'/question/type/coderunner/tests/fixtures/test-sandbox-config.php');
+    }
+
+    /**
+     * Enables the CodeRunner webservice for testing purposes.
+     *
+     * @Given /^the CodeRunner webservice is enabled/
+     */
     public function the_coderunner_webservice_is_enabled() {
         set_config('wsenabled', 1, 'qtype_coderunner');
     }
 
+    /**
+     * Disables the Jobe sandbox. Currently unused/untested.
+     *
+     * @Given /^the Jobe sandbox is disabled/
+     */
+    public function the_jobe_sandbox_is_disabled() {
+        set_config('jobesandbox_enabled', 0, 'qtype_coderunner');
+    }
+
     /**
      * Checks that a given string appears within answer textarea.
      * Intended for checking UI serialization
diff --git a/tests/behat/check_graph_question_types.feature b/tests/behat/check_graph_question_types.feature
index 8b65a60a..f7807101 100644
--- a/tests/behat/check_graph_question_types.feature
+++ b/tests/behat/check_graph_question_types.feature
@@ -5,7 +5,8 @@ Feature: Check that the directed and undirected graph question types work.
   I should be able to write simple graph questions and have them work correctly
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/check_python_template_params.feature b/tests/behat/check_python_template_params.feature
index 14955b0c..1ea50445 100644
--- a/tests/behat/check_python_template_params.feature
+++ b/tests/behat/check_python_template_params.feature
@@ -5,7 +5,8 @@ Feature: Check that Python and other languages can be used instead of Twig as a
   I should be able to write a function that prints the seed and my username it should be marked right
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname       | lastname  | email            |
       | teacher1 | Teacher         | Last      | teacher1@asd.com |
       | student1 | Student First   | O'Connell | student@asd.com  |
diff --git a/tests/behat/check_stepinfo.feature b/tests/behat/check_stepinfo.feature
index 69ebbc82..36c91e91 100644
--- a/tests/behat/check_stepinfo.feature
+++ b/tests/behat/check_stepinfo.feature
@@ -5,7 +5,8 @@ Feature: Check that the QUESTION.stepinfo record is working.
   I should be able to write a question that gives different feedback for different submissions.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
       | student1 | Student   | 1        | student@asd.com  |
diff --git a/tests/behat/check_twig_student_variable.feature b/tests/behat/check_twig_student_variable.feature
index 7d011155..1338840b 100644
--- a/tests/behat/check_twig_student_variable.feature
+++ b/tests/behat/check_twig_student_variable.feature
@@ -5,7 +5,8 @@ Feature: Check the STUDENT Twig variable allows access to current username in Co
   I should be able to write a function that prints my username it should be marked right
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
       | student1 | Student   | 1        | student@asd.com  |
diff --git a/tests/behat/create_python3_sqr_function.feature b/tests/behat/create_python3_sqr_function.feature
index e459fc9e..b6d0d9df 100644
--- a/tests/behat/create_python3_sqr_function.feature
+++ b/tests/behat/create_python3_sqr_function.feature
@@ -5,7 +5,8 @@ Feature: Create a CodeRunner question (the sqr function example)
   I need to create a new CodeRunner question
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
@@ -27,8 +28,10 @@ Feature: Create a CodeRunner question (the sqr function example)
       | id_questiontext   | Write a sqr function    |
       | id_testcode_0     | print(sqr(-7))          |
       | id_expected_0     | 49                      |
+      | id_ordering_0     | 20                      |
       | id_testcode_1     | print(sqr(11))          |
       | id_expected_1     | 120                     |
+      | id_ordering_1     | 10                      |
     Then I should see "Failed 1 test(s)"
     And I should see "Click on the << button to replace the expected output of this testcase with actual output."
     And the field "Customise" matches value "0"
diff --git a/tests/behat/duplicate_prototype.feature b/tests/behat/duplicate_prototype.feature
index 7ff68ced..5b2e7cb9 100644
--- a/tests/behat/duplicate_prototype.feature
+++ b/tests/behat/duplicate_prototype.feature
@@ -5,7 +5,8 @@ Feature: duplicate_prototypes
   I should see an informative error message and be able to fix by editing the duplicates
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/edit.feature b/tests/behat/edit.feature
index db79a174..fe2430d6 100644
--- a/tests/behat/edit.feature
+++ b/tests/behat/edit.feature
@@ -5,7 +5,8 @@ Feature: Test editing a CodeRunner question
   I need to edit them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/edit_question_precheck.feature b/tests/behat/edit_question_precheck.feature
index 29ee74e7..7d5559b3 100644
--- a/tests/behat/edit_question_precheck.feature
+++ b/tests/behat/edit_question_precheck.feature
@@ -5,7 +5,8 @@ Feature: edit_question_precheck
   I should get informative error messages if saving was unsuccessful
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/edit_table.feature b/tests/behat/edit_table.feature
index 51371477..71ce0260 100644
--- a/tests/behat/edit_table.feature
+++ b/tests/behat/edit_table.feature
@@ -5,7 +5,8 @@ Feature: Test editing a CodeRunner question using the Table UI
   I should be able to set the table headers and see the table in the edit form.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/export.feature b/tests/behat/export.feature
index 2180f485..c37f4ff3 100644
--- a/tests/behat/export.feature
+++ b/tests/behat/export.feature
@@ -5,7 +5,8 @@ Feature: Export CodeRunner questions
   I need to export them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username |
       | teacher  |
     And the following "courses" exist:
diff --git a/tests/behat/gapfiller_ui.feature b/tests/behat/gapfiller_ui.feature
index 58e6c783..f036aa40 100644
--- a/tests/behat/gapfiller_ui.feature
+++ b/tests/behat/gapfiller_ui.feature
@@ -5,7 +5,8 @@ Feature: Test the GapFiller_UI
   I should be able specify the required gaps in the global extra or test0 fields
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/grading_scenarios.feature b/tests/behat/grading_scenarios.feature
index cb7602a3..f3306fa5 100644
--- a/tests/behat/grading_scenarios.feature
+++ b/tests/behat/grading_scenarios.feature
@@ -5,7 +5,8 @@ Feature: Check grading with the Python 3 sqr function CodeRunner question
   I must be able to preview them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/html_ui.feature b/tests/behat/html_ui.feature
index b235bbcd..56de0933 100644
--- a/tests/behat/html_ui.feature
+++ b/tests/behat/html_ui.feature
@@ -5,7 +5,8 @@ Feature: Test the HTML_UI
   I should be able specify the required html in either globalextra or prototypeextra
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/import.feature b/tests/behat/import.feature
index 01062935..d2818f70 100644
--- a/tests/behat/import.feature
+++ b/tests/behat/import.feature
@@ -5,7 +5,8 @@ Feature: Import CodeRunner questions
   I need to import them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username |
       | teacher  |
     And the following "courses" exist:
diff --git a/tests/behat/make_combinator_prototype.feature b/tests/behat/make_combinator_prototype.feature
index bfbc2d6b..4c195b8a 100644
--- a/tests/behat/make_combinator_prototype.feature
+++ b/tests/behat/make_combinator_prototype.feature
@@ -5,7 +5,8 @@ Feature: make_combinator_prototype
   I must be able to create new combinator templates
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
@@ -50,8 +51,19 @@ Feature: make_combinator_prototype
       | id_questiontext   | Write the inevitable sqr function |
       | id_testcode_0     | sqr(-11)                          |
       | id_expected_0     | 121                               |
+      | id_ordering_0     | 20                                |
       | id_testcode_1     | sqr(9)                            |
-      | id_expected_1     | 81                                |
+      | id_expected_1     | 80                                |
+      | id_ordering_1     | 10                                |
+      | id_answer         | def sqr(n): return n * n          |
+
+    Then I should see "Failed 1 test(s)"
+    And I should see "Click on the << button to replace the expected output of this testcase with actual output."
+    When I press "<<"
+    And I press "id_submitbutton"
+    Then I should not see "Save changes"
+    And I should not see "Write a sqr function"
+    And I should see "Combinator prototype tester"
 
   Scenario: As a teacher, I get marked right (using combinator template) if I submit a correct answer to a CodeRunner question
     When I choose "Preview" action for "Combinator prototype tester" in the question bank
diff --git a/tests/behat/make_prototype.feature b/tests/behat/make_prototype.feature
index b1b1617f..8d5cbcd3 100644
--- a/tests/behat/make_prototype.feature
+++ b/tests/behat/make_prototype.feature
@@ -5,7 +5,8 @@ Feature: make_prototype
   I must be able to create new question templates
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/missing_prototype.feature b/tests/behat/missing_prototype.feature
index 9eb2687b..db01f4ff 100644
--- a/tests/behat/missing_prototype.feature
+++ b/tests/behat/missing_prototype.feature
@@ -5,7 +5,8 @@ Feature: missing_prototype
   I should see an informative error message and be able to fix by editing
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/reset_button.feature b/tests/behat/reset_button.feature
index fc215c1d..3945795a 100644
--- a/tests/behat/reset_button.feature
+++ b/tests/behat/reset_button.feature
@@ -5,7 +5,8 @@ Feature: Preview the Python 3 sqr function CodeRunner question with a preload
   I should see a Reset answer button that resets the preload,
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/run_python3_sqr_function.feature b/tests/behat/run_python3_sqr_function.feature
index 47c424af..34448259 100644
--- a/tests/behat/run_python3_sqr_function.feature
+++ b/tests/behat/run_python3_sqr_function.feature
@@ -5,7 +5,8 @@ Feature: Preview the Python 3 sqr function CodeRunner question
   I must be able to preview them
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/run_python3_sqr_function_templated.feature b/tests/behat/run_python3_sqr_function_templated.feature
index b4092269..f7755100 100644
--- a/tests/behat/run_python3_sqr_function_templated.feature
+++ b/tests/behat/run_python3_sqr_function_templated.feature
@@ -5,7 +5,8 @@ Feature: Combinator template is called test-by-test if a runtime error occurs wh
   I need the combinator template to be used test-by-test if my answer gives a runtime error
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/sandbox_webservice.feature b/tests/behat/sandbox_webservice.feature
index f013dd35..6018f149 100644
--- a/tests/behat/sandbox_webservice.feature
+++ b/tests/behat/sandbox_webservice.feature
@@ -4,7 +4,8 @@ Feature: Test sandbox web service
   server (Jobe) via Ajax.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email           |
       | teacher  | Teacher   | 1        | teacher@asd.com |
       | student  | Student   | 1        | student@asd.com |
diff --git a/tests/behat/scratchpad_ui.feature b/tests/behat/scratchpad_ui.feature
index 762c0a25..42ff8a8c 100644
--- a/tests/behat/scratchpad_ui.feature
+++ b/tests/behat/scratchpad_ui.feature
@@ -5,7 +5,9 @@ Feature: Test the Scratchpad UI
   I should be able specify the required html in either globalextra or prototypeextra
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the CodeRunner webservice is enabled
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
@@ -20,7 +22,6 @@ Feature: Test the Scratchpad UI
     And the following "questions" exist:
       | questioncategory | qtype      | name         | template |
       | Test questions   | coderunner | Print answer | printans |
-    And the CodeRunner webservice is enabled
 
   Scenario: Edit a CodeRunner question into a Scratchpad UI question
     When I am on the "Print answer" "core_question > edit" page logged in as teacher1
diff --git a/tests/behat/scratchpad_ui_params.feature b/tests/behat/scratchpad_ui_params.feature
index eddd52bf..eef70b32 100644
--- a/tests/behat/scratchpad_ui_params.feature
+++ b/tests/behat/scratchpad_ui_params.feature
@@ -5,7 +5,9 @@ Feature: Test the Scratchpad UI, UI Params
   I should be able specify the UI Parameters to change the Scratchpad UI
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the CodeRunner webservice is enabled
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
@@ -20,7 +22,6 @@ Feature: Test the Scratchpad UI, UI Params
     And the following "questions" exist:
       | questioncategory | qtype      | name         | template |
       | Test questions   | coderunner | Print answer | printans |
-    And the CodeRunner webservice is enabled
 
     And I am on the "Print answer" "core_question > edit" page logged in as teacher1
     And I set the field "id_validateonsave" to ""
diff --git a/tests/behat/set_uiplugin.feature b/tests/behat/set_uiplugin.feature
index 53f60b1d..913c61bb 100644
--- a/tests/behat/set_uiplugin.feature
+++ b/tests/behat/set_uiplugin.feature
@@ -5,7 +5,8 @@ Feature: Check that a selected UI plugin is saved
   I should be able to select a UI plugin and save the form
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/showdifferences_button.feature b/tests/behat/showdifferences_button.feature
index c244167e..b5d1d2ec 100644
--- a/tests/behat/showdifferences_button.feature
+++ b/tests/behat/showdifferences_button.feature
@@ -5,7 +5,8 @@ Feature: Show differences in CodeRunner questions
   clicked.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/template_params_error.feature b/tests/behat/template_params_error.feature
index bce67654..cfe26cc4 100644
--- a/tests/behat/template_params_error.feature
+++ b/tests/behat/template_params_error.feature
@@ -5,7 +5,8 @@ Feature: template_params_error
   I should get informative template parameter error messages
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:
diff --git a/tests/behat/twigprefix.feature b/tests/behat/twigprefix.feature
index 86eb7563..5099d1cc 100644
--- a/tests/behat/twigprefix.feature
+++ b/tests/behat/twigprefix.feature
@@ -5,7 +5,8 @@ Feature: twigprefix
   I must be able to use the Twig prefix data in a question.
 
   Background:
-    Given the following "users" exist:
+    Given the CodeRunner test configuration file is loaded
+    And the following "users" exist:
       | username | firstname | lastname | email            |
       | teacher1 | Teacher   | 1        | teacher1@asd.com |
     And the following "courses" exist:

From d7ba97b87217ce4bf6bc1380c5360f5c09bc48e1 Mon Sep 17 00:00:00 2001
From: Anupama Sarjoshi <81178902+AnupamaSarjoshi@users.noreply.github.com>
Date: Wed, 20 Mar 2024 22:07:41 +0000
Subject: [PATCH 4/6] Add Load-balancing cookie support (#206)

These changes support AWS load-balancer stickiness,
by saving and sending back the cookie set by the load balancer
using curl.
---
 classes/jobesandbox.php | 54 ++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/classes/jobesandbox.php b/classes/jobesandbox.php
index 848bcb28..239544bb 100644
--- a/classes/jobesandbox.php
+++ b/classes/jobesandbox.php
@@ -50,6 +50,7 @@ class qtype_coderunner_jobesandbox extends qtype_coderunner_sandbox {
      *  - with haproxy try "balance hdr(X-CodeRunner-Job-Id)" (not tested)
      *  - with a Netscaler use rule-based persistence with expression
      *    HTTP.REQ.HEADER(“X-CodeRunner-Job-Id”)
+     *  - with cookies support
      */
     private $currentjobid = null;
 
@@ -137,6 +138,8 @@ public function get_languages() {
      */
 
     public function execute($sourcecode, $language, $input, $files = null, $params = null) {
+        global $CFG;
+
         $language = strtolower($language);
         if (is_null($input)) {
             $input = '';
@@ -202,21 +205,36 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
         $postbody = ['run_spec' => $runspec];
         $this->currentjobid = sprintf('%08x', mt_rand());
 
+        // Create a single curl object here, with support for cookies, and use it for all requests.
+        // This supports Jobe back-ends that use cookies for sticky load-balancing.
+        // Make a place to store the cookies.
+        make_temp_directory('qtype_coderunner');
+        $cookiefile = $CFG->tempdir . '/qtype_coderunner/session_cookies_' . $this->currentjobid . '.txt';
+
+        $curl = new curl();
+        $curl->setopt([
+                'CURLOPT_COOKIEJAR' => $cookiefile,
+                'CURLOPT_COOKIEFILE' => $cookiefile,
+        ]);
+
         // Try submitting the job. If we get a 404, try again after
         // putting all the files on the server. Anything else is an error.
-        $httpcode = $this->submit($postbody);
+        $httpcode = $this->submit($postbody, $curl);
         if ($httpcode == 404) { // If it's a file not found error ...
             foreach ($files as $filename => $contents) {
-                if (($httpcode = $this->put_file($contents)) != 204) {
+                if (($httpcode = $this->put_file($contents, $curl)) != 204) {
                     break;
                 }
             }
             if ($httpcode == 204) {
                 // Try again if put_files all worked.
-                $httpcode = $this->submit($postbody);
+                $httpcode = $this->submit($postbody, $curl);
             }
         }
 
+        // Delete the cookie file.
+        unlink($cookiefile);
+
         $runresult = [];
         $runresult['sandboxinfo'] = [
             'jobeserver' => $this->jobeserver,
@@ -328,11 +346,10 @@ private function get_error_code($httpcode) {
     }
 
     // Put the given file to the server, using its MD5 checksum as the id.
+    // If you pass a curl object, this will be used to make the request.
     // Returns the HTTP response code, or -1 if the HTTP request fails
     // altogether.
-    // Moodle curl class doesn't support an appropriate form of PUT so
-    // we use raw PHP curl here.
-    private function put_file($contents) {
+    private function put_file($contents, $curl) {
         $id = md5($contents);
         $contentsb64 = base64_encode($contents);
         $resource = "files/$id";
@@ -340,16 +357,9 @@ private function put_file($contents) {
         [$url, $headers] = $this->get_jobe_connection_info($resource);
 
         $body = ['file_contents' => $contentsb64];
-        $curl = curl_init();
-        curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
-        curl_setopt($curl, CURLOPT_CUSTOMREQUEST, "PUT");
-        curl_setopt($curl, CURLOPT_URL, $url);
-        curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($body));
-        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-        $result = curl_exec($curl);
-        $info = curl_getinfo($curl);
-        curl_close($curl);
-        return $result === false ? -1 : $info['http_code'];
+        $result = $curl->put($url, json_encode($body));
+        $returncode = $curl->info['http_code'];
+        return $result === false ? -1 : $returncode;
     }
 
     /**
@@ -403,8 +413,9 @@ private function get_jobe_connection_info($resource) {
     // response was 400 Bad Parameter.
     // We don't at this stage deal with Jobe servers that may defer requests
     // i.e. that return 202 Accepted rather than 200 OK.
-    private function submit($job) {
-        [$returncode, $response] = $this->http_request('runs', self::HTTP_POST, $job);
+    // If you pass a curl object, this will be used to make the request.
+    private function submit($job, $curl) {
+        [$returncode, $response] = $this->http_request('runs', self::HTTP_POST, $job, $curl);
         $this->response = $response;
         return $returncode;
     }
@@ -419,10 +430,13 @@ private function submit($job) {
     // Note that the Moodle curl class documentation lies when it says the
     // return value from get and post is a bool. It's either the value false
     // if the request failed or the actual string response, otherwise.
-    private function http_request($resource, $method, $body = null) {
+    // If you pass a curl object, this will be used to make the request.
+    private function http_request($resource, $method, $body = null, $curl = null) {
         [$url, $headers] = $this->get_jobe_connection_info($resource);
 
-        $curl = new curl();
+        if ($curl == null) {
+            $curl = new curl();
+        }
         $curl->setHeader($headers);
 
         if ($method === self::HTTP_GET) {

From 260736af195d5fe92236406043576f1a8821316e Mon Sep 17 00:00:00 2001
From: Richard Lobb <trampgeek@gmail.com>
Date: Thu, 21 Mar 2024 13:42:01 +1300
Subject: [PATCH 5/6] Testcases marked as Precheck Only were not being
 validating on save.

---
 edit_coderunner_form.php |  2 +-
 question.php             | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/edit_coderunner_form.php b/edit_coderunner_form.php
index 7f6f1978..30f81d52 100644
--- a/edit_coderunner_form.php
+++ b/edit_coderunner_form.php
@@ -1633,7 +1633,7 @@ private function validate_sample_answer() {
             if ($error) {
                 return $error;
             }
-            [$mark, , $cachedata] = $this->formquestion->grade_response($response);
+            [$mark, , $cachedata] = $this->formquestion->grade_response($response, false, true);
         } catch (Exception $e) {
             return $e->getMessage();
         }
diff --git a/question.php b/question.php
index b6d6e141..3fb0d8a8 100644
--- a/question.php
+++ b/question.php
@@ -818,13 +818,15 @@ public function display_feedback() {
      * the history of prior submissions.
      * @param bool $isprecheck true iff this grading is occurring because the
      * student clicked the precheck button
+     * @param bool $isvalidationrun true iff this is a validation run when saving 
+     * a question.
      * @return 3-element array of the mark (0 - 1), the question_state (
      * gradedright, gradedwrong, gradedpartial, invalid) and the full
      * qtype_coderunner_testing_outcome object to be cached. The invalid
      * state is used when a sandbox error occurs.
      * @throws coding_exception
      */
-    public function grade_response(array $response, bool $isprecheck = false) {
+    public function grade_response(array $response, bool $isprecheck = false, $isvalidationrun = false) {
         if ($isprecheck && empty($this->precheck)) {
             throw new coding_exception("Unexpected precheck");
         }
@@ -847,7 +849,11 @@ public function grade_response(array $response, bool $isprecheck = false) {
             // filenames and values being file contents.
             $code = $response['answer'];
             $attachments = $this->get_attached_files($response);
-            $testcases = $this->filter_testcases($isprecheck, $this->precheck);
+            if ($isvalidationrun) {
+                $testcases = $this->testcases;
+            } else {
+                $testcases = $this->filter_testcases($isprecheck, $this->precheck);
+            }
             $runner = new qtype_coderunner_jobrunner();
             $this->stepinfo = self::step_info($response);
             if (isset($response['graderstate'])) {

From d262ff9c3e0aceb81a58375343a4d3878e02c440 Mon Sep 17 00:00:00 2001
From: Richard Lobb <trampgeek@gmail.com>
Date: Thu, 21 Mar 2024 19:40:25 +1300
Subject: [PATCH 6/6] Fix silly grunt error objecting to multiple blank lines.

---
 tests/behat/ace_scratchpad_compatibility.feature | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/behat/ace_scratchpad_compatibility.feature b/tests/behat/ace_scratchpad_compatibility.feature
index 0a36b15d..9c10e433 100644
--- a/tests/behat/ace_scratchpad_compatibility.feature
+++ b/tests/behat/ace_scratchpad_compatibility.feature
@@ -23,7 +23,6 @@ Feature: Ace UI convert to Scratchpad UI questions with one click
       | questioncategory | qtype      | name            |
       | Test questions   | coderunner | Square function |
 
-
     When I am on the "Square function" "core_question > edit" page logged in as teacher1
     And I set the following fields to these values:
       | id_customise | 1                        |