Development - grade result cache with on/off switch for local testing

classes/external/run_in_sandbox.php

@@ -184,7 +184,8 @@ public static function execute(
             if ($jobehostws !== '') {
                 $paramsarray['jobeserver'] = $jobehostws;
             }
-            $runresult = $sandbox->execute($sourcecode, $language, $stdin, $filesarray, $paramsarray);
+            // usecache set to false for these runs as we will never regrade them.
+            $runresult = $sandbox->execute($sourcecode, $language, $stdin, $filesarray, $paramsarray, false);
         } catch (Exception $ex) {
             throw new qtype_coderunner_exception("Attempt to run job failed with error {$ex->message}");
         }

classes/jobesandbox.php

@@ -30,8 +30,6 @@
 global $CFG;
 require_once($CFG->libdir . '/filelib.php'); // Needed when run as web service.
 
-const READ_FROM_CACHE = true;
-const WRITE_TO_CACHE = true;
 
 class qtype_coderunner_jobesandbox extends qtype_coderunner_sandbox {
     const DEBUGGING = 0;
@@ -75,8 +73,10 @@ public function __construct() {
         global $CFG;
         qtype_coderunner_sandbox::__construct();
         $this->jobeserver = get_config('qtype_coderunner', 'jobe_host');
-        if (qtype_coderunner_sandbox::is_canterbury_server($this->jobeserver)
-                && qtype_coderunner_sandbox::is_using_test_sandbox()) {
+        if (
+            qtype_coderunner_sandbox::is_canterbury_server($this->jobeserver)
+                && qtype_coderunner_sandbox::is_using_test_sandbox()
+        ) {
             throw new Exception("Please don't use the Canterbury jobe server for test runs");
         }
         $this->apikey = get_config('qtype_coderunner', 'jobe_apikey');
@@ -112,11 +112,24 @@ public function get_languages() {
      * @param string $language  One of the languages recognised by the sandbox
      * @param string $input A string to use as standard input during execution
      * @param associative array $files either null or a map from filename to
-     *         file contents, defining a file context at execution time
+     *         file contents, defining a file context at execution time.
+     * @param bool $usecache Will read and write to cache if set to True. Otherwise
+     *         won't use the cache. NOTE: global settings for cache <read>
+     *         are also factored in here.
      * @param associative array $params Sandbox parameters, depends on
      *         particular sandbox but most sandboxes should recognise
      *         at least cputime (secs) and memorylimit (Megabytes).
      *         If the $params array is null, sandbox defaults are used.
+     * @param bool $usecache determines if result caching (both reads and writes)
+     *         will be used. This is true by default (which is usually for normal
+     *         grading runs) but is typically set to false by calls from the
+     *         web-service/sandbox (eg, scratchpad or try-it box runs, as these are never
+     *         going to be regraded). Note, the coderunner setting for enabling
+     *         cache are combined with this argument to determine
+     *         if cache reading and writing are done. For example, if usecache
+     *         is true and enablegradecache is true then this function
+     *         will try to read the result from the grading cache but if
+     *         either is false then it won't.
      * @return an object with at least the attribute 'error'.
      *         The error attribute is one of the
      *         values 0 through 9 (OK to UNKNOWN_SERVER_ERROR, OVERLOAD)
@@ -140,7 +153,7 @@ public function get_languages() {
      *         showing which jobeserver was used and what key was used (if any).
      */
 
-    public function execute($sourcecode, $language, $input, $files = null, $params = null) {
+    public function execute($sourcecode, $language, $input, $files = null, $params = null, $usecache = true) {
         global $CFG;
 
         $language = strtolower($language);
@@ -208,22 +221,14 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
 
         $cache = cache::make('qtype_coderunner', 'coderunner_grading_cache');
         $runresult = null;
-        if (READ_FROM_CACHE) {
+        if (get_config('qtype_coderunner', 'enablegradecache') && $usecache) {
             // NOTE: Changing jobeserver setting will effectively flush the cache
             // eg, adding another jobeserver to a list of servers will mean the
             // jobeserver parameter has changed and therefore the key will change.
 
-            // QUESTION: Do we want the cache to ignore the jobeserver setting?
-            // eg, adding a new, presumeably equal jobeserver to the mix shouldn't
-            // change the result (unless it isn't equal!)
-            // But, remember that the server is chosen at random from the pool!
-
             $key = hash("md5", serialize($runspec));
             // Debugger: echo '<pre>' . serialize($runspec) . '</pre>';.
             $runresult = $cache->get($key);  // Unserializes the returned value :) false if not found.
-            if ($runresult) {
-                // echo $key . '----------->   FOUND' . '<br>'; .
-            }
         }
 
         if (!$runresult) {  // if cache read failed regrade, to be safe.
@@ -293,10 +298,9 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
                 $runresult['output'] = $this->filter_file_path($this->response->stdout);
 
                 // Got a useable result from Jobe server so cache it if required.
-                if (WRITE_TO_CACHE) {
+                if (get_config('qtype_coderunner', 'enablegradecache') && $usecache) {
                     $key = hash("md5", serialize($runspec));
                     $cache->set($key, $runresult); // Set serializes the result, get will unserialize.
-                    // echo 'CACHE WRITE for ---> ' . $key . '<br>';
                 }
             }
         }

lang/en/qtype_coderunner.php

@@ -1343,7 +1343,5 @@ function should be applied, e.g. <code>{{STUDENT_ANSWER | e(\'py\')}}</code> is
 $string['wssubmissionrateexceeded'] = 'You have exceeded the maximum hourly \'Try it!\' submission rate. Request denied.';
 
 $string['xmlcoderunnerformaterror'] = 'XML format error in coderunner question';
-$string['coderunner_grading_cache'] = 'Caches grading results so we can avoid going to Jobe so often';
-$string['cachedef_coderunner_grading_cache'] = 'Caches grading results so we can avoid going to Jobe so often';
-$string['cachegradingresultsenable'] = 'Cache results when grading answers.';
-$string['cachegradingresults_desc'] = 'Uses a local Moodle cache (currently file cache) to store results of grading questions. Mainly to speed up regrading by using cached results for steps where the same quesiton and answer have already been graded.';
+$string['enablegradecache'] = 'Enable reading/writing of job,result pairs from/to the Coderunner grading cache.';
+$string['enablegradecache_desc'] = 'Experimental. The cache is a local Moodle cache (currently file cache) to store results of grading questions. Mainly to speed up regrading by using cached results for jobe runs where the same jobe submission has already been graded. Currently WS jobs (eg, try-it boxes and scratchpad runs) will never be cached. NOTE: If you turn off grade caching then it is usually good to empty the Coderunner grade cache before you turn it on again so you have a known state for the cache. You should also clear the cache if you change the Jobe back-end (eg, installing a new version of Python there) as results may now differ from what is in the cache.';

settings.php

@@ -78,20 +78,30 @@
     constants::JOBE_HOST_DEFAULT_API_KEY
 ));
 
+$settings->add(new admin_setting_configcheckbox(
+    "qtype_coderunner/enablegradecache",
+    get_string('enablegradecache', 'qtype_coderunner'),
+    get_string('enablegradecache_desc', 'qtype_coderunner'),
+    false
+));
+
+
 $settings->add(new admin_setting_configtext(
     "qtype_coderunner/ideone_user",
     get_string('ideone_user', 'qtype_coderunner'),
     get_string('ideone_user_desc', 'qtype_coderunner'),
     ''
 ));
 
+
 $settings->add(new admin_setting_configtext(
     "qtype_coderunner/ideone_password",
     get_string('ideone_pass', 'qtype_coderunner'),
     get_string('ideone_pass_desc', 'qtype_coderunner'),
     ''
 ));
 
+
 $settings->add(new admin_setting_heading(
     'codeRunnerwssettings',
     get_string('coderunnerwssettings', 'qtype_coderunner'),
@@ -134,14 +144,3 @@
     get_string('wsmaxcputime_desc', 'qtype_coderunner'),
     '5'
 ));
-
-/*
-Currently left out so we can test
-
-$settings->add(new admin_setting_configcheckbox(
-    "qtype_coderunner/cachegradingresults",
-    get_string('cachegradingresultsenable', 'qtype_coderunner'),
-    get_string('cachegradingresults_desc', 'qtype_coderunner'),
-    false
-));
-*/