From 7266ee7bc0e1e3b63fd7dfbed435409b7ba3692e Mon Sep 17 00:00:00 2001 From: Richard Lobb Date: Sun, 19 May 2024 20:20:38 +1200 Subject: [PATCH] Reimplement web-service throttle to use leaky bucket algorithm, and to operate independently of logging. --- classes/external/run_in_sandbox.php | 28 ++++++++-------- classes/wsthrottle.php | 51 +++++++++++++---------------- lang/en/qtype_coderunner.php | 6 ++-- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/classes/external/run_in_sandbox.php b/classes/external/run_in_sandbox.php index a224b1620..d4cb77e96 100644 --- a/classes/external/run_in_sandbox.php +++ b/classes/external/run_in_sandbox.php @@ -143,22 +143,22 @@ public static function execute( throw new qtype_coderunner_exception(get_string('wsnolanguage', 'qtype_coderunner', $language)); } - if (get_config('qtype_coderunner', 'wsloggingenabled')) { - // Check if need to throttle this user, and if not or if rate - // sufficiently low, allow the request and log it. - $maxhourlyrate = intval(get_config('qtype_coderunner', 'wsmaxhourlyrate')); - if ($maxhourlyrate > 0) { // Throttling enabled? - if (!isset($SESSION->throttle)) { - $throttle = new \qtype_coderunner_wsthrottle(); - } else { - $throttle = unserialize($SESSION->throttle); - } - if (!$throttle->logrunok()) { - throw new qtype_coderunner_exception(get_string('wssubmissionrateexceeded', 'qtype_coderunner')); - } - $SESSION->throttle = serialize($throttle); + // Check if need to throttle this submission. + $maxhourlyrate = intval(get_config('qtype_coderunner', 'wsmaxhourlyrate')); + if ($maxhourlyrate > 0) { // Throttling enabled? + if (!isset($SESSION->throttle)) { + $throttle = new \qtype_coderunner_wsthrottle(); + } else { + $throttle = unserialize($SESSION->throttle); } + if (!$throttle->logrunok()) { + throw new qtype_coderunner_exception(get_string('wssubmissionrateexceeded', 'qtype_coderunner')); + } + $SESSION->throttle = serialize($throttle); + } + // If logging enabled, log the run. + if (get_config('qtype_coderunner', 'wsloggingenabled')) { $event = \qtype_coderunner\event\sandbox_webservice_exec::create([ 'contextid' => $context->id]); $event->trigger(); diff --git a/classes/wsthrottle.php b/classes/wsthrottle.php index 8afce7c38..a421d7493 100644 --- a/classes/wsthrottle.php +++ b/classes/wsthrottle.php @@ -31,12 +31,16 @@ /** * Class to manage the throttling of an individual webservice user to the rate given * in the wsmaxhourlyrate config parameter. + * Uses the "leaky bucket" algorithm (https://en.wikipedia.org/wiki/Leaky_bucket). This allows + * a surge of run requests equal to the maxhourlyrate but thereafter the rate is limited + * to maxhourlyrate / 3600 runs per second. + * This class must be instantiated within the SESSION variable. */ class qtype_coderunner_wsthrottle { private $timestamps; private $maxhourlyrate; - private $head; - private $tail; + private $bucketlevel; + private $timestamp; public function __construct() { $this->init(); @@ -44,12 +48,13 @@ public function __construct() { private function init() { $this->maxhourlyrate = intval(get_config('qtype_coderunner', 'wsmaxhourlyrate')); - $this->timestamps = array_fill(0, $this->maxhourlyrate, 0); - $this->head = $this->tail = 0; // Head and tail indices for circular list. + $this->bucketlevel = 0; // Current level of virtual fluid in the bucket (runs in last hour). + $this->timestamp = time(); // When the bucket level was last updated. } + /** - * Add a log entry to the circular list of timestamps, clearing any - * expired entries (i.e. entries more than 1 hour ago). + * Allow any drainage to occur from the bucket. Then, if it's still full, + * disallow the run, otherwise allow it. * Return true if logging succeeds, false if user has reached their limit. */ public function logrunok() { @@ -57,30 +62,20 @@ public function logrunok() { // Rate has been changed. Restart throttle. $this->init(); } - $now = strtotime('now'); - // Purge any non-zero entries older than 1 hour. - while ($this->expired($this->timestamps[$this->tail], $now)) { - $this->timestamps[$this->tail] = 0; - $this->tail = ($this->tail + 1) % $this->maxhourlyrate; - } - if ($this->timestamps[$this->head] == 0) { // Empty entry available? - $this->timestamps[$this->head] = $now; - $this->head = ($this->head + 1) % $this->maxhourlyrate; + // Allow any fluid to drain since the last time the level was computed. + $now = time(); + $elapsedseconds = $now - $this->timestamp; + $drainage = $this->maxhourlyrate * $elapsedseconds / (60 * 60); // Change in bucket level. + $this->bucketlevel = max(0.0, $this->bucketlevel - $drainage); + $this->timestamp = $now; + + // Now if there's enough space for 1 more run, allow it. + if ($this->bucketlevel + 1 <= $this->maxhourlyrate) { // Enough space in bucket? + $this->bucketlevel += 1; // Yes, another quantum of fluid gets added. return true; - } else { - // List of timestamps is full. Need to throttle user. + } else { // Not enough space. return false; } } - - /** - * - * @param int $timestamp the timestamp of interest - * @param int $now current timestamp - * @return bool true if the timestamp is non-zero and older than 1 hour - */ - private function expired($timestamp, $now) { - return ($timestamp !== 0) && ($now - $timestamp) > 3600; - } -} +} \ No newline at end of file diff --git a/lang/en/qtype_coderunner.php b/lang/en/qtype_coderunner.php index a1851f6b8..d9163fda9 100644 --- a/lang/en/qtype_coderunner.php +++ b/lang/en/qtype_coderunner.php @@ -1333,14 +1333,14 @@ function should be applied, e.g. {{STUDENT_ANSWER | e(\'py\')}} is $string['wscputimeexcess'] = 'CPU time specified exceeds set maximum CPU time'; $string['wsdisabled'] = 'Sandbox web service disabled. Talk to a sysadmin'; $string['wsloggingenable'] = 'Log sandbox web service usage'; -$string['wsloggingenable_desc'] = 'If this option is checked, every code execution via the sandbox web service will be logged. This option must be enabled if user rate throttling is to work.'; +$string['wsloggingenable_desc'] = 'If this option is checked, every code execution via the sandbox web service will be logged.'; $string['wsmaxcputime'] = 'Max CPU time (secs)'; $string['wsmaxcputime_desc'] = 'Limits the maximum CPU time that a web service job can use, even if it explicitly sets the CPU time sandbox parameter.'; $string['wsmaxhourlyrate'] = 'Max hourly rate of submissions'; -$string['wsmaxhourlyrate_desc'] = 'If a user attempts to exceed this rate of submissions in any given hour their submissions will be disallowed. 0 for no rate throttling.'; +$string['wsmaxhourlyrate_desc'] = 'A burst of submissions of up to this value will be accepted but thereafter, if a user attempts to exceed this average rate of submissions, their submissions will be rejected. 0 for no rate throttling.'; $string['wsnoaccess'] = 'Only logged-in non-guest users can access this functionality'; $string['wsnolanguage'] = 'Language "{$a}" is not known'; -$string['wssubmissionrateexceeded'] = 'You have exceeded the maximum hourly \'Try it!\' submission rate. Request denied.'; +$string['wssubmissionrateexceeded'] = 'You have exceeded the maximum web-service job submission rate. Request denied.'; $string['xmlcoderunnerformaterror'] = 'XML format error in coderunner question'; $string['coderunner_grading_cache'] = 'Caches grading results so we can avoid going to Jobe so often';