#5 add configuration

catalyst · Dec 9, 2016 · b8f9769 · b8f9769
1 parent 9f16f83
commit b8f9769
Show file tree

Hide file tree

Showing 8 changed files with 281 additions and 45 deletions.
diff --git a/classes/form/settings_form.php b/classes/form/settings_form.php
@@ -0,0 +1,132 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Task that pushes files to S3.
+ *
+ * @package   tool_sssfs
+ * @author    Kenneth Hendricks <kennethhendricks@catalyst-au.net>
+ * @copyright Catalyst IT
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace tool_sssfs\form;
+
+require_once($CFG->libdir . "/formslib.php");
+
+/**
+ * Form for editing an Enviroment bar.
+ *
+ * @copyright Catalyst IT
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class settings_form extends \moodleform {
+
+    /**
+     * {@inheritDoc}
+     * @see moodleform::definition()
+     */
+    public function definition() {
+        $mform = $this->_form;
+
+        $config = $this->_customdata['config'];
+
+        $regionoptions = array ('us-east-1',
+                                'us-east-2',
+                                'us-west-1',
+                                'us-west-2',
+                                'ap-northeast-2',
+                                'ap-southeast-1',
+                                'ap-southeast-2',
+                                'ap-northeast-1',
+                                'eu-central-1',
+                                'eu-west-1');
+
+        $mform->addElement('advcheckbox', 'enabled', get_string('settings:enabled', 'tool_sssfs'));
+        $mform->addHelpButton('enabled', 'settings:enabled', 'tool_sssfs');
+        if (isset($config->enabled)) {
+            $mform->setDefault('enabled', $config->enabled);
+        }
+
+        $mform->addElement('header', 'awsheader', get_string('settings:awsheader', 'tool_sssfs'));
+
+        $mform->addElement('text', 'key', get_string('settings:key', 'tool_sssfs'));
+        $mform->addHelpButton('key', 'settings:key', 'tool_sssfs');
+        $mform->setType("key", PARAM_TEXT);
+        if (isset($config->key)) {
+            $mform->setDefault('key', $config->key);
+        }
+
+        $mform->addElement('text', 'secret', get_string('settings:secret', 'tool_sssfs'));
+        $mform->addHelpButton('secret', 'settings:secret', 'tool_sssfs');
+        $mform->setType("secret", PARAM_TEXT);
+        if (isset($config->secret)) {
+            $mform->setDefault('secret', $config->secret);
+        }
+
+        $mform->addElement('text', 'bucket', get_string('settings:bucket', 'tool_sssfs'));
+        $mform->addHelpButton('bucket', 'settings:bucket', 'tool_sssfs');
+        $mform->setType("bucket", PARAM_TEXT);
+        if (isset($config->bucket)) {
+            $mform->setDefault('bucket', $config->bucket);
+        }
+
+        $mform->addElement('select', 'region', get_string('settings:region', 'tool_sssfs'), $regionoptions);
+        $mform->addHelpButton('region', 'settings:region', 'tool_sssfs');
+        if (isset($config->region)) {
+            $mform->setDefault('region', $config->region);
+        }
+
+        $mform->addElement('button', 'checkconnection', get_string('settings:checkconnenction', 'tool_sssfs'));
+
+        $mform->addElement('header', 'filetransferheader', get_string('settings:filetransferheader', 'tool_sssfs'));
+
+        $mform->addElement('text', 'sizethreshold', get_string('settings:sizethreshold', 'tool_sssfs'));
+        $mform->addHelpButton('sizethreshold', 'settings:sizethreshold', 'tool_sssfs');
+        $mform->setType("sizethreshold", PARAM_INT);
+        if (isset($config->sizethreshold)) {
+            $mform->setDefault('sizethreshold', $config->sizethreshold);
+        }
+
+
+        $mform->addElement('duration', 'minimumage', get_string('settings:minimumage', 'tool_sssfs'));
+        $mform->addHelpButton('minimumage', 'settings:minimumage', 'tool_sssfs');
+        $mform->setType("minimumage", PARAM_INT);
+        if (isset($config->minimumage)) {
+            $mform->setDefault('minimumage', $config->minimumage);
+        }
+
+        $mform->addElement('text', 'consistancydelay', get_string('settings:consistancydelay', 'tool_sssfs'));
+        $mform->addHelpButton('consistancydelay', 'settings:consistancydelay', 'tool_sssfs');
+        $mform->setType("consistancydelay", PARAM_INT);
+        if (isset($config->consistancydelay)) {
+            $mform->setDefault('consistancydelay', $config->consistancydelay);
+        }
+
+        $mform->addElement('header', 'loggingheader', get_string('settings:loggingheader', 'tool_sssfs'));
+
+        $mform->addElement('text', 'logginglocation', get_string('settings:logginglocation', 'tool_sssfs'));
+        $mform->addHelpButton('logginglocation', 'settings:logginglocation', 'tool_sssfs');
+        $mform->setType("logginglocation", PARAM_SAFEPATH);
+        if (isset($config->logginglocation)) {
+            $mform->setDefault('logginglocation', $config->logginglocation);
+        }
+
+        $this->add_action_buttons();
+    }
+
+}
+
diff --git a/classes/sss_client.php b/classes/sss_client.php
@@ -32,35 +32,27 @@
 defined('MOODLE_INTERNAL') || die();
 
 define('AWS_API_VERSION', '2006-03-01');
-define('AWS_REGION', 'ap-southeast-2');
 
 class sss_client {
 
-    private $s3client;
-    private $bucketname;
+    private $client;
+    private $bucket;
 
-    public function __construct() {
-        global $CFG;
-
-        $config = $CFG->sss_config;
-
-        $this->bucketname = $config['bucket'];
-
-        $this->s3client = S3Client::factory(array(
-                'credentials' => array('key' => $config['key'], 'secret' => $config['secret']),
-                'version' => AWS_API_VERSION,
-                'region' => AWS_REGION
+    public function __construct($config) {
+        $this->bucket = $config->bucket;
+        $this->client = S3Client::factory(array(
+                'credentials' => array('key' => $config->key, 'secret' => $config->secret),
+                'region' => $config->region,
+                'version' => AWS_API_VERSION
             ));
     }
 
     public function push_file($filekey, $filecontent) {
-
-        $result = $this->s3client->putObject(array(
-                'Bucket' => $this->bucketname,
+        $result = $this->client->putObject(array(
+                'Bucket' => $this->bucket,
                 'Key' => $filekey,
                 'Body' => $filecontent
             ));
-
         return $result;
     }
 }
diff --git a/classes/sss_file_pusher.php b/classes/sss_file_pusher.php
@@ -24,29 +24,40 @@
  */
 
 namespace tool_sssfs;
+require_once($CFG->dirroot . '/admin/tool/sssfs/lib.php');
 
 defined('MOODLE_INTERNAL') || die();
 
 class sss_file_pusher {
-
     private $client;
     private $filesystem;
     private $sizethreshold;
+    private $minimumage;
+    private $maxruntime;
 
-    public function __construct($client, $filesystem) {
+    public function __construct($client, $filesystem, $config) {
         $this->client = $client;
         $this->filesystem = $filesystem;
-        $this->sizethreshold = 1000;
+        $this->sizethreshold = $config->sizethreshold;
+        $this->minimumage = $config->minimumage;
+        $this->maxruntime = 60; // Seconds.
     }
 
     public function push() {
         global $DB;
 
-        $contenthashestopush = get_content_hashes_over_threshold($this->sizethreshold);
+        $finishtime = time() + $this->maxruntime;
+
+        // This Should filter down the file list the most so we do this first.
+        // TODO: refactor into get_push_candidate_content_hashes.
+        $contenthashestopush = $this->get_content_hashes_over_threshold($this->sizethreshold);
 
-        $ssscontenthashes = get_content_hashes_in_sss();
+        $ssscontenthashes = $this->get_content_hashes_in_sss();
 
         foreach ($contenthashestopush as $contenthash) {
+            if (time() > $finishtime) {
+                break;
+            }
 
             if (in_array($contenthash, $ssscontenthashes)) {
                 continue;
@@ -57,10 +68,24 @@ public function push() {
             if ($filecontent) {
                 // TODO: deal with response.
                 $response = $this->client->push_file($contenthash, $filecontent);
-                $logger->log_file_state($contenthash, SSS_FILE_STATE_DUPLICATED);
+                log_file_state($contenthash, SSS_FILE_STATE_DUPLICATED);
             }
         }
     }
+
+    private function get_content_hashes_over_threshold($threshold) {
+        global $DB;
+        $sql = "SELECT DISTINCT contenthash FROM {files} WHERE filesize > ?";
+        $contenthashes = $DB->get_fieldset_sql($sql, array($threshold));
+        return $contenthashes;
+    }
+
+    private function get_content_hashes_in_sss() {
+        global $DB;
+        $sql = 'SELECT contenthash FROM {tool_sssfs_filestate} WHERE STATE in (?, ?)';
+        $ssscontenthashes = $DB->get_fieldset_sql($sql, array(SSS_FILE_STATE_DUPLICATED, SSS_FILE_STATE_EXTERNAL));
+        return $ssscontenthashes;
+    }
 }
 
 
diff --git a/classes/task/push_to_sss.php b/classes/task/push_to_sss.php
@@ -44,12 +44,15 @@ public function get_name() {
      * Execute task
      */
     public function execute() {
-        $client = new sss_client();
-        $filesystem = sss_file_system::instance();
 
-        $filepusher = new sss_file_pusher($client, $filesystem);
+        $config = get_config('tool_sssfs');
 
-        $filepusher->push();
+        if ($config->enabled) {
+            $client = new sss_client($config);
+            $filesystem = sss_file_system::instance();
+            $filepusher = new sss_file_pusher($client, $filesystem, $config);
+            $filepusher->push();
+        }
     }
 }
 

diff --git a/index.php b/index.php
@@ -0,0 +1,51 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * File status page - stats on where files are b/w local file system and s3
+ *
+ * @package   tool_sssfs
+ * @author    Kenneth Hendricks <kennethhendricks@catalyst-au.net>
+ * @copyright Catalyst IT
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+use \tool_sssfs\form\settings_form;
+
+require_once(__DIR__ . '/../../../config.php');
+require_once( __DIR__ . '/lib.php');
+require_once($CFG->libdir.'/adminlib.php');
+
+admin_externalpage_setup('tool_sssfs_settings');
+
+$output = $PAGE->get_renderer('tool_sssfs');
+
+$config = get_config('tool_sssfs');
+
+$form = new settings_form(null, array('config' => $config));
+
+if ($data = $form->get_data()) {
+    save_config_data($data);
+    redirect(new moodle_url('/admin/tool/sssfs/index.php'));
+}
+
+echo $output->header();
+echo $output->heading(get_string('pluginname', 'tool_sssfs'));
+$form->display();
+echo $output->footer();
+
+
+
diff --git a/lang/en/tool_sssfs.php b/lang/en/tool_sssfs.php
@@ -36,3 +36,26 @@
 $string['file_status:state:external'] = 'External';
 $string['file_status:state:unknown'] = 'Unknown';
 
+
+$string['settings:enabled'] = 'Enable S3 file system';
+$string['settings:enabled_help'] = 'Enable or disable the S3 file system.';
+$string['settings:awsheader'] = 'AWS S3 Settings';
+$string['settings:key'] = 'Key';
+$string['settings:key_help'] = 'AWS key credential.';
+$string['settings:secret'] = 'Secret';
+$string['settings:secret_help'] = 'AWS secret credential.';
+$string['settings:bucket'] = 'Bucket';
+$string['settings:bucket_help'] = 'AWS bucket to store files in.';
+$string['settings:region'] = 'AWS region';
+$string['settings:region_help'] = 'AWS API gateway region.';
+$string['settings:checkconnenction'] = 'Check Connection';
+$string['settings:filetransferheader'] = 'File Transfer Settings';
+$string['settings:sizethreshold'] = 'Size threshold (KB)';
+$string['settings:sizethreshold_help'] = 'Size threshold for transfering files to S3. If files are over this size they will be transfered to S3.';
+$string['settings:minimumage'] = 'Minimum age';
+$string['settings:minimumage_help'] = 'Minimum age that a file must exist on the local file system before it will be considered for transfer.';
+$string['settings:consistancydelay'] = 'Consistancy delay';
+$string['settings:consistancydelay_help'] = 'How long a file must existed after being transfered to S3 before they are a candidate for deletion locally. A value of -1 will never delete local files.';
+$string['settings:loggingheader'] = 'Logging Settings';
+$string['settings:logginglocation'] = 'Logging location';
+$string['settings:logginglocation_help'] = 'Location of the file access log file.';
diff --git a/lib.php b/lib.php
@@ -29,20 +29,6 @@
 define('SSS_FILE_STATE_DUPLICATED', 1);
 define('SSS_FILE_STATE_EXTERNAL', 2);
 
-function get_content_hashes_over_threshold($threshold) {
-    global $DB;
-    $sql = "SELECT DISTINCT contenthash FROM {files} WHERE filesize > ?";
-    $contenthashes = $DB->get_fieldset_sql($sql, array($threshold));
-    return $contenthashes;
-}
-
-function get_content_hashes_in_sss() {
-    global $DB;
-    $sql = 'SELECT contenthash FROM {tool_sssfs_filestate} WHERE STATE in (?, ?)';
-    $ssscontenthashes = $DB->get_fieldset_sql($sql, array(SSS_FILE_STATE_DUPLICATED, SSS_FILE_STATE_EXTERNAL));
-    return $ssscontenthashes;
-}
-
 function log_file_state($contenthash, $state) {
     global $DB;
 
@@ -60,4 +46,23 @@ function log_file_state($contenthash, $state) {
     } else {
         $DB->insert_record('tool_sssfs_filestate', $logrecord);
     }
+}
+
+function save_config_data($data) {
+    $config = new stdClass();
+    $config->enabled = $data->enabled;
+    $config->key = $data->key;
+    $config->secret = $data->secret;
+    $config->bucket = $data->bucket;
+    $config->region = $data->region;
+    $config->sizethreshold = $data->sizethreshold;
+    $config->minimumage = $data->minimumage;
+    $config->consistancydelay = $data->consistancydelay;
+    $config->logginglocation = $data->logginglocation;
+
+    // TODO: Encrypt credentials.
+
+    foreach (get_object_vars($config) as $key => $value) {
+        set_config($key, $value, 'tool_sssfs');
+    }
 }