Configurable Chunk Size and optimized Timeout (opencast-ilias#205)

* [FEATURE] Configurable Chunk Size

* [FIX] timout in fileuploads now use max_execution_time

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## [5.3.0]
+- Implemented #193: The Chunk Size for Uploads is now configurable. The default value is 20MB, this can be changes in the plugin configuration.
+- Fix #176: Fixed a static timeout in fileuploads, this now uses max_execution_time of the server. 
+
 ## [5.2.0]
 - Version 5.2.0 contains a large number of refactorings. The ILIAS 7 compatible version is continuously refactored so that an update to ILIAS 8 is easier possible. For example, libraries that are no longer compatible have to be removed. In this first step the internal use of srg/dic was removed.
 - With the new release the plugin uses the new php-library `elan-ev/opencast-api` in version 1.4.0 for all API calls to Opencast.

classes/Conf/class.xoctConfFormGUI.php

@@ -202,10 +202,8 @@ protected function initAPISection()
         $this->addItem($te);
     }
 
-    /**
-     *
-     */
-    protected function initEventsSection()
+
+    protected function initEventsSection(): void
     {
         $h = new ilFormSectionHeaderGUI();
         $h->setTitle($this->parent_gui->txt('events'));
@@ -221,7 +219,17 @@ protected function initEventsSection()
             PluginConfig::F_CURL_MAX_UPLOADSIZE
         );
         $te->setInfo($this->parent_gui->txt(PluginConfig::F_CURL_MAX_UPLOADSIZE . '_info'));
-        $te->setRequired(false);
+        $te->setRequired(true);
+        $this->addItem($te);
+
+        $te = new ilNumberInputGUI(
+            $this->parent_gui->txt(PluginConfig::F_CURL_CHUNK_SIZE),
+            PluginConfig::F_CURL_CHUNK_SIZE
+        );
+        $te->setInfo($this->parent_gui->txt(PluginConfig::F_CURL_CHUNK_SIZE . '_info'));
+        $te->setRequired(true);
+        $te->setMinValue(1, true);
+        $te->setMaxValue(\ilUtil::getUploadSizeLimitBytes() / 1024 / 1024 / 2, true);
         $this->addItem($te);
 
         $te = new ilTextInputGUI(

lang/ilias_de.lang

@@ -18,6 +18,8 @@ config_api_version#:#API-Version
 config_api_version_info#:#Format: vX.Y.Z - z.B.: v1.0.0
 config_curl_max_upload_size#:#Maximale Upload-Größe in MB
 config_curl_max_upload_size_info#:#Definiert die maximal mögliche Uploadgröße für Video-Dateien in MB
+config_curl_chunk_size#:#Chunk-Größe in MB
+config_curl_chunk_size_info#:#Definiert die Grösse der einzelnen Upload-Chunks in MB. Grosse Dateien werden in einzelnen Teilen (Chunks) hochgeladen. Kleinere Chunks sind besser bei langsamen Internet-Verbindungen, grössere Chunks sind besser bei schnellen Internetverbindungen.
 config_audio_allowed#:#Audio-Dateien
 config_cancel#:#Abbrechen
 config_create_scheduled_allowed#:#"Aufzeichnungstermin(e) planen" erlauben

lang/ilias_en.lang

@@ -18,6 +18,8 @@ config_api_version#:#API Version
 config_api_version_info#:#Format: vX.Y.Z - e.g.: v1.0.0
 config_curl_max_upload_size#:#Max. File-Size in MB
 config_curl_max_upload_size_info#:#Defines the maximum possible upload size for video files in MB
+config_curl_chunk_size#:#Chunk size in MB
+config_curl_chunk_size_info#:#Defines the size of individual upload chunks in MB. Large files are uploaded in single parts (chunks). Smaller chunks are better for slow internet connections, larger chunks are better for fast internet connections.
 config_audio_allowed#:#Audio Files
 config_cancel#:#Cancel
 config_create_scheduled_allowed#:#Activate "Schedule Event(s)"

plugin.php

@@ -1,7 +1,7 @@
 <?php
 
 $id = 'xoct';
-$version = '5.2.0';
+$version = '5.3.0';
 $version_check = '44ac530093a998b525b0a73ba536e64f03bbaff47446cf99e1a31d6a042a4549';
 $ilias_min_version = '6.0';
 $ilias_max_version = '7.999';

sql/dbupdate.php

@@ -469,3 +469,15 @@
 <?php
 $ilDB->manipulate('update xoct_data set intro_text = "" where intro_text is null');
 ?>
+<#41>
+<?php
+/** @var $ilDB ilDBInterface */
+$res = $ilDB->queryF('SELECT value FROM xoct_config WHERE name = %s', ['text'], ['curl_chunk_size']);
+if ($res->rowCount() === 0) {
+    $ilDB->insert('xoct_config', [
+        'name' => ['text', 'curl_chunk_size'],
+        'value' => ['text', '20']
+    ]);
+}
+?>
+

src/Model/Config/PluginConfig.php

@@ -31,6 +31,7 @@ class PluginConfig extends ActiveRecord
     public const F_CURL_USERNAME = 'curl_username';
     public const F_CURL_PASSWORD = 'curl_password';
     public const F_CURL_MAX_UPLOADSIZE = 'curl_max_upload_size';
+    public const F_CURL_CHUNK_SIZE = 'curl_chunk_size';
     public const F_WORKFLOW = 'workflow';
     public const F_WORKFLOW_UNPUBLISH = 'workflow_unpublish';
     public const F_EULA = 'eula';

src/UI/EventFormBuilder.php

@@ -168,9 +168,14 @@ public function upload(string $form_action, bool $with_terms_of_use, int $obj_id
             ? $configured_upload_limit * self::MB_IN_B
             : self::DEFAULT_UPLOAD_LIMIT_IN_MIB * self::MB_IN_B;
 
+        // Chunk Size
+        $chunk_size = (int) PluginConfig::getConfig(PluginConfig::F_CURL_CHUNK_SIZE);
+        $chunk_size = $chunk_size > 0 ? $chunk_size * 1024 * 1024 : \ilUtil::getUploadSizeLimitBytes();
+
         $file_input = $file_input->withAcceptedMimeTypes($this->getMimeTypes())
                                  ->withRequired(true)
                                  ->withMaxFileSize($upload_limit)
+                                 ->withChunkSizeInBytes($chunk_size)
                                  ->withAdditionalTransformation(
                                      $this->refinery_factory->custom()->transformation(
                                          function ($file) use ($upload_storage_service): array {

src/UI/Form/ChunkedFile.php

@@ -13,6 +13,8 @@
  */
 class ChunkedFile extends File
 {
+    protected $chunk_size = 1;
+
     public function __construct(
         DataFactory $data_factory,
         Factory $refinery,
@@ -42,6 +44,18 @@ public static function getInstance(
         ));
     }
 
+    public function withChunkSizeInBytes(int $chunk_size_in_bytes): self
+    {
+        $clone = clone $this;
+        $clone->chunk_size = $chunk_size_in_bytes;
+        return $clone;
+    }
+
+    public function getChunkSizeInBytes(): int
+    {
+        return $this->chunk_size;
+    }
+
     protected function isClientSideValueOk($value): bool
     {
         if (is_null($value)) {

src/UI/Form/ChunkedFileRenderer.php

@@ -31,6 +31,7 @@ public function registerResources(ResourceRegistry $registry): void
     }
 
     /**
+     * @param ChunkedFile         $component
      * @throws ilTemplateException
      */
     public function render(Component $component, RendererInterface $default_renderer): string
@@ -59,13 +60,12 @@ public function render(Component $component, RendererInterface $default_renderer
         $settings->accepted_files = implode(',', $component->getAcceptedMimeTypes());
         $settings->existing_file_ids = $component->getValue();
         $settings->existing_files = $handler->getInfoForExistingFiles($component->getValue() ?? []);
+        $settings->timeout = (int) ini_get('max_execution_time') * 1000; // dropzone.js expects milliseconds
 
         $upload_limit = \ilUtil::getUploadSizeLimitBytes();
         $settings->chunked_upload = $handler->supportsChunkedUploads();
-        $settings->chunk_size = min(
-            $upload_limit / 2,
-            20 * self::MB_IN_B
-        ); // we use 20MB as default chunk size which seems to be a good compromise for slow connections
+        $settings->chunk_size = $component->getChunkSizeInBytes();
+
         if (!$settings->chunked_upload) {
             $max_file_size = $component->getMaxFileFize() === -1
                 ? $upload_limit

templates/default/chunked_file.js

@@ -66,7 +66,8 @@ il.UI.Input = il.UI.Input || {};
         forceChunking: true,
         acceptedFiles: settings.accepted_files,
         dictInvalidFileType: settings.dictInvalidFileType,
-        progress_storage: 0
+        progress_storage: 0,
+        timeout: settings.timeout,
       });
 
       myDropzone.on('uploadprogress', function (file, progress, bytesSent) {