From 89d17dc3fc9f7448fb9ce55052ae7b5532fed979 Mon Sep 17 00:00:00 2001 From: Rello Date: Tue, 30 Jul 2024 10:50:42 +0200 Subject: [PATCH] wrong decimal handling for certain numbers #397 --- CHANGELOG.md | 1 + lib/Datasource/LocalJson.php | 267 +++++++++++++++++---------------- lib/Service/StorageService.php | 12 +- 3 files changed, 142 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e326d200..801ed4af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - excel data source: remove all-null columns - code style - dark mode problems #400 +- wrong decimal handling for certain numbers #397 ## 4.14.0 - 2024-06-20 ### Added diff --git a/lib/Datasource/LocalJson.php b/lib/Datasource/LocalJson.php index b680fc9c..64235dda 100644 --- a/lib/Datasource/LocalJson.php +++ b/lib/Datasource/LocalJson.php @@ -12,58 +12,62 @@ use OCP\IL10N; use Psr\Log\LoggerInterface; -class LocalJson implements IDatasource -{ +class LocalJson implements IDatasource { private $rootFolder; - private LoggerInterface $logger; - private IL10N $l10n; - - public function __construct( - IL10N $l10n, - IRootFolder $rootFolder, - LoggerInterface $logger - ) - { - $this->l10n = $l10n; + private LoggerInterface $logger; + private IL10N $l10n; + + public function __construct( + IL10N $l10n, + IRootFolder $rootFolder, + LoggerInterface $logger + ) { + $this->l10n = $l10n; $this->rootFolder = $rootFolder; - $this->logger = $logger; - } - - /** - * @return string Display Name of the datasource - */ - public function getName(): string - { - return $this->l10n->t('Local') . ': JSON'; - } - - /** - * @return int digit unique datasource id - */ - public function getId(): int - { - return 6; - } - - /** - * @return array available options of the datasoure - */ - public function getTemplate(): array - { - $template = array(); - $template[] = ['id' => 'link', 'name' => $this->l10n->t('File'), 'placeholder' => $this->l10n->t('File'), 'type' => 'filePicker']; - $template[] = ['id' => 'path', 'name' => $this->l10n->t('Object path'), 'placeholder' => 'x/y/z']; - $template[] = ['id' => 'timestamp', 'name' => $this->l10n->t('Timestamp of data load'), 'placeholder' => 'true-' . $this->l10n->t('Yes') . '/false-' . $this->l10n->t('No'), 'type' => 'tf']; - return $template; - } - - /** - * Read the Data - * @param $option - * @return array available options of the datasoure - */ - public function readData($option): array - { + $this->logger = $logger; + } + + /** + * @return string Display Name of the datasource + */ + public function getName(): string { + return $this->l10n->t('Local') . ': JSON'; + } + + /** + * @return int digit unique datasource id + */ + public function getId(): int { + return 6; + } + + /** + * @return array available options of the datasoure + */ + public function getTemplate(): array { + $template = array(); + $template[] = [ + 'id' => 'link', + 'name' => $this->l10n->t('File'), + 'placeholder' => $this->l10n->t('File'), + 'type' => 'filePicker' + ]; + $template[] = ['id' => 'path', 'name' => $this->l10n->t('Object path'), 'placeholder' => 'x/y/z']; + $template[] = [ + 'id' => 'timestamp', + 'name' => $this->l10n->t('Timestamp of data load'), + 'placeholder' => 'true-' . $this->l10n->t('Yes') . '/false-' . $this->l10n->t('No'), + 'type' => 'tf' + ]; + return $template; + } + + /** + * Read the Data + * @param $option + * @return array available options of the datasoure + */ + public function readData($option): array { $error = 0; $path = $option['path']; $file = $this->rootFolder->getUserFolder($option['user_id'])->get($option['link']); @@ -71,86 +75,85 @@ public function readData($option): array $json = json_decode($rawResult, true); - // check if a specific array of values should be extracted - // e.g. {BTC,tmsp,price} - preg_match_all("/(?<={).*(?=})/", $path, $matches); - if (count($matches[0]) > 0) { - // array extraction - - // check if absolute path is in front of the array - // e.g. data/data{from,to,intensity/forecast} - $firstArray = strpos($path, '{'); - if ($firstArray && $firstArray !== 0) { - $singlePath = substr($path, 0, $firstArray); - $json = $this->get_nested_array_value($json, $singlePath); - } - - // separate the fields of the array {BTC,tmsp,price} - $paths = explode(',', $matches[0][0]); - // fill up with dummies in case of missing columns - while (count($paths) < 3) { - array_unshift($paths, 'empty'); - } - foreach ($json as $rowArray) { - // get the array fields from the json - // if no match is not found, the field name will be used as a constant string - $dim1 = $this->get_nested_array_value($rowArray, $paths[0]) ?: $paths[0]; - $dim2 = $this->get_nested_array_value($rowArray, $paths[1]) ?: $paths[1]; - $val = $this->get_nested_array_value($rowArray, $paths[2]) ?: $paths[2]; - $data[] = [$dim1, $dim2, $val]; - } - } else { - // single value extraction - // e.g. data/currentHashrate,data/averageHashrate - $paths = explode(',', $path); - foreach ($paths as $singlePath) { - // e.g. data/currentHashrate - $array = $this->get_nested_array_value($json, $singlePath); - - if (is_array($array)) { - // if the tartet is an array itself - foreach ($array as $key => $value) { - $pathArray = explode('/', $singlePath); - $group = end($pathArray); - $data[] = [$group, $key, $value]; - } - } else { - $pathArray = explode('/', $singlePath); - $key = end($pathArray); - $data[] = ['', $key, $array]; - } - } - } - - $header = array(); - $header[0] = ''; - $header[1] = 'Key'; - $header[2] = 'Value'; - - return [ - 'header' => $header, - 'dimensions' => array_slice($header, 0, count($header) - 1), - 'data' => $data, - 'rawdata' => $rawResult, - 'error' => $error, - ]; - } - - /** - * get array object from string - * - * @NoAdminRequired - * @param $array - * @param $path - * @return array|string|null - */ - private function get_nested_array_value(&$array, $path) - { - $pathParts = explode('/', $path); - $current = &$array; - foreach ($pathParts as $key) { - $current = &$current[$key]; - } - return $current; - } + // check if a specific array of values should be extracted + // e.g. {BTC,tmsp,price} + preg_match_all("/(?<={).*(?=})/", $path, $matches); + if (count($matches[0]) > 0) { + // array extraction + + // check if absolute path is in front of the array + // e.g. data/data{from,to,intensity/forecast} + $firstArray = strpos($path, '{'); + if ($firstArray && $firstArray !== 0) { + $singlePath = substr($path, 0, $firstArray); + $json = $this->get_nested_array_value($json, $singlePath); + } + + // separate the fields of the array {BTC,tmsp,price} + $paths = explode(',', $matches[0][0]); + // fill up with dummies in case of missing columns + while (count($paths) < 3) { + array_unshift($paths, 'empty'); + } + foreach ($json as $rowArray) { + // get the array fields from the json + // if no match is not found, the field name will be used as a constant string + $dim1 = $this->get_nested_array_value($rowArray, $paths[0]) ?: $paths[0]; + $dim2 = $this->get_nested_array_value($rowArray, $paths[1]) ?: $paths[1]; + $val = $this->get_nested_array_value($rowArray, $paths[2]) ?: $paths[2]; + $data[] = [$dim1, $dim2, $val]; + } + } else { + // single value extraction + // e.g. data/currentHashrate,data/averageHashrate + $paths = explode(',', $path); + foreach ($paths as $singlePath) { + // e.g. data/currentHashrate + $array = $this->get_nested_array_value($json, $singlePath); + + if (is_array($array)) { + // if the target is an array itself + foreach ($array as $key => $value) { + $pathArray = explode('/', $singlePath); + $group = end($pathArray); + $data[] = [$group, $key, $value]; + } + } else { + $pathArray = explode('/', $singlePath); + $key = end($pathArray); + $data[] = ['', $key, $array]; + } + } + } + + $header = array(); + $header[0] = ''; + $header[1] = 'Key'; + $header[2] = 'Value'; + + return [ + 'header' => $header, + 'dimensions' => array_slice($header, 0, count($header) - 1), + 'data' => $data, + 'rawdata' => $rawResult, + 'error' => $error, + ]; + } + + /** + * get array object from string + * + * @NoAdminRequired + * @param $array + * @param $path + * @return array|string|null + */ + private function get_nested_array_value(&$array, $path) { + $pathParts = explode('/', $path); + $current = &$array; + foreach ($pathParts as $key) { + $current = &$current[$key]; + } + return $current; + } } \ No newline at end of file diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index 6c6f6a00..c417800b 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -21,12 +21,12 @@ class StorageService { private $VariableService; public function __construct( - LoggerInterface $logger, - StorageMapper $StorageMapper, - DatasetService $DatasetService, + LoggerInterface $logger, + StorageMapper $StorageMapper, + DatasetService $DatasetService, ThresholdService $ThresholdService, - VariableService $VariableService, - ReportService $ReportService + VariableService $VariableService, + ReportService $ReportService ) { $this->logger = $logger; $this->StorageMapper = $StorageMapper; @@ -216,7 +216,7 @@ public function getRecordCount(int $datasetId, string $user_id = null) { private function floatvalue($val) { // if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal - if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) { + if (preg_match('/(?<=\b0),(?=\d{3}\b)/', $val) === 1 && preg_match('/(?<=\b0).(?=\d{3}\b)/', $val) === 1) { // remove , as 1000 separator $val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val); // remove . as 1000 separator