Skip to content

Commit

Permalink
wrong decimal handling for certain numbers #397
Browse files Browse the repository at this point in the history
  • Loading branch information
@Rello
Rello committed Jul 30, 2024
1 parent 6dab145 commit 89d17dc
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 138 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
- excel data source: remove all-null columns
- code style
- dark mode problems #400
- wrong decimal handling for certain numbers #397

## 4.14.0 - 2024-06-20
### Added
Expand Down
267 changes: 135 additions & 132 deletions lib/Datasource/LocalJson.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,145 +12,148 @@
use OCP\IL10N;
use Psr\Log\LoggerInterface;

class LocalJson implements IDatasource
{
class LocalJson implements IDatasource {
private $rootFolder;
private LoggerInterface $logger;
private IL10N $l10n;

public function __construct(
IL10N $l10n,
IRootFolder $rootFolder,
LoggerInterface $logger
)
{
$this->l10n = $l10n;
private LoggerInterface $logger;
private IL10N $l10n;

public function __construct(
IL10N $l10n,
IRootFolder $rootFolder,
LoggerInterface $logger
) {
$this->l10n = $l10n;
$this->rootFolder = $rootFolder;
$this->logger = $logger;
}

/**
* @return string Display Name of the datasource
*/
public function getName(): string
{
return $this->l10n->t('Local') . ': JSON';
}

/**
* @return int digit unique datasource id
*/
public function getId(): int
{
return 6;
}

/**
* @return array available options of the datasoure
*/
public function getTemplate(): array
{
$template = array();
$template[] = ['id' => 'link', 'name' => $this->l10n->t('File'), 'placeholder' => $this->l10n->t('File'), 'type' => 'filePicker'];
$template[] = ['id' => 'path', 'name' => $this->l10n->t('Object path'), 'placeholder' => 'x/y/z'];
$template[] = ['id' => 'timestamp', 'name' => $this->l10n->t('Timestamp of data load'), 'placeholder' => 'true-' . $this->l10n->t('Yes') . '/false-' . $this->l10n->t('No'), 'type' => 'tf'];
return $template;
}

/**
* Read the Data
* @param $option
* @return array available options of the datasoure
*/
public function readData($option): array
{
$this->logger = $logger;
}

/**
* @return string Display Name of the datasource
*/
public function getName(): string {
return $this->l10n->t('Local') . ': JSON';
}

/**
* @return int digit unique datasource id
*/
public function getId(): int {
return 6;
}

/**
* @return array available options of the datasoure
*/
public function getTemplate(): array {
$template = array();
$template[] = [
'id' => 'link',
'name' => $this->l10n->t('File'),
'placeholder' => $this->l10n->t('File'),
'type' => 'filePicker'
];
$template[] = ['id' => 'path', 'name' => $this->l10n->t('Object path'), 'placeholder' => 'x/y/z'];
$template[] = [
'id' => 'timestamp',
'name' => $this->l10n->t('Timestamp of data load'),
'placeholder' => 'true-' . $this->l10n->t('Yes') . '/false-' . $this->l10n->t('No'),
'type' => 'tf'
];
return $template;
}

/**
* Read the Data
* @param $option
* @return array available options of the datasoure
*/
public function readData($option): array {
$error = 0;
$path = $option['path'];
$file = $this->rootFolder->getUserFolder($option['user_id'])->get($option['link']);
$rawResult = $file->getContent();

$json = json_decode($rawResult, true);

// check if a specific array of values should be extracted
// e.g. {BTC,tmsp,price}
preg_match_all("/(?<={).*(?=})/", $path, $matches);
if (count($matches[0]) > 0) {
// array extraction

// check if absolute path is in front of the array
// e.g. data/data{from,to,intensity/forecast}
$firstArray = strpos($path, '{');
if ($firstArray && $firstArray !== 0) {
$singlePath = substr($path, 0, $firstArray);
$json = $this->get_nested_array_value($json, $singlePath);
}

// separate the fields of the array {BTC,tmsp,price}
$paths = explode(',', $matches[0][0]);
// fill up with dummies in case of missing columns
while (count($paths) < 3) {
array_unshift($paths, 'empty');
}
foreach ($json as $rowArray) {
// get the array fields from the json
// if no match is not found, the field name will be used as a constant string
$dim1 = $this->get_nested_array_value($rowArray, $paths[0]) ?: $paths[0];
$dim2 = $this->get_nested_array_value($rowArray, $paths[1]) ?: $paths[1];
$val = $this->get_nested_array_value($rowArray, $paths[2]) ?: $paths[2];
$data[] = [$dim1, $dim2, $val];
}
} else {
// single value extraction
// e.g. data/currentHashrate,data/averageHashrate
$paths = explode(',', $path);
foreach ($paths as $singlePath) {
// e.g. data/currentHashrate
$array = $this->get_nested_array_value($json, $singlePath);

if (is_array($array)) {
// if the tartet is an array itself
foreach ($array as $key => $value) {
$pathArray = explode('/', $singlePath);
$group = end($pathArray);
$data[] = [$group, $key, $value];
}
} else {
$pathArray = explode('/', $singlePath);
$key = end($pathArray);
$data[] = ['', $key, $array];
}
}
}

$header = array();
$header[0] = '';
$header[1] = 'Key';
$header[2] = 'Value';

return [
'header' => $header,
'dimensions' => array_slice($header, 0, count($header) - 1),
'data' => $data,
'rawdata' => $rawResult,
'error' => $error,
];
}

/**
* get array object from string
*
* @NoAdminRequired
* @param $array
* @param $path
* @return array|string|null
*/
private function get_nested_array_value(&$array, $path)
{
$pathParts = explode('/', $path);
$current = &$array;
foreach ($pathParts as $key) {
$current = &$current[$key];
}
return $current;
}
// check if a specific array of values should be extracted
// e.g. {BTC,tmsp,price}
preg_match_all("/(?<={).*(?=})/", $path, $matches);
if (count($matches[0]) > 0) {
// array extraction

// check if absolute path is in front of the array
// e.g. data/data{from,to,intensity/forecast}
$firstArray = strpos($path, '{');
if ($firstArray && $firstArray !== 0) {
$singlePath = substr($path, 0, $firstArray);
$json = $this->get_nested_array_value($json, $singlePath);
}

// separate the fields of the array {BTC,tmsp,price}
$paths = explode(',', $matches[0][0]);
// fill up with dummies in case of missing columns
while (count($paths) < 3) {
array_unshift($paths, 'empty');
}
foreach ($json as $rowArray) {
// get the array fields from the json
// if no match is not found, the field name will be used as a constant string
$dim1 = $this->get_nested_array_value($rowArray, $paths[0]) ?: $paths[0];
$dim2 = $this->get_nested_array_value($rowArray, $paths[1]) ?: $paths[1];
$val = $this->get_nested_array_value($rowArray, $paths[2]) ?: $paths[2];
$data[] = [$dim1, $dim2, $val];
}
} else {
// single value extraction
// e.g. data/currentHashrate,data/averageHashrate
$paths = explode(',', $path);
foreach ($paths as $singlePath) {
// e.g. data/currentHashrate
$array = $this->get_nested_array_value($json, $singlePath);

if (is_array($array)) {
// if the target is an array itself
foreach ($array as $key => $value) {
$pathArray = explode('/', $singlePath);
$group = end($pathArray);
$data[] = [$group, $key, $value];
}
} else {
$pathArray = explode('/', $singlePath);
$key = end($pathArray);
$data[] = ['', $key, $array];
}
}
}

$header = array();
$header[0] = '';
$header[1] = 'Key';
$header[2] = 'Value';

return [
'header' => $header,
'dimensions' => array_slice($header, 0, count($header) - 1),
'data' => $data,
'rawdata' => $rawResult,
'error' => $error,
];
}

/**
* get array object from string
*
* @NoAdminRequired
* @param $array
* @param $path
* @return array|string|null
*/
private function get_nested_array_value(&$array, $path) {
$pathParts = explode('/', $path);
$current = &$array;
foreach ($pathParts as $key) {
$current = &$current[$key];
}
return $current;
}
}
12 changes: 6 additions & 6 deletions lib/Service/StorageService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ class StorageService {
private $VariableService;

public function __construct(
LoggerInterface $logger,
StorageMapper $StorageMapper,
DatasetService $DatasetService,
LoggerInterface $logger,
StorageMapper $StorageMapper,
DatasetService $DatasetService,
ThresholdService $ThresholdService,
VariableService $VariableService,
ReportService $ReportService
VariableService $VariableService,
ReportService $ReportService
) {
$this->logger = $logger;
$this->StorageMapper = $StorageMapper;
Expand Down Expand Up @@ -216,7 +216,7 @@ public function getRecordCount(int $datasetId, string $user_id = null) {

private function floatvalue($val) {
// if value is a 3 digit comma number with one leading zero like 0,111, it should not go through the 1000 separator removal
if (preg_match('/(?<=\b0)\,(?=\d{3}\b)/', $val) === 0 && preg_match('/(?<=\b0)\.(?=\d{3}\b)/', $val) === 0) {
if (preg_match('/(?<=\b0),(?=\d{3}\b)/', $val) === 1 && preg_match('/(?<=\b0).(?=\d{3}\b)/', $val) === 1) {
// remove , as 1000 separator
$val = preg_replace('/(?<=\d)\,(?=\d{3}\b)/', '', $val);
// remove . as 1000 separator
Expand Down

0 comments on commit 89d17dc

Please sign in to comment.