Skip to content

Commit

Permalink
Added ArrayLoader with usage example
Browse files Browse the repository at this point in the history
  • Loading branch information
norberttech committed Sep 16, 2024
1 parent 29e875e commit 55f0664
Show file tree
Hide file tree
Showing 9 changed files with 117 additions and 3 deletions.
23 changes: 23 additions & 0 deletions examples/topics/data_sink/array/code.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?php

declare(strict_types=1);

use function Flow\ETL\DSL\{data_frame, from_array, to_array};

require __DIR__ . '/../../../autoload.php';

$array = [];

data_frame()
->read(from_array([
['id' => 1],
['id' => 2],
['id' => 3],
['id' => 4],
['id' => 5],
]))
->collect()
->write(to_array($array))
->run();

\file_put_contents(__DIR__ . '/output.txt', \var_export($array, true));
2 changes: 2 additions & 0 deletions examples/topics/data_sink/array/description.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Write datasets directly to an associative array. Please pay attention to the memory usage when using array loader.
Large datasets may cause memory overflow.
22 changes: 22 additions & 0 deletions examples/topics/data_sink/array/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
array (
0 =>
array (
'id' => 1,
),
1 =>
array (
'id' => 2,
),
2 =>
array (
'id' => 3,
),
3 =>
array (
'id' => 4,
),
4 =>
array (
'id' => 5,
),
)
1 change: 1 addition & 0 deletions examples/topics/data_sink/array/priority.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1
2 changes: 1 addition & 1 deletion examples/topics/data_sink/parquet/priority.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1
2
14 changes: 13 additions & 1 deletion src/core/etl/src/Flow/ETL/DSL/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
When
};
use Flow\ETL\Loader\StreamLoader\Output;
use Flow\ETL\Loader\{CallbackLoader, MemoryLoader, StreamLoader, TransformerLoader};
use Flow\ETL\Loader\{ArrayLoader, CallbackLoader, MemoryLoader, StreamLoader, TransformerLoader};
use Flow\ETL\Memory\Memory;
use Flow\ETL\PHP\Type\Logical\List\ListElement;
use Flow\ETL\PHP\Type\Logical\Map\{MapKey, MapValue};
Expand Down Expand Up @@ -282,6 +282,18 @@ function to_memory(Memory $memory) : MemoryLoader
return new MemoryLoader($memory);
}

/**
* Convert rows to an array and store them in passed array variable.
*
* @param array<array<mixed>> $array
*/
#[DocumentationDSL(module: Module::CORE, type: DSLType::LOADER)]
#[DocumentationExample(topic: 'data_sink', example: 'array')]
function to_array(array &$array) : ArrayLoader
{
return new ArrayLoader($array);

Check failure on line 294 in src/core/etl/src/Flow/ETL/DSL/functions.php

View workflow job for this annotation

GitHub Actions / Static Analyze (locked, 8.1, ubuntu-latest)

ReferenceConstraintViolation

src/core/etl/src/Flow/ETL/DSL/functions.php:294:5: ReferenceConstraintViolation: Variable $array is limited to values of type array<array-key, array<array-key, mixed>> because it is passed by reference, array<array-key, mixed> type found. Use @param-out to specify a different output type (see https://psalm.dev/086)
}

#[DocumentationDSL(module: Module::CORE, type: DSLType::LOADER)]
function to_output(int|bool $truncate = 20, Output $output = Output::rows, Formatter $formatter = new Formatter\AsciiTableFormatter(), SchemaFormatter $schemaFormatter = new ASCIISchemaFormatter()) : StreamLoader
{
Expand Down
22 changes: 22 additions & 0 deletions src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?php

declare(strict_types=1);

namespace Flow\ETL\Loader;

use Flow\ETL\{FlowContext, Loader, Rows};

final class ArrayLoader implements Loader
{
public function __construct(private array &$memory)
{
}

public function load(Rows $rows, FlowContext $context) : void
{
$this->memory = \array_merge(
$this->memory,
$rows->toArray()
);
}
}
32 changes: 32 additions & 0 deletions src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/ArrayLoaderTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
<?php

declare(strict_types=1);

namespace Flow\ETL\Tests\Unit\Loader;

use function Flow\ETL\DSL\{flow_context, int_entry, row, rows, str_entry, to_array};
use PHPUnit\Framework\TestCase;

final class ArrayLoaderTest extends TestCase
{
public function test_loads_rows_data_into_memory() : void
{
$rows1 = rows(
row(int_entry('number', 1), str_entry('name', 'one')),
row(int_entry('number', 2), str_entry('name', 'two')),
);

$rows2 = rows(
row(int_entry('number', 3), str_entry('name', 'three')),
row(int_entry('number', 4), str_entry('name', 'four')),
);

$array = [];

$loader = to_array($array);
$loader->load($rows1, flow_context());
$loader->load($rows2, flow_context());

self::assertEquals($rows1->merge($rows2)->toArray(), $array);
}
}
2 changes: 1 addition & 1 deletion web/landing/resources/dsl.json

Large diffs are not rendered by default.

0 comments on commit 55f0664

Please sign in to comment.