From a09e99faeb4dfac51e542ea138ba101eacd89df9 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz <1921950+norberttech@users.noreply.github.com> Date: Sun, 31 Dec 2023 14:25:04 +0100 Subject: [PATCH] List select function (#893) * List select function * Added missing classes --- src/core/etl/src/Flow/ETL/CLI/CLI.php | 8 -- src/core/etl/src/Flow/ETL/CLI/Input.php | 40 ------ src/core/etl/src/Flow/ETL/DSL/functions.php | 8 +- .../src/Flow/ETL/Function/ListFunctions.php | 17 +++ .../etl/src/Flow/ETL/Function/ListSelect.php | 53 ++++++++ .../etl/src/Flow/ETL/Row/EntryReference.php | 6 + .../Integration/Function/ListSelectTest.php | 32 +++++ .../Structure/StructureSelectTest.php | 4 +- .../Flow/ETL/Tests/Unit/CLI/InputTest.php | 41 ------ .../Tests/Unit/Function/ListSelectTest.php | 121 ++++++++++++++++++ ...reKeepTest.php => StructureSelectTest.php} | 2 +- 11 files changed, 239 insertions(+), 93 deletions(-) delete mode 100644 src/core/etl/src/Flow/ETL/CLI/CLI.php delete mode 100644 src/core/etl/src/Flow/ETL/CLI/Input.php create mode 100644 src/core/etl/src/Flow/ETL/Function/ListFunctions.php create mode 100644 src/core/etl/src/Flow/ETL/Function/ListSelect.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ListSelectTest.php delete mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/CLI/InputTest.php create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php rename src/core/etl/tests/Flow/ETL/Tests/Unit/Function/{StructureKeepTest.php => StructureSelectTest.php} (98%) diff --git a/src/core/etl/src/Flow/ETL/CLI/CLI.php b/src/core/etl/src/Flow/ETL/CLI/CLI.php deleted file mode 100644 index 1e992c030..000000000 --- a/src/core/etl/src/Flow/ETL/CLI/CLI.php +++ /dev/null @@ -1,8 +0,0 @@ - $argv - */ - public function __construct(private readonly array $argv) - { - } - - /** - * @return array - */ - public function argv() : array - { - return $this->argv; - } - - public function optionValue(string $name, ?string $default = null) : ?string - { - foreach ($this->argv as $arg) { - $parts = \explode('=', $arg); - - if (\count($parts) !== 2) { - continue; - } - - if ($parts[0] === '--' . \strtolower($name)) { - return $parts[1]; - } - } - - return $default; - } -} diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index e04ba8cb8..175c6ed78 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -49,6 +49,7 @@ use Flow\ETL\Function\First; use Flow\ETL\Function\Hash; use Flow\ETL\Function\Last; +use Flow\ETL\Function\ListFunctions; use Flow\ETL\Function\Literal; use Flow\ETL\Function\Max; use Flow\ETL\Function\Min; @@ -485,11 +486,16 @@ function ref(string $entry) : EntryReference return new EntryReference($entry); } -function structure(string $entry) : StructureFunctions +function structure_ref(string $entry) : StructureFunctions { return ref($entry)->structure(); } +function list_ref(string $entry) : ListFunctions +{ + return ref($entry)->list(); +} + function refs(string|Reference ...$entries) : References { return new References(...$entries); diff --git a/src/core/etl/src/Flow/ETL/Function/ListFunctions.php b/src/core/etl/src/Flow/ETL/Function/ListFunctions.php new file mode 100644 index 000000000..952cae180 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/ListFunctions.php @@ -0,0 +1,17 @@ +ref, ...$refs); + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/ListSelect.php b/src/core/etl/src/Flow/ETL/Function/ListSelect.php new file mode 100644 index 000000000..68d9e082d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/ListSelect.php @@ -0,0 +1,53 @@ +ref = EntryReference::init($ref); + $this->refs = References::init(...$refs); + } + + public function eval(Row $row) : array|null + { + if (!$row->has($this->ref)) { + return null; + } + + $list = $row->get($this->ref); + + if (!$list instanceof ListEntry) { + return null; + } + + $output = []; + + foreach ($list->value() as $index => $element) { + $output[$index] = []; + + foreach ($this->refs as $ref) { + if (\is_array($element) && \array_key_exists($ref->to(), $element)) { + $output[$index][$ref->name()] = $element[$ref->to()]; + } else { + $output[$index][$ref->name()] = null; + } + } + } + + return $output; + } +} diff --git a/src/core/etl/src/Flow/ETL/Row/EntryReference.php b/src/core/etl/src/Flow/ETL/Row/EntryReference.php index 6c091dee3..ec7039a17 100644 --- a/src/core/etl/src/Flow/ETL/Row/EntryReference.php +++ b/src/core/etl/src/Flow/ETL/Row/EntryReference.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Row; +use Flow\ETL\Function\ListFunctions; use Flow\ETL\Function\ScalarFunctionChain; use Flow\ETL\Function\StructureFunctions; use Flow\ETL\Row; @@ -85,6 +86,11 @@ public function is(Reference $ref) : bool return $this->name() === $ref->name(); } + public function list() : ListFunctions + { + return new ListFunctions($this); + } + public function name() : string { return $this->alias ?? $this->entry; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ListSelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ListSelectTest.php new file mode 100644 index 000000000..0447a3c26 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ListSelectTest.php @@ -0,0 +1,32 @@ +read( + from_array([ + ['list' => [['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test2'], ['id' => 3, 'name' => 'test3']]], + ['list' => [['id' => 4, 'name' => 'test4'], ['id' => 5, 'name' => 'test5'], ['id' => 6, 'name' => 'test6']]], + ]) + ) + ->withEntry('list', list_ref('list')->select('id')) + ->fetch(); + + $this->assertEquals( + [ + ['list' => [['id' => 1], ['id' => 2], ['id' => 3]]], + ['list' => [['id' => 4], ['id' => 5], ['id' => 6]]], + ], + $rows->toArray() + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureSelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureSelectTest.php index 4581341f0..b167f1350 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureSelectTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/Structure/StructureSelectTest.php @@ -4,7 +4,7 @@ use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; -use function Flow\ETL\DSL\structure; +use function Flow\ETL\DSL\structure_ref; use PHPUnit\Framework\TestCase; final class StructureSelectTest extends TestCase @@ -30,7 +30,7 @@ public function test_structure_keep() : void ] ) ) - ->withEntry('user', structure('user')->select('id', 'email', 'tags')) + ->withEntry('user', structure_ref('user')->select('id', 'email', 'tags')) ->fetch(); $this->assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/CLI/InputTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/CLI/InputTest.php deleted file mode 100644 index 82346ba69..000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/CLI/InputTest.php +++ /dev/null @@ -1,41 +0,0 @@ -assertSame( - 'default', - $input->optionValue('host', 'default') - ); - } - - public function test_option_value() : void - { - $input = new Input(['bin/worker', '--host=127.0.0.1']); - - $this->assertSame( - '127.0.0.1', - $input->optionValue('host', 'default') - ); - } - - public function test_option_value_with_invalid_option_format() : void - { - $input = new Input(['bin/worker', 'host=127.0.0.1']); - - $this->assertSame( - 'default', - $input->optionValue('host', 'default') - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php new file mode 100644 index 000000000..5cef3634a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php @@ -0,0 +1,121 @@ + 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + type_list(struct_type( + struct_element('id', type_int()), + struct_element('name', type_string()), + )) + ) + ); + + $this->assertEquals( + [ + ['id' => 1, 'mail' => null], + ['id' => 2, 'mail' => null], + ['id' => 3, 'mail' => null], + ], + (new ListSelect(ref('list'), ref('id'), ref('mail')))->eval($list) + ); + } + + public function test_selecting_value_from_list() : void + { + $list = row( + list_entry( + 'list', + [ + ['id' => 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + type_list(struct_type( + struct_element('id', type_int()), + struct_element('name', type_string()), + )) + ) + ); + + $this->assertEquals( + [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], + (new ListSelect(ref('list'), 'id'))->eval($list) + ); + } + + public function test_selecting_value_from_list_using_alias() : void + { + $list = row( + list_entry( + 'list', + [ + ['id' => 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + type_list(struct_type( + struct_element('id', type_int()), + struct_element('name', type_string()), + )) + ) + ); + + $this->assertEquals( + [ + ['new_id' => 1], + ['new_id' => 2], + ['new_id' => 3], + ], + (new ListSelect(ref('list'), ref('id')->as('new_id')))->eval($list) + ); + } + + public function test_selecting_value_from_simple_list() : void + { + $list = row( + list_entry( + 'list', + [ + 'a', 'b', 'c', 'd', + ], + type_list(type_string()) + ) + ); + + $this->assertEquals( + [ + ['id' => null], + ['id' => null], + ['id' => null], + ['id' => null], + ], + (new ListSelect(ref('list'), ref('id')))->eval($list) + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureKeepTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php similarity index 98% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureKeepTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php index 33b98d7e9..cbd66770a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureKeepTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php @@ -14,7 +14,7 @@ use Flow\ETL\Function\StructureSelect; use PHPUnit\Framework\TestCase; -final class StructureKeepTest extends TestCase +final class StructureSelectTest extends TestCase { public function test_selecting_multiple_values_from_structure() : void {