-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Added batchSize method to DataFrame * Updated UPGRADE.md
- Loading branch information
1 parent
247bd2c
commit 85e3db4
Showing
8 changed files
with
306 additions
and
114 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
<?php declare(strict_types=1); | ||
|
||
namespace Flow\ETL\Pipeline; | ||
|
||
use Flow\ETL\DSL\From; | ||
use Flow\ETL\Exception\InvalidArgumentException; | ||
use Flow\ETL\Extractor; | ||
use Flow\ETL\FlowContext; | ||
use Flow\ETL\Loader; | ||
use Flow\ETL\Pipeline; | ||
use Flow\ETL\Transformer; | ||
|
||
final class BatchingPipeline implements Pipeline | ||
{ | ||
private readonly Pipeline $nextPipeline; | ||
|
||
/** | ||
* @param Pipeline $pipeline | ||
* @param int<1, max> $size | ||
* | ||
* @throws InvalidArgumentException | ||
*/ | ||
public function __construct(private readonly Pipeline $pipeline, private readonly int $size) | ||
{ | ||
$this->nextPipeline = $pipeline->cleanCopy(); | ||
|
||
/** | ||
* @psalm-suppress DocblockTypeContradiction | ||
* | ||
* @phpstan-ignore-next-line | ||
*/ | ||
if ($this->size <= 0) { | ||
throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->size); | ||
} | ||
} | ||
|
||
public function add(Loader|Transformer $pipe) : self | ||
{ | ||
$this->nextPipeline->add($pipe); | ||
|
||
return $this; | ||
} | ||
|
||
public function cleanCopy() : Pipeline | ||
{ | ||
return $this->pipeline->cleanCopy(); | ||
} | ||
|
||
public function closure(FlowContext $context) : void | ||
{ | ||
$this->pipeline->closure($context); | ||
} | ||
|
||
public function has(string $transformerClass) : bool | ||
{ | ||
return $this->pipeline->has($transformerClass); | ||
} | ||
|
||
public function isAsync() : bool | ||
{ | ||
return $this->pipeline->isAsync(); | ||
} | ||
|
||
public function process(FlowContext $context) : \Generator | ||
{ | ||
$this->nextPipeline->source( | ||
From::chunks_from( | ||
From::pipeline($this->pipeline), | ||
$this->size | ||
) | ||
); | ||
|
||
return $this->nextPipeline->process($context); | ||
} | ||
|
||
public function source(Extractor $extractor) : self | ||
{ | ||
$this->pipeline->source($extractor); | ||
|
||
return $this; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
132 changes: 132 additions & 0 deletions
132
src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/BatchingPipelineTest.php
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,132 @@ | ||
<?php declare(strict_types=1); | ||
|
||
namespace Flow\ETL\Tests\Integration\Pipeline; | ||
|
||
use Flow\ETL\Config; | ||
use Flow\ETL\DSL\From; | ||
use Flow\ETL\FlowContext; | ||
use Flow\ETL\Pipeline\BatchingPipeline; | ||
use Flow\ETL\Pipeline\SynchronousPipeline; | ||
use Flow\ETL\Rows; | ||
use PHPUnit\Framework\TestCase; | ||
|
||
final class BatchingPipelineTest extends TestCase | ||
{ | ||
public function test_batching_rows() : void | ||
{ | ||
$pipeline = new BatchingPipeline(new SynchronousPipeline(), size: 10); | ||
$pipeline->source(From::chain( | ||
From::array([ | ||
['id' => 1], | ||
['id' => 2], | ||
['id' => 3], | ||
['id' => 4], | ||
['id' => 5], | ||
]), | ||
From::array([ | ||
['id' => 6], | ||
['id' => 7], | ||
['id' => 8], | ||
['id' => 9], | ||
['id' => 10], | ||
]) | ||
)); | ||
|
||
$this->assertCount( | ||
1, | ||
\iterator_to_array($pipeline->process(new FlowContext(Config::default()))) | ||
); | ||
} | ||
|
||
public function test_that_rows_are_not_lost() : void | ||
{ | ||
$pipeline = new BatchingPipeline(new SynchronousPipeline(), $batchSize = 7); | ||
$pipeline->source(From::chain( | ||
From::array([ | ||
['id' => 1], | ||
['id' => 2], | ||
['id' => 3], | ||
['id' => 4], | ||
['id' => 5], | ||
['id' => 6], | ||
['id' => 7], | ||
['id' => 8], | ||
['id' => 9], | ||
['id' => 10], | ||
]) | ||
)); | ||
|
||
$this->assertEquals( | ||
[ | ||
[ | ||
['id' => 1], | ||
['id' => 2], | ||
['id' => 3], | ||
['id' => 4], | ||
['id' => 5], | ||
['id' => 6], | ||
['id' => 7], | ||
], | ||
[ | ||
['id' => 8], | ||
['id' => 9], | ||
['id' => 10], | ||
], | ||
], | ||
\array_map( | ||
static fn (Rows $r) => $r->toArray(), | ||
\iterator_to_array($pipeline->process(new FlowContext(Config::default()))) | ||
) | ||
); | ||
} | ||
|
||
public function test_using_bigger_batch_size_than_total_number_of_rows() : void | ||
{ | ||
$pipeline = new BatchingPipeline(new SynchronousPipeline(), size: 11); | ||
$pipeline->source(From::chain( | ||
From::array([ | ||
['id' => 1], | ||
['id' => 2], | ||
['id' => 3], | ||
['id' => 4], | ||
['id' => 5], | ||
]), | ||
From::array([ | ||
['id' => 6], | ||
['id' => 7], | ||
['id' => 8], | ||
['id' => 9], | ||
['id' => 10], | ||
]) | ||
)); | ||
|
||
$this->assertCount( | ||
1, | ||
\iterator_to_array($pipeline->process(new FlowContext(Config::default()))) | ||
); | ||
} | ||
|
||
public function test_using_smaller_batch_size_than_total_number_of_rows() : void | ||
{ | ||
$pipeline = new BatchingPipeline(new SynchronousPipeline(), size: 5); | ||
$pipeline->source(From::chain( | ||
From::array([ | ||
['id' => 1], | ||
['id' => 2], | ||
['id' => 3], | ||
['id' => 4], | ||
['id' => 5], | ||
['id' => 6], | ||
['id' => 7], | ||
['id' => 8], | ||
['id' => 9], | ||
['id' => 10], | ||
]) | ||
)); | ||
|
||
$this->assertCount( | ||
2, | ||
\iterator_to_array($pipeline->process(new FlowContext(Config::default()))) | ||
); | ||
} | ||
} |
Oops, something went wrong.