Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unify passing options to loaders/extractors through fluent interface #1208

Merged
45 changes: 45 additions & 0 deletions UPGRADE.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,51 @@ Please follow the instructions for your specific version to ensure a smooth upgr

---

## Upgrading from 0.8.x to 0.10.x

### 1) Providing multiple paths to single extractor

From now in order to read from multiple locations use `from_all(Extractor ...$extractors) : Exctractor` extractor.

Before:
```php
<?php

from_parquet([
path(__DIR__ . '/data/1.parquet'),
path(__DIR__ . '/data/2.parquet'),
]);
```

After:
```php
<?php

from_all(
from_parquet(path(__DIR__ . '/data/1.parquet')),
from_parquet(path(__DIR__ . '/data/2.parquet')),
);
```

### 2) Passing optional arguments to extractors/loaders

From now all extractors/loaders are accepting only mandatory arguments,
all optional arguments should be passed through `with*` methods and fluent interface.

Before:
```php
<?php

from_parquet(path(__DIR__ . '/data/1.parquet'), schema: $schema);
```

After:
```php
<?php

from_parquet(path(__DIR__ . '/data/1.parquet'))->withSchema($schema);
```

## Upgrading from 0.7.x to 0.8.x

### 1) Joins
Expand Down
2 changes: 1 addition & 1 deletion examples/topics/data_source/csv/code.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
__DIR__ . '/input/dataset.csv',
with_header: true,
empty_to_null: true,
delimiter: ',',
separator: ',',
enclosure: '"',
escape: '\\',
characters_read_in_line: 1000
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@

final class ChartJSLoader implements Closure, Loader
{
public function __construct(
private readonly Chart $type,
private readonly ?Path $output = null,
private readonly Path $template = new Path(__DIR__ . '/Resources/template/full_page.html'),
private ?array &$outputVar = null
) {
private ?Path $output = null;

private ?array $outputVar = null;

private Path $template;

public function __construct(private readonly Chart $type)
{
$this->template = new Path(__DIR__ . '/Resources/template/full_page.html');
}

public function closure(FlowContext $context) : void
Expand Down Expand Up @@ -60,4 +63,25 @@ public function load(Rows $rows, FlowContext $context) : void

$this->type->collect($rows);
}

public function withOutputPath(Path $output) : self
{
$this->output = $output;

return $this;
}

public function withOutputVar(array &$outputVar) : self
{
$this->outputVar = &$outputVar;

return $this;
}

public function withTemplate(Path $template) : self
{
$this->template = $template;

return $this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,47 @@ function pie_chart(EntryReference $label, References $datasets) : PieChart
return new PieChart($label, $datasets);
}

#[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
function to_chartjs(Chart $type) : ChartJSLoader
{
return new ChartJSLoader($type);
}

/**
* @param Chart $type
* @param null|Path|string $output - @deprecated use $loader->withOutputPath() instead
* @param null|Path|string $template - @deprecated use $loader->withTemplate() instead
*/
#[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
function to_chartjs_file(Chart $type, Path|string|null $output = null, Path|string|null $template = null) : ChartJSLoader
{
if (\is_string($output)) {
$output = Path::realpath($output);
}

if (null === $template) {
return new ChartJSLoader($type, $output);
}

if (\is_string($template)) {
$template = Path::realpath($template);
}

return new ChartJSLoader($type, output: $output, template: $template);
$loader = new ChartJSLoader($type);

if ($template) {
$loader->withTemplate($template);
}

if ($output !== null) {
$loader->withOutputPath($output);
}

return $loader;
}

/**
* @param Chart $type
* @param array $output - @deprecated use $loader->withOutputVar() instead
*/
#[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
function to_chartjs_var(Chart $type, array &$output) : ChartJSLoader
{
/** @psalm-suppress ReferenceConstraintViolation */
return new ChartJSLoader($type, outputVar: $output);
return (new ChartJSLoader($type))->withOutputVar($output);
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@

namespace Flow\ETL\Adapter\ChartJS\Tests\Integration;

use function Flow\ETL\Adapter\ChartJS\{bar_chart, line_chart, pie_chart, to_chartjs_file, to_chartjs_var};
use function Flow\ETL\Adapter\ChartJS\{bar_chart, line_chart, pie_chart, to_chartjs, to_chartjs_file, to_chartjs_var};
use function Flow\ETL\DSL\{df, first, from_array, lit, ref, refs, sum};
use function Flow\Filesystem\DSL\path;
use PHPUnit\Framework\TestCase;

final class ChartJSLoaderTest extends TestCase
Expand All @@ -25,7 +26,7 @@ public function test_loading_data_to_bar_chart() : void
->read(from_array($data))
->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2)))
->write(
to_chartjs_file(
to_chartjs(
$chart = bar_chart(
ref('Date'),
refs(
Expand All @@ -37,8 +38,7 @@ public function test_loading_data_to_bar_chart() : void
ref('Profit'),
)
),
$output = __DIR__ . '/Output/bar_chart.html'
)
)->withOutputPath(path($output = __DIR__ . '/Output/bar_chart.html'))
)
->run();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
use function Flow\ETL\DSL\array_to_rows;
use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal};
use Flow\ETL\Row\Schema;
use Flow\ETL\{Extractor, FlowContext};
use Flow\ETL\{Exception\InvalidArgumentException, Extractor, FlowContext};
use Flow\Filesystem\Path;

final class CSVExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor
Expand All @@ -16,18 +16,24 @@ final class CSVExtractor implements Extractor, FileExtractor, LimitableExtractor
use PathFiltering;

/**
* @param ?int<1, max> $charactersReadInLine
* @var null|int<1, max>
*/
public function __construct(
private readonly Path $path,
private readonly bool $withHeader = true,
private readonly bool $emptyToNull = true,
private readonly ?string $separator = null,
private readonly ?string $enclosure = null,
private readonly ?string $escape = null,
private readonly ?int $charactersReadInLine = null,
private readonly ?Schema $schema = null
) {
private ?int $charactersReadInLine = null;

private bool $emptyToNull = true;

private ?string $enclosure = null;

private ?string $escape = null;

private ?Schema $schema = null;

private ?string $separator = null;

private bool $withHeader = true;

public function __construct(private readonly Path $path)
{
$this->resetLimit();
}

Expand Down Expand Up @@ -115,4 +121,60 @@ public function source() : Path
{
return $this->path;
}

/**
* @param int<1, max> $charactersReadInLine
*/
public function withCharactersReadInLine(int $charactersReadInLine) : self
{
if ($charactersReadInLine < 1) {
throw new InvalidArgumentException('Characters read in line must be greater than 0');
}

$this->charactersReadInLine = $charactersReadInLine;

return $this;
}

public function withEmptyToNull(bool $emptyToNull) : self
{
$this->emptyToNull = $emptyToNull;

return $this;
}

public function withEnclosure(string $enclosure) : self
{
$this->enclosure = $enclosure;

return $this;
}

public function withEscape(string $escape) : self
{
$this->escape = $escape;

return $this;
}

public function withHeader(bool $withHeader) : self
{
$this->withHeader = $withHeader;

return $this;
}

public function withSchema(Schema $schema) : self
{
$this->schema = $schema;

return $this;
}

public function withSeparator(string $separator) : self
{
$this->separator = $separator;

return $this;
}
}
60 changes: 54 additions & 6 deletions src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,20 @@

final class CSVLoader implements Closure, Loader, Loader\FileLoader
{
private string $dateTimeFormat = \DateTimeInterface::ATOM;

private string $enclosure = '"';

private string $escape = '\\';

private bool $header = true;

private string $newLineSeparator = PHP_EOL;

private string $separator = ',';

public function __construct(
private readonly Path $path,
private bool $header = true,
private string $separator = ',',
private string $enclosure = '"',
private string $escape = '\\',
private string $newLineSeparator = PHP_EOL,
private string $dateTimeFormat = \DateTimeInterface::ATOM
) {
}

Expand Down Expand Up @@ -50,6 +56,48 @@ public function load(Rows $rows, FlowContext $context) : void
}
}

public function withDateTimeFormat(string $dateTimeFormat) : self
{
$this->dateTimeFormat = $dateTimeFormat;

return $this;
}

public function withEnclosure(string $enclosure) : self
{
$this->enclosure = $enclosure;

return $this;
}

public function withEscape(string $escape) : self
{
$this->escape = $escape;

return $this;
}

public function withHeader(bool $header) : self
{
$this->header = $header;

return $this;
}

public function withNewLineSeparator(string $newLineSeparator) : self
{
$this->newLineSeparator = $newLineSeparator;

return $this;
}

public function withSeparator(string $separator) : self
{
$this->separator = $separator;

return $this;
}

/**
* @param array<Partition> $partitions
*/
Expand Down
Loading