Unify passing options to loaders/extractors through fluent interface (#…

…1208) * Simplified XML Extractor * Unified extractor/loader options in CSV adapters * Unified extractor/loader options in ChartJS adapters * Unified extractor/loader options in Doctrine adapter * Unified extractor/loader options in Doctrine adapter - bug fixes * Unified extractor/loader options in Elasticsearch adapter * Unified extractor/loader options in Google sheet adapter * Unified extractor/loader options in http adapter * Unified extractor/loader options in json adapter * Unified extractor/loader options in parquet adapter * Unified extractor/loader options in text adapter * Added upgrade document
flow-php · Sep 5, 2024 · 5fe9605 · 5fe9605
1 parent b09cfd4
commit 5fe9605
Show file tree

Hide file tree

Showing 39 changed files with 857 additions and 328 deletions.
diff --git a/UPGRADE.md b/UPGRADE.md
@@ -5,6 +5,51 @@ Please follow the instructions for your specific version to ensure a smooth upgr
 
 ---
 
+## Upgrading from 0.8.x to 0.10.x
+
+### 1) Providing multiple paths to single extractor
+
+From now in order to read from multiple locations use `from_all(Extractor ...$extractors) : Exctractor` extractor. 
+
+Before:
+```php
+<?php
+
+from_parquet([
+    path(__DIR__ . '/data/1.parquet'),
+    path(__DIR__ . '/data/2.parquet'),
+]);
+```
+
+After:
+```php
+<?php
+
+from_all(
+    from_parquet(path(__DIR__ . '/data/1.parquet')),
+    from_parquet(path(__DIR__ . '/data/2.parquet')),
+);
+```
+
+### 2) Passing optional arguments to extractors/loaders
+
+From now all extractors/loaders are accepting only mandatory arguments, 
+all optional arguments should be passed through `with*` methods and fluent interface. 
+
+Before:
+```php
+<?php
+
+from_parquet(path(__DIR__ . '/data/1.parquet'), schema: $schema);
+```
+
+After:
+```php
+<?php
+
+from_parquet(path(__DIR__ . '/data/1.parquet'))->withSchema($schema);
+```
+
 ## Upgrading from 0.7.x to 0.8.x
 
 ### 1) Joins 

diff --git a/examples/topics/data_source/csv/code.php b/examples/topics/data_source/csv/code.php
@@ -12,7 +12,7 @@
         __DIR__ . '/input/dataset.csv',
         with_header: true,
         empty_to_null: true,
-        delimiter: ',',
+        separator: ',',
         enclosure: '"',
         escape: '\\',
         characters_read_in_line: 1000

diff --git a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php
@@ -10,12 +10,15 @@
 
 final class ChartJSLoader implements Closure, Loader
 {
-    public function __construct(
-        private readonly Chart $type,
-        private readonly ?Path $output = null,
-        private readonly Path $template = new Path(__DIR__ . '/Resources/template/full_page.html'),
-        private ?array &$outputVar = null
-    ) {
+    private ?Path $output = null;
+
+    private ?array $outputVar = null;
+
+    private Path $template;
+
+    public function __construct(private readonly Chart $type)
+    {
+        $this->template = new Path(__DIR__ . '/Resources/template/full_page.html');
     }
 
     public function closure(FlowContext $context) : void
@@ -60,4 +63,25 @@ public function load(Rows $rows, FlowContext $context) : void
 
         $this->type->collect($rows);
     }
+
+    public function withOutputPath(Path $output) : self
+    {
+        $this->output = $output;
+
+        return $this;
+    }
+
+    public function withOutputVar(array &$outputVar) : self
+    {
+        $this->outputVar = &$outputVar;
+
+        return $this;
+    }
+
+    public function withTemplate(Path $template) : self
+    {
+        $this->template = $template;
+
+        return $this;
+    }
 }
diff --git a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php
@@ -27,27 +27,47 @@ function pie_chart(EntryReference $label, References $datasets) : PieChart
     return new PieChart($label, $datasets);
 }
 
+#[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
+function to_chartjs(Chart $type) : ChartJSLoader
+{
+    return new ChartJSLoader($type);
+}
+
+/**
+ * @param Chart $type
+ * @param null|Path|string $output - @deprecated use $loader->withOutputPath() instead
+ * @param null|Path|string $template - @deprecated use $loader->withTemplate() instead
+ */
 #[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
 function to_chartjs_file(Chart $type, Path|string|null $output = null, Path|string|null $template = null) : ChartJSLoader
 {
     if (\is_string($output)) {
         $output = Path::realpath($output);
     }
 
-    if (null === $template) {
-        return new ChartJSLoader($type, $output);
-    }
-
     if (\is_string($template)) {
         $template = Path::realpath($template);
     }
 
-    return new ChartJSLoader($type, output: $output, template: $template);
+    $loader = new ChartJSLoader($type);
+
+    if ($template) {
+        $loader->withTemplate($template);
+    }
+
+    if ($output !== null) {
+        $loader->withOutputPath($output);
+    }
+
+    return $loader;
 }
 
+/**
+ * @param Chart $type
+ * @param array $output - @deprecated use $loader->withOutputVar() instead
+ */
 #[DocumentationDSL(module: Module::CHART_JS, type: Type::LOADER)]
 function to_chartjs_var(Chart $type, array &$output) : ChartJSLoader
 {
-    /** @psalm-suppress ReferenceConstraintViolation */
-    return new ChartJSLoader($type, outputVar: $output);
+    return (new ChartJSLoader($type))->withOutputVar($output);
 }
diff --git a/...tl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php b/...tl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php
@@ -4,8 +4,9 @@
 
 namespace Flow\ETL\Adapter\ChartJS\Tests\Integration;
 
-use function Flow\ETL\Adapter\ChartJS\{bar_chart, line_chart, pie_chart, to_chartjs_file, to_chartjs_var};
+use function Flow\ETL\Adapter\ChartJS\{bar_chart, line_chart, pie_chart, to_chartjs, to_chartjs_file, to_chartjs_var};
 use function Flow\ETL\DSL\{df, first, from_array, lit, ref, refs, sum};
+use function Flow\Filesystem\DSL\path;
 use PHPUnit\Framework\TestCase;
 
 final class ChartJSLoaderTest extends TestCase
@@ -25,7 +26,7 @@ public function test_loading_data_to_bar_chart() : void
             ->read(from_array($data))
             ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2)))
             ->write(
-                to_chartjs_file(
+                to_chartjs(
                     $chart = bar_chart(
                         ref('Date'),
                         refs(
@@ -37,8 +38,7 @@ public function test_loading_data_to_bar_chart() : void
                             ref('Profit'),
                         )
                     ),
-                    $output = __DIR__ . '/Output/bar_chart.html'
-                )
+                )->withOutputPath(path($output = __DIR__ . '/Output/bar_chart.html'))
             )
             ->run();
 

diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php
@@ -7,7 +7,7 @@
 use function Flow\ETL\DSL\array_to_rows;
 use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal};
 use Flow\ETL\Row\Schema;
-use Flow\ETL\{Extractor, FlowContext};
+use Flow\ETL\{Exception\InvalidArgumentException, Extractor, FlowContext};
 use Flow\Filesystem\Path;
 
 final class CSVExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor
@@ -16,18 +16,24 @@ final class CSVExtractor implements Extractor, FileExtractor, LimitableExtractor
     use PathFiltering;
 
     /**
-     * @param ?int<1, max> $charactersReadInLine
+     * @var null|int<1, max>
      */
-    public function __construct(
-        private readonly Path $path,
-        private readonly bool $withHeader = true,
-        private readonly bool $emptyToNull = true,
-        private readonly ?string $separator = null,
-        private readonly ?string $enclosure = null,
-        private readonly ?string $escape = null,
-        private readonly ?int $charactersReadInLine = null,
-        private readonly ?Schema $schema = null
-    ) {
+    private ?int $charactersReadInLine = null;
+
+    private bool $emptyToNull = true;
+
+    private ?string $enclosure = null;
+
+    private ?string $escape = null;
+
+    private ?Schema $schema = null;
+
+    private ?string $separator = null;
+
+    private bool $withHeader = true;
+
+    public function __construct(private readonly Path $path)
+    {
         $this->resetLimit();
     }
 
@@ -115,4 +121,60 @@ public function source() : Path
     {
         return $this->path;
     }
+
+    /**
+     * @param int<1, max> $charactersReadInLine
+     */
+    public function withCharactersReadInLine(int $charactersReadInLine) : self
+    {
+        if ($charactersReadInLine < 1) {
+            throw new InvalidArgumentException('Characters read in line must be greater than 0');
+        }
+
+        $this->charactersReadInLine = $charactersReadInLine;
+
+        return $this;
+    }
+
+    public function withEmptyToNull(bool $emptyToNull) : self
+    {
+        $this->emptyToNull = $emptyToNull;
+
+        return $this;
+    }
+
+    public function withEnclosure(string $enclosure) : self
+    {
+        $this->enclosure = $enclosure;
+
+        return $this;
+    }
+
+    public function withEscape(string $escape) : self
+    {
+        $this->escape = $escape;
+
+        return $this;
+    }
+
+    public function withHeader(bool $withHeader) : self
+    {
+        $this->withHeader = $withHeader;
+
+        return $this;
+    }
+
+    public function withSchema(Schema $schema) : self
+    {
+        $this->schema = $schema;
+
+        return $this;
+    }
+
+    public function withSeparator(string $separator) : self
+    {
+        $this->separator = $separator;
+
+        return $this;
+    }
 }
diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php
@@ -12,14 +12,20 @@
 
 final class CSVLoader implements Closure, Loader, Loader\FileLoader
 {
+    private string $dateTimeFormat = \DateTimeInterface::ATOM;
+
+    private string $enclosure = '"';
+
+    private string $escape = '\\';
+
+    private bool $header = true;
+
+    private string $newLineSeparator = PHP_EOL;
+
+    private string $separator = ',';
+
     public function __construct(
         private readonly Path $path,
-        private bool $header = true,
-        private string $separator = ',',
-        private string $enclosure = '"',
-        private string $escape = '\\',
-        private string $newLineSeparator = PHP_EOL,
-        private string $dateTimeFormat = \DateTimeInterface::ATOM
     ) {
     }
 
@@ -50,6 +56,48 @@ public function load(Rows $rows, FlowContext $context) : void
         }
     }
 
+    public function withDateTimeFormat(string $dateTimeFormat) : self
+    {
+        $this->dateTimeFormat = $dateTimeFormat;
+
+        return $this;
+    }
+
+    public function withEnclosure(string $enclosure) : self
+    {
+        $this->enclosure = $enclosure;
+
+        return $this;
+    }
+
+    public function withEscape(string $escape) : self
+    {
+        $this->escape = $escape;
+
+        return $this;
+    }
+
+    public function withHeader(bool $header) : self
+    {
+        $this->header = $header;
+
+        return $this;
+    }
+
+    public function withNewLineSeparator(string $newLineSeparator) : self
+    {
+        $this->newLineSeparator = $newLineSeparator;
+
+        return $this;
+    }
+
+    public function withSeparator(string $separator) : self
+    {
+        $this->separator = $separator;
+
+        return $this;
+    }
+
     /**
      * @param array<Partition> $partitions
      */