Skip to content

Commit

Permalink
Small additions to DSL
Browse files Browse the repository at this point in the history
  • Loading branch information
norberttech committed Dec 12, 2023
1 parent 372ca5e commit d8048b6
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
use function Flow\ETL\DSL\float_entry;
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\from_rows;
use function Flow\ETL\DSL\ignore;
use function Flow\ETL\DSL\int_entry;
use function Flow\ETL\DSL\json_entry;
use function Flow\ETL\DSL\json_object_entry;
Expand Down Expand Up @@ -331,7 +332,7 @@ public function test_writing_twice_to_the_same_location_with_ignore_mode() : voi
}, \range(1, 100))
)
))
->mode(SaveMode::Ignore)
->saveMode(ignore())
->write(to_avro($path))
->run();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
use function Flow\ETL\Adapter\CSV\to_csv;
use function Flow\ETL\DSL\array_entry;
use function Flow\ETL\DSL\int_entry;
use function Flow\ETL\DSL\overwrite;
use function Flow\ETL\DSL\str_entry;
use Flow\ETL\Filesystem\Path;
use Flow\ETL\Filesystem\SaveMode;
use Flow\ETL\Flow;
use Flow\ETL\Row;
use Flow\ETL\Rows;
Expand Down Expand Up @@ -207,7 +207,7 @@ public function test_loading_overwrite_csv() : void
Row::create(int_entry('id', 3), str_entry('name', 'Dawid')),
)
)
->mode(SaveMode::Overwrite)
->saveMode(overwrite())
->load(to_csv($path))
->run();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
use function Flow\ETL\Adapter\Json\from_json;
use function Flow\ETL\Adapter\Json\to_json;
use function Flow\ETL\DSL\df;
use function Flow\ETL\DSL\exception_if_exists;
use function Flow\ETL\DSL\from_array;
use function Flow\ETL\DSL\ignore;
use function Flow\ETL\DSL\ref;
use Flow\ETL\Adapter\JSON\JsonLoader;
use Flow\ETL\Config;
Expand Down Expand Up @@ -259,7 +261,7 @@ public function test_save_mode_throw_exception_on_partitioned_rows() : void
['id' => 5, 'partition' => 'b'],
]))
->partitionBy(ref('partition'))
->mode(SaveMode::ExceptionIfExists)
->saveMode(exception_if_exists())
->write(to_json($path))
->run();

Expand Down Expand Up @@ -291,7 +293,7 @@ public function test_save_with_ignore_mode() : void
['id' => 2],
['id' => 3],
]))
->mode(SaveMode::Ignore)
->saveMode(ignore())
->write(to_json($path))
->run();

Expand Down
21 changes: 21 additions & 0 deletions src/core/etl/src/Flow/ETL/DSL/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
use Flow\ETL\ErrorHandler\SkipRows;
use Flow\ETL\ErrorHandler\ThrowError;
use Flow\ETL\Extractor;
use Flow\ETL\Filesystem\SaveMode;
use Flow\ETL\Filesystem\Stream\Mode;
use Flow\ETL\Flow;
use Flow\ETL\FlowContext;
Expand Down Expand Up @@ -988,3 +989,23 @@ function config_builder() : ConfigBuilder
{
return new ConfigBuilder();
}

function overwrite() : SaveMode
{
return SaveMode::Overwrite;
}

function ignore() : SaveMode
{
return SaveMode::Ignore;
}

function exception_if_exists() : SaveMode
{
return SaveMode::ExceptionIfExists;
}

function append() : SaveMode
{
return SaveMode::Append;
}
10 changes: 10 additions & 0 deletions src/core/etl/src/Flow/ETL/DataFrame.php
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,16 @@ public function run(?callable $callback = null) : void
}
}

/**
* Alias for DataFrame::mode.
*
* @lazy
*/
public function saveMode(SaveMode $mode) : self
{
return $this->mode($mode);
}

/**
* @lazy
* Keep only given entries.
Expand Down

0 comments on commit d8048b6

Please sign in to comment.