Skip to content

Commit

Permalink
Added library version to parquet created_by metadata (#693)
Browse files Browse the repository at this point in the history
* Added library version to parquet created_by metadata

* Added composer-runtime-api dependency in order to read parquet library version
  • Loading branch information
norberttech authored Oct 31, 2023
1 parent d4e3fdc commit 1356d23
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 29 deletions.
1 change: 1 addition & 0 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"ext-mbstring": "*",
"ext-xmlreader": "*",
"ext-zlib": "*",
"composer-runtime-api": "^2.1",
"amphp/process": "^2",
"amphp/socket": "^2",
"doctrine/dbal": "^3.6",
Expand Down
53 changes: 27 additions & 26 deletions composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions src/lib/parquet/composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
"php": "~8.1 || ~8.2",
"ext-bcmath": "*",
"ext-zlib": "*",
"psr/log": "^2.0 || ^3.0",
"composer-runtime-api": "^2.1",
"flow-php/dremel": "1.x-dev",
"flow-php/snappy": "1.x-dev"
},
Expand Down
11 changes: 8 additions & 3 deletions src/lib/parquet/src/Flow/Parquet/Writer.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Flow\Parquet;

use Composer\InstalledVersions;
use Flow\Parquet\Data\DataConverter;
use Flow\Parquet\Exception\InvalidArgumentException;
use Flow\Parquet\Exception\RuntimeException;
Expand Down Expand Up @@ -108,8 +109,7 @@ public function open(string $path, Schema $schema) : void
\fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER);
$this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER);

$this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-parquet'));

$this->initMetadata($schema);
$this->initGroupBuilder($schema);
}

Expand Down Expand Up @@ -141,7 +141,7 @@ public function openForStream($resource, Schema $schema) : void
\fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER);
$this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER);

$this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-parquet'));
$this->initMetadata($schema);

$this->initGroupBuilder($schema);
}
Expand Down Expand Up @@ -211,6 +211,11 @@ private function initGroupBuilder(Schema $schema) : void
}
}

private function initMetadata(Schema $schema) : void
{
$this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-php parquet version ' . InstalledVersions::getRootPackage()['pretty_version']));
}

private function metadata() : Metadata
{
if ($this->metadata === null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Flow\Parquet\Tests\Integration\IO;

use Composer\InstalledVersions;
use Faker\Factory;
use Flow\Parquet\Consts;
use Flow\Parquet\Exception\InvalidArgumentException;
Expand All @@ -24,6 +25,21 @@ public function test_closing_not_open_writer() : void
$writer->close();
}

public function test_created_by_metadata() : void
{
$writer = new Writer();

$path = \sys_get_temp_dir() . '/test-writer' . \uniqid('parquet-test-', true) . '.parquet';

$schema = $this->createSchema();
$writer->open($path, $schema);
$writer->close();

$metadata = (new Reader())->read($path)->metadata();

$this->assertSame('flow-php parquet version ' . InstalledVersions::getRootPackage()['pretty_version'], $metadata->createdBy());
}

public function test_opening_already_open_writer() : void
{
$writer = new Writer();
Expand Down

0 comments on commit 1356d23

Please sign in to comment.