From 8d3ee8a6f3c25722af83043db5b8a0cf218fd803 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Tue, 31 Oct 2023 15:40:54 +0100 Subject: [PATCH 1/2] Added library version to parquet created_by metadata --- src/lib/parquet/composer.json | 1 + src/lib/parquet/src/Flow/Parquet/Writer.php | 11 ++++++++--- .../Parquet/Tests/Integration/IO/WriterTest.php | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/lib/parquet/composer.json b/src/lib/parquet/composer.json index ffefe6fc8..84237230c 100644 --- a/src/lib/parquet/composer.json +++ b/src/lib/parquet/composer.json @@ -14,6 +14,7 @@ "php": "~8.1 || ~8.2", "ext-bcmath": "*", "ext-zlib": "*", + "psr/log": "^2.0 || ^3.0", "flow-php/dremel": "1.x-dev", "flow-php/snappy": "1.x-dev" }, diff --git a/src/lib/parquet/src/Flow/Parquet/Writer.php b/src/lib/parquet/src/Flow/Parquet/Writer.php index 4ad07859b..b50232f1d 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer.php @@ -2,6 +2,7 @@ namespace Flow\Parquet; +use Composer\InstalledVersions; use Flow\Parquet\Data\DataConverter; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Exception\RuntimeException; @@ -108,8 +109,7 @@ public function open(string $path, Schema $schema) : void \fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER); $this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER); - $this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-parquet')); - + $this->initMetadata($schema); $this->initGroupBuilder($schema); } @@ -141,7 +141,7 @@ public function openForStream($resource, Schema $schema) : void \fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER); $this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER); - $this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-parquet')); + $this->initMetadata($schema); $this->initGroupBuilder($schema); } @@ -211,6 +211,11 @@ private function initGroupBuilder(Schema $schema) : void } } + private function initMetadata(Schema $schema) : void + { + $this->metadata = (new Metadata($schema, new RowGroups([]), 0, self::VERSION, 'flow-php parquet version ' . InstalledVersions::getRootPackage()['pretty_version'])); + } + private function metadata() : Metadata { if ($this->metadata === null) { diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php index 853069b2f..c0a47dfb9 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php @@ -2,6 +2,7 @@ namespace Flow\Parquet\Tests\Integration\IO; +use Composer\InstalledVersions; use Faker\Factory; use Flow\Parquet\Consts; use Flow\Parquet\Exception\InvalidArgumentException; @@ -24,6 +25,21 @@ public function test_closing_not_open_writer() : void $writer->close(); } + public function test_created_by_metadata() : void + { + $writer = new Writer(); + + $path = \sys_get_temp_dir() . '/test-writer' . \uniqid('parquet-test-', true) . '.parquet'; + + $schema = $this->createSchema(); + $writer->open($path, $schema); + $writer->close(); + + $metadata = (new Reader())->read($path)->metadata(); + + $this->assertSame('flow-php parquet version ' . InstalledVersions::getRootPackage()['pretty_version'], $metadata->createdBy()); + } + public function test_opening_already_open_writer() : void { $writer = new Writer(); From 90aa692c4ebf4fa926dae14d34f82c180435427e Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Tue, 31 Oct 2023 18:37:38 +0100 Subject: [PATCH 2/2] Added composer-runtime-api dependency in order to read parquet library version --- composer.json | 1 + composer.lock | 53 ++++++++++++++++++----------------- src/lib/parquet/composer.json | 1 + 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/composer.json b/composer.json index 00e411382..87b75c83f 100644 --- a/composer.json +++ b/composer.json @@ -18,6 +18,7 @@ "ext-mbstring": "*", "ext-xmlreader": "*", "ext-zlib": "*", + "composer-runtime-api": "^2.1", "amphp/process": "^2", "amphp/socket": "^2", "doctrine/dbal": "^3.6", diff --git a/composer.lock b/composer.lock index fc604e96d..76ecaa866 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6aaf0a11cbf58a79a4beef08cd111bd4", + "content-hash": "01dbc7fd37119b8ab1b0188ee0171f2e", "packages": [ { "name": "amphp/amp", @@ -1561,16 +1561,16 @@ }, { "name": "google/apiclient-services", - "version": "v0.321.0", + "version": "v0.322.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "bf0aa8b1cb272f45c1f73270ad113047625c5937" + "reference": "e892db67c88eb119de0c12aa2419b48401555ded" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/bf0aa8b1cb272f45c1f73270ad113047625c5937", - "reference": "bf0aa8b1cb272f45c1f73270ad113047625c5937", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/e892db67c88eb119de0c12aa2419b48401555ded", + "reference": "e892db67c88eb119de0c12aa2419b48401555ded", "shasum": "" }, "require": { @@ -1599,9 +1599,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.321.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.322.0" }, - "time": "2023-10-23T01:08:38+00:00" + "time": "2023-10-29T01:10:18+00:00" }, { "name": "google/auth", @@ -2552,16 +2552,16 @@ }, { "name": "monolog/monolog", - "version": "3.4.0", + "version": "3.5.0", "source": { "type": "git", "url": "https://github.com/Seldaek/monolog.git", - "reference": "e2392369686d420ca32df3803de28b5d6f76867d" + "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/e2392369686d420ca32df3803de28b5d6f76867d", - "reference": "e2392369686d420ca32df3803de28b5d6f76867d", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/c915e2634718dbc8a4a15c61b0e62e7a44e14448", + "reference": "c915e2634718dbc8a4a15c61b0e62e7a44e14448", "shasum": "" }, "require": { @@ -2637,7 +2637,7 @@ ], "support": { "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.4.0" + "source": "https://github.com/Seldaek/monolog/tree/3.5.0" }, "funding": [ { @@ -2649,7 +2649,7 @@ "type": "tidelift" } ], - "time": "2023-06-21T08:46:11+00:00" + "time": "2023-10-27T15:32:31+00:00" }, { "name": "packaged/thrift", @@ -5163,16 +5163,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.283.13", + "version": "3.283.15", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "853eecfb21e8d623fa1b32e597b0a75912e8a404" + "reference": "44b70c15c2a5d0fd2cfc74ee529a7877f1e7457d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/853eecfb21e8d623fa1b32e597b0a75912e8a404", - "reference": "853eecfb21e8d623fa1b32e597b0a75912e8a404", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/44b70c15c2a5d0fd2cfc74ee529a7877f1e7457d", + "reference": "44b70c15c2a5d0fd2cfc74ee529a7877f1e7457d", "shasum": "" }, "require": { @@ -5252,9 +5252,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.283.13" + "source": "https://github.com/aws/aws-sdk-php/tree/3.283.15" }, - "time": "2023-10-26T18:14:40+00:00" + "time": "2023-10-30T18:09:22+00:00" }, { "name": "brick/math", @@ -6479,16 +6479,16 @@ }, { "name": "symfony/dotenv", - "version": "v6.3.0", + "version": "v6.3.7", "source": { "type": "git", "url": "https://github.com/symfony/dotenv.git", - "reference": "ceadb434fe2a6763a03d2d110441745834f3dd1e" + "reference": "7dfbe2976f3c1b7cfa8fac2212a050bfa9bd7d9e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dotenv/zipball/ceadb434fe2a6763a03d2d110441745834f3dd1e", - "reference": "ceadb434fe2a6763a03d2d110441745834f3dd1e", + "url": "https://api.github.com/repos/symfony/dotenv/zipball/7dfbe2976f3c1b7cfa8fac2212a050bfa9bd7d9e", + "reference": "7dfbe2976f3c1b7cfa8fac2212a050bfa9bd7d9e", "shasum": "" }, "require": { @@ -6533,7 +6533,7 @@ "environment" ], "support": { - "source": "https://github.com/symfony/dotenv/tree/v6.3.0" + "source": "https://github.com/symfony/dotenv/tree/v6.3.7" }, "funding": [ { @@ -6549,7 +6549,7 @@ "type": "tidelift" } ], - "time": "2023-04-21T14:41:17+00:00" + "time": "2023-10-26T18:15:14+00:00" }, { "name": "symfony/finder", @@ -6859,7 +6859,8 @@ "ext-json": "*", "ext-mbstring": "*", "ext-xmlreader": "*", - "ext-zlib": "*" + "ext-zlib": "*", + "composer-runtime-api": "^2.1" }, "platform-dev": [], "plugin-api-version": "2.6.0" diff --git a/src/lib/parquet/composer.json b/src/lib/parquet/composer.json index 84237230c..9ab1cb3c7 100644 --- a/src/lib/parquet/composer.json +++ b/src/lib/parquet/composer.json @@ -15,6 +15,7 @@ "ext-bcmath": "*", "ext-zlib": "*", "psr/log": "^2.0 || ^3.0", + "composer-runtime-api": "^2.1", "flow-php/dremel": "1.x-dev", "flow-php/snappy": "1.x-dev" },