diff --git a/.github/workflows/monorepo-split.yml b/.github/workflows/monorepo-split.yml index ef3f4931e..7ac6b963c 100644 --- a/.github/workflows/monorepo-split.yml +++ b/.github/workflows/monorepo-split.yml @@ -26,6 +26,8 @@ jobs: - local_path: 'src/lib/array-dot' split_repository: 'array-dot' + - local_path: 'src/lib/azure-sdk' + split_repository: 'azure-sdk' - local_path: 'src/lib/doctrine-dbal-bulk' split_repository: 'doctrine-dbal-bulk' - local_path: 'src/lib/parquet' @@ -34,6 +36,8 @@ jobs: split_repository: 'parquet-viewer' - local_path: 'src/lib/dremel' split_repository: 'dremel' + - local_path: 'src/lib/filesystem' + split_repository: 'filesystem' - local_path: 'src/lib/rdsl' split_repository: 'rdsl' - local_path: 'src/lib/snappy' @@ -49,8 +53,6 @@ jobs: split_repository: 'etl-adapter-doctrine' - local_path: 'src/adapter/etl-adapter-elasticsearch' split_repository: 'etl-adapter-elasticsearch' - - local_path: 'src/adapter/etl-adapter-filesystem' - split_repository: 'etl-adapter-filesystem' - local_path: 'src/adapter/etl-adapter-meilisearch' split_repository: 'etl-adapter-meilisearch' - local_path: 'src/adapter/etl-adapter-google-sheet' @@ -68,6 +70,11 @@ jobs: - local_path: 'src/adapter/etl-adapter-xml' split_repository: 'etl-adapter-xml' + - local_path: 'src/bridge/filesystem/azure' + split_repository: 'filesystem-azure-bridge' + - local_path: 'src/bridge/monolog/http' + split_repository: 'monolog-http-bridge' + - local_path: 'src/tools/homebrew' split_repository: 'homebrew-flow' diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 74fb5eff5..6e8c59ca1 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -96,6 +96,20 @@ jobs: - name: "List PHP configuration" run: php -i + - name: Set up Node.js + uses: actions/setup-node@v1 + with: + node-version: '14' + + - name: Install Azurite storage emulator + run: npm install -g azurite + + - name: Start Azurite blob endpoint + shell: bash + run: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 & + env: + AZURITE_ACCOUNTS: flowphpaccount01:flowphpkey01 + - name: "Get Composer Cache Directory" id: composer-cache run: | diff --git a/.php-cs-fixer.php b/.php-cs-fixer.php index 89b7bc30c..1de9c1a09 100644 --- a/.php-cs-fixer.php +++ b/.php-cs-fixer.php @@ -12,6 +12,7 @@ __DIR__ . '/src/core/**/src', __DIR__ . '/src/core/**/tests', __DIR__ . '/src/adapter/**/src', + __DIR__ . '/src/bridge/**/**/src', __DIR__ . '/src/adapter/**/tests', __DIR__ . '/src/lib/**/src', __DIR__ . '/src/lib/**/tests', diff --git a/compose.yml.dist b/compose.yml.dist index 01735e2e4..293cbfdae 100644 --- a/compose.yml.dist +++ b/compose.yml.dist @@ -36,3 +36,15 @@ services: environment: - MEILI_MASTER_KEY=masterKey - MEILI_NO_ANALYTICS=true + azurite: + image: mcr.microsoft.com/azure-storage/azurite + container_name: flow-php-azurite + hostname: azurite + restart: always + command: "azurite --loose --blobHost 0.0.0.0 --blobPort 10000 --location /workspace --debug /workspace/debug.log" + environment: + - AZURITE_ACCOUNTS=flowphpaccount01:flowphpkey01 + ports: + - 10000:10000 + volumes: + - ./var/azurite:/workspace diff --git a/composer.json b/composer.json index c882a5767..16fb15b85 100644 --- a/composer.json +++ b/composer.json @@ -24,11 +24,12 @@ "elasticsearch/elasticsearch": "^7.6|^8.0", "google/apiclient": "^2.13", "halaxa/json-machine": "^1.0", - "league/flysystem": "^3.0", "meilisearch/meilisearch-php": "^1.1", - "monolog/monolog": "^3.0", + "monolog/monolog": "^2.0||^3.0", "packaged/thrift": "^0.15.0", + "php-http/discovery": "^1.0", "psr/http-client": "^1.0", + "psr/http-message": "^1.0", "psr/log": "^2.0 || ^3.0", "psr/simple-cache": "^1.0 || ^2.0 || ^3.0", "symfony/console": "^6.3 || ^7.0", @@ -39,8 +40,6 @@ "fakerphp/faker": "^1.23", "fig/log-test": "^1.1", "jawira/case-converter": "^3.4", - "league/flysystem-aws-s3-v3": "^3.0", - "league/flysystem-azure-blob-storage": "^3.0", "moneyphp/money": "^4", "nyholm/psr7": "^1.8", "php-http/curl-client": "^2.2", @@ -53,21 +52,23 @@ }, "autoload": { "files": [ - "src/functions.php", "src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php", "src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php", "src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php", "src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php", "src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php", - "src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/functions.php", "src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php", "src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php", "src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php", "src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php", "src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php", "src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php", + "src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php", "src/core/etl/src/Flow/ETL/DSL/functions.php", + "src/functions.php", "src/lib/array-dot/src/Flow/ArrayDot/array_dot.php", + "src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php", + "src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php", "src/lib/parquet/src/Flow/Parquet/functions.php", "src/lib/snappy/polyfill.php" ], @@ -78,7 +79,6 @@ "src/adapter/etl-adapter-csv/src/Flow", "src/adapter/etl-adapter-doctrine/src/Flow", "src/adapter/etl-adapter-elasticsearch/src/Flow", - "src/adapter/etl-adapter-filesystem/src/Flow", "src/adapter/etl-adapter-google-sheet/src/Flow", "src/adapter/etl-adapter-http/src/Flow", "src/adapter/etl-adapter-json/src/Flow", @@ -87,10 +87,14 @@ "src/adapter/etl-adapter-parquet/src/Flow", "src/adapter/etl-adapter-text/src/Flow", "src/adapter/etl-adapter-xml/src/Flow", + "src/bridge/filesystem/azure/src/Flow", + "src/bridge/monolog/http/src/Flow", "src/core/etl/src/Flow", "src/lib/array-dot/src/Flow", + "src/lib/azure-sdk/src/Flow", "src/lib/doctrine-dbal-bulk/src/Flow", "src/lib/dremel/src/Flow", + "src/lib/filesystem/src/Flow", "src/lib/parquet-viewer/src/Flow", "src/lib/parquet/src/Flow", "src/lib/rdsl/src/Flow", @@ -112,7 +116,6 @@ "src/adapter/etl-adapter-csv/tests/Flow", "src/adapter/etl-adapter-doctrine/tests/Flow", "src/adapter/etl-adapter-elasticsearch/tests/Flow", - "src/adapter/etl-adapter-filesystem/tests/Flow", "src/adapter/etl-adapter-google-sheet/tests/Flow", "src/adapter/etl-adapter-http/tests/Flow", "src/adapter/etl-adapter-json/tests/Flow", @@ -121,10 +124,14 @@ "src/adapter/etl-adapter-parquet/tests/Flow", "src/adapter/etl-adapter-text/tests/Flow", "src/adapter/etl-adapter-xml/tests/Flow", + "src/bridge/filesystem/azure/tests/Flow", + "src/bridge/monolog/http/tests/Flow", "src/core/etl/tests/Flow", "src/lib/array-dot/tests/Flow", + "src/lib/azure-sdk/tests/Flow", "src/lib/doctrine-dbal-bulk/tests/Flow", "src/lib/dremel/tests/Flow", + "src/lib/filesystem/tests/Flow", "src/lib/parquet-viewer/tests/Flow", "src/lib/parquet/tests/Flow", "src/lib/rdsl/tests/Flow", @@ -145,6 +152,7 @@ }, "replace": { "flow-php/array-dot": "self.version", + "flow-php/azure-sdk": "self.version", "flow-php/doctrine-dbal-bulk": "self.version", "flow-php/doctrine-dbal-bulk-tools": "self.version", "flow-php/dremel": "self.version", @@ -165,6 +173,9 @@ "flow-php/etl-adapter-parquet": "self.version", "flow-php/etl-adapter-text": "self.version", "flow-php/etl-adapter-xml": "self.version", + "flow-php/filesystem": "self.version", + "flow-php/filesytem-azure-bridge": "self.version", + "flow-php/monolog-http-bridge": "self.version", "flow-php/parquet": "self.version", "flow-php/parquet-viewer": "self.version", "flow-php/rdsl": "self.version", @@ -204,7 +215,7 @@ "tools/phpbench/vendor/bin/phpbench run --report=flow-report --group=transformer" ], "test:mutation": [ - "tools/infection/vendor/bin/infection -j2" + "tools/infection/vendor/bin/infection --threads=max" ], "test:monorepo": "tools/monorepo/vendor/bin/monorepo-builder validate", "static:analyze": [ diff --git a/composer.lock b/composer.lock index b946284ec..46a007d16 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8ab08e266c725b7fbecd8c417042c5fc", + "content-hash": "1b9eec1abb65e88a59758d60c617aa5d", "packages": [ { "name": "aeon-php/calendar", @@ -582,16 +582,16 @@ }, { "name": "google/apiclient-services", - "version": "v0.361.0", + "version": "v0.362.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "f90e9a059ce5a6076b4fc8571a4fac6564012782" + "reference": "c5016217c8aba823a8ebeed6ccd75d93522e3311" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/f90e9a059ce5a6076b4fc8571a4fac6564012782", - "reference": "f90e9a059ce5a6076b4fc8571a4fac6564012782", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/c5016217c8aba823a8ebeed6ccd75d93522e3311", + "reference": "c5016217c8aba823a8ebeed6ccd75d93522e3311", "shasum": "" }, "require": { @@ -620,9 +620,9 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.361.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.362.0" }, - "time": "2024-06-23T01:02:19+00:00" + "time": "2024-06-28T01:06:13+00:00" }, { "name": "google/auth", @@ -1068,206 +1068,18 @@ ], "time": "2023-11-28T21:12:40+00:00" }, - { - "name": "league/flysystem", - "version": "3.28.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/flysystem.git", - "reference": "e611adab2b1ae2e3072fa72d62c62f52c2bf1f0c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/e611adab2b1ae2e3072fa72d62c62f52c2bf1f0c", - "reference": "e611adab2b1ae2e3072fa72d62c62f52c2bf1f0c", - "shasum": "" - }, - "require": { - "league/flysystem-local": "^3.0.0", - "league/mime-type-detection": "^1.0.0", - "php": "^8.0.2" - }, - "conflict": { - "async-aws/core": "<1.19.0", - "async-aws/s3": "<1.14.0", - "aws/aws-sdk-php": "3.209.31 || 3.210.0", - "guzzlehttp/guzzle": "<7.0", - "guzzlehttp/ringphp": "<1.1.1", - "phpseclib/phpseclib": "3.0.15", - "symfony/http-client": "<5.2" - }, - "require-dev": { - "async-aws/s3": "^1.5 || ^2.0", - "async-aws/simple-s3": "^1.1 || ^2.0", - "aws/aws-sdk-php": "^3.295.10", - "composer/semver": "^3.0", - "ext-fileinfo": "*", - "ext-ftp": "*", - "ext-mongodb": "^1.3", - "ext-zip": "*", - "friendsofphp/php-cs-fixer": "^3.5", - "google/cloud-storage": "^1.23", - "guzzlehttp/psr7": "^2.6", - "microsoft/azure-storage-blob": "^1.1", - "mongodb/mongodb": "^1.2", - "phpseclib/phpseclib": "^3.0.36", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5.11|^10.0", - "sabre/dav": "^4.6.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\Flysystem\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frank de Jonge", - "email": "info@frankdejonge.nl" - } - ], - "description": "File storage abstraction for PHP", - "keywords": [ - "WebDAV", - "aws", - "cloud", - "file", - "files", - "filesystem", - "filesystems", - "ftp", - "s3", - "sftp", - "storage" - ], - "support": { - "issues": "https://github.com/thephpleague/flysystem/issues", - "source": "https://github.com/thephpleague/flysystem/tree/3.28.0" - }, - "time": "2024-05-22T10:09:12+00:00" - }, - { - "name": "league/flysystem-local", - "version": "3.28.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/flysystem-local.git", - "reference": "13f22ea8be526ea58c2ddff9e158ef7c296e4f40" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-local/zipball/13f22ea8be526ea58c2ddff9e158ef7c296e4f40", - "reference": "13f22ea8be526ea58c2ddff9e158ef7c296e4f40", - "shasum": "" - }, - "require": { - "ext-fileinfo": "*", - "league/flysystem": "^3.0.0", - "league/mime-type-detection": "^1.0.0", - "php": "^8.0.2" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\Flysystem\\Local\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frank de Jonge", - "email": "info@frankdejonge.nl" - } - ], - "description": "Local filesystem adapter for Flysystem.", - "keywords": [ - "Flysystem", - "file", - "files", - "filesystem", - "local" - ], - "support": { - "source": "https://github.com/thephpleague/flysystem-local/tree/3.28.0" - }, - "time": "2024-05-06T20:05:52+00:00" - }, - { - "name": "league/mime-type-detection", - "version": "1.15.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/mime-type-detection.git", - "reference": "ce0f4d1e8a6f4eb0ddff33f57c69c50fd09f4301" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/mime-type-detection/zipball/ce0f4d1e8a6f4eb0ddff33f57c69c50fd09f4301", - "reference": "ce0f4d1e8a6f4eb0ddff33f57c69c50fd09f4301", - "shasum": "" - }, - "require": { - "ext-fileinfo": "*", - "php": "^7.4 || ^8.0" - }, - "require-dev": { - "friendsofphp/php-cs-fixer": "^3.2", - "phpstan/phpstan": "^0.12.68", - "phpunit/phpunit": "^8.5.8 || ^9.3 || ^10.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\MimeTypeDetection\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frank de Jonge", - "email": "info@frankdejonge.nl" - } - ], - "description": "Mime-type detection for Flysystem", - "support": { - "issues": "https://github.com/thephpleague/mime-type-detection/issues", - "source": "https://github.com/thephpleague/mime-type-detection/tree/1.15.0" - }, - "funding": [ - { - "url": "https://github.com/frankdejonge", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/league/flysystem", - "type": "tidelift" - } - ], - "time": "2024-01-28T23:22:08+00:00" - }, { "name": "meilisearch/meilisearch-php", - "version": "v1.8.0", + "version": "v1.9.0", "source": { "type": "git", "url": "https://github.com/meilisearch/meilisearch-php.git", - "reference": "77058e5cd0c9ee1236eaf8dfabdde2b339370b21" + "reference": "57f15d3cc13305c09d7d218720340b5f71157ae3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/meilisearch/meilisearch-php/zipball/77058e5cd0c9ee1236eaf8dfabdde2b339370b21", - "reference": "77058e5cd0c9ee1236eaf8dfabdde2b339370b21", + "url": "https://api.github.com/repos/meilisearch/meilisearch-php/zipball/57f15d3cc13305c09d7d218720340b5f71157ae3", + "reference": "57f15d3cc13305c09d7d218720340b5f71157ae3", "shasum": "" }, "require": { @@ -1282,7 +1094,7 @@ "guzzlehttp/guzzle": "^7.1", "http-interop/http-factory-guzzle": "^1.0", "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "1.10.67", + "phpstan/phpstan": "1.11.5", "phpstan/phpstan-deprecation-rules": "^1.0", "phpstan/phpstan-phpunit": "^1.0", "phpstan/phpstan-strict-rules": "^1.1", @@ -1320,22 +1132,22 @@ ], "support": { "issues": "https://github.com/meilisearch/meilisearch-php/issues", - "source": "https://github.com/meilisearch/meilisearch-php/tree/v1.8.0" + "source": "https://github.com/meilisearch/meilisearch-php/tree/v1.9.0" }, - "time": "2024-05-06T13:58:08+00:00" + "time": "2024-07-01T11:36:46+00:00" }, { "name": "monolog/monolog", - "version": "3.6.0", + "version": "3.7.0", "source": { "type": "git", "url": "https://github.com/Seldaek/monolog.git", - "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654" + "reference": "f4393b648b78a5408747de94fca38beb5f7e9ef8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/4b18b21a5527a3d5ffdac2fd35d3ab25a9597654", - "reference": "4b18b21a5527a3d5ffdac2fd35d3ab25a9597654", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f4393b648b78a5408747de94fca38beb5f7e9ef8", + "reference": "f4393b648b78a5408747de94fca38beb5f7e9ef8", "shasum": "" }, "require": { @@ -1411,7 +1223,7 @@ ], "support": { "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.6.0" + "source": "https://github.com/Seldaek/monolog/tree/3.7.0" }, "funding": [ { @@ -1423,7 +1235,7 @@ "type": "tidelift" } ], - "time": "2024-04-12T21:02:21+00:00" + "time": "2024-06-28T09:40:51+00:00" }, { "name": "packaged/thrift", @@ -2228,16 +2040,16 @@ }, { "name": "psr/http-message", - "version": "2.0", + "version": "1.1", "source": { "type": "git", "url": "https://github.com/php-fig/http-message.git", - "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71" + "reference": "cb6ce4845ce34a8ad9e68117c10ee90a29919eba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71", - "reference": "402d35bcb92c70c026d1a6a9883f06b2ead23d71", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/cb6ce4845ce34a8ad9e68117c10ee90a29919eba", + "reference": "cb6ce4845ce34a8ad9e68117c10ee90a29919eba", "shasum": "" }, "require": { @@ -2246,7 +2058,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.0.x-dev" + "dev-master": "1.1.x-dev" } }, "autoload": { @@ -2261,7 +2073,7 @@ "authors": [ { "name": "PHP-FIG", - "homepage": "https://www.php-fig.org/" + "homepage": "http://www.php-fig.org/" } ], "description": "Common interface for HTTP messages", @@ -2275,9 +2087,9 @@ "response" ], "support": { - "source": "https://github.com/php-fig/http-message/tree/2.0" + "source": "https://github.com/php-fig/http-message/tree/1.1" }, - "time": "2023-04-04T09:54:51+00:00" + "time": "2023-04-04T09:50:52+00:00" }, { "name": "psr/log", @@ -2426,16 +2238,16 @@ }, { "name": "symfony/console", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91" + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/be5854cee0e8c7b110f00d695d11debdfa1a2a91", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91", + "url": "https://api.github.com/repos/symfony/console/zipball/6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", "shasum": "" }, "require": { @@ -2500,7 +2312,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.8" + "source": "https://github.com/symfony/console/tree/v6.4.9" }, "funding": [ { @@ -2516,7 +2328,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -3135,16 +2947,16 @@ }, { "name": "symfony/string", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d" + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/a147c0f826c4a1f3afb763ab8e009e37c877a44d", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d", + "url": "https://api.github.com/repos/symfony/string/zipball/76792dbd99690a5ebef8050d9206c60c59e681d7", + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7", "shasum": "" }, "require": { @@ -3201,7 +3013,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.8" + "source": "https://github.com/symfony/string/tree/v6.4.9" }, "funding": [ { @@ -3217,7 +3029,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:25:38+00:00" }, { "name": "symfony/translation", @@ -3515,155 +3327,6 @@ } ], "packages-dev": [ - { - "name": "aws/aws-crt-php", - "version": "v1.2.6", - "source": { - "type": "git", - "url": "https://github.com/awslabs/aws-crt-php.git", - "reference": "a63485b65b6b3367039306496d49737cf1995408" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/awslabs/aws-crt-php/zipball/a63485b65b6b3367039306496d49737cf1995408", - "reference": "a63485b65b6b3367039306496d49737cf1995408", - "shasum": "" - }, - "require": { - "php": ">=5.5" - }, - "require-dev": { - "phpunit/phpunit": "^4.8.35||^5.6.3||^9.5", - "yoast/phpunit-polyfills": "^1.0" - }, - "suggest": { - "ext-awscrt": "Make sure you install awscrt native extension to use any of the functionality." - }, - "type": "library", - "autoload": { - "classmap": [ - "src/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "AWS SDK Common Runtime Team", - "email": "aws-sdk-common-runtime@amazon.com" - } - ], - "description": "AWS Common Runtime for PHP", - "homepage": "https://github.com/awslabs/aws-crt-php", - "keywords": [ - "amazon", - "aws", - "crt", - "sdk" - ], - "support": { - "issues": "https://github.com/awslabs/aws-crt-php/issues", - "source": "https://github.com/awslabs/aws-crt-php/tree/v1.2.6" - }, - "time": "2024-06-13T17:21:28+00:00" - }, - { - "name": "aws/aws-sdk-php", - "version": "3.314.8", - "source": { - "type": "git", - "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "cec444ca2e86dade32886d586ac55838779e2ae2" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/cec444ca2e86dade32886d586ac55838779e2ae2", - "reference": "cec444ca2e86dade32886d586ac55838779e2ae2", - "shasum": "" - }, - "require": { - "aws/aws-crt-php": "^1.2.3", - "ext-json": "*", - "ext-pcre": "*", - "ext-simplexml": "*", - "guzzlehttp/guzzle": "^6.5.8 || ^7.4.5", - "guzzlehttp/promises": "^1.4.0 || ^2.0", - "guzzlehttp/psr7": "^1.9.1 || ^2.4.5", - "mtdowling/jmespath.php": "^2.6", - "php": ">=7.2.5", - "psr/http-message": "^1.0 || ^2.0" - }, - "require-dev": { - "andrewsville/php-token-reflection": "^1.4", - "aws/aws-php-sns-message-validator": "~1.0", - "behat/behat": "~3.0", - "composer/composer": "^1.10.22", - "dms/phpunit-arraysubset-asserts": "^0.4.0", - "doctrine/cache": "~1.4", - "ext-dom": "*", - "ext-openssl": "*", - "ext-pcntl": "*", - "ext-sockets": "*", - "nette/neon": "^2.3", - "paragonie/random_compat": ">= 2", - "phpunit/phpunit": "^5.6.3 || ^8.5 || ^9.5", - "psr/cache": "^1.0", - "psr/simple-cache": "^1.0", - "sebastian/comparator": "^1.2.3 || ^4.0", - "yoast/phpunit-polyfills": "^1.0" - }, - "suggest": { - "aws/aws-php-sns-message-validator": "To validate incoming SNS notifications", - "doctrine/cache": "To use the DoctrineCacheAdapter", - "ext-curl": "To send requests using cURL", - "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", - "ext-sockets": "To use client-side monitoring" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "Aws\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Amazon Web Services", - "homepage": "http://aws.amazon.com" - } - ], - "description": "AWS SDK for PHP - Use Amazon Web Services in your PHP project", - "homepage": "http://aws.amazon.com/sdkforphp", - "keywords": [ - "amazon", - "aws", - "cloud", - "dynamodb", - "ec2", - "glacier", - "s3", - "sdk" - ], - "support": { - "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", - "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.314.8" - }, - "time": "2024-06-25T18:13:28+00:00" - }, { "name": "brick/math", "version": "0.12.1", @@ -3900,197 +3563,6 @@ }, "time": "2022-08-14T11:40:18+00:00" }, - { - "name": "league/flysystem-aws-s3-v3", - "version": "3.28.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", - "reference": "22071ef1604bc776f5ff2468ac27a752514665c8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/22071ef1604bc776f5ff2468ac27a752514665c8", - "reference": "22071ef1604bc776f5ff2468ac27a752514665c8", - "shasum": "" - }, - "require": { - "aws/aws-sdk-php": "^3.295.10", - "league/flysystem": "^3.10.0", - "league/mime-type-detection": "^1.0.0", - "php": "^8.0.2" - }, - "conflict": { - "guzzlehttp/guzzle": "<7.0", - "guzzlehttp/ringphp": "<1.1.1" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\Flysystem\\AwsS3V3\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frank de Jonge", - "email": "info@frankdejonge.nl" - } - ], - "description": "AWS S3 filesystem adapter for Flysystem.", - "keywords": [ - "Flysystem", - "aws", - "file", - "files", - "filesystem", - "s3", - "storage" - ], - "support": { - "source": "https://github.com/thephpleague/flysystem-aws-s3-v3/tree/3.28.0" - }, - "time": "2024-05-06T20:05:52+00:00" - }, - { - "name": "league/flysystem-azure-blob-storage", - "version": "3.28.0", - "source": { - "type": "git", - "url": "https://github.com/thephpleague/flysystem-azure-blob-storage.git", - "reference": "7856cc57ae74b0cf673b924e05c236c4f5dbdcb0" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem-azure-blob-storage/zipball/7856cc57ae74b0cf673b924e05c236c4f5dbdcb0", - "reference": "7856cc57ae74b0cf673b924e05c236c4f5dbdcb0", - "shasum": "" - }, - "require": { - "league/flysystem": "^3.10.0", - "microsoft/azure-storage-blob": "^1.1", - "php": "^8.0.2" - }, - "type": "library", - "autoload": { - "psr-4": { - "League\\Flysystem\\AzureBlobStorage\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Frank de Jonge", - "email": "info@frankdejonge.nl" - } - ], - "support": { - "source": "https://github.com/thephpleague/flysystem-azure-blob-storage/tree/3.28.0" - }, - "time": "2024-05-06T20:05:52+00:00" - }, - { - "name": "microsoft/azure-storage-blob", - "version": "1.5.4", - "source": { - "type": "git", - "url": "https://github.com/Azure/azure-storage-blob-php.git", - "reference": "1023ce1dbf062351a32ca5ec72ad1fd4a504f1bf" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Azure/azure-storage-blob-php/zipball/1023ce1dbf062351a32ca5ec72ad1fd4a504f1bf", - "reference": "1023ce1dbf062351a32ca5ec72ad1fd4a504f1bf", - "shasum": "" - }, - "require": { - "microsoft/azure-storage-common": "~1.5", - "php": ">=5.6.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "MicrosoftAzure\\Storage\\Blob\\": "src/Blob" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Azure Storage PHP Client Library", - "email": "dmsh@microsoft.com" - } - ], - "description": "This project provides a set of PHP client libraries that make it easy to access Microsoft Azure Storage Blob APIs.", - "keywords": [ - "azure", - "blob", - "php", - "sdk", - "storage" - ], - "support": { - "issues": "https://github.com/Azure/azure-storage-blob-php/issues", - "source": "https://github.com/Azure/azure-storage-blob-php/tree/v1.5.4" - }, - "time": "2022-09-02T02:13:06+00:00" - }, - { - "name": "microsoft/azure-storage-common", - "version": "1.5.2", - "source": { - "type": "git", - "url": "https://github.com/Azure/azure-storage-common-php.git", - "reference": "8ca7b1bf4c9ca7c663e75a02a0035b05b37196a0" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Azure/azure-storage-common-php/zipball/8ca7b1bf4c9ca7c663e75a02a0035b05b37196a0", - "reference": "8ca7b1bf4c9ca7c663e75a02a0035b05b37196a0", - "shasum": "" - }, - "require": { - "guzzlehttp/guzzle": "~6.0|^7.0", - "php": ">=5.6.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "MicrosoftAzure\\Storage\\Common\\": "src/Common" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Azure Storage PHP Client Library", - "email": "dmsh@microsoft.com" - } - ], - "description": "This project provides a set of common code shared by Azure Storage Blob, Table, Queue and File PHP client libraries.", - "keywords": [ - "azure", - "common", - "php", - "sdk", - "storage" - ], - "support": { - "issues": "https://github.com/Azure/azure-storage-common-php/issues", - "source": "https://github.com/Azure/azure-storage-common-php/tree/v1.5.2" - }, - "time": "2021-10-09T03:03:47+00:00" - }, { "name": "moneyphp/money", "version": "v4.5.0", @@ -4179,72 +3651,6 @@ }, "time": "2024-02-15T19:47:21+00:00" }, - { - "name": "mtdowling/jmespath.php", - "version": "2.7.0", - "source": { - "type": "git", - "url": "https://github.com/jmespath/jmespath.php.git", - "reference": "bbb69a935c2cbb0c03d7f481a238027430f6440b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/jmespath/jmespath.php/zipball/bbb69a935c2cbb0c03d7f481a238027430f6440b", - "reference": "bbb69a935c2cbb0c03d7f481a238027430f6440b", - "shasum": "" - }, - "require": { - "php": "^7.2.5 || ^8.0", - "symfony/polyfill-mbstring": "^1.17" - }, - "require-dev": { - "composer/xdebug-handler": "^3.0.3", - "phpunit/phpunit": "^8.5.33" - }, - "bin": [ - "bin/jp.php" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.7-dev" - } - }, - "autoload": { - "files": [ - "src/JmesPath.php" - ], - "psr-4": { - "JmesPath\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Graham Campbell", - "email": "hello@gjcampbell.co.uk", - "homepage": "https://github.com/GrahamCampbell" - }, - { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" - } - ], - "description": "Declaratively specify how to extract elements from a JSON document", - "keywords": [ - "json", - "jsonpath" - ], - "support": { - "issues": "https://github.com/jmespath/jmespath.php/issues", - "source": "https://github.com/jmespath/jmespath.php/tree/2.7.0" - }, - "time": "2023-08-25T10:54:48+00:00" - }, { "name": "nyholm/psr7", "version": "1.8.1", @@ -5096,16 +4502,16 @@ }, { "name": "symfony/var-exporter", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/var-exporter.git", - "reference": "792ca836f99b340f2e9ca9497c7953948c49a504" + "reference": "f9a060622e0d93777b7f8687ec4860191e16802e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/var-exporter/zipball/792ca836f99b340f2e9ca9497c7953948c49a504", - "reference": "792ca836f99b340f2e9ca9497c7953948c49a504", + "url": "https://api.github.com/repos/symfony/var-exporter/zipball/f9a060622e0d93777b7f8687ec4860191e16802e", + "reference": "f9a060622e0d93777b7f8687ec4860191e16802e", "shasum": "" }, "require": { @@ -5153,7 +4559,7 @@ "serialize" ], "support": { - "source": "https://github.com/symfony/var-exporter/tree/v6.4.8" + "source": "https://github.com/symfony/var-exporter/tree/v6.4.9" }, "funding": [ { @@ -5169,7 +4575,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-24T15:53:56+00:00" } ], "aliases": [], diff --git a/docs/components/adapters/chartjs.md b/docs/components/adapters/chartjs.md index 45b46c2c7..3c2fcf9ef 100644 --- a/docs/components/adapters/chartjs.md +++ b/docs/components/adapters/chartjs.md @@ -18,4 +18,43 @@ with the robust and adaptable framework of the Flow PHP ecosystem. ``` composer require flow-php/etl-adapter-chartjs +``` + +## Usage + +```php + '2023-01-01', 'Revenue' => 10000.53, 'CM' => 5000.12, 'Ads Spends' => 2000.78, 'Storage Costs' => 1000.34, 'Shipping Costs' => 1500.45, 'Currency' => 'USD'], + ['Date' => '2023-01-02', 'Revenue' => 10234.56, 'CM' => 5102.23, 'Ads Spends' => 2050.12, 'Storage Costs' => 1050.78, 'Shipping Costs' => 1550.99, 'Currency' => 'USD'], + ['Date' => '2023-01-03', 'Revenue' => 11000.98, 'CM' => 5200.32, 'Ads Spends' => 2100.67, 'Storage Costs' => 1100.87, 'Shipping Costs' => 1600.34, 'Currency' => 'USD'], + ['Date' => '2023-01-04', 'Revenue' => 10890.34, 'CM' => 5300.98, 'Ads Spends' => 2150.56, 'Storage Costs' => 1150.67, 'Shipping Costs' => 1650.87, 'Currency' => 'USD'], + ['Date' => '2023-01-05', 'Revenue' => 13750.12, 'CM' => 5950.78, 'Ads Spends' => 2750.78, 'Storage Costs' => 1750.78, 'Shipping Costs' => 2250.12, 'Currency' => 'USD'], + ['Date' => '2023-02-06', 'Revenue' => 14000.23, 'CM' => 6000.89, 'Ads Spends' => 2800.89, 'Storage Costs' => 1800.89, 'Shipping Costs' => 2300.23, 'Currency' => 'USD'], +]; + +df() + ->read(from_array($data)) + ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2))) + ->write( + to_chartjs_file( + $chart = bar_chart( + ref('Date'), + refs( + ref('Revenue'), + ref('CM'), + ref('Ads Spends'), + ref('Storage Costs'), + ref('Shipping Costs'), + ref('Profit'), + ) + ), + $output = __DIR__ . '/Output/bar_chart.html' + ) + ) + ->run(); ``` \ No newline at end of file diff --git a/docs/components/bridges/filesystem-azure-bridge.md b/docs/components/bridges/filesystem-azure-bridge.md new file mode 100644 index 000000000..b22a911cb --- /dev/null +++ b/docs/components/bridges/filesystem-azure-bridge.md @@ -0,0 +1,45 @@ +# Filesystem Azure Bridge + +- [⬅️️ Back](../../introduction.md) + +The Filesystem Azure Bridge is a bridge that allows you to use the Azure Blob Storage as a filesystem in your application. + +## Installation + +```bash +composer require flow-php/filesystem-azure-bridge +``` + +## Usage + +> [!NOTE] +> Since the Azure SDK is not providing any http client or factories, you need to install them manually. +> The following example uses the `php-http/discovery` package to find the factories in your project existing dependencies. +> Use below links to find the implementations for client and the factories: + +- [Http Client](https://packagist.org/providers/psr/http-client-implementation) +- [Http Factories](https://packagist.org/providers/psr/http-factory-implementation) + + +```php +pushProcessor( + new PSR7Processor( + new Config( + request: new RequestConfig(withBody: true, bodySizeLimit: 200), + response: new ResponseConfig(withBody: true) + ) + ) +); +$logger->pushHandler(new StreamHandler(__DIR__ . '/logs.txt', LogLevel::DEBUG)); +``` + +## Configuration + +The Processor can be configured to normalize Request/Response objects in different ways. + +For more details, please refer to the following classes: + + - `Flow\Bridge\Monolog\Http\Config\RequestConfig` + - `Flow\Bridge\Monolog\Http\Config\ResponseConfig` diff --git a/docs/components/libs/azure-sdk.md b/docs/components/libs/azure-sdk.md new file mode 100644 index 000000000..abcd73d29 --- /dev/null +++ b/docs/components/libs/azure-sdk.md @@ -0,0 +1,72 @@ +# Azure SDK + +- [⬅️️ Back](../../introduction.md) + +Simple, lightweight, dependency-free and efficient Azure SDK for PHP. + +## Installation + +```bash +composer require flow-php/azure-sdk +``` + +> [!NOTE] +> Since the Azure SDK is not providing any http client or factories, you need to install them manually. +> The following example uses the `php-http/discovery` package to find the factories in your project existing dependencies. +> Use below links to find the implementations for client and the factories: + +- [Http Client](https://packagist.org/providers/psr/http-client-implementation) +- [Http Factories](https://packagist.org/providers/psr/http-factory-implementation) + +> [!TIP] +> To fully benefit from SDK features, you need to install the following packages: +> `composer require flow-php/monolog-http-bridge` that will normalize request/response objects from logs context + +> [!WARNING] +> This implementation is not fully covering Azure SDK, only Storage related services are implemented. +> Feel free to contribute to the project and add more services. + +## Usage + +The absolute minimum configuration to start using the SDK is to provide the account name and the account key. + +```php +for(protocol('azure-blob'))->writeTo(path('azure-blob://orders.csv')); + +$stream->append('id,name,active'); +$stream->append('1,norbert,true'); +$stream->append('2,john,true'); +$stream->append('3,jane,true'); +$stream->close(); +``` \ No newline at end of file diff --git a/docs/components/libs/parquet-viewer.md b/docs/components/libs/parquet-viewer.md index 3d4d8ac43..c43e2b321 100644 --- a/docs/components/libs/parquet-viewer.md +++ b/docs/components/libs/parquet-viewer.md @@ -7,3 +7,12 @@ ``` composer require flow-php/parquet-viewer ``` + +Parquet Viewer is a simple CLI tool to inspect and view the content and metadata of parquet files. + +## Usage + +```bash +./vendor/bin/parquet.php read:data /path/to/file.parquet +./vendor/bin/parquet.php read:metadata /path/to/file.parquet --columns --row-groups --column-chunks --statistics --page-headers +``` \ No newline at end of file diff --git a/docs/components/libs/parquet.md b/docs/components/libs/parquet.md index 2b1c9d1b5..20a44e156 100644 --- a/docs/components/libs/parquet.md +++ b/docs/components/libs/parquet.md @@ -229,7 +229,7 @@ $writer->close(); We can also open a file for a resource: ```php -$writer->openForStream($resource, $schema); +$writer->openForStream($stream, $schema); ``` ### Writing a single row @@ -244,29 +244,6 @@ $writer->writeRow($row); $writer->close(); ``` -### Appending data to existing file - -Like with writing to the file we can append entire dataset or batch or single row. - -```php -$writter->append($path, $rows); -``` - -First we need to reopen a file or stream: - -```php -$writer->reopen($path); -$writer->reopenForStream(\fopen($path, 'rb+')); - -$writer->writeBatch([$row, $row]); -$writer->writeBatch([$row, $row]); -$writer->writeBatch([$row, $row]); -$writer->writeBatch([$row, $row]); -$writer->writeBatch([$row, $row]); -``` - -As we can see, we don't need to provide a schema as it is already stored in the file. - > [!WARNING] > At this point, schema evolution is not yet supported. > We need to make sure that schema is the same as the one used to create a file. diff --git a/docs/installation.md b/docs/installation.md index 872cc4c9c..2ccaedd90 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -26,9 +26,13 @@ this will reduce the number of unnecessary dependencies in your project (less ma - [array-dot](components/libs/array-dot.md) - [doctrine-dbal-bulk](components/libs/doctrine-dbal-bulk.md) - [dremel](components/libs/dremel.md) + - [filesystem](components/libs/filesystem.md) - [parquet](components/libs/parquet.md) - [parquet-viewer](components/libs/parquet-viewer.md) - [snappy](components/libs/snappy.md) +- Bridges + - [filesystem-azure](components/bridges/filesystem-azure-bridge.md) + - [monolog-http](components/bridges/monolog-http-bridge.md) For example, if you want to work with JSON/CSV files here are the dependencies you will need to install: diff --git a/examples/topics/cloud_storage/aws_s3/.env.dist b/examples/topics/cloud_storage/aws_s3/.env.dist deleted file mode 100644 index 78ac4e1c4..000000000 --- a/examples/topics/cloud_storage/aws_s3/.env.dist +++ /dev/null @@ -1,2 +0,0 @@ -AWS_S3_KEY="xxxx" -AWS_S3_SECRET="xxxx" \ No newline at end of file diff --git a/examples/topics/cloud_storage/aws_s3/code.php b/examples/topics/cloud_storage/aws_s3/code.php deleted file mode 100644 index 6ce92fa21..000000000 --- a/examples/topics/cloud_storage/aws_s3/code.php +++ /dev/null @@ -1,45 +0,0 @@ -load(__DIR__ . '/.env'); - -$s3_client_option = [ - 'client' => [ - 'credentials' => [ - 'key' => $_ENV['AWS_S3_KEY'], - 'secret' => $_ENV['AWS_S3_SECRET'], - ], - 'region' => 'eu-west-2', - 'version' => 'latest', - ], - 'bucket' => 'flow-php', -]; - -AwsS3Stream::register(); - -data_frame() - ->read(from_array([ - ['id' => 1, 'name' => 'test'], - ['id' => 2, 'name' => 'test'], - ['id' => 3, 'name' => 'test'], - ['id' => 4, 'name' => 'test'], - ])) - ->saveMode(overwrite()) - ->write(to_csv(new Path('flow-aws-s3://test.csv', $s3_client_option))) - ->run(); diff --git a/examples/topics/cloud_storage/azure/.env.dist b/examples/topics/cloud_storage/azure/.env.dist index 4432ce8f6..42b9bdec6 100644 --- a/examples/topics/cloud_storage/azure/.env.dist +++ b/examples/topics/cloud_storage/azure/.env.dist @@ -1,2 +1,3 @@ -AZURE_CONNECTION_STRING="xxxx" -AZURE_CONTAINER="xxxx" \ No newline at end of file +AZURE_ACCOUNT="xxx" +AZURE_ACCOUNT_KEY="xxxx" +AZURE_CONTAINER="xxx" \ No newline at end of file diff --git a/examples/topics/cloud_storage/azure/code.php b/examples/topics/cloud_storage/azure/code.php index 516d83895..db89ff3e4 100644 --- a/examples/topics/cloud_storage/azure/code.php +++ b/examples/topics/cloud_storage/azure/code.php @@ -2,10 +2,11 @@ declare(strict_types=1); +use function Flow\Azure\SDK\DSL\{azure_blob_service, azure_blob_service_config, azure_shared_key_authorization_factory}; use function Flow\ETL\Adapter\CSV\to_csv; -use function Flow\ETL\DSL\{data_frame, from_array, overwrite}; -use Flow\ETL\Adapter\Filesystem\AzureBlobStream; -use Flow\ETL\Filesystem\Path; +use function Flow\ETL\DSL\{config_builder, data_frame, from_array, overwrite}; +use function Flow\Filesystem\Bridge\Azure\DSL\azure_filesystem; +use function Flow\Filesystem\DSL\path; use Symfony\Component\Dotenv\Dotenv; require __DIR__ . '/../../../autoload.php'; @@ -19,14 +20,23 @@ $dotenv = new Dotenv(); $dotenv->load(__DIR__ . '/.env'); -$azure_client_option = [ - 'connection-string' => $_ENV['AZURE_CONNECTION_STRING'], - 'container' => $_ENV['AZURE_CONTAINER'], -]; - -AzureBlobStream::register(); - -data_frame() +$config = config_builder() + ->mount( + azure_filesystem( + azure_blob_service( + azure_blob_service_config( + $_ENV['AZURE_ACCOUNT'], + $_ENV['AZURE_CONTAINER'] + ), + azure_shared_key_authorization_factory( + $_ENV['AZURE_ACCOUNT'], + $_ENV['AZURE_ACCOUNT_KEY'] + ), + ) + ) + ); + +data_frame($config) ->read(from_array([ ['id' => 1, 'name' => 'test'], ['id' => 2, 'name' => 'test'], @@ -34,5 +44,5 @@ ['id' => 4, 'name' => 'test'], ])) ->saveMode(overwrite()) - ->write(to_csv(new Path('flow-azure-blob://test.csv', $azure_client_option))) + ->write(to_csv(path('azure-blob://test.csv'))) ->run(); diff --git a/monorepo-builder.php b/monorepo-builder.php index 4b8f6a442..c9032daaa 100644 --- a/monorepo-builder.php +++ b/monorepo-builder.php @@ -10,6 +10,7 @@ $config->packageDirectories([ __DIR__ . '/src/core', __DIR__ . '/src/adapter', + __DIR__ . '/src/bridge', __DIR__ . '/src/lib', ]); diff --git a/phpbench.json b/phpbench.json index 4700ecb31..3231d07a0 100644 --- a/phpbench.json +++ b/phpbench.json @@ -16,7 +16,6 @@ } }, "runner.path": [ - "src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/", "src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Benchmark/", "src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Benchmark/", "src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Benchmark/", diff --git a/phpstan.neon b/phpstan.neon index cf5bec63c..d515aeed0 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -10,7 +10,6 @@ parameters: - src/adapter/etl-adapter-csv/src - src/adapter/etl-adapter-doctrine/src - src/adapter/etl-adapter-elasticsearch/src - - src/adapter/etl-adapter-filesystem/src - src/adapter/etl-adapter-google-sheet/src - src/adapter/etl-adapter-http/src - src/adapter/etl-adapter-json/src @@ -19,12 +18,17 @@ parameters: - src/adapter/etl-adapter-parquet/src - src/adapter/etl-adapter-text/src - src/adapter/etl-adapter-xml/src + - src/bridge/filesystem/azure/src + - src/bridge/monolog/http/src - src/lib/array-dot/src + - src/lib/azure-sdk/src - src/lib/doctrine-dbal-bulk/src - src/lib/dremel/src + - src/lib/filesystem/src - src/lib/parquet/src - - src/lib/snappy/src + - src/lib/parquet-viewer/src - src/lib/rdsl/src + - src/lib/snappy/src - examples/topics excludePaths: diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 41b8073f2..fc81726fd 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -15,6 +15,10 @@ + + + + @@ -27,12 +31,14 @@ src/adapter/**/**/**/**/**/**/Tests/Unit + src/bridge/**/**/**/**/**/**/Tests/Unit src/core/etl/tests/Flow/ETL/Tests/Unit src/lib/**/**/**/**/Tests/Unit src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Unit src/adapter/**/**/**/**/**/**/Tests/Integration + src/bridge/**/**/**/**/**/**/Tests/Integration src/core/etl/tests/Flow/ETL/Tests/Integration src/lib/**/**/**/**/Tests/Integration src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Integration @@ -41,6 +47,7 @@ src/adapter/**/src + src/bridge/**/src src/core/**/src src/lib/**/src diff --git a/psalm.xml b/psalm.xml index d2e8839ab..acc62d7aa 100644 --- a/psalm.xml +++ b/psalm.xml @@ -11,6 +11,7 @@ > + @@ -24,9 +25,6 @@ - - - @@ -36,9 +34,11 @@ + + diff --git a/src/adapter/etl-adapter-avro/README.md b/src/adapter/etl-adapter-avro/README.md index 1be590881..3c0404ca2 100644 --- a/src/adapter/etl-adapter-avro/README.md +++ b/src/adapter/etl-adapter-avro/README.md @@ -3,7 +3,7 @@ Avro integration was temporarily abandoned due to the lack of availability of good libraries for PHP. If you are interested in this integration, please let us know by creating an issue in the repository. -At some point in the future we are going to write our own Avro library for PHP, but it is not a priority at the moment. +At some point in the future, we are going to write our own Avro library for PHP, but it is not a priority at the moment. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/avro.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-avro/composer.json b/src/adapter/etl-adapter-avro/composer.json index b7cc61fca..38f2357a2 100644 --- a/src/adapter/etl-adapter-avro/composer.json +++ b/src/adapter/etl-adapter-avro/composer.json @@ -13,7 +13,6 @@ "require": { "php": "~8.1.0 || ~8.2.0 || ~8.3.0", "ext-json": "*", - "flix-tech/avro-php": "~4.2.0 || ~4.3.0", "flow-php/etl": "^0.7 || 1.x-dev" }, "config": { diff --git a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php index 172cf7b12..17bc4e75a 100644 --- a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php +++ b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroExtractor.php @@ -4,14 +4,14 @@ namespace Flow\ETL\Adapter\Avro\FlixTech; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering}; -use Flow\ETL\Filesystem\Path; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor}; use Flow\ETL\{Exception\RuntimeException, Extractor, FlowContext}; +use Flow\Filesystem\Path; final class AvroExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { + use Extractor\PathFiltering; use Limitable; - use PartitionFiltering; public function __construct(private readonly Path $path) { diff --git a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php index 7cdfd7006..275328de6 100644 --- a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php +++ b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/AvroLoader.php @@ -4,10 +4,10 @@ namespace Flow\ETL\Adapter\Avro\FlixTech; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Loader\Closure; use Flow\ETL\Row\Schema; use Flow\ETL\{Exception\RuntimeException, FlowContext, Loader, Rows}; +use Flow\Filesystem\Path; final class AvroLoader implements Closure, Loader, Loader\FileLoader { diff --git a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php index 64a5aadb8..395e61d0a 100644 --- a/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php +++ b/src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php @@ -7,8 +7,8 @@ use function Flow\ETL\DSL\from_all; use Flow\ETL\Adapter\Avro\FlixTech\{AvroExtractor, AvroLoader}; use Flow\ETL\Extractor; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Row\Schema; +use Flow\Filesystem\Path; function from_avro(Path|string|array $path) : Extractor { diff --git a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroExtractorBench.php b/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroExtractorBench.php deleted file mode 100644 index 8babdcc27..000000000 --- a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroExtractorBench.php +++ /dev/null @@ -1,24 +0,0 @@ -context = new FlowContext(Config::default()); - } - - public function bench_extract_10k() : void - { - foreach (from_avro(__DIR__ . '/../Fixtures/orders_flow.avro')->extract($this->context) as $rows) { - } - } -} diff --git a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroLoaderBench.php b/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroLoaderBench.php deleted file mode 100644 index ec6c8307e..000000000 --- a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Benchmark/AvroLoaderBench.php +++ /dev/null @@ -1,42 +0,0 @@ -context = new FlowContext(Config::default()); - $this->outputPath = \tempnam(\sys_get_temp_dir(), 'etl_avro_loader_bench') . '.avro'; - $this->rows = new Rows(); - - foreach (from_avro(__DIR__ . '/../Fixtures/orders_flow.avro')->extract($this->context) as $rows) { - $this->rows = $this->rows->merge($rows); - } - } - - public function __destruct() - { - if (!\file_exists($this->outputPath)) { - throw new \RuntimeException("Benchmark failed, \"{$this->outputPath}\" doesn't exist"); - } - - \unlink($this->outputPath); - } - - public function bench_load_10k() : void - { - to_avro($this->outputPath)->load($this->rows, $this->context); - } -} diff --git a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php b/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php index 45ac8ce83..32fefb323 100644 --- a/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php +++ b/src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php @@ -4,13 +4,11 @@ namespace Flow\ETL\Adapter\Avro\Tests\Integration; -use function Flow\ETL\DSL\Adapter\Avro\{from_avro, to_avro}; -use function Flow\ETL\DSL\{df, from_array, lit, type_map, type_string}; +use function Flow\ETL\DSL\Adapter\Avro\{to_avro}; use Flow\ETL\Adapter\Avro\FlixTech\AvroExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\Tests\Double\FakeExtractor; -use Flow\ETL\{Config, Flow, FlowContext}; +use Flow\ETL\{Config, FlowContext}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class AvroTest extends TestCase @@ -22,18 +20,7 @@ protected function setUp() : void public function test_limit() : void { - $path = \sys_get_temp_dir() . '/avro_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - df() - ->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_avro($path)) - ->run(); - - $extractor = new AvroExtractor(Path::realpath($path)); + $extractor = new AvroExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro')); $extractor->changeLimit(2); self::assertCount( @@ -44,28 +31,14 @@ public function test_limit() : void public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/avro_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - df() - ->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_avro($path)) - ->run(); - - $extractor = new AvroExtractor(Path::realpath($path)); + $extractor = new AvroExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.avro')); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame([['id' => 1]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 2]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 3]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); @@ -77,40 +50,4 @@ public function test_using_pattern_path() : void to_avro(new Path('/path/*/pattern.avro')); } - - public function test_writing_and_reading_avro_with_all_supported_types() : void - { - $this->removeFile($path = \sys_get_temp_dir() . '/file.avro'); - - df() - ->read(new FakeExtractor(100)) - ->drop('null', 'array', 'object', 'enum', 'map') - // avro maps support only string keys - ->withEntry('map', lit(['0' => 'zero', '1' => 'one'])->cast(type_map(type_string(), type_string()))) - ->batchSize(10) - ->write(to_avro($path)) - ->run(); - - self::assertFileExists($path); - - self::assertEquals( - 100, - Flow::setUp(Config::builder()->putInputIntoRows()->build()) - ->read(from_avro($path)) - ->drop('_input_file_uri') - ->count() - ); - - $this->removeFile($path); - } - - /** - * @param string $path - */ - private function removeFile(string $path) : void - { - if (\file_exists($path) && !\is_dir($path)) { - \unlink($path); - } - } } diff --git a/src/adapter/etl-adapter-chartjs/README.md b/src/adapter/etl-adapter-chartjs/README.md index c9249f761..b72abd230 100644 --- a/src/adapter/etl-adapter-chartjs/README.md +++ b/src/adapter/etl-adapter-chartjs/README.md @@ -12,5 +12,5 @@ data visualization in large-scale and data-intensive environments. With Flow PHP rendering and interaction within your ETL workflows becomes a more refined and efficient endeavor, harmoniously aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/chartjs.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php index 31c513575..d5d8e7b03 100644 --- a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php +++ b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/ChartJSLoader.php @@ -4,9 +4,9 @@ namespace Flow\ETL\Adapter\ChartJS; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Loader\Closure; use Flow\ETL\{FlowContext, Loader, Rows}; +use Flow\Filesystem\Path; final class ChartJSLoader implements Closure, Loader { @@ -33,8 +33,7 @@ public function closure(FlowContext $context) : void $templateStream = $context->streams()->read($this->template); - /** @var string $template */ - $template = \stream_get_contents($templateStream->resource()); + $template = \implode('', \iterator_to_array($templateStream->readLines())); $templateStream->close(); $content = \str_replace( @@ -43,7 +42,7 @@ public function closure(FlowContext $context) : void $template ); - \fwrite($output->resource(), $content); + $output->append($content); $context->streams()->closeWriters($this->output); } diff --git a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php index 0b7bacc9a..b8db21951 100644 --- a/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php +++ b/src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Adapter\ChartJS; use Flow\ETL\Adapter\ChartJS\Chart\{BarChart, LineChart, PieChart}; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Row\{EntryReference, References}; +use Flow\Filesystem\Path; function bar_chart(EntryReference $label, References $datasets) : BarChart { diff --git a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php index 1d4f86348..a81102e87 100644 --- a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php +++ b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/ChartJSLoaderTest.php @@ -5,8 +5,7 @@ namespace Flow\ETL\Adapter\ChartJS\Tests\Integration; use function Flow\ETL\Adapter\ChartJS\{bar_chart, line_chart, pie_chart, to_chartjs_file, to_chartjs_var}; -use function Flow\ETL\DSL\{df, first, from_memory, lit, ref, refs, sum}; -use Flow\ETL\Memory\ArrayMemory; +use function Flow\ETL\DSL\{df, first, from_array, lit, ref, refs, sum}; use PHPUnit\Framework\TestCase; final class ChartJSLoaderTest extends TestCase @@ -23,7 +22,7 @@ public function test_loading_data_to_bar_chart() : void ]; df() - ->read(from_memory(new ArrayMemory($data))) + ->read(from_array($data)) ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2))) ->write( to_chartjs_file( @@ -95,7 +94,7 @@ public function test_loading_data_to_bar_chart_output_variable() : void $output = []; df() - ->read(from_memory(new ArrayMemory($data))) + ->read(from_array($data)) ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2))) ->write( to_chartjs_var( @@ -164,7 +163,7 @@ public function test_loading_data_to_line_chart() : void ]; df() - ->read(from_memory(new ArrayMemory($data))) + ->read(from_array($data)) ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2))) ->write( to_chartjs_file( @@ -247,7 +246,7 @@ public function test_loading_data_to_pie_chart() : void ->setOptions(['label' => 'PnL']); df() - ->read(from_memory(new ArrayMemory($data))) + ->read(from_array($data)) ->withEntry('Profit', ref('Revenue')->minus(ref('CM'))->minus(ref('Ads Spends'))->minus(ref('Storage Costs'))->minus(ref('Shipping Costs'))->round(lit(2))) ->aggregate( first(ref('Date')->as('Date')), diff --git a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/bar_chart.html b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/bar_chart.html index e1b6c02f3..ee7bbd722 100644 --- a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/bar_chart.html +++ b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/bar_chart.html @@ -1,17 +1 @@ - - - - Flow PHP - ChartJS - - - - -
- -
- - \ No newline at end of file + Flow PHP - ChartJS
\ No newline at end of file diff --git a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/line_chart.html b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/line_chart.html index 2e05fdca7..33ee0fd4f 100644 --- a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/line_chart.html +++ b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/line_chart.html @@ -1,17 +1 @@ - - - - Flow PHP - ChartJS - - - - -
- -
- - \ No newline at end of file + Flow PHP - ChartJS
\ No newline at end of file diff --git a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/pie_chart.html b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/pie_chart.html index 30510b499..f0a544138 100644 --- a/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/pie_chart.html +++ b/src/adapter/etl-adapter-chartjs/tests/Flow/ETL/Adapter/ChartJS/Tests/Integration/Output/pie_chart.html @@ -1,17 +1 @@ - - - - Flow PHP - ChartJS - - - - -
- -
- - \ No newline at end of file + Flow PHP - ChartJS
\ No newline at end of file diff --git a/src/adapter/etl-adapter-csv/README.md b/src/adapter/etl-adapter-csv/README.md index c35fcbce7..57ff3fcb3 100644 --- a/src/adapter/etl-adapter-csv/README.md +++ b/src/adapter/etl-adapter-csv/README.md @@ -11,5 +11,5 @@ making it a prime choice for developers dealing with CSV data in large-scale and PHP's Adapter CSV, managing CSV data within your ETL workflows becomes a more simplified and efficient task, perfectly aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/csv.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDetector.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDetector.php index 031da5bbe..c9fd46c85 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDetector.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVDetector.php @@ -7,6 +7,7 @@ use Flow\ETL\Adapter\CSV\Detector\{Option, Options}; use Flow\ETL\Adapter\CSV\Exception\CantDetectCSVOptions; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\Filesystem\SourceStream; final class CSVDetector { @@ -14,34 +15,15 @@ final class CSVDetector private Options $options; - /** - * @var resource - */ - private $resource; - - private int $startingPosition; + private SourceStream $stream; - /** - * @param resource $resource - */ - public function __construct($resource, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null) + public function __construct(SourceStream $stream, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null) { - if (!\is_resource($resource)) { - throw new InvalidArgumentException('Argument must be a valid resource'); - } - - $this->resource = $resource; - /** @phpstan-ignore-next-line */ - $this->startingPosition = \ftell($resource); + $this->stream = $stream; $this->options = $options ?? Options::all(); $this->fallback = $fallback; } - public function __destruct() - { - \fseek($this->resource, $this->startingPosition); - } - /** * @throws CantDetectCSVOptions|InvalidArgumentException */ @@ -53,7 +35,7 @@ public function detect(int $lines = 5) : Option $readLines = 1; - while ($line = \fgets($this->resource)) { + foreach ($this->stream->readLines() as $line) { $this->options->parse($line); if ($readLines++ >= $lines) { diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php index e1f56289e..fc60d4fce 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVExtractor.php @@ -5,18 +5,18 @@ namespace Flow\ETL\Adapter\CSV; use function Flow\ETL\DSL\array_to_rows; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering, Signal}; -use Flow\ETL\Filesystem\Path; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal}; use Flow\ETL\Row\Schema; use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\Path; final class CSVExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; /** - * @param int<0, max> $charactersReadInLine + * @param ?int<1, max> $charactersReadInLine */ public function __construct( private readonly Path $path, @@ -25,7 +25,7 @@ public function __construct( private readonly ?string $separator = null, private readonly ?string $enclosure = null, private readonly ?string $escape = null, - private readonly int $charactersReadInLine = 1000, + private readonly ?int $charactersReadInLine = null, private readonly ?Schema $schema = null ) { $this->resetLimit(); @@ -35,9 +35,9 @@ public function extract(FlowContext $context) : \Generator { $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows(); - foreach ($context->streams()->scan($this->path, $this->partitionFilter()) as $stream) { + foreach ($context->streams()->list($this->path, $this->filter()) as $stream) { - $option = \Flow\ETL\Adapter\CSV\csv_detect_separator($stream->resource()); + $option = \Flow\ETL\Adapter\CSV\csv_detect_separator($stream); $separator = $this->separator ?? $option->separator; $enclosure = $this->enclosure ?? $option->enclosure; @@ -45,22 +45,24 @@ public function extract(FlowContext $context) : \Generator $headers = []; - if ($this->withHeader && \count($headers) === 0) { - /** @var array $headers */ - $headers = \fgetcsv($stream->resource(), $this->charactersReadInLine, $separator, $enclosure, $escape); - } + foreach ($stream->readLines(length: $this->charactersReadInLine) as $csvLine) { + if ($this->withHeader && \count($headers) === 0) { + /** @var array $headers */ + $headers = \str_getcsv($csvLine, $separator, $enclosure, $escape); - /** @var array $rowData */ - $rowData = \fgetcsv($stream->resource(), $this->charactersReadInLine, $separator, $enclosure, $escape); + continue; + } - if (!\count($headers)) { - $headers = \array_map(fn (int $e) : string => 'e' . \str_pad((string) $e, 2, '0', STR_PAD_LEFT), \range(0, \count($rowData) - 1)); - } + /** @var array $rowData */ + $rowData = \str_getcsv($csvLine, $separator, $enclosure, $escape); - $headers = \array_map(fn (string $header) : string => \trim($header), $headers); - $headers = \array_map(fn (string $header, int $index) : string => $header !== '' ? $header : 'e' . \str_pad((string) $index, 2, '0', STR_PAD_LEFT), $headers, \array_keys($headers)); + if (!\count($headers)) { + $headers = \array_map(fn (int $e) : string => 'e' . \str_pad((string) $e, 2, '0', STR_PAD_LEFT), \range(0, \count($rowData) - 1)); + } + + $headers = \array_map(fn (string $header) : string => \trim($header), $headers); + $headers = \array_map(fn (string $header, int $index) : string => $header !== '' ? $header : 'e' . \str_pad((string) $index, 2, '0', STR_PAD_LEFT), $headers, \array_keys($headers)); - while (\is_array($rowData)) { if (\count($headers) > \count($rowData)) { \array_push( $rowData, @@ -84,8 +86,6 @@ public function extract(FlowContext $context) : \Generator } if (\count($headers) !== \count($rowData)) { - $rowData = \fgetcsv($stream->resource(), $this->charactersReadInLine, $separator, $enclosure, $escape); - continue; } @@ -96,7 +96,7 @@ public function extract(FlowContext $context) : \Generator } $signal = yield array_to_rows($row, $context->entryFactory(), $stream->path()->partitions(), $this->schema); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $context->streams()->closeWriters($this->path); @@ -104,7 +104,7 @@ public function extract(FlowContext $context) : \Generator return; } - $rowData = \fgetcsv($stream->resource(), $this->charactersReadInLine, $separator, $enclosure, $escape); + $rowData = \str_getcsv($csvLine, $separator, $enclosure, $escape); } $stream->close(); diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php index ce3f08627..8f7404831 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/CSVLoader.php @@ -5,11 +5,10 @@ namespace Flow\ETL\Adapter\CSV; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\Filesystem\Stream\FileStream; use Flow\ETL\Loader\Closure; use Flow\ETL\Row\Entry; -use Flow\ETL\{FlowContext, Loader, Partition, Rows}; +use Flow\ETL\{FlowContext, Loader, Rows}; +use Flow\Filesystem\{DestinationStream, Partition, Path}; final class CSVLoader implements Closure, Loader, Loader\FileLoader { @@ -71,7 +70,7 @@ public function write(Rows $nextRows, array $headers, FlowContext $context, arra } } - private function writeCSV(array $row, FileStream $destination) : void + private function writeCSV(array $row, DestinationStream $stream) : void { /** * @var string $entry @@ -83,13 +82,27 @@ private function writeCSV(array $row, FileStream $destination) : void } } + $tmpHandle = fopen('php://temp/maxmemory:' . (5 * 1024 * 1024), 'rb+'); + + if ($tmpHandle === false) { + throw new RuntimeException('Failed to open temporary stream for CSV row'); + } + \fputcsv( - stream: $destination->resource(), + stream: $tmpHandle, fields: $row, separator: $this->separator, enclosure: $this->enclosure, escape: $this->escape, eol: $this->newLineSeparator ); + $csvRowData = \stream_get_contents($tmpHandle, offset: 0); + \fclose($tmpHandle); + + if ($csvRowData === false) { + throw new RuntimeException('Failed to read temporary stream for CSV row'); + } + + $stream->append($csvRowData); } } diff --git a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php index ceb8d3893..a65390f6b 100644 --- a/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php +++ b/src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php @@ -6,12 +6,12 @@ use function Flow\ETL\DSL\from_all; use Flow\ETL\Adapter\CSV\Detector\{Option, Options}; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Row\Schema; use Flow\ETL\{Extractor, Loader}; +use Flow\Filesystem\{Path, SourceStream}; /** - * @param int<0, max> $characters_read_in_line + * @param int<1, max> $characters_read_in_line */ function from_csv( string|Path|array $path, @@ -73,12 +73,12 @@ function to_csv( } /** - * @param resource $resource - valid resource to CSV file opened with 'r' mode + * @param SourceStream $stream - valid resource to CSV file * @param int<1, max> $lines - number of lines to read from CSV file, default 5, more lines means more accurate detection but slower detection * @param null|Option $fallback - fallback option to use when no best option can be detected, default is Option(',', '"', '\\') * @param null|Options $options - options to use for detection, default is Options::all() */ -function csv_detect_separator($resource, int $lines = 5, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null) : Option +function csv_detect_separator(SourceStream $stream, int $lines = 5, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null) : Option { - return (new CSVDetector($resource, $fallback, $options))->detect($lines); + return (new CSVDetector($stream, $fallback, $options))->detect($lines); } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVDetectorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVDetectorTest.php index f66ab8ef6..66185d813 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVDetectorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVDetectorTest.php @@ -5,6 +5,8 @@ namespace Flow\ETL\Adapter\CSV\Tests\Integration; use Flow\ETL\Adapter\CSV\CSVDetector; +use Flow\Filesystem\SourceStream; +use Flow\Filesystem\Stream\MemorySourceStream; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -63,10 +65,7 @@ public function test_detecting_separators(string $separator) : void self::assertSame($separator, $detector->detect()->separator); } - /** - * @return resource - */ - private function createResource(string $separator = ',', string $enclosure = '"') + private function createResource(string $separator = ',', string $enclosure = '"') : SourceStream { $data = [ ['id', 'name', 'email'], @@ -89,8 +88,9 @@ private function createResource(string $separator = ',', string $enclosure = '"' \fputcsv($resource, $line, $separator, $enclosure); } - \rewind($resource); + $csv = \stream_get_contents($resource, offset: 0); + \fclose($resource); - return $resource; + return new MemorySourceStream($csv); } } diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php index 33c908f78..344313f2b 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVExtractorTest.php @@ -4,12 +4,12 @@ namespace Flow\ETL\Adapter\CSV\Tests\Integration; -use function Flow\ETL\Adapter\CSV\{from_csv, to_csv}; -use function Flow\ETL\DSL\{df, from_array, print_schema, ref}; +use function Flow\ETL\Adapter\CSV\{from_csv}; +use function Flow\ETL\DSL\{df, print_schema, ref}; use Flow\ETL\Adapter\CSV\CSVExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\{LocalFilesystem, Path}; -use Flow\ETL\{Config, ConfigBuilder, Exception\FileNotFoundException, Flow, FlowContext, Row, Rows}; +use Flow\ETL\{Config, FlowContext, Row, Rows}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class CSVExtractorTest extends TestCase @@ -289,7 +289,7 @@ public function test_extracting_csv_with_more_headers_than_columns() : void public function test_extracting_csv_with_more_than_1000_characters_per_line_splits_rows() : void { self::assertCount( - 2, + 1, df() ->read(from_csv(__DIR__ . '/../Fixtures/more_than_1000_characters_per_line.csv')) ->fetch() @@ -312,17 +312,8 @@ public function test_extracting_csv_with_more_than_1000_characters_per_line_with public function test_limit() : void { - $path = \sys_get_temp_dir() . '/csv_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - df()->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_csv($path)) - ->run(); - $extractor = new CSVExtractor(Path::realpath($path)); + $extractor = new CSVExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv')); $extractor->changeLimit(2); self::assertCount( @@ -331,14 +322,6 @@ public function test_limit() : void ); } - public function test_load_not_existing_file_throws_exception() : void - { - $this->expectException(FileNotFoundException::class); - $extractor = from_csv(Path::realpath('not_existing_file.csv')); - $generator = $extractor->extract(new FlowContext(Config::default())); - \iterator_to_array($generator); - } - public function test_loading_data_from_all_partitions() : void { self::assertSame( @@ -361,51 +344,16 @@ public function test_loading_data_from_all_partitions() : void ); } - public function test_loading_data_from_all_with_local_fs() : void - { - self::assertSame( - [ - ['group' => '1', 'id' => 1, 'value' => 'a'], - ['group' => '1', 'id' => 2, 'value' => 'b'], - ['group' => '1', 'id' => 3, 'value' => 'c'], - ['group' => '1', 'id' => 4, 'value' => 'd'], - ['group' => '2', 'id' => 5, 'value' => 'e'], - ['group' => '2', 'id' => 6, 'value' => 'f'], - ['group' => '2', 'id' => 7, 'value' => 'g'], - ['group' => '2', 'id' => 8, 'value' => 'h'], - ], - (new Flow((new ConfigBuilder())->filesystem(new LocalFilesystem()))) - ->read(from_csv(__DIR__ . '/../Fixtures/partitioned/group=*/*.csv')) - ->withEntry('id', ref('id')->cast('int')) - ->sortBy(ref('id')) - ->fetch() - ->toArray() - ); - } - public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/csv_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - df()->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_csv($path)) - ->run(); - - $extractor = new CSVExtractor(Path::realpath($path)); + $extractor = new CSVExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv')); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame([['id' => '1']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => '2']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => '3']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); diff --git a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVTest.php b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVTest.php index e93a477a7..9ad75048c 100644 --- a/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVTest.php +++ b/src/adapter/etl-adapter-csv/tests/Flow/ETL/Adapter/CSV/Tests/Integration/CSVTest.php @@ -5,43 +5,40 @@ namespace Flow\ETL\Adapter\CSV\Tests\Integration; use function Flow\ETL\Adapter\CSV\{from_csv, to_csv}; -use function Flow\ETL\DSL\{array_entry, df, int_entry, ref, row, rows}; -use Flow\ETL\Filesystem\Path; +use function Flow\ETL\DSL\{array_entry, df, int_entry, overwrite, ref, row, rows}; use Flow\ETL\Flow; use Flow\ETL\Tests\Double\FakeExtractor; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class CSVTest extends TestCase { - public function test_loading_array_entry() : void + protected function setUp() : void { - $path = \sys_get_temp_dir() . '/flow_php_etl_csv_loader' . bin2hex(random_bytes(16)) . '.csv'; - - if (\file_exists($path)) { - \unlink($path); + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); } + } + public function test_loading_array_entry() : void + { $this->expectExceptionMessage('Entry "data" is an list|array, please cast to string before writing to CSV. Easiest way to cast arrays to string is to use Transform::to_json transformer.'); (new Flow()) ->process(rows(row(int_entry('id', 1), array_entry('data', ['foo' => 'bar'])))) - ->write(to_csv($path)) + ->write(to_csv(__DIR__ . '/var/test_loading_array_entry.csv')) ->run(); } public function test_loading_csv_files() : void { - $path = \sys_get_temp_dir() . '/flow_php_etl_csv_loader' . bin2hex(random_bytes(16)) . '.csv'; - - if (\file_exists($path)) { - \unlink($path); - } df() ->read(new FakeExtractor(100)) ->drop('array', 'list', 'map', 'struct', 'object', 'enum', 'list_of_datetimes') ->withEntry('datetime', ref('datetime')->dateFormat('Y-m-d H:i:s')) - ->load(to_csv($path)) + ->saveMode(overwrite()) + ->load(to_csv($path = __DIR__ . '/var/test_loading_csv_files.csv')) ->run(); self::assertEquals( diff --git a/src/adapter/etl-adapter-doctrine/README.md b/src/adapter/etl-adapter-doctrine/README.md index 9f5681a35..502a25f09 100644 --- a/src/adapter/etl-adapter-doctrine/README.md +++ b/src/adapter/etl-adapter-doctrine/README.md @@ -12,5 +12,5 @@ operations in large-scale and data-intensive environments. With Flow PHP's Adapt interactions within your ETL workflows becomes a more simplified and efficient endeavor, perfectly aligning with the robust and adaptable nature of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/doctrine.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalDataFrameFactory.php b/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalDataFrameFactory.php index c11f37ea0..bc03383ad 100644 --- a/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalDataFrameFactory.php +++ b/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalDataFrameFactory.php @@ -58,9 +58,6 @@ public function from(Rows $rows) : DataFrame private function connection() : Connection { if ($this->connection === null) { - /** - * @psalm-suppress ArgumentTypeCoercion - */ $this->connection = DriverManager::getConnection($this->connectionParams); } diff --git a/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLoader.php b/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLoader.php index 6deb81e76..08d64a582 100644 --- a/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLoader.php +++ b/src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/DbalLoader.php @@ -79,9 +79,6 @@ public function load(Rows $rows, FlowContext $context) : void private function connection() : Connection { if ($this->connection === null) { - /** - * @psalm-suppress ArgumentTypeCoercion - */ $this->connection = DriverManager::getConnection($this->connectionParams); } diff --git a/src/adapter/etl-adapter-elasticsearch/README.md b/src/adapter/etl-adapter-elasticsearch/README.md index b8290cc20..06297d273 100644 --- a/src/adapter/etl-adapter-elasticsearch/README.md +++ b/src/adapter/etl-adapter-elasticsearch/README.md @@ -12,5 +12,5 @@ making it an excellent choice for developers dealing with Elasticsearch in large With Flow PHP's Adapter Elasticsearch, managing Elasticsearch data within your ETL workflows becomes a more refined and efficient endeavor, harmoniously aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/elasticsearch.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-filesystem/.gitattributes b/src/adapter/etl-adapter-filesystem/.gitattributes deleted file mode 100644 index e2534369b..000000000 --- a/src/adapter/etl-adapter-filesystem/.gitattributes +++ /dev/null @@ -1,12 +0,0 @@ -*.php text eol=lf -/composer.lock export-ignore -/.github/ export-ignore -/docs export-ignore -/tests export-ignore -/tools/ export-ignore -/.gitattributes export-ignore -/.gitignore export-ignore -/.php-cs-fixer.php export-ignore -/phpstan.neon export-ignore -/phpunit.xml export-ignore -/psalm.xml export-ignore \ No newline at end of file diff --git a/src/adapter/etl-adapter-filesystem/.gitignore b/src/adapter/etl-adapter-filesystem/.gitignore deleted file mode 100644 index df866db47..000000000 --- a/src/adapter/etl-adapter-filesystem/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -vendor -*.cache -var \ No newline at end of file diff --git a/src/adapter/etl-adapter-filesystem/README.md b/src/adapter/etl-adapter-filesystem/README.md deleted file mode 100644 index 521594513..000000000 --- a/src/adapter/etl-adapter-filesystem/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# ETL Adapter: Filesystem - -Filesystem adapter integrates Flow Filesystem with Flysystem library allowing for reading/writing into external filesystems. - -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) -- 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) - diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AwsS3Stream.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AwsS3Stream.php deleted file mode 100644 index a674ba6b4..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AwsS3Stream.php +++ /dev/null @@ -1,59 +0,0 @@ -context)) { - throw new RuntimeException(__CLASS__ . ' requires context in order to initialize filesystem'); - } - - if ($this->filesystem === null) { - /** - * @psalm-suppress PossiblyNullArgument - * @psalm-suppress UndefinedThisPropertyFetch - */ - $contextOptions = \stream_context_get_options($this->context); - - $clientOptions = $contextOptions[self::PROTOCOL]['client']; - - /** @var string $bucket */ - $bucket = $contextOptions[self::PROTOCOL]['bucket']; - - /** - * @psalm-suppress PossiblyNullArrayAccess - * @psalm-suppress PossiblyNullArgument - */ - $this->filesystem = (new Filesystem(new AwsS3V3Adapter(new S3Client($clientOptions), $bucket))); - } - - return $this->filesystem; - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AzureBlobStream.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AzureBlobStream.php deleted file mode 100644 index 62c1eb252..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/AzureBlobStream.php +++ /dev/null @@ -1,70 +0,0 @@ -context)) { - throw new RuntimeException(__CLASS__ . ' requires context in order to initialize filesystem'); - } - - if ($this->filesystem === null) { - /** - * @psalm-suppress PossiblyNullArgument - * @psalm-suppress UndefinedThisPropertyFetch - */ - $contextOptions = \stream_context_get_options($this->context); - - /** - * @var array{connection-string: string} $clientOptions - */ - $clientOptions = ['connection-string' => $contextOptions[self::PROTOCOL]['connection-string']]; - - /** @var string $container */ - $container = $contextOptions[self::PROTOCOL]['container']; - - /** - * @psalm-suppress PossiblyNullArrayAccess - * @psalm-suppress PossiblyNullArgument - */ - $this->filesystem = (new Filesystem( - new AzureBlobStorageAdapter( - BlobRestProxy::createBlobService($clientOptions['connection-string']), - $container - ) - )); - } - - return $this->filesystem; - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFS.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFS.php deleted file mode 100644 index 18b972b9c..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFS.php +++ /dev/null @@ -1,180 +0,0 @@ -factory->create($path); - - if ($path->isPattern()) { - return false; - } - - return $fs->directoryExists($path->path()); - } - - public function exists(Path $path) : bool - { - $fs = $this->factory->create($path); - - if ($path->isPattern()) { - $anyFileExistsInPattern = false; - - foreach ($this->scan($path, new NoopFilter()) as $nextPath) { - $anyFileExistsInPattern = true; - - break; - } - - return $anyFileExistsInPattern; - } - - return $fs->fileExists($path->path()) || $fs->directoryExists($path->path()); - } - - public function fileExists(Path $path) : bool - { - $fs = $this->factory->create($path); - - if ($path->isPattern()) { - $anyFileExistsInPattern = false; - - foreach ($this->scan($path, new NoopFilter()) as $nextPath) { - $anyFileExistsInPattern = true; - - break; - } - - return $anyFileExistsInPattern; - } - - return $fs->fileExists($path->path()); - } - - public function mv(Path $from, Path $to) : void - { - if ($from->isPattern() || $to->isPattern()) { - throw new InvalidArgumentException("Pattern paths can't be moved: " . $from->uri() . ' -> ' . $to->uri()); - } - - if ($from->scheme() !== $to->scheme()) { - throw new InvalidArgumentException("Can't move path from different schemes: " . $from->scheme() . ' -> ' . $to->scheme()); - } - - if ($this->fileExists($to)) { - $this->rm($to); - } - - $this->factory->create($from)->move($from->path(), $to->path()); - } - - public function open(Path $path, Mode $mode) : FileStream - { - if ($path->isPattern()) { - throw new InvalidArgumentException("Pattern paths can't be open: " . $path->uri()); - } - - if ($path->isLocal()) { - $fs = $this->factory->create($path); - - if (!$fs->directoryExists($path->parentDirectory()->path())) { - $fs->createDirectory($path->parentDirectory()->path()); - } - - return new FileStream($path, \fopen($path->path(), $mode->value, false, $path->context()->resource()) ?: null); - } - - return new FileStream($path, \fopen($path->uri(), $mode->value, false, $path->context()->resource()) ?: null); - } - - public function rm(Path $path) : void - { - $fs = $this->factory->create($path); - - if ($path->isPattern()) { - foreach ($this->scan($path, new NoopFilter()) as $nextPath) { - if ($fs->fileExists($nextPath->path())) { - $fs->delete($nextPath->path()); - } else { - $fs->deleteDirectory($nextPath->path()); - } - } - - return; - } - - if ($fs->fileExists($path->path())) { - $fs->delete($path->path()); - - return; - } - - if ($fs->directoryExists($path->path())) { - $fs->deleteDirectory($path->path()); - - return; - } - - throw new FileNotFoundException($path); - } - - /** - * @throws InvalidArgumentException - * @throws MissingDependencyException - * @throws FilesystemException - * - * @return \Generator - */ - public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilter()) : \Generator - { - if (!$path->isPattern() && !$this->fileExists($path)) { - throw new FileNotFoundException($path); - } - - $fs = $this->factory->create($path); - - if ($fs->fileExists($path->path())) { - yield $path; - - return; - } - - $filter = function (FileAttributes|DirectoryAttributes $file) use ($path, $partitionFilter) : bool { - if ($file instanceof DirectoryAttributes) { - return false; - } - - if ($path->isPattern()) { - if (!$path->matches(new Path($path->scheme() . '://' . $file->path(), $path->options()))) { - return false; - } - } - - return $partitionFilter->keep(...(new Path(DIRECTORY_SEPARATOR . $file->path()))->partitions()->toArray()); - }; - - /** - * @psalm-suppress ArgumentTypeCoercion - * - * @phpstan-ignore-next-line - */ - foreach ($fs->listContents($path->staticPart()->path(), Flysystem::LIST_DEEP)->sortByPath()->filter($filter) as $file) { - yield new Path($path->scheme() . '://' . $file->path(), $path->options()); - } - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFactory.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFactory.php deleted file mode 100644 index 76d21bbf2..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFactory.php +++ /dev/null @@ -1,82 +0,0 @@ -isLocal()) { - return $this->local(); - } - - return match ($path->scheme()) { - AwsS3Stream::PROTOCOL => $this->aws($path), - AzureBlobStream::PROTOCOL => $this->azure($path), - 'file' => $this->local(), - default => throw new InvalidArgumentException('Unexpected scheme: ' . $path->scheme()) - }; - } - - /** - * @throws MissingDependencyException - * @throws InvalidArgumentException - */ - private function aws(Path $path) : Filesystem - { - AwsS3Stream::register(); - - $options = $path->options(); - - $expectedOptions = '["client" => ["credentials" => ["key" => "__key__", "secret" => "__secret__"], "region" => "eu-west-2", "version" => "latest"], "bucket" => "__name__"]'; - - if (!\array_key_exists('client', $options)) { - throw new InvalidArgumentException("Missing AWS client in Path options, expected options: {$expectedOptions}"); - } - - if (!\array_key_exists('bucket', $options)) { - throw new InvalidArgumentException("Missing AWS bucket in Path options, expected options: {$expectedOptions}"); - } - - return new Filesystem(new AwsS3V3Adapter(new S3Client($options['client']), $options['bucket'])); - } - - /** - * @throws MissingDependencyException - * @throws InvalidArgumentException - */ - private function azure(Path $path) : Filesystem - { - AzureBlobStream::register(); - - $options = $path->options(); - - $expectedOptions = '["connection-string" => "__connection_string___", "container" => "__container__"]'; - - if (!\array_key_exists('connection-string', $options)) { - throw new InvalidArgumentException("Missing Azure Blob connection-string in Path options, expected options: {$expectedOptions}"); - } - - if (!\array_key_exists('container', $options)) { - throw new InvalidArgumentException("Missing Azure Blob container in Path options, expected options: {$expectedOptions}"); - } - - return new Filesystem(new AzureBlobStorageAdapter(BlobRestProxy::createBlobService($options['connection-string']), $options['container'])); - } - - private function local() : Filesystem - { - return new Filesystem(new LocalFilesystemAdapter(DIRECTORY_SEPARATOR, linkHandling: LocalFilesystemAdapter::SKIP_LINKS)); - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemWrapper.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemWrapper.php deleted file mode 100644 index a566fac92..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemWrapper.php +++ /dev/null @@ -1,230 +0,0 @@ -stream)) { - /** @psalm-suppress InvalidPropertyAssignmentValue */ - \fclose($this->stream); - } - } - - public function stream_eof() : bool - { - if ($this->stream === null) { - return false; - } - - $this->openRead(); - - /** - * @psalm-suppress PossiblyNullArgument - * - * @phpstan-ignore-next-line - */ - return \feof($this->stream); - } - - public function stream_flush() : bool - { - $this->filesystem()->writeStream($this->path(), $this->buffer()->stream()); - - $this->buffer()->release(); - - return true; - } - - public function stream_lock(int $operation) : bool - { - /** - * @psalm-suppress PossiblyNullArgument - * - * @phpstan-ignore-next-line - */ - return \flock($this->stream, $operation); - } - - public function stream_open(string $path, string $mode, int $options, ?string &$opened_path) : bool - { - $this->path = $path; - /** - * @psalm-suppress InvalidPropertyAssignmentValue - * - * @phpstan-ignore-next-line - */ - $this->url = \parse_url($this->path); - - $this->stream = match ($mode) { - 'r' => $this->filesystem()->readStream($this->path()), - default => null - }; - - if ($this->stream) { - $this->streamMedata = \stream_get_meta_data($this->stream); - } - - return true; - } - - public function stream_read(int $count) : string|false - { - $this->openRead(); - - /** - * @psalm-suppress PossiblyNullArgument - * - * @phpstan-ignore-next-line - */ - return \fread($this->stream, $count); - } - - public function stream_seek(int $offset, int $whence = SEEK_SET) : bool - { - if ($this->stream === null) { - return false; - } - - if ($this->streamMedata === null) { - return false; - } - - if ($this->streamMedata['seekable'] === false) { - throw new RuntimeException('Remote streams are not seekable'); - } - - $this->buffer()->seek($offset, $whence); - - return true; - } - - public function stream_stat() : array|false - { - if (!$this->filesystem()->fileExists($this->path())) { - return false; - } - - return [ - 'dev' => 0, - 'ino' => 0, - 'mode' => 0, - 'nlink' => 0, - 'uid' => 0, - 'gid' => 0, - 'rdev' => 0, - 'size' => $this->filesystem()->fileSize($this->path()), - 'atime' => 0, - 'ctime' => 0, - 'blksize' => 0, - 'blocks' => 0, - ]; - } - - public function stream_tell() : int|false - { - if ($this->stream === null) { - return $this->buffer()->tell(); - } - - return \ftell($this->stream); - } - - public function stream_write(string $data) : int - { - $this->buffer()->write($data); - - return \strlen($data); - } - - public function url_stat(string $path, int $flags) : array|false - { - if (!$this->filesystem()->fileExists($path)) { - return false; - } - - return [ - 'dev' => 0, - 'ino' => 0, - 'mode' => 0, - 'nlink' => 0, - 'uid' => 0, - 'gid' => 0, - 'rdev' => 0, - 'size' => $this->filesystem()->fileSize($path), - 'atime' => 0, - 'mtime' => $this->filesystem()->lastModified($path), - 'ctime' => 0, - 'blksize' => 0, - 'blocks' => 0, - ]; - } - - abstract protected function filesystem() : Filesystem; - - protected function openRead() : void - { - if ($this->stream === null) { - $this->stream = $this->filesystem()->readStream($this->path()); - } - } - - private function buffer() : LocalBuffer - { - if ($this->buffer === null) { - $this->buffer = new TmpfileBuffer(); - } - - return $this->buffer; - } - - private function path() : string - { - if ($this->url === null) { - return ''; - } - - return ($this->url['host'] ?? '') . ($this->url['path'] ?? ''); - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/RemoteFileListExtractor.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/RemoteFileListExtractor.php deleted file mode 100644 index 7bf954b42..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/RemoteFileListExtractor.php +++ /dev/null @@ -1,73 +0,0 @@ -directory->isLocal()) { - throw new InvalidArgumentException('Path must point to a remote directory, got local path ' . $this->directory->uri() . ' instead'); - } - - if ($this->directory->isPattern()) { - throw new InvalidArgumentException('RemoteFileListExtractor does not support glob paths, got ' . $this->directory->path()); - } - } - - public function extract(FlowContext $context) : \Generator - { - $fs = $this->factory->create($this->directory); - - /** @var DirectoryAttributes|FileAttributes $file */ - foreach ($fs->listContents($this->directory->path(), $this->recursive) as $file) { - $path = new Path($this->directory->scheme() . '://' . $file->path(), $this->directory->options()); - - /** - * @psalm-suppress PossiblyUndefinedMethod - */ - $signal = yield array_to_rows([ - 'path' => $file->path(), - 'uri' => $path->uri(), - 'scheme' => $path->scheme(), - 'base_name' => $path->basename(), - 'file_name' => $path->filename(), - 'extension' => $path->extension(), - 'is_file' => $file->isFile(), - 'is_dir' => $file->isDir(), - /** @phpstan-ignore-next-line */ - 'size' => $file->isFile() ? $file->fileSize() : null, - 'visibility' => $file->visibility(), - 'metadata' => $file->extraMetadata(), - /** @phpstan-ignore-next-line */ - 'mime_type' => $file->isFile() ? $file->mimeType() : null, - 'last_modified' => $file->lastModified(), - ]); - - $this->countRow(); - - if ($signal === Signal::STOP || $this->reachedLimit()) { - return; - } - } - } - - public function source() : Path - { - return $this->directory; - } -} diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/functions.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/functions.php deleted file mode 100644 index b168664fc..000000000 --- a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/functions.php +++ /dev/null @@ -1,12 +0,0 @@ -open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/append.txt'), Mode::APPEND_WRITE); - \fwrite($stream->resource(), "some data to make file not empty\n"); - $stream->close(); - - $appendStream = $fs->open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/append.txt'), Mode::APPEND_WRITE); - \fwrite($appendStream->resource(), "some more data to make file not empty\n"); - $appendStream->close(); - - self::assertStringContainsString( - <<<'STRING' -some data to make file not empty -some more data to make file not empty -STRING, - \file_get_contents($appendStream->path()->path()) - ); - - $fs->rm($stream->path()); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_dir_exists() : void - { - self::assertTrue((new FlysystemFS())->exists(new Path(__DIR__))); - self::assertFalse((new FlysystemFS())->exists(new Path(__DIR__ . '/not_existing_directory'))); - } - - public function test_fie_exists() : void - { - self::assertTrue((new FlysystemFS())->exists(new Path(__FILE__))); - self::assertFalse((new FlysystemFS())->exists(new Path(__DIR__ . '/not_existing_file.php'))); - } - - public function test_file_pattern_exists() : void - { - self::assertTrue((new FlysystemFS())->exists(new Path(__DIR__ . '/**/*.txt'))); - self::assertFalse((new FlysystemFS())->exists(new Path(__DIR__ . '/**/*.pdf'))); - } - - public function test_open_file_stream_for_existing_file() : void - { - $stream = (new FlysystemFS())->open(new Path(__FILE__), Mode::READ); - - self::assertIsResource($stream->resource()); - self::assertSame( - \file_get_contents(__FILE__), - \stream_get_contents($stream->resource()) - ); - } - - public function test_open_file_stream_for_non_existing_file() : void - { - $path = \sys_get_temp_dir() . '/flow_php_test_file_' . bin2hex(random_bytes(16)) . '.txt'; - - $stream = (new FlysystemFS())->open(new Path($path), Mode::WRITE); - - self::assertIsResource($stream->resource()); - } - - public function test_reading_multi_partitioned_path() : void - { - $paths = \iterator_to_array( - (new FlysystemFS()) - ->scan( - new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), - new ScalarFunctionFilter( - all( - ref('country')->equals(lit('pl')), - all( - ref('date')->cast('date')->greaterThanEqual(lit(new \DateTimeImmutable('2022-01-02 00:00:00'))), - ref('date')->cast('date')->lessThan(lit(new \DateTimeImmutable('2022-01-04 00:00:00'))) - ) - ), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ) - ) - ); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'), - ], - $paths - ); - } - - public function test_reading_partitioned_folder() : void - { - $paths = \iterator_to_array((new FlysystemFS())->scan(new Path(__DIR__ . '/Fixtures/partitioned**/*.txt'), new NoopFilter())); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - $paths - ); - } - - public function test_reading_partitioned_folder_with_partitions_filtering() : void - { - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - \iterator_to_array( - (new FlysystemFS()) - ->scan( - new Path(__DIR__ . '/Fixtures/partitioned/**/*.txt'), - new ScalarFunctionFilter(ref('partition_01')->equals(lit('b')), new NativeEntryFactory(), new AutoCaster(Caster::default())) - ) - ) - ); - } - - public function test_reading_partitioned_folder_with_pattern() : void - { - $paths = \iterator_to_array((new FlysystemFS())->scan(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=*/*.txt'), new NoopFilter())); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - $paths - ); - } - - public function test_remove_directory_with_content_when_exists() : void - { - $fs = new FlysystemFS(); - - $dirPath = Path::realpath(\sys_get_temp_dir() . '/flow-fs-test-directory/'); - - $stream = $fs->open(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($dirPath)); - self::assertTrue($fs->exists($stream->path())); - $fs->rm($dirPath); - self::assertFalse($fs->exists($dirPath)); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_remove_file_when_exists() : void - { - $fs = new FlysystemFS(); - - $stream = $fs->open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($stream->path())); - $fs->rm($stream->path()); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_remove_pattern() : void - { - $fs = new FlysystemFS(); - - $dirPath = Path::realpath(\sys_get_temp_dir() . '/flow-fs-test-directory/'); - - $stream = $fs->open(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($dirPath)); - self::assertTrue($fs->exists($stream->path())); - $fs->rm(Path::realpath($dirPath->path() . '/*.txt')); - self::assertTrue($fs->exists($dirPath)); - self::assertFalse($fs->exists($stream->path())); - $fs->rm($dirPath); - } - - public function test_that_scan_sort_files_by_path_names() : void - { - $paths = \iterator_to_array( - (new FlysystemFS()) - ->scan( - new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), - ) - ); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt'), - ], - $paths - ); - } -} diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/RegisterWrapperTest.php b/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/RegisterWrapperTest.php deleted file mode 100644 index bf86d8281..000000000 --- a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/RegisterWrapperTest.php +++ /dev/null @@ -1,34 +0,0 @@ -getValues() ?? []; @@ -97,7 +95,7 @@ function (array $rowData) use ($headers, $shouldPutInputIntoRows) { foreach ($rows as $row) { $signal = yield array_to_rows($row, $context->entryFactory()); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { return; @@ -113,8 +111,6 @@ function (array $rowData) use ($headers, $shouldPutInputIntoRows) { $response = $this->service->spreadsheets_values->get($this->spreadsheetId, $cellsRange->toString(), $this->options); /** * @var array[] $values - * - * @psalm-suppress RedundantConditionGivenDocblockType */ $values = $response->getValues() ?? []; } diff --git a/src/adapter/etl-adapter-http/README.md b/src/adapter/etl-adapter-http/README.md index 208906fad..5f5426d9c 100644 --- a/src/adapter/etl-adapter-http/README.md +++ b/src/adapter/etl-adapter-http/README.md @@ -12,5 +12,5 @@ large-scale and data-intensive environments. With Flow PHP's Adapter HTTP, navig workflows becomes a more refined and efficient endeavor, harmoniously aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/http.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-json/README.md b/src/adapter/etl-adapter-json/README.md index f208beb42..434f58df1 100644 --- a/src/adapter/etl-adapter-json/README.md +++ b/src/adapter/etl-adapter-json/README.md @@ -11,5 +11,5 @@ data processing solutions, making it a prime choice for developers dealing with data-intensive environments. With Flow PHP's Adapter JSON, managing JSON data within your ETL workflows becomes a more simplified and efficient task, perfectly aligning with the robust and adaptable nature of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/json.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php index 94ac43b0b..1e5b8f321 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JSONMachine/JsonExtractor.php @@ -5,17 +5,17 @@ namespace Flow\ETL\Adapter\JSON\JSONMachine; use function Flow\ETL\DSL\array_to_rows; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering, Signal}; -use Flow\ETL\Filesystem\Path; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal}; use Flow\ETL\Row\Schema; use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\Path; use JsonMachine\Items; use JsonMachine\JsonDecoder\ExtJsonDecoder; final class JsonExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; public function __construct( private readonly Path $path, @@ -29,12 +29,12 @@ public function extract(FlowContext $context) : \Generator { $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows(); - foreach ($context->streams()->scan($this->path, $this->partitionFilter()) as $stream) { + foreach ($context->streams()->list($this->path, $this->filter()) as $stream) { /** * @var array|object $rowData */ - foreach (Items::fromStream($stream->resource(), $this->readerOptions())->getIterator() as $rowData) { + foreach ((new Items($stream->iterate(8 * 1024), $this->readerOptions()))->getIterator() as $rowData) { $row = (array) $rowData; if ($shouldPutInputIntoRows) { @@ -42,7 +42,7 @@ public function extract(FlowContext $context) : \Generator } $signal = yield array_to_rows($row, $context->entryFactory(), $stream->path()->partitions(), $this->schema); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $context->streams()->closeWriters($this->path); diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonLoader.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonLoader.php index 85d3f2c4c..bf7cbfc04 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonLoader.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/JsonLoader.php @@ -5,10 +5,9 @@ namespace Flow\ETL\Adapter\JSON; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\Filesystem\Stream\FileStream; use Flow\ETL\Loader\Closure; -use Flow\ETL\{FlowContext, Loader, Partition, Rows}; +use Flow\ETL\{FlowContext, Loader, Rows}; +use Flow\Filesystem\{DestinationStream, Partition, Path}; final class JsonLoader implements Closure, Loader, Loader\FileLoader { @@ -28,7 +27,7 @@ public function closure(FlowContext $context) : void { foreach ($context->streams() as $stream) { if ($stream->path()->extension() === 'json') { - \fwrite($stream->resource(), ']'); + $stream->append(']'); } } @@ -63,7 +62,7 @@ public function write(Rows $nextRows, array $partitions, FlowContext $context) : $this->writes[$stream->path()->path()] = 0; } - \fwrite($stream->resource(), '['); + $stream->append('['); } else { $stream = $streams->writeTo($this->path, $partitions); } @@ -73,12 +72,12 @@ public function write(Rows $nextRows, array $partitions, FlowContext $context) : /** * @param Rows $rows - * @param FileStream $stream + * @param DestinationStream $stream * * @throws RuntimeException * @throws \JsonException */ - public function writeJSON(Rows $rows, FileStream $stream) : void + public function writeJSON(Rows $rows, DestinationStream $stream) : void { if (!\count($rows)) { return; @@ -87,7 +86,7 @@ public function writeJSON(Rows $rows, FileStream $stream) : void $json = \substr(\substr(\json_encode($rows->toArray(), JSON_THROW_ON_ERROR), 0, -1), 1); $json = ($this->writes[$stream->path()->path()] > 0) ? ',' . $json : $json; - \fwrite($stream->resource(), $json); + $stream->append($json); $this->writes[$stream->path()->path()]++; } diff --git a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php index 01ddc12c1..91f2c127a 100644 --- a/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php +++ b/src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php @@ -6,9 +6,9 @@ use function Flow\ETL\DSL\from_all; use Flow\ETL\Adapter\JSON\JSONMachine\JsonExtractor; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Row\Schema; use Flow\ETL\{Extractor, Loader}; +use Flow\Filesystem\Path; /** * @param array|Path|string $path - string is internally turned into stream diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php index dfcac3314..4b9d27cb8 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JSONMachine/JsonExtractorTest.php @@ -5,11 +5,11 @@ namespace Flow\ETL\Adapter\JSON\Tests\Integration\JSONMachine; use function Flow\ETL\Adapter\JSON\{from_json, to_json}; -use function Flow\ETL\DSL\{df, from_array, print_schema}; +use function Flow\ETL\DSL\{df, print_schema}; use Flow\ETL\Adapter\JSON\JSONMachine\JsonExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\Path; use Flow\ETL\{Config, Flow, FlowContext, Row, Rows}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class JsonExtractorTest extends TestCase @@ -131,17 +131,7 @@ public function test_extracting_json_from_local_file_string_uri() : void public function test_limit() : void { - $path = \sys_get_temp_dir() . '/json_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_json($path)) - ->run(); - - $extractor = new JsonExtractor(Path::realpath($path)); + $extractor = new JsonExtractor(\Flow\Filesystem\DSL\path(__DIR__ . '/../../Fixtures/timezones.json')); $extractor->changeLimit(2); self::assertCount( @@ -152,27 +142,14 @@ public function test_limit() : void public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/json_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_json($path)) - ->run(); - - $extractor = new JsonExtractor(Path::realpath($path)); + $extractor = new JsonExtractor(\Flow\Filesystem\DSL\path(__DIR__ . '/../../Fixtures/timezones.json')); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame([['id' => 1]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 2]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 3]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); diff --git a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JsonTest.php b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JsonTest.php index 0edf07a2e..e5cd62741 100644 --- a/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JsonTest.php +++ b/src/adapter/etl-adapter-json/tests/Flow/ETL/Adapter/JSON/Tests/Integration/JsonTest.php @@ -7,8 +7,8 @@ use function Flow\ETL\Adapter\JSON\from_json; use function Flow\ETL\Adapter\Json\to_json; use function Flow\ETL\DSL\df; +use function Flow\Filesystem\DSL\path; use Flow\ETL\Adapter\JSON\JsonLoader; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Tests\Double\FakeExtractor; use Flow\ETL\{Config, FlowContext, Rows}; use PHPUnit\Framework\TestCase; @@ -17,15 +17,10 @@ final class JsonTest extends TestCase { public function test_json_loader() : void { - $path = \sys_get_temp_dir() . '/flow_php_etl_json_loader' . bin2hex(random_bytes(16)) . '.json'; - - if (\file_exists($path)) { - \unlink($path); - } df() ->read(new FakeExtractor(100)) - ->write(to_json($path)) + ->write(to_json($path = __DIR__ . '/var/test_json_loader.json')) ->run(); self::assertEquals( @@ -40,9 +35,7 @@ public function test_json_loader() : void public function test_json_loader_loading_empty_string() : void { - $stream = \sys_get_temp_dir() . '/flow_php_etl_json_loader' . bin2hex(random_bytes(16)) . '.json'; - - $loader = new JsonLoader(Path::realpath($stream)); + $loader = new JsonLoader(path($path = __DIR__ . '/var/test_json_loader_loading_empty_string.json')); $loader->load(new Rows(), $context = new FlowContext(Config::default())); @@ -53,11 +46,11 @@ public function test_json_loader_loading_empty_string() : void [ ] JSON, - \file_get_contents($stream) + \file_get_contents($path) ); - if (\file_exists($stream)) { - \unlink($stream); + if (\file_exists($path)) { + \unlink($path); } } } diff --git a/src/adapter/etl-adapter-logger/README.md b/src/adapter/etl-adapter-logger/README.md index 151ee67bb..49bd704dc 100644 --- a/src/adapter/etl-adapter-logger/README.md +++ b/src/adapter/etl-adapter-logger/README.md @@ -2,5 +2,5 @@ ETL Adapter that provides PSR Logger support for ETL. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/logger.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/adapter/etl-adapter-meilisearch/README.md b/src/adapter/etl-adapter-meilisearch/README.md index 3fc86af99..0d193678d 100644 --- a/src/adapter/etl-adapter-meilisearch/README.md +++ b/src/adapter/etl-adapter-meilisearch/README.md @@ -13,5 +13,5 @@ indexing operations in large-scale and data-intensive environments. With Flow PH search and indexing tasks within your ETL workflows becomes a more streamlined and efficient endeavor, harmoniously aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/meilisearch.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/adapter/etl-adapter-parquet/README.md b/src/adapter/etl-adapter-parquet/README.md index bb9bc1281..2ce407761 100644 --- a/src/adapter/etl-adapter-parquet/README.md +++ b/src/adapter/etl-adapter-parquet/README.md @@ -12,5 +12,5 @@ for developers dealing with Parquet data in large-scale and data-intensive scena managing Parquet data within your ETL workflows becomes a more simplified and efficient task, perfectly aligning with the robust and adaptable nature of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/parquet.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php index a2e87a234..50ba9a3b8 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetExtractor.php @@ -5,16 +5,15 @@ namespace Flow\ETL\Adapter\Parquet; use function Flow\ETL\DSL\array_to_rows; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering, Signal}; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\Filesystem\Stream\FileStream; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal}; use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\{Path, SourceStream}; use Flow\Parquet\{ByteOrder, Options, ParquetFile, Reader}; final class ParquetExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; private SchemaConverter $schemaConverter; @@ -57,7 +56,7 @@ public function extract(FlowContext $context) : \Generator $signal = yield array_to_rows($row, $context->entryFactory(), $fileData['stream']->path()->partitions(), $flowSchema); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $context->streams()->closeWriters($this->path); @@ -77,14 +76,14 @@ public function source() : Path } /** - * @return \Generator + * @return \Generator */ private function readers(FlowContext $context) : \Generator { - foreach ($context->streams()->scan($this->path, $this->partitionFilter()) as $stream) { + foreach ($context->streams()->list($this->path, $this->filter()) as $stream) { yield [ 'file' => (new Reader(byteOrder: $this->byteOrder, options: $this->options)) - ->readStream($stream->resource()), + ->readStream($stream), 'stream' => $stream, ]; } diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php index da38526a0..2d04c6726 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php @@ -4,11 +4,11 @@ namespace Flow\ETL\Adapter\Parquet; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Loader\Closure; use Flow\ETL\PHP\Type\Caster; use Flow\ETL\Row\Schema; use Flow\ETL\{FlowContext, Loader, Rows}; +use Flow\Filesystem\Path; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\{Options, Writer}; @@ -74,7 +74,7 @@ public function load(Rows $rows, FlowContext $context) : void options: $this->options ); - $this->writers[$stream->path()->uri()]->openForStream($stream->resource(), $this->converter->toParquet($this->schema())); + $this->writers[$stream->path()->uri()]->openForStream($stream, $this->converter->toParquet($this->schema())); } $this->writers[$stream->path()->uri()]->writeBatch($this->normalizer->normalize($rows, $this->schema())); @@ -87,7 +87,7 @@ public function load(Rows $rows, FlowContext $context) : void options: $this->options ); - $this->writers[$stream->path()->uri()]->openForStream($stream->resource(), $this->converter->toParquet($this->schema())); + $this->writers[$stream->path()->uri()]->openForStream($stream, $this->converter->toParquet($this->schema())); } $this->writers[$stream->path()->uri()]->writeBatch($this->normalizer->normalize($rows, $this->schema())); diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php index b299cb323..41db68faa 100644 --- a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php +++ b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php @@ -6,9 +6,9 @@ use function Flow\ETL\DSL\from_all; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Row\Schema; use Flow\ETL\{Extractor, Loader}; +use Flow\Filesystem\Path; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\{ByteOrder, Options}; diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php index 2da0d7416..041c14e87 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/PaginationTest.php @@ -6,7 +6,7 @@ use function Flow\ETL\DSL\{config, flow_context}; use Flow\ETL\Adapter\Parquet\ParquetExtractor; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; use Flow\Parquet\{Options, Reader}; use PHPUnit\Framework\TestCase; diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php index c36e1cbd1..f6ff13e8e 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetExtractorTest.php @@ -4,12 +4,10 @@ namespace Flow\ETL\Adapter\Parquet\Tests\Integration; -use function Flow\ETL\Adapter\Parquet\to_parquet; -use function Flow\ETL\DSL\from_array; use Flow\ETL\Adapter\Parquet\ParquetExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\{Config, Flow, FlowContext}; +use Flow\ETL\{Config, FlowContext}; +use Flow\Filesystem\Path; use Flow\Parquet\{Options, Reader}; use PHPUnit\Framework\TestCase; @@ -17,17 +15,7 @@ final class ParquetExtractorTest extends TestCase { public function test_limit() : void { - $path = \sys_get_temp_dir() . '/parquet_extractor_signal_stop.parquet'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_parquet($path)) - ->run(); - - $extractor = new ParquetExtractor(Path::realpath($path), Options::default()); + $extractor = new ParquetExtractor(\Flow\Filesystem\DSL\path(__DIR__ . '/../Fixtures/orders_flow.parquet'), Options::default()); $extractor->changeLimit(2); self::assertCount( @@ -54,27 +42,14 @@ public function test_reading_file_from_given_offset() : void public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/parquet_extractor_signal_stop.parquet'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_parquet($path)) - ->run(); - - $extractor = new ParquetExtractor(Path::realpath($path), Options::default()); + $extractor = new ParquetExtractor(\Flow\Filesystem\DSL\path(__DIR__ . '/../Fixtures/orders_flow.parquet'), Options::default()); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame([['id' => 1]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 2]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['id' => 3]], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetTest.php index 286401b9c..f3a29bb4e 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ParquetTest.php @@ -17,7 +17,7 @@ final class ParquetTest extends TestCase { public function test_writing_to_file() : void { - $path = \sys_get_temp_dir() . '/file.snappy.parquet'; + $path = __DIR__ . '/var/file.snappy.parquet'; $this->removeFile($path); df() @@ -46,7 +46,7 @@ public function test_writing_to_file() : void public function test_writing_with_provided_schema() : void { - $path = \sys_get_temp_dir() . '/file_schema.snappy.parquet'; + $path = __DIR__ . '/var/file_schema.snappy.parquet'; $this->removeFile($path); df() diff --git a/src/adapter/etl-adapter-text/README.md b/src/adapter/etl-adapter-text/README.md index a4a64f364..639acaacb 100644 --- a/src/adapter/etl-adapter-text/README.md +++ b/src/adapter/etl-adapter-text/README.md @@ -12,5 +12,5 @@ text data in large-scale and data-intensive projects. With Flow PHP's Adapter Te workflows becomes a more refined and efficient endeavor, perfectly aligning with the robust and adaptable framework of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/text.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php index b8a018545..7911c183e 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php @@ -5,14 +5,14 @@ namespace Flow\ETL\Adapter\Text; use function Flow\ETL\DSL\array_to_rows; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering, Signal}; -use Flow\ETL\Filesystem\Path; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal}; use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\Path; final class TextExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; public function __construct( private readonly Path $path, @@ -24,15 +24,9 @@ public function extract(FlowContext $context) : \Generator { $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows(); - foreach ($context->streams()->scan($this->path, $this->partitionFilter()) as $stream) { + foreach ($context->streams()->list($this->path, $this->filter()) as $stream) { - $rowData = \fgets($stream->resource()); - - if ($rowData === false) { - return; - } - - while ($rowData !== false) { + foreach ($stream->readLines() as $rowData) { if ($shouldPutInputIntoRows) { $row = [['text' => \rtrim($rowData), '_input_file_uri' => $stream->path()->uri()]]; } else { @@ -41,15 +35,13 @@ public function extract(FlowContext $context) : \Generator $signal = yield array_to_rows($row, $context->entryFactory(), $stream->path()->partitions()); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $context->streams()->closeWriters($this->path); return; } - - $rowData = \fgets($stream->resource()); } $stream->close(); diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php index 4ab4be2db..6493762e8 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Adapter\Text; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Path; use Flow\ETL\Loader\Closure; use Flow\ETL\{FlowContext, Loader, Rows}; +use Flow\Filesystem\Path; final class TextLoader implements Closure, Loader, Loader\FileLoader { @@ -38,8 +38,7 @@ public function load(Rows $rows, FlowContext $context) : void throw new RuntimeException(\sprintf('Text data loader supports only a single entry rows, and you have %d rows.', $row->entries()->count())); } - \fwrite( - $context->streams()->writeTo($this->path, $rows->partitions()->toArray())->resource(), + $context->streams()->writeTo($this->path, $rows->partitions()->toArray())->append( $row->entries()->all()[0]->toString() . $this->newLineSeparator ); } @@ -49,8 +48,7 @@ public function load(Rows $rows, FlowContext $context) : void throw new RuntimeException(\sprintf('Text data loader supports only a single entry rows, and you have %d rows.', $row->entries()->count())); } - \fwrite( - $context->streams()->writeTo($this->path)->resource(), + $context->streams()->writeTo($this->path)->append( $row->entries()->all()[0]->toString() . $this->newLineSeparator ); } diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php index 91ddbbce8..034406bda 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php @@ -4,8 +4,8 @@ namespace Flow\ETL\Adapter\Text; -use Flow\ETL\Filesystem\Path; use Flow\ETL\{Extractor, Loader}; +use Flow\Filesystem\Path; /** * @param array|Path|string $path diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php index 0d4a07f3c..a0ccf9f56 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php @@ -4,12 +4,11 @@ namespace Flow\ETL\Adapter\Text\Tests\Integration; -use function Flow\ETL\Adapter\Text\{from_text, to_text}; -use function Flow\ETL\DSL\from_array; +use function Flow\ETL\Adapter\Text\{from_text}; use Flow\ETL\Adapter\Text\TextExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\Path; use Flow\ETL\{Config, Flow, FlowContext, Row, Rows}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class TextExtractorTest extends TestCase @@ -53,16 +52,7 @@ public function test_extracting_text_files_from_directory() : void public function test_limit() : void { - $path = \sys_get_temp_dir() . '/text_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_text($path)) - ->run(); - $extractor = new TextExtractor(Path::realpath($path)); + $extractor = new TextExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv')); $extractor->changeLimit(2); self::assertCount( @@ -73,26 +63,14 @@ public function test_limit() : void public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/text_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - (new Flow())->read(from_array([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]])) - ->write(to_text($path)) - ->run(); + $extractor = new TextExtractor(Path::realpath(__DIR__ . '/../Fixtures/orders_flow.csv')); - $extractor = new TextExtractor(Path::realpath($path)); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame([['text' => '1']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['text' => '2']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame([['text' => '3']], $generator->current()->toArray()); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php index 2ff65bbb6..4001741f4 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php @@ -5,15 +5,15 @@ namespace Flow\ETL\Adapter\Text\Tests\Integration; use function Flow\ETL\Adapter\Text\to_text; -use Flow\ETL\Filesystem\Path; use Flow\ETL\{Flow, Row, Rows}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class TextTest extends TestCase { public function test_loading_text_files() : void { - $path = \sys_get_temp_dir() . '/flow_php_etl_csv_loader' . bin2hex(random_bytes(16)) . '.csv'; + $path = __DIR__ . '/var/flow_php_etl_csv_loader' . bin2hex(random_bytes(16)) . '.csv'; (new Flow()) ->process( diff --git a/src/adapter/etl-adapter-xml/README.md b/src/adapter/etl-adapter-xml/README.md index d267081a9..044660e1c 100644 --- a/src/adapter/etl-adapter-xml/README.md +++ b/src/adapter/etl-adapter-xml/README.md @@ -11,5 +11,5 @@ making it a prime choice for developers dealing with XML data in large-scale and PHP's Adapter XML, managing XML data within your ETL workflows becomes a more simplified and efficient endeavor, aligning perfectly with the robust and adaptable nature of the Flow PHP ecosystem. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/adapters/xml.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php index 4a5c60d17..94e0214f0 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php @@ -5,14 +5,14 @@ namespace Flow\ETL\Adapter\XML; use function Flow\ETL\DSL\array_to_rows; -use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PartitionFiltering, Signal}; -use Flow\ETL\Filesystem\Path; +use Flow\ETL\Extractor\{FileExtractor, Limitable, LimitableExtractor, PartitionExtractor, PathFiltering, Signal}; use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\Path; final class XMLReaderExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; /** * In order to iterate only over nodes us root/elements/element. @@ -40,7 +40,7 @@ public function extract(FlowContext $context) : \Generator { $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows(); - foreach ($context->streams()->scan($this->path, $this->partitionFilter()) as $stream) { + foreach ($context->streams()->list($this->path, $this->filter()) as $stream) { $xmlReader = new \XMLReader(); $xmlReader->open($stream->path()->path()); @@ -80,7 +80,7 @@ public function extract(FlowContext $context) : \Generator $signal = yield array_to_rows($rowData, $context->entryFactory(), $stream->path()->partitions()); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $xmlReader->close(); diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php index 914d5784f..1162c371f 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php @@ -6,7 +6,7 @@ use function Flow\ETL\DSL\from_all; use Flow\ETL\Extractor; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; /** * @param array|Path|string $path diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php index 0d9630452..fb4f43769 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php @@ -8,43 +8,15 @@ use function Flow\ETL\DSL\xml_entry; use Flow\ETL\Adapter\XML\XMLReaderExtractor; use Flow\ETL\Extractor\Signal; -use Flow\ETL\Filesystem\Path; use Flow\ETL\{Config, Flow, FlowContext, Row, Rows}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class XMLReaderExtractorTest extends TestCase { public function test_limit() : void { - $path = \sys_get_temp_dir() . '/xml_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - \file_put_contents($path, <<<'XML' - - - - 1 - - - 2 - - - 3 - - - 4 - - - 5 - - - -XML); - - $extractor = new XMLReaderExtractor(Path::realpath($path), 'items/item'); + $extractor = new XMLReaderExtractor(Path::realpath(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'); $extractor->changeLimit(2); self::assertCount( @@ -147,45 +119,14 @@ public function test_reading_xml_from_path() : void public function test_signal_stop() : void { - $path = \sys_get_temp_dir() . '/xml_extractor_signal_stop.csv'; - - if (\file_exists($path)) { - \unlink($path); - } - - \file_put_contents($path, <<<'XML' - - - - 1 - - - 2 - - - 3 - - - 4 - - - 5 - - - -XML); - - $extractor = new XMLReaderExtractor(Path::realpath($path), 'items/item'); + $extractor = new XMLReaderExtractor(Path::realpath(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'); $generator = $extractor->extract(new FlowContext(Config::default())); - self::assertSame('1', $generator->current()->first()->valueOf('node')->getElementsByTagName('id')[0]->nodeValue); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame('2', $generator->current()->first()->valueOf('node')->getElementsByTagName('id')[0]->nodeValue); self::assertTrue($generator->valid()); $generator->next(); - self::assertSame('3', $generator->current()->first()->valueOf('node')->getElementsByTagName('id')[0]->nodeValue); self::assertTrue($generator->valid()); $generator->send(Signal::STOP); self::assertFalse($generator->valid()); diff --git a/src/bridge/filesystem/azure/.gitattributes b/src/bridge/filesystem/azure/.gitattributes new file mode 100644 index 000000000..e02097205 --- /dev/null +++ b/src/bridge/filesystem/azure/.gitattributes @@ -0,0 +1,9 @@ +*.php text eol=lf + +/.github export-ignore +/tests export-ignore + +/README.md export-ignore + +/.gitattributes export-ignore +/.gitignore export-ignore diff --git a/src/adapter/etl-adapter-filesystem/.github/workflows/readonly.yaml b/src/bridge/filesystem/azure/.github/workflows/readonly.yaml similarity index 100% rename from src/adapter/etl-adapter-filesystem/.github/workflows/readonly.yaml rename to src/bridge/filesystem/azure/.github/workflows/readonly.yaml diff --git a/src/adapter/etl-adapter-filesystem/CONTRIBUTING.md b/src/bridge/filesystem/azure/CONTRIBUTING.md similarity index 100% rename from src/adapter/etl-adapter-filesystem/CONTRIBUTING.md rename to src/bridge/filesystem/azure/CONTRIBUTING.md diff --git a/src/adapter/etl-adapter-filesystem/LICENSE b/src/bridge/filesystem/azure/LICENSE similarity index 100% rename from src/adapter/etl-adapter-filesystem/LICENSE rename to src/bridge/filesystem/azure/LICENSE diff --git a/src/bridge/filesystem/azure/README.md b/src/bridge/filesystem/azure/README.md new file mode 100644 index 000000000..71784e19f --- /dev/null +++ b/src/bridge/filesystem/azure/README.md @@ -0,0 +1,6 @@ +# Dremel + +## Installation + +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/bridges/filesystem-azure-bridge.md) +- 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/bridge/filesystem/azure/composer.json b/src/bridge/filesystem/azure/composer.json new file mode 100644 index 000000000..fe9612e65 --- /dev/null +++ b/src/bridge/filesystem/azure/composer.json @@ -0,0 +1,43 @@ +{ + "name": "flow-php/filesytem-azure-bridge", + "type": "library", + "description": "PHP ETL - Flow Filesystem Azure Bridge", + "keywords": [ + "filesystem", + "azure", + "blob", + "read", + "range", + "stream", + "remote", + "storage", + "cloud" + ], + "require": { + "php": "~8.1.0 || ~8.2.0 || ~8.3.0", + "flow-php/filesystem": "^0.7 || 1.x-dev", + "flow-php/azure-sdk": "^0.7 || 1.x-dev" + }, + "config": { + "optimize-autoloader": true, + "sort-packages": true + }, + "license": "MIT", + "autoload": { + "psr-4": { + "Flow\\": [ + "src/Flow" + ] + }, + "files": [ + "src/Flow/Filesystem/Bridge/Azure/DSL/functions.php" + ] + }, + "autoload-dev": { + "psr-4": { + "Flow\\": "tests/Flow" + } + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream.php b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream.php new file mode 100644 index 000000000..96ebd567d --- /dev/null +++ b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream.php @@ -0,0 +1,114 @@ +blocks->append($data); + + return $this; + } + + public function close() : void + { + if ($this->blocks->size() === 0) { + $this->blobService->putBlockBlob($this->path->path()); + $this->closed = true; + + return; + } + + if ($this->blocks->count() > 1) { + $this->blocks->done(); + $this->blobService->putBlockBlobBlockList( + $this->path->path(), + $this->blockList + ); + } else { + $handle = \fopen($this->blocks->block()->path()->path(), 'rb'); + + if ($handle === false) { + throw new RuntimeException('Cannot open block file for reading'); + } + + $this->blobService->putBlockBlob($this->path->path(), $handle, $this->blocks->block()->size()); + + /** @psalm-suppress RedundantCondition */ + if (\is_resource($handle)) { + \fclose($handle); + } + + \unlink($this->blocks->block()->path()->path()); + } + + $this->closed = true; + } + + public function fromResource($resource) : self + { + if (!\is_resource($resource)) { + throw new InvalidArgumentException('DestinationStream::fromResource expects resource type, given: ' . \gettype($resource)); + } + + $meta = \stream_get_meta_data($resource); + + if ($meta['seekable']) { + \rewind($resource); + } + + $this->blocks->fromResource($resource); + + return $this; + } + + public function isOpen() : bool + { + return !$this->closed; + } + + public function path() : Path + { + return $this->path; + } +} diff --git a/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream/AzureBlobBlockLifecycle.php b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream/AzureBlobBlockLifecycle.php new file mode 100644 index 000000000..4d0a69809 --- /dev/null +++ b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobDestinationStream/AzureBlobBlockLifecycle.php @@ -0,0 +1,59 @@ +blockList->all())) { + $this->initialized = true; + } + } + + public function filled(Block $block) : void + { + if (!$this->initialized) { + $this->blobService->putBlockBlob($this->path->path()); + $this->initialized = true; + } + + $handle = \fopen($block->path()->path(), 'rb'); + + if ($handle === false) { + throw new RuntimeException('Cannot open block file for reading'); + } + + $this->blobService->putBlockBlobBlock( + $this->path->path(), + $block->id(), + $handle, + $block->size(), + ); + + /** @psalm-suppress RedundantCondition */ + if (\is_resource($handle)) { + \fclose($handle); + } + + \unlink($block->path()->path()); + + $this->blockList->append(new \Flow\Azure\SDK\BlobService\BlockBlob\Block( + $block->id(), + BlockState::UNCOMMITTED, + )); + } +} diff --git a/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobFilesystem.php b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobFilesystem.php new file mode 100644 index 000000000..d9b531346 --- /dev/null +++ b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobFilesystem.php @@ -0,0 +1,159 @@ +protocol()->validateScheme($path); + + if ($path->isPattern()) { + $prefix = \ltrim($path->staticPart()->path(), DIRECTORY_SEPARATOR); + } else { + $prefix = \ltrim($path->path(), DIRECTORY_SEPARATOR); + } + + $options = $this->options->listBlobOptions(); + + if ($prefix) { + $options->withPrefix($prefix); + } + + foreach ($this->blobService->listBlobs($options) as $blob) { + $blobPath = new Path($path->protocol()->scheme() . DIRECTORY_SEPARATOR . \ltrim($blob->name(), DIRECTORY_SEPARATOR), $path->options()); + $blobFileStatus = new FileStatus($blobPath, (bool) $blobPath->extension()); + + if ($path->isPattern() && !$path->matches($blobPath)) { + continue; + } + + if ($pathFilter->accept($blobFileStatus)) { + yield $blobFileStatus; + } + } + } + + public function mv(Path $from, Path $to) : bool + { + $this->protocol()->validateScheme($from); + $this->protocol()->validateScheme($to); + + $this->blobService->copyBlob($from->path(), $to->path()); + $this->blobService->deleteBlob($from->path()); + + return true; + } + + public function protocol() : Protocol + { + return new Protocol('azure-blob'); + } + + public function readFrom(Path $path) : SourceStream + { + return new AzureBlobSourceStream($path, $this->blobService); + } + + public function rm(Path $path) : bool + { + $this->protocol()->validateScheme($path); + + if ($path->isPattern()) { + $deletedCount = 0; + + foreach ($this->list($path) as $fileStatus) { + $this->blobService->deleteBlob($fileStatus->path->path()); + $deletedCount++; + } + + return (bool) $deletedCount; + } + + try { + $this->blobService->deleteBlob($path->path()); + + return true; + } catch (\Exception $e) { + /** + * Since AzureBlobStorage doesn't have a concept of folders, before we check if the intention is not to delete + * entire path, like for example azure-blob://nested/folder we need to first add / at the end, to accidentally + * not delete files that would also match the prefix, like: azure-blob://nested/folder_but_file.txt. + */ + $folderPath = new Path(\trim($path->uri(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR, $path->options()); + $blobProperties = $this->blobService->getBlobProperties($folderPath->path()); + + if ($blobProperties === null) { + $deletedCount = 0; + + foreach ($this->list($folderPath) as $fileStatus) { + $this->blobService->deleteBlob($fileStatus->path->path()); + $deletedCount++; + } + + return (bool) $deletedCount; + } + + return false; + } + } + + public function status(Path $path) : ?FileStatus + { + $this->protocol()->validateScheme($path); + + if (!$path->isPattern()) { + if ($path->path() === '/') { + return new FileStatus($path, false); + } + + $blobProperties = $this->blobService->getBlobProperties(\ltrim($path->path(), DIRECTORY_SEPARATOR)); + + if ($blobProperties === null) { + /** + * Since AzureBlobStorage doesn't have a concept of folders, before we check if the intention is not to delete + * entire path, like for example azure-blob://nested/folder we need to first add / at the end, to accidentally + * not match files that would also match the prefix, like: azure-blob://nested/folder_but_file.txt. + */ + $folderPath = new Path(trim($path->uri(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR, $path->options()); + + foreach ($this->list($folderPath) as $fileStatus) { + return new FileStatus($folderPath, false); + } + + return null; + } + + return new FileStatus($path, true); + } + + foreach ($this->list($path) as $fileStatus) { + return $fileStatus; + } + + return null; + } + + public function writeTo(Path $path) : DestinationStream + { + $this->protocol()->validateScheme($path); + + return AzureBlobDestinationStream::openBlank( + $this->blobService, + $path, + $this->options->blockFactory(), + $this->options->blockSize() + ); + } +} diff --git a/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobSourceStream.php b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobSourceStream.php new file mode 100644 index 000000000..a81754811 --- /dev/null +++ b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/AzureBlobSourceStream.php @@ -0,0 +1,117 @@ +blobService->getBlob($this->path->path())->content(); + } + + public function isOpen() : bool + { + return true; + } + + public function iterate(int $length = 1) : \Generator + { + $offset = 0; + + while ($offset < $this->size()) { + yield $this->read($length, $offset); + $offset += $length; + } + } + + public function path() : Path + { + return $this->path; + } + + public function read(int $length, int $offset) : string + { + $offset = $offset < 0 ? $this->size() + $offset : $offset; + + return $this->blobService->getBlob( + $this->path->path(), + (new GetBlobOptions())->withRange(new BlobService\GetBlob\Range($offset, $offset + $length - 1)) + )->content(); + } + + /** + * @psalm-suppress PossiblyFalseArgument + * @psalm-suppress PossiblyFalseOperand + */ + public function readLines(string $separator = "\n", ?int $length = null) : \Generator + { + $offset = 0; + $content = ''; + + while ($offset < $this->size()) { + // Read a chunk of the file + $chunk = $this->read($length ?? 1024 * 1024 * 9, $offset); + $offset += \strlen($chunk); + $content .= $chunk; + + // no separators found in the chunk, we are still processing single line + if (!\str_contains($content, $separator)) { + continue; + } + + if (\substr_count($content, $separator) > 1) { + /** + * @phpstan-ignore-next-line + */ + $lines = \array_filter(\explode($separator, $content)); + + // Yield all lines except the last one + foreach (\array_slice($lines, 0, -1) as $line) { + yield $line; + } + + // The last line is incomplete, so we need to keep it for the next iteration + $content = \end($lines); + } elseif (\substr_count($content, $separator) === 1) { + // Split the content by the separator + /** + * @phpstan-ignore-next-line + */ + yield \substr($content, 0, \strpos($content, $separator)); + $content = \substr($content, \strpos($content, $separator) + 1); + } + } + + // Yield the remaining content if it's not empty + if ($content) { + yield $content; + } + } + + public function size() : ?int + { + if ($this->blobProperties === null) { + $this->blobProperties = $this->blobService->getBlobProperties($this->path->path()); + } + + return $this->blobProperties?->size(); + } +} diff --git a/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php new file mode 100644 index 000000000..feee2e99a --- /dev/null +++ b/src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php @@ -0,0 +1,18 @@ + + */ + private ?array $listBlobInclude = null; + + private ?int $listBlobMaxResults = null; + + private ?OptionShowOnly $listBlobShowOnly = null; + + public function __construct() + { + $this->blockFactory = new NativeLocalFileBlocksFactory(); + } + + public function blockFactory() : BlockFactory + { + return $this->blockFactory; + } + + public function blockSize() : int + { + return $this->blockSize; + } + + public function listBlobOptions() : ListBlobOptions + { + $listBlobOptions = new ListBlobOptions(); + + if ($this->listBlobInclude !== null) { + $listBlobOptions->withInclude(...$this->listBlobInclude); + } + + if ($this->listBlobMaxResults !== null) { + $listBlobOptions->withMaxResults($this->listBlobMaxResults); + } + + if ($this->listBlobShowOnly !== null) { + $listBlobOptions->withShowOnly($this->listBlobShowOnly); + } + + return $listBlobOptions; + } + + public function withBlockFactory(BlockFactory $blockFactory) : self + { + $this->blockFactory = $blockFactory; + + return $this; + } + + public function withBlockSize(int $blockSize) : self + { + $this->blockSize = $blockSize; + + return $this; + } + + public function withListBlobInclude(OptionInclude ...$listBlobInclude) : self + { + $this->listBlobInclude = $listBlobInclude; + + return $this; + } + + public function withListBlobMaxResults(int $listBlobMaxResults) : self + { + $this->listBlobMaxResults = $listBlobMaxResults; + + return $this; + } + + public function withListBlobShowOnly(OptionShowOnly $listBlobShowOnly) : self + { + $this->listBlobShowOnly = $listBlobShowOnly; + + return $this; + } +} diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobDestinationStreamTest.php b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobDestinationStreamTest.php new file mode 100644 index 000000000..d424a9120 --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobDestinationStreamTest.php @@ -0,0 +1,66 @@ +blobService('flow-php')); + $stream = $fs->writeTo(new Path('azure-blob://file.txt')); + self::assertTrue($stream->isOpen()); + $stream->close(); + self::assertFalse($stream->isOpen()); + } + + public function test_writing_content_bigger_than_block_size_to_azure() : void + { + $fs = azure_filesystem($this->blobService('flow-php'), (new Options())->withBlockSize(100)); + + $stream = $fs->writeTo(new Path('azure-blob://file.txt')); + $stream->append($content = \str_repeat('a', 200)); + $stream->close(); + + self::assertTrue($fs->status(new Path('azure-blob://file.txt'))->isFile()); + self::assertFalse($fs->status(new Path('azure-blob://file.txt'))->isDirectory()); + self::assertSame($content, $fs->readFrom(new Path('azure-blob://file.txt'))->content()); + + $fs->rm(new Path('azure-blob://file.txt')); + } + + public function test_writing_content_from_resource() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $stream = $fs->writeTo(new Path('azure-blob://orders.csv')); + $stream->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $stream->close(); + + self::assertTrue($fs->status(new Path('azure-blob://orders.csv'))->isFile()); + self::assertFalse($fs->status(new Path('azure-blob://orders.csv'))->isDirectory()); + self::assertSame(\file_get_contents(__DIR__ . '/Fixtures/orders.csv'), $fs->readFrom(new Path('azure-blob://orders.csv'))->content()); + + $fs->rm(new Path('azure-blob://orders.csv')); + } + + public function test_writing_content_smaller_than_block_size_to_azure() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $stream = $fs->writeTo(new Path('azure-blob://file.txt')); + $stream->append('Hello, World!'); + $stream->close(); + + self::assertTrue($fs->status(new Path('azure-blob://file.txt'))->isFile()); + self::assertFalse($fs->status(new Path('azure-blob://file.txt'))->isDirectory()); + self::assertSame('Hello, World!', $fs->readFrom(new Path('azure-blob://file.txt'))->content()); + + $fs->rm(new Path('azure-blob://file.txt')); + } +} diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobFilesystemTest.php b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobFilesystemTest.php new file mode 100644 index 000000000..dcd262f26 --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobFilesystemTest.php @@ -0,0 +1,169 @@ +blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + + self::assertTrue($fs->status(new Path('azure-blob://file.txt'))->isFile()); + } + + public function test_file_status_on_existing_folder() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + + self::assertTrue($fs->status(new Path('azure-blob://nested/orders'))->isDirectory()); + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/'))->isDirectory()); + } + + public function test_file_status_on_non_existing_file() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + self::assertNull($fs->status(new Path('azure-blob://non-existing-file.txt'))); + } + + public function test_file_status_on_non_existing_folder() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + self::assertNull($fs->status(new Path('azure-blob://non-existing-folder/'))); + } + + public function test_file_status_on_non_existing_pattern() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + self::assertNull($fs->status(new Path('azure-blob://non-existing-folder/*'))); + } + + public function test_file_status_on_partial_path() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://some_path_to/file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertNull($fs->status(new Path('azure-blob://some_path'))); + } + + public function test_file_status_on_pattern() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://some_path_to/file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path('azure-blob://some_path_to/*.txt'))->isFile()); + self::assertSame('azure-blob://some_path_to/file.txt', $fs->status(new Path('azure-blob://some_path_to/*.txt'))->path->uri()); + } + + public function test_file_status_on_root_folder() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + self::assertTrue($fs->status(new Path('azure-blob://'))->isDirectory()); + } + + public function test_move_blob() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://file.txt'))->append('Hello, World!')->close(); + + $fs->mv(new Path('azure-blob://file.txt'), new Path('azure-blob://file_mv.txt')); + + self::assertNull($fs->status(new Path('azure-blob://file.txt'))); + self::assertSame('Hello, World!', $fs->readFrom(new Path('azure-blob://file_mv.txt'))->content()); + } + + public function test_not_removing_a_content_when_its_not_a_full_folder_path_pattern() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders_01.csv'))->isFile()); + + self::assertFalse($fs->rm(new Path('azure-blob://nested/orders/ord'))); + } + + public function test_removing_folder() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders_01.csv'))->isFile()); + + $fs->rm(new Path('azure-blob://nested/orders')); + + self::assertTrue($fs->status(new Path('azure-blob://orders.csv'))->isFile()); + self::assertNull($fs->status(new Path('azure-blob://nested/orders/orders.csv'))); + self::assertNull($fs->status(new Path('azure-blob://nested/orders/orders_01.csv'))); + } + + public function test_removing_folder_pattern() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $fs->writeTo(new Path('azure-blob://nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + $fs->writeTo(new Path('azure-blob://nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'))->close(); + + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders_01.csv'))->isFile()); + + $fs->rm(new Path('azure-blob://nested/orders/*.csv')); + + self::assertTrue($fs->status(new Path('azure-blob://nested/orders/orders.txt'))->isFile()); + self::assertNull($fs->status(new Path('azure-blob://nested/orders/orders.csv'))); + self::assertNull($fs->status(new Path('azure-blob://nested/orders/orders_01.csv'))); + } + + public function test_writing_to_azure_blob_storage() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $stream = $fs->writeTo(new Path('azure-blob://file.txt')); + $stream->append('Hello, World!'); + $stream->close(); + + self::assertTrue($fs->status(new Path('azure-blob://file.txt'))->isFile()); + self::assertFalse($fs->status(new Path('azure-blob://file.txt'))->isDirectory()); + self::assertSame('Hello, World!', $fs->readFrom(new Path('azure-blob://file.txt'))->content()); + + $fs->rm(new Path('azure-blob://file.txt')); + } + + public function test_writing_to_to_azure_from_resources() : void + { + $fs = azure_filesystem($this->blobService('flow-php')); + + $stream = $fs->writeTo(new Path('azure-blob://orders.csv')); + $stream->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $stream->close(); + + self::assertTrue($fs->status(new Path('azure-blob://orders.csv'))->isFile()); + self::assertFalse($fs->status(new Path('azure-blob://orders.csv'))->isDirectory()); + self::assertSame(\file_get_contents(__DIR__ . '/Fixtures/orders.csv'), $fs->readFrom(new Path('azure-blob://orders.csv'))->content()); + + $fs->rm(new Path('azure-blob://orders.csv')); + } +} diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobServiceTestCase.php b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobServiceTestCase.php new file mode 100644 index 000000000..ca0136e24 --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobServiceTestCase.php @@ -0,0 +1,59 @@ + + */ + private array $containers = []; + + protected function tearDown() : void + { + foreach ($this->containers as $container) { + $this->blobService($container)->deleteContainer(); + } + } + + public function givenFileExists(string $container, string $path, string $content) : void + { + $this->blobService($container)->putBlockBlob($path, $content, \strlen($content)); + } + + public function givenFileExistsFromPath(string $container, string $path, string $sourcePath) : void + { + $this->blobService($container)->putBlockBlob($path, fopen($sourcePath, 'rb'), \filesize($sourcePath)); + } + + protected function blobService(string $container) : BlobService + { + $blobService = azure_blob_service( + azure_blob_service_config($_ENV['AZURITE_ACCOUNT_NAME'], $container), + azure_shared_key_authorization_factory($_ENV['AZURITE_ACCOUNT_NAME'], $_ENV['AZURITE_ACCOUNT_KEY']), + Psr18ClientDiscovery::find(), + azure_http_factory(Psr17FactoryDiscovery::findRequestFactory(), Psr17FactoryDiscovery::findStreamFactory()), + azurite_url_factory($_ENV['AZURITE_HOST'], $_ENV['AZURITE_BLOB_PORT'], false) + ); + + $properties = $blobService->getContainerProperties(); + + if (!$properties) { + $blobService->putContainer(); + $properties = $blobService->getContainerProperties(); + + if (!\in_array($container, $this->containers, true)) { + $this->containers[] = $container; + } + } + + return $blobService; + } +} diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobSourceStreamTest.php b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobSourceStreamTest.php new file mode 100644 index 000000000..a672f3baf --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/AzureBlobSourceStreamTest.php @@ -0,0 +1,88 @@ +givenFileExists('flow-php', 'file.txt', $content); + + $stream = azure_filesystem($this->blobService('flow-php'))->readFrom(new Path('azure-blob://file.txt')); + + self::assertSame($content, \implode('', \iterator_to_array($stream->iterate()))); + + $stream->close(); + } + + public function test_reading_from_blob_by_limit_and_offset() : void + { + $content = <<<'TEXT' +This is some +multi line file +that we are storing on azure blob +TEXT; + $this->givenFileExists('flow-php', 'file.txt', $content); + + $stream = azure_filesystem($this->blobService('flow-php'))->readFrom(new Path('azure-blob://file.txt')); + + self::assertSame($content, $stream->content()); + + self::assertSame('This is some', $stream->read(12, 0)); + self::assertSame(12, \strlen($stream->read(12, 0))); + self::assertSame('multi line file', $stream->read(15, 13)); + self::assertSame(15, \strlen($stream->read(15, 13))); + self::assertSame('that we are storing on azure blob', $stream->read(33, 29)); + self::assertSame(33, \strlen($stream->read(33, 29))); + + $stream->close(); + } + + #[DataProvider('line_lengths')] + public function test_reading_lines_from_blob(int $lineLength) : void + { + $content = <<<'TEXT' +This is some +multi line file +that we are storing on azure blob +TEXT; + $this->givenFileExists('flow-php', 'file.txt', $content); + + $stream = azure_filesystem($this->blobService('flow-php'))->readFrom(new Path('azure-blob://file.txt')); + + self::assertSame($content, $stream->content()); + + $lines = $stream->readLines(length: $lineLength); + self::assertSame('This is some', $lines->current()); + $lines->next(); + self::assertSame('multi line file', $lines->current()); + $lines->next(); + self::assertSame('that we are storing on azure blob', $lines->current()); + $lines->next(); + self::assertNull($lines->current()); + + $stream->close(); + } +} diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/Fixtures/orders.csv b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/Fixtures/orders.csv new file mode 100644 index 000000000..78f967e1c --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Integration/Fixtures/orders.csv @@ -0,0 +1,44 @@ +order_id,created_at,updated_at,discount,address,notes,items +e13d7098-5a78-3389-9289-022812e9ffa9,2024-06-17T19:24:49+00:00,2024-06-17T19:24:49+00:00,12.45,"{""street"":""9742 Jaskolski Forks Suite 585"",""city"":""South Lucianoside"",""zip"":""90339-5731"",""country"":""Saint Vincent and the Grenadines""}","[""Doloremque cum et adipisci sunt maiores qui."",""Distinctio fuga neque ut est et velit."",""Laborum consequuntur dolores quia quos eveniet."",""Voluptatem ipsam et quaerat atque hic quia maxime hic."",""Modi omnis non dolorem illo.""]","[{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +947df050-3abb-3f5a-92c5-5b6ceb49f0d6,2024-02-23T19:18:53+00:00,2024-02-23T19:18:53+00:00,,"{""street"":""37051 Alejandrin Orchard"",""city"":""Lebsackhaven"",""zip"":""85928-9879"",""country"":""Taiwan""}","[""Neque dolor et minima nulla aliquid."",""Est quas quod exercitationem ducimus nulla ut."",""Aut tempora quia quod aperiam vitae veritatis placeat.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55}]" +6315f9e2-86bf-3321-afa7-dfed4a31f10d,2024-04-02T11:30:25+00:00,2024-04-02T11:30:25+00:00,47.1,"{""street"":""792 Golda Estates Suite 028"",""city"":""Rubymouth"",""zip"":""50549"",""country"":""Turkey""}","[""Et porro fugiat fugiat culpa vitae dolores."",""Sit quibusdam minus consequuntur quo id distinctio aut tempora."",""Incidunt vel consequuntur beatae delectus."",""Cupiditate rerum minus iste ea illo et.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75}]" +4cccb632-fade-34e2-890c-ba557e227b5a,2024-05-06T00:17:57+00:00,2024-05-06T00:17:57+00:00,19.76,"{""street"":""30203 Wallace Plain"",""city"":""North Gennaro"",""zip"":""95802-1226"",""country"":""Zambia""}","[""Aliquam saepe iste suscipit officiis fuga."",""Blanditiis eaque cupiditate pariatur amet iure."",""Est repellat distinctio nesciunt dolorum ipsum recusandae."",""Qui vero nisi qui blanditiis consequatur magnam."",""Eius odio eveniet dolor est rem quia.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":9,""price"":397.55},{""sku"":""SKU_0003"",""quantity"":3,""price"":203.16}]" +82384f8c-9adb-38be-92b5-3c362a5f733e,2024-05-10T11:17:41+00:00,2024-05-10T11:17:41+00:00,,"{""street"":""757 Tobin Ports Suite 557"",""city"":""North Edison"",""zip"":""88755"",""country"":""Benin""}","[""Beatae nesciunt aut quia a.""]","[{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":1,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55}]" +e3fcf736-0f8c-3d97-bc5a-ffde26c0e2c1,2024-01-25T20:14:24+00:00,2024-01-25T20:14:24+00:00,,"{""street"":""9088 Tracey Ville Suite 686"",""city"":""Rosschester"",""zip"":""45566"",""country"":""Spain""}","[""Provident quam cum reprehenderit eius.""]","[{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0003"",""quantity"":7,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":6,""price"":246.3}]" +b987a49a-b4c5-37de-b5d9-2ec119cebedb,2024-06-03T23:22:13+00:00,2024-06-03T23:22:13+00:00,,"{""street"":""6867 Tyrell Flats"",""city"":""Hudsonmouth"",""zip"":""86961-2954"",""country"":""Bangladesh""}","[""Quibusdam maiores ex earum vel accusamus necessitatibus."",""Ex corporis laboriosam id optio qui beatae."",""Officia in at eveniet est architecto."",""Ullam porro ipsa eos dignissimos nihil recusandae dignissimos.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":10,""price"":247.75}]" +663523a9-713b-3354-8f0e-8f7142816aaf,2024-03-22T23:31:26+00:00,2024-03-22T23:31:26+00:00,25.88,"{""street"":""1577 Terence Tunnel"",""city"":""Berniecebury"",""zip"":""12143"",""country"":""Singapore""}","[""In rem maxime iure natus ipsam dolores."",""Sit hic voluptatibus facilis quibusdam consequuntur omnis aut est.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +6259fa2c-ec68-36a9-8476-dbd5d916465f,2024-05-10T10:12:52+00:00,2024-05-10T10:12:52+00:00,21.67,"{""street"":""987 Lloyd Radial"",""city"":""Zulaufberg"",""zip"":""18542-5386"",""country"":""Greece""}","[""Voluptatem non atque ad ipsum provident ut quia quas."",""Aspernatur maxime est quas debitis mollitia."",""Voluptatem et expedita ducimus inventore."",""Nihil dolorum distinctio qui facilis illo autem occaecati provident.""]","[{""sku"":""SKU_0003"",""quantity"":10,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":5,""price"":247.75},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55}]" +f7153c83-34b6-3769-95de-2109d932d925,2024-02-26T09:20:45+00:00,2024-02-26T09:20:45+00:00,18.93,"{""street"":""2039 Nova Summit Apt. 838"",""city"":""West Florian"",""zip"":""67943-5978"",""country"":""Turkmenistan""}","[""Culpa error recusandae fugit consequatur nam."",""Natus aspernatur aut dolor beatae ut.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":10,""price"":397.55}]" +966b91b5-e252-3787-96f9-c487a1ba3067,2024-05-10T11:34:49+00:00,2024-05-10T11:34:49+00:00,8.97,"{""street"":""518 Leannon Dam"",""city"":""Ziemannshire"",""zip"":""23503"",""country"":""Gambia""}","[""Ipsum adipisci veniam eaque quas."",""Vitae ad possimus sed dignissimos est occaecati.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":8,""price"":247.75}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" \ No newline at end of file diff --git a/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/AzureBlobDestinationStreamTest.php b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/AzureBlobDestinationStreamTest.php new file mode 100644 index 000000000..e739b6a3a --- /dev/null +++ b/src/bridge/filesystem/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/AzureBlobDestinationStreamTest.php @@ -0,0 +1,100 @@ +createMock(BlockFactory::class); + $blockFactory->method('create') + ->willReturnCallback( + function () use ($blockSize) { + return new Block($id = \bin2hex(\random_bytes(16)), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt')); + } + ); + + $stream = AzureBlobDestinationStream::openBlank( + $blobService = $this->createMock(BlobServiceInterface::class), + new Path('azure-blob://file.txt'), + $blockFactory, + $blockSize + ); + + $blobService->expects(self::once()) + ->method('putBlockBlob') + ->with( + '/file.txt', + self::isNull(), + self::isNull(), + self::isInstanceOf(PutBlockBlobOptions::class) + ); + + $blobService->expects(self::exactly(2)) + ->method('putBlockBlobBlock') + ->with( + '/file.txt', + self::isType('string'), + self::isType('resource'), + self::isType('int'), + self::isInstanceOf(PutBlockBlobBlockOptions::class) + ); + + $blobService->expects(self::once()) + ->method('putBlockBlobBlockList') + ->with( + '/file.txt', + self::isInstanceOf(BlockList::class), + self::isInstanceOf(PutBlockBlobBlockListOptions::class) + ); + + $stream->append(\str_repeat('a', 150)); + + $stream->close(); + } + + public function test_using_put_blob_with_content_when_data_is_smaller_than_block_size() : void + { + $blockSize = 100; + $blockFactory = $this->createMock(BlockFactory::class); + $blockFactory->method('create') + ->willReturnCallback( + function () use ($blockSize) { + return new Block($id = \bin2hex(\random_bytes(16)), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt')); + } + ); + $stream = AzureBlobDestinationStream::openBlank( + $blobService = $this->createMock(BlobServiceInterface::class), + new Path('azure-blob://file.txt'), + $blockFactory, + $blockSize + ); + + $blobService->expects(self::once()) + ->method('putBlockBlob') + ->with( + '/file.txt', + self::isType('resource'), + \strlen('Hello, World!'), + self::isInstanceOf(PutBlockBlobOptions::class) + ); + + $stream->append('Hello, World!'); + + $stream->close(); + } +} diff --git a/src/bridge/monolog/http/.gitattributes b/src/bridge/monolog/http/.gitattributes new file mode 100644 index 000000000..e02097205 --- /dev/null +++ b/src/bridge/monolog/http/.gitattributes @@ -0,0 +1,9 @@ +*.php text eol=lf + +/.github export-ignore +/tests export-ignore + +/README.md export-ignore + +/.gitattributes export-ignore +/.gitignore export-ignore diff --git a/src/bridge/monolog/http/.github/workflows/readonly.yaml b/src/bridge/monolog/http/.github/workflows/readonly.yaml new file mode 100644 index 000000000..da596bcdd --- /dev/null +++ b/src/bridge/monolog/http/.github/workflows/readonly.yaml @@ -0,0 +1,17 @@ +name: Readonly + +on: + pull_request_target: + types: [opened] + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: superbrothers/close-pull-request@v3 + with: + comment: | + Hi, thank you for your contribution. + Unfortunately, this repository is read-only. It's a split from our main monorepo repository. + In order to proceed with this PR please open it against https://github.com/flow-php/flow repository. + Thank you. \ No newline at end of file diff --git a/src/bridge/monolog/http/CONTRIBUTING.md b/src/bridge/monolog/http/CONTRIBUTING.md new file mode 100644 index 000000000..a2d0671c7 --- /dev/null +++ b/src/bridge/monolog/http/CONTRIBUTING.md @@ -0,0 +1,6 @@ +## Contributing + +This repo is **READ ONLY**, in order to contribute to Flow PHP project, please +open PR against [flow](https://github.com/flow-php/flow) monorepo. + +Changes merged to monorepo are automatically propagated into sub repositories. \ No newline at end of file diff --git a/src/bridge/monolog/http/LICENSE b/src/bridge/monolog/http/LICENSE new file mode 100644 index 000000000..bc3cc4d08 --- /dev/null +++ b/src/bridge/monolog/http/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020-present Flow PHP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/bridge/monolog/http/README.md b/src/bridge/monolog/http/README.md new file mode 100644 index 000000000..af84049bf --- /dev/null +++ b/src/bridge/monolog/http/README.md @@ -0,0 +1,6 @@ +# Dremel + +## Installation + +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/bridges/monolog-http-bridge.md) +- 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/bridge/monolog/http/composer.json b/src/bridge/monolog/http/composer.json new file mode 100644 index 000000000..6272f9abb --- /dev/null +++ b/src/bridge/monolog/http/composer.json @@ -0,0 +1,38 @@ +{ + "name": "flow-php/monolog-http-bridge", + "type": "library", + "description": "PHP ETL - Monolog HTTP bridge", + "keywords": [ + "logs", + "monolog", + "http", + "bridge" + ], + "require": { + "php": "~8.1.0 || ~8.2.0 || ~8.3.0", + "monolog/monolog": "^2.0||^3.0", + "psr/http-message": "^1.0" + }, + "require-dev": { + "nyholm/psr7": "^1.8" + }, + "config": { + "optimize-autoloader": true, + "sort-packages": true + }, + "license": "MIT", + "autoload": { + "psr-4": { + "Flow\\": [ + "src/Flow" + ] + } + }, + "autoload-dev": { + "psr-4": { + "Flow\\": "tests/Flow" + } + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config.php b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config.php new file mode 100644 index 000000000..b93cfb17a --- /dev/null +++ b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config.php @@ -0,0 +1,17 @@ +bodySizeLimit; + } + + public function includeBody() : bool + { + return $this->withBody; + } + + /** + * @return array + */ + public function includeHeaders() : array + { + return $this->headers; + } + + public function includeMethod() : bool + { + return $this->withMethod; + } + + public function includeUri() : bool + { + return $this->withUri; + } +} diff --git a/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config/ResponseConfig.php b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config/ResponseConfig.php new file mode 100644 index 000000000..4bef6f542 --- /dev/null +++ b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/Config/ResponseConfig.php @@ -0,0 +1,63 @@ + + * @param array $headers + */ + public function __construct( + private readonly bool $withReasonPhrase = true, + private readonly bool $withStatus = true, + private readonly bool $withBody = false, + private readonly int $bodySizeLimit = 1024 * 1024 * 32, + private readonly array $withoutStatusCodes = [], + private readonly array $headers = ['cache-control', 'location', 'set-cookie', 'server', 'expires', 'content-type', 'content-length', 'last-modified', 'kee-alive', 'referrer-policy', 'etag'] + ) { + + } + + public function bodySizeLimit() : int + { + return $this->bodySizeLimit; + } + + /** + * @return array + */ + public function excludeStatusCodes() : array + { + return $this->withoutStatusCodes; + } + + public function includeBody() : bool + { + return $this->withBody; + } + + /** + * @return array + */ + public function includeHeaders() : array + { + return $this->headers; + } + + public function includeReasonPhrase() : bool + { + return $this->withReasonPhrase; + } + + public function includeStatus() : bool + { + return $this->withStatus; + } +} diff --git a/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/PSR7Processor.php b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/PSR7Processor.php new file mode 100644 index 000000000..a5048349a --- /dev/null +++ b/src/bridge/monolog/http/src/Flow/Bridge/Monolog/Http/PSR7Processor.php @@ -0,0 +1,119 @@ +context; + + foreach ($context as $key => $val) { + if ($val instanceof RequestInterface) { + $context[$key] = $this->normalizeRequest($val); + + if (empty($context[$key])) { + unset($context[$key]); + } + } + + if ($val instanceof ResponseInterface) { + $context[$key] = $this->normalizeResponse($val); + + if (empty($context[$key])) { + unset($context[$key]); + } + } + } + + if (\is_array($record)) { + $record['context'] = $context; + + return $record; + } + + return $record->with(context: $context); + } + + private function normalizeRequest(RequestInterface $request) : array + { + $requestData = []; + + if ($this->config->request->includeMethod()) { + $requestData['method'] = $request->getMethod(); + } + + if ($this->config->request->includeUri()) { + $requestData['uri'] = (string) $request->getUri(); + } + + if ($this->config->request->includeBody()) { + $requestData['body'] = \substr($request->getBody()->getContents(), 0, $this->config->request->bodySizeLimit()); + $request->getBody()->rewind(); + + if ($requestData['body'] === '') { + unset($requestData['body']); + } + } + + if ($this->config->request->includeHeaders()) { + $requestData['headers'] = \array_filter( + $request->getHeaders(), + fn (string $header) => \in_array(\strtolower($header), $this->config->request->includeHeaders(), true), + ARRAY_FILTER_USE_KEY + ); + } + + return $requestData; + } + + private function normalizeResponse(ResponseInterface $response) : array + { + $responseData = []; + + if (\in_array($response->getStatusCode(), $this->config->response->excludeStatusCodes(), true)) { + return $responseData; + } + + if ($this->config->response->includeStatus()) { + $responseData['status'] = $response->getStatusCode(); + } + + if ($this->config->response->includeReasonPhrase()) { + $responseData['reason_phrase'] = $response->getReasonPhrase(); + } + + if ($this->config->response->includeBody()) { + $responseData['body'] = \substr($response->getBody()->getContents(), 0, $this->config->response->bodySizeLimit()); + $response->getBody()->rewind(); + + if ($responseData['body'] === '') { + unset($responseData['body']); + } + } + + if ($this->config->response->includeHeaders()) { + $responseData['headers'] = \array_filter( + $response->getHeaders(), + fn (string $header) => \in_array(\strtolower($header), $this->config->response->includeHeaders(), true), + ARRAY_FILTER_USE_KEY + ); + } + + return $responseData; + } +} diff --git a/src/bridge/monolog/http/tests/Flow/Bridge/Monolog/Http/Tests/Unit/PSR7ProcessorTest.php b/src/bridge/monolog/http/tests/Flow/Bridge/Monolog/Http/Tests/Unit/PSR7ProcessorTest.php new file mode 100644 index 000000000..f367c8954 --- /dev/null +++ b/src/bridge/monolog/http/tests/Flow/Bridge/Monolog/Http/Tests/Unit/PSR7ProcessorTest.php @@ -0,0 +1,253 @@ +createRequest('GET', 'https://example.com/api/v1/users?limit=10#page=1') + ->withHeader('User-Agent', 'Flow/1.0') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Authorization', 'Bearer 123') + ->withBody($psr17->createStream('Hello World!')); + + $processor = new PSR7Processor(); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Request', 'context' => ['request' => $request]]); + + self::assertEquals( + [ + 'request' => [ + 'method' => 'GET', + 'uri' => 'https://example.com/api/v1/users?limit=10#page=1', + 'headers' => [ + 'User-Agent' => ['Flow/1.0'], + 'Host' => ['example.com'], + ], + ], + ], + $record['context'], + ); + } + + public function test_normalizing_http_request_with_body() : void + { + $psr17 = new Psr17Factory(); + + $request = $psr17->createRequest('POST', 'https://example.com/api/v1/users') + ->withHeader('User-Agent', 'Flow/1.0') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Authorization', 'Bearer 123') + ->withBody($psr17->createStream('Hello World!')); + + $processor = new PSR7Processor((new Config(new Config\RequestConfig(withBody: true)))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Request', 'context' => ['request' => $request]]); + + self::assertEquals( + [ + 'request' => [ + 'method' => 'POST', + 'uri' => 'https://example.com/api/v1/users', + 'headers' => [ + 'User-Agent' => ['Flow/1.0'], + 'Host' => ['example.com'], + ], + 'body' => 'Hello World!', + ], + ], + $record['context'], + ); + } + + public function test_normalizing_http_request_with_limited_body() : void + { + $psr17 = new Psr17Factory(); + + $request = $psr17->createRequest('POST', 'https://example.com/api/v1/users') + ->withHeader('User-Agent', 'Flow/1.0') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Authorization', 'Bearer 123') + ->withBody($psr17->createStream('Hello World!')); + + $processor = new PSR7Processor((new Config(new Config\RequestConfig(withBody: true, bodySizeLimit: 5)))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Request', 'context' => ['request' => $request]]); + + self::assertEquals( + [ + 'request' => [ + 'method' => 'POST', + 'uri' => 'https://example.com/api/v1/users', + 'headers' => [ + 'User-Agent' => ['Flow/1.0'], + 'Host' => ['example.com'], + ], + 'body' => 'Hello', + ], + ], + $record['context'], + ); + } + + public function test_normalizing_http_request_without_headers() : void + { + $psr17 = new Psr17Factory(); + + $request = $psr17->createRequest('POST', 'https://example.com/api/v1/users') + ->withBody($psr17->createStream('Hello World!')); + + $processor = new PSR7Processor((new Config(new Config\RequestConfig(headers: [])))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Request', 'context' => ['request' => $request]]); + + self::assertEquals( + [ + 'request' => [ + 'method' => 'POST', + 'uri' => 'https://example.com/api/v1/users', + ], + ], + $record['context'], + ); + } + + public function test_normalizing_http_response() : void + { + $psr17 = new Psr17Factory(); + + $response = $psr17->createResponse(200, 'OK') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Content-Length', '42') + ->withBody($psr17->createStream('{"message":"Hello, World!"}')); + + $processor = new PSR7Processor(); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Response', 'context' => ['response' => $response]]); + + self::assertEquals( + [ + 'response' => [ + 'status' => 200, + 'reason_phrase' => 'OK', + 'headers' => [ + 'Content-Type' => ['application/json'], + 'Content-Length' => ['42'], + ], + ], + ], + $record['context'] + ); + } + + public function test_normalizing_http_response_when_status_code_is_excluded() : void + { + $psr17 = new Psr17Factory(); + + $response = $psr17->createResponse(404, 'Not Found') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Content-Length', '42') + ->withBody($psr17->createStream('{"message":"Not Found!"}')); + + $processor = new PSR7Processor((new Config(response: new ResponseConfig(withoutStatusCodes: [404])))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Response', 'context' => ['response' => $response]]); + + self::assertEquals([], $record['context']); + } + + public function test_normalizing_http_response_with_body() : void + { + $psr17 = new Psr17Factory(); + + $response = $psr17->createResponse(200, 'OK') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Content-Length', '42') + ->withBody($psr17->createStream('{"message":"Hello, World!"}')); + + $processor = new PSR7Processor((new Config(response: new ResponseConfig(withBody: true)))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Response', 'context' => ['response' => $response]]); + + self::assertEquals( + [ + 'response' => [ + 'status' => 200, + 'reason_phrase' => 'OK', + 'headers' => [ + 'Content-Type' => ['application/json'], + 'Content-Length' => ['42'], + ], + 'body' => '{"message":"Hello, World!"}', + ], + ], + $record['context'] + ); + } + + public function test_normalizing_http_response_with_body_limit() : void + { + $psr17 = new Psr17Factory(); + + $response = $psr17->createResponse(200, 'OK') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Content-Length', '42') + ->withBody($psr17->createStream('{"message":"Hello, World!"}')); + + $processor = new PSR7Processor((new Config(response: new ResponseConfig(withBody: true, bodySizeLimit: 5)))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Response', 'context' => ['response' => $response]]); + + self::assertEquals( + [ + 'response' => [ + 'status' => 200, + 'reason_phrase' => 'OK', + 'headers' => [ + 'Content-Type' => ['application/json'], + 'Content-Length' => ['42'], + ], + 'body' => '{"mes', + ], + ], + $record['context'] + ); + } + + public function test_normalizing_http_response_without_available_body() : void + { + $psr17 = new Psr17Factory(); + + $response = $psr17->createResponse(200, 'OK') + ->withHeader('Content-Type', 'application/json') + ->withHeader('Content-Length', '42'); + + $processor = new PSR7Processor((new Config(response: new ResponseConfig(withBody: true)))); + + $record = $processor(['datetime' => new \DateTimeImmutable, 'channel' => 'http', 'level_name' => 'debug', 'message' => 'HTTP Response', 'context' => ['response' => $response]]); + + self::assertEquals( + [ + 'response' => [ + 'status' => 200, + 'reason_phrase' => 'OK', + 'headers' => [ + 'Content-Type' => ['application/json'], + 'Content-Length' => ['42'], + ], + ], + ], + $record['context'] + ); + } +} diff --git a/src/core/etl/README.md b/src/core/etl/README.md index f8469b4f7..9b01b57ab 100644 --- a/src/core/etl/README.md +++ b/src/core/etl/README.md @@ -12,5 +12,5 @@ for managing large-scale data processing tasks and building scalable web systems Whether you are dealing with data transformation or orchestrating complex data flows, Flow PHP is tailored to meet the demands of modern web infrastructures. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/core/core.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/core/etl/composer.json b/src/core/etl/composer.json index 61a389184..6bc2e0867 100644 --- a/src/core/etl/composer.json +++ b/src/core/etl/composer.json @@ -14,6 +14,7 @@ "ext-mbstring": "*", "flow-php/array-dot": "^0.7 || 1.x-dev", "flow-php/rdsl": "^0.7 || 1.x-dev", + "flow-php/filesystem": "^0.7 || 1.x-dev", "psr/simple-cache": "^1.0 || ^2.0 || ^3.0", "webmozart/glob": "^3.0 || ^4.0" }, diff --git a/src/core/etl/src/Flow/ETL/Config.php b/src/core/etl/src/Flow/ETL/Config.php index 68361efb5..0ddf33d40 100644 --- a/src/core/etl/src/Flow/ETL/Config.php +++ b/src/core/etl/src/Flow/ETL/Config.php @@ -9,6 +9,7 @@ use Flow\ETL\PHP\Type\Caster; use Flow\ETL\Pipeline\Optimizer; use Flow\ETL\Row\EntryFactory; +use Flow\Filesystem\FilesystemTable; use Flow\Serializer\Serializer; /** @@ -29,6 +30,7 @@ public function __construct( private readonly Serializer $serializer, private readonly Cache $cache, private readonly ExternalSort $externalSort, + private readonly FilesystemTable $filesystemTable, private readonly FilesystemStreams $filesystemStreams, private readonly Optimizer $optimizer, private readonly Caster $caster, @@ -84,6 +86,11 @@ public function filesystemStreams() : FilesystemStreams return $this->filesystemStreams; } + public function fstab() : FilesystemTable + { + return $this->filesystemTable; + } + public function id() : string { return $this->id; diff --git a/src/core/etl/src/Flow/ETL/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/ConfigBuilder.php index f656e90d0..768beab6c 100644 --- a/src/core/etl/src/Flow/ETL/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/ConfigBuilder.php @@ -4,14 +4,16 @@ namespace Flow\ETL; +use function Flow\Filesystem\DSL\fstab; use Flow\ETL\Cache\LocalFilesystemCache; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\ExternalSort\MemorySort; -use Flow\ETL\Filesystem\{FilesystemStreams, LocalFilesystem}; +use Flow\ETL\Filesystem\FilesystemStreams; use Flow\ETL\Monitoring\Memory\Unit; use Flow\ETL\PHP\Type\Caster; use Flow\ETL\Pipeline\Optimizer; use Flow\ETL\Row\Factory\NativeEntryFactory; +use Flow\Filesystem\{Filesystem, FilesystemTable}; use Flow\Serializer\{Base64Serializer, NativePHPSerializer, Serializer}; final class ConfigBuilder @@ -27,7 +29,7 @@ final class ConfigBuilder private ?ExternalSort $externalSort; - private ?Filesystem $filesystem; + private ?FilesystemTable $fstab; private ?string $id; @@ -43,7 +45,7 @@ public function __construct() $this->serializer = null; $this->cache = null; $this->externalSort = null; - $this->filesystem = null; + $this->fstab = null; $this->putInputIntoRows = false; $this->optimizer = null; $this->caster = null; @@ -75,19 +77,6 @@ public function build() : Config \is_string(\getenv(Config::EXTERNAL_SORT_MAX_MEMORY_ENV)) ? Unit::fromString(\getenv(Config::EXTERNAL_SORT_MAX_MEMORY_ENV)) : Unit::fromMb(200) ); - // We need to keep it as a string in order to avoid circular dependency between etl and flysystem adapter - $flysystemFSClass = '\Flow\ETL\Adapter\Filesystem\FlysystemFS'; - - if (!$this->filesystem instanceof Filesystem) { - if (\class_exists($flysystemFSClass)) { - /** @var Filesystem $flysystemFS */ - $flysystemFS = new $flysystemFSClass(); - $this->filesystem = $flysystemFS; - } else { - $this->filesystem = new LocalFilesystem(); - } - } - $this->optimizer ??= new Optimizer( new Optimizer\LimitOptimization(), new Optimizer\BatchSizeOptimization(batchSize: 1000) @@ -100,7 +89,8 @@ public function build() : Config $this->serializer, $this->cache, $this->externalSort, - new FilesystemStreams($this->filesystem), + $this->fstab(), + new FilesystemStreams($this->fstab()), $this->optimizer, $this->caster, $this->putInputIntoRows, @@ -144,16 +134,16 @@ public function externalSort(ExternalSort $externalSort) : self return $this; } - public function filesystem(Filesystem $filesystem) : self + public function id(string $id) : self { - $this->filesystem = $filesystem; + $this->id = $id; return $this; } - public function id(string $id) : self + public function mount(Filesystem $filesystem) : self { - $this->id = $id; + $this->fstab()->mount($filesystem); return $this; } @@ -187,4 +177,20 @@ public function serializer(Serializer $serializer) : self return $this; } + + public function unmount(Filesystem $filesystem) : self + { + $this->fstab()->unmount($filesystem); + + return $this; + } + + private function fstab() : FilesystemTable + { + if ($this->fstab === null) { + $this->fstab = fstab(); + } + + return $this->fstab; + } } diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index c303eff50..7d0402a15 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -8,9 +8,8 @@ use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException, SchemaDefinitionNotFoundException}; -use Flow\ETL\Extractor\LocalFileListExtractor; -use Flow\ETL\Filesystem\Stream\Mode; -use Flow\ETL\Filesystem\{Path, SaveMode}; +use Flow\ETL\Extractor\FilesExtractor; +use Flow\ETL\Filesystem\{SaveMode}; use Flow\ETL\Function\ArrayExpand\ArrayExpand; use Flow\ETL\Function\ArraySort\Sort; use Flow\ETL\Function\Between\Boundary; @@ -48,13 +47,13 @@ Join\Comparison\Identical, Join\Expression, Loader, - Partition, - Partitions, Pipeline, Row, Rows, Transformer, Window}; +use Flow\Filesystem\Stream\Mode; +use Flow\Filesystem\{Partition, Partitions, Path}; /** * Alias for data_frame() : Flow. @@ -69,14 +68,14 @@ function data_frame(Config|ConfigBuilder|null $config = null) : Flow return new Flow($config); } -function from_rows(Rows ...$rows) : Extractor\ProcessExtractor +function from_rows(Rows ...$rows) : Extractor\RowsExtractor { - return new Extractor\ProcessExtractor(...$rows); + return new Extractor\RowsExtractor(...$rows); } function from_path_partitions(Path|string $path) : Extractor\PathPartitionsExtractor { - return new Extractor\PathPartitionsExtractor(\is_string($path) ? Path::realpath($path) : $path); + return new Extractor\PathPartitionsExtractor(\is_string($path) ? \Flow\Filesystem\DSL\path($path) : $path); } function from_array(iterable $array, ?Schema $schema = null) : Extractor\ArrayExtractor @@ -99,9 +98,9 @@ function from_memory(Memory $memory) : Extractor\MemoryExtractor return new Extractor\MemoryExtractor($memory); } -function local_files(string|Path $directory, bool $recursive = false) : LocalFileListExtractor +function files(string|Path $directory) : FilesExtractor { - return new LocalFileListExtractor(\is_string($directory) ? Path::realpath($directory) : $directory, $recursive); + return new FilesExtractor(\is_string($directory) ? \Flow\Filesystem\DSL\path($directory) : $directory); } /** @@ -124,7 +123,6 @@ function from_data_frame(DataFrame $data_frame) : Extractor\DataFrameExtractor function from_sequence_date_period(string $entry_name, \DateTimeInterface $start, \DateInterval $interval, \DateTimeInterface $end, int $options = 0) : Extractor\SequenceExtractor { - /** @psalm-suppress ArgumentTypeCoercion */ return new Extractor\SequenceExtractor( new Extractor\SequenceGenerator\DatePeriodSequenceGenerator(new \DatePeriod($start, $interval, $end, $options)), $entry_name @@ -133,7 +131,6 @@ function from_sequence_date_period(string $entry_name, \DateTimeInterface $start function from_sequence_date_period_recurrences(string $entry_name, \DateTimeInterface $start, \DateInterval $interval, int $recurrences, int $options = 0) : Extractor\SequenceExtractor { - /** @psalm-suppress ArgumentTypeCoercion */ return new Extractor\SequenceExtractor( new Extractor\SequenceGenerator\DatePeriodSequenceGenerator(new \DatePeriod($start, $interval, $recurrences, $options)), $entry_name @@ -448,29 +445,6 @@ function rows(Row ...$row) : Rows return new Rows(...$row); } -function partition(string $name, string $value) : Partition -{ - return new Partition($name, $value); -} - -function partitions(Partition ...$partition) : Partitions -{ - return new Partitions(...$partition); -} - -/** - * @param array $options - */ -function path(string $path, array $options = []) : Path -{ - return new Path($path, $options); -} - -function path_real(string $path, array $options = []) : Path -{ - return Path::realpath($path, $options); -} - function rows_partitioned(array $rows, array|Partitions $partitions) : Rows { return Rows::partitioned($rows, $partitions); diff --git a/src/core/etl/src/Flow/ETL/DataFrame.php b/src/core/etl/src/Flow/ETL/DataFrame.php index 93eba5d8d..49e2a3591 100644 --- a/src/core/etl/src/Flow/ETL/DataFrame.php +++ b/src/core/etl/src/Flow/ETL/DataFrame.php @@ -9,13 +9,12 @@ use Flow\ETL\Dataset\{Report, Statistics}; use Flow\ETL\Exception\{InvalidArgumentException, InvalidFileFormatException, RuntimeException}; use Flow\ETL\Extractor\PartitionExtractor; -use Flow\ETL\Filesystem\SaveMode; +use Flow\ETL\Filesystem\{SaveMode, ScalarFunctionFilter}; use Flow\ETL\Formatter\AsciiTableFormatter; use Flow\ETL\Function\{AggregatingFunction, ScalarFunction, WindowFunction}; use Flow\ETL\Join\{Expression, Join}; use Flow\ETL\Loader\SchemaValidationLoader; use Flow\ETL\Loader\StreamLoader\Output; -use Flow\ETL\Partition\ScalarFunctionFilter; use Flow\ETL\PHP\Type\{AutoCaster}; use Flow\ETL\Pipeline\{BatchingPipeline, CachingPipeline, @@ -50,6 +49,7 @@ UntilTransformer, WindowFunctionTransformer }; +use Flow\Filesystem\Path\Filter; use Flow\RDSL\AccessControl\{AllowAll, AllowList, DenyAll}; use Flow\RDSL\Attribute\DSLMethod; use Flow\RDSL\{Builder, DSLNamespace, Executor, Finder}; @@ -360,7 +360,7 @@ public function filter(ScalarFunction $function) : self * * @throws RuntimeException */ - public function filterPartitions(Partition\PartitionFilter|ScalarFunction $filter) : self + public function filterPartitions(Filter|ScalarFunction $filter) : self { $extractor = $this->pipeline->source(); @@ -368,13 +368,13 @@ public function filterPartitions(Partition\PartitionFilter|ScalarFunction $filte throw new RuntimeException('filterPartitions can be used only with extractors that implement PartitionsExtractor interface'); } - if ($filter instanceof Partition\PartitionFilter) { - $extractor->addPartitionFilter($filter); + if ($filter instanceof Filter) { + $extractor->addFilter($filter); return $this; } - $extractor->addPartitionFilter( + $extractor->addFilter( new ScalarFunctionFilter( $filter, $this->context->entryFactory(), diff --git a/src/core/etl/src/Flow/ETL/Exception/FileNotFoundException.php b/src/core/etl/src/Flow/ETL/Exception/FileNotFoundException.php deleted file mode 100644 index 399068c89..000000000 --- a/src/core/etl/src/Flow/ETL/Exception/FileNotFoundException.php +++ /dev/null @@ -1,19 +0,0 @@ -path->uri()), - 0, - $previous - ); - } -} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php index af6add5f7..fe0a0ff94 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php @@ -4,9 +4,14 @@ namespace Flow\ETL\Extractor; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; +use Flow\Filesystem\Path\Filter; interface FileExtractor { + public function addFilter(Filter $filter) : void; + + public function filter() : Filter; + public function source() : Path; } diff --git a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php new file mode 100644 index 000000000..bd1f6a954 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php @@ -0,0 +1,46 @@ +filesystem($this->path)->list($this->path, $this->filter()) as $fileStatus) { + $signal = yield array_to_rows([ + 'path' => $fileStatus->path->path(), + 'protocol' => $fileStatus->path->protocol()->name, + 'file_name' => $fileStatus->path->filename(), + 'base_name' => $fileStatus->path->basename(), + 'is_file' => $fileStatus->isFile(), + 'is_dir' => $fileStatus->isDirectory(), + 'extension' => $fileStatus->path->extension(), + ], $context->entryFactory()); + + $this->incrementReturnedRows(); + + if ($signal === Signal::STOP || $this->reachedLimit()) { + return; + } + } + } + + public function source() : Path + { + return $this->path; + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/Limitable.php b/src/core/etl/src/Flow/ETL/Extractor/Limitable.php index 65b23b6a5..c80cacc29 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/Limitable.php +++ b/src/core/etl/src/Flow/ETL/Extractor/Limitable.php @@ -21,7 +21,7 @@ public function changeLimit(int $limit) : void $this->limit = $limit; } - public function countRow() : void + public function incrementReturnedRows() : void { $this->yieldedRows++; } diff --git a/src/core/etl/src/Flow/ETL/Extractor/LocalFileListExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/LocalFileListExtractor.php deleted file mode 100644 index 89527c4f5..000000000 --- a/src/core/etl/src/Flow/ETL/Extractor/LocalFileListExtractor.php +++ /dev/null @@ -1,82 +0,0 @@ -directory->isLocal()) { - throw new InvalidArgumentException('Path must point to a local directory, got ' . $this->directory->uri() . ' instead'); - } - - if ($this->directory->isPattern()) { - throw new InvalidArgumentException('LocalFileListExtractor does not support glob paths, got ' . $this->directory->path()); - } - } - - public function extract(FlowContext $context) : \Generator - { - if ($this->recursive) { - $files = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator( - $this->directory->path(), - \RecursiveDirectoryIterator::SKIP_DOTS - ), - \RecursiveIteratorIterator::SELF_FIRST - ); - } else { - $files = new \DirectoryIterator($this->directory->path()); - } - - /** @var \SplFileInfo $file */ - foreach ($files as $file) { - $signal = yield array_to_rows([ - 'path' => $file->getPath(), - 'real_path' => $file->getRealPath(), - 'path_name' => $file->getPathname(), - 'file_name' => $file->getFilename(), - 'base_name' => $file->getBasename(), - 'is_file' => $file->isFile(), - 'is_dir' => $file->isDir(), - 'is_link' => $file->isLink(), - 'is_executable' => $file->isExecutable(), - 'is_readable' => $file->isReadable(), - 'is_writable' => $file->isWritable(), - 'link_target' => $file->isLink() ? $file->getLinkTarget() : null, - 'owner' => $file->getOwner(), - 'group' => $file->getGroup(), - 'permissions' => $file->getPerms(), - 'inode' => $file->getInode(), - 'file_type' => $file->getType(), - 'extension' => $file->getExtension(), - 'size' => $file->getSize(), - 'last_accessed' => $file->getATime(), - 'last_inode_change_time' => $file->getCTime(), - 'last_modified' => $file->getMTime(), - ], $context->entryFactory()); - - $this->countRow(); - - if ($signal === Signal::STOP || $this->reachedLimit()) { - return; - } - } - } - - public function source() : Path - { - return $this->directory; - } -} diff --git a/src/core/etl/src/Flow/ETL/Extractor/PartitionExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/PartitionExtractor.php index 4e8252fb5..38db1757f 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PartitionExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/PartitionExtractor.php @@ -4,11 +4,11 @@ namespace Flow\ETL\Extractor; -use Flow\ETL\Partition\PartitionFilter; +use Flow\Filesystem\Path\Filter; interface PartitionExtractor { - public function addPartitionFilter(PartitionFilter $filter) : void; + public function addFilter(Filter $filter) : void; - public function partitionFilter() : ?PartitionFilter; + public function filter() : Filter; } diff --git a/src/core/etl/src/Flow/ETL/Extractor/PartitionFiltering.php b/src/core/etl/src/Flow/ETL/Extractor/PartitionFiltering.php deleted file mode 100644 index 7b1eb59d4..000000000 --- a/src/core/etl/src/Flow/ETL/Extractor/PartitionFiltering.php +++ /dev/null @@ -1,34 +0,0 @@ -partitionFilter === null) { - $this->partitionFilter = $filter; - - return; - } - - if ($this->partitionFilter instanceof FiltersCollection) { - $this->partitionFilter = new FiltersCollection([...$this->partitionFilter->filters, $filter]); - - return; - } - - $this->partitionFilter = new FiltersCollection([$this->partitionFilter, $filter]); - } - - public function partitionFilter() : PartitionFilter - { - return $this->partitionFilter ?? new NoopFilter(); - } -} diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php b/src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php new file mode 100644 index 000000000..e5b0bd854 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php @@ -0,0 +1,35 @@ +filter === null) { + $this->filter = $filter; + + return; + } + + if ($this->filter instanceof Filters) { + $this->filter = $this->filter->add($filter); + + return; + } + + $this->filter = new Filters($this->filter, $filter); + } + + public function filter() : Filter + { + return $this->filter ?? new OnlyFiles(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php index c41aee736..5503b7f8a 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php @@ -5,32 +5,31 @@ namespace Flow\ETL\Extractor; use function Flow\ETL\DSL\{array_entry, row, rows, string_entry}; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\{Extractor, FlowContext, Partition}; +use Flow\ETL\{Extractor, FlowContext}; +use Flow\Filesystem\{Partition, Path}; final class PathPartitionsExtractor implements Extractor, FileExtractor, LimitableExtractor, PartitionExtractor { use Limitable; - use PartitionFiltering; + use PathFiltering; public function __construct(private readonly Path $path) { - } public function extract(FlowContext $context) : \Generator { - foreach ($context->config->filesystemStreams()->scan($this->path, $this->partitionFilter()) as $stream) { - $partitions = $stream->path()->partitions(); + foreach ($context->filesystem($this->path)->list($this->path, $this->filter()) as $fileStatus) { + $partitions = $fileStatus->path->partitions(); $row = row( - string_entry('path', $stream->path()->uri()), + string_entry('path', $fileStatus->path->uri()), array_entry('partitions', \array_merge(...\array_values(\array_map(static fn (Partition $p) => [$p->name => $p->value], $partitions->toArray())))) ); $signal = yield rows($row); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { $context->streams()->closeWriters($this->path); diff --git a/src/core/etl/src/Flow/ETL/Extractor/ProcessExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php similarity index 91% rename from src/core/etl/src/Flow/ETL/Extractor/ProcessExtractor.php rename to src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php index 0ad0a832c..a22b9a3e1 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/ProcessExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php @@ -9,7 +9,7 @@ /** * @internal */ -final class ProcessExtractor implements Extractor +final class RowsExtractor implements Extractor { /** * @var array diff --git a/src/core/etl/src/Flow/ETL/Filesystem.php b/src/core/etl/src/Flow/ETL/Filesystem.php deleted file mode 100644 index 8c5c7e501..000000000 --- a/src/core/etl/src/Flow/ETL/Filesystem.php +++ /dev/null @@ -1,29 +0,0 @@ - - */ - public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilter()) : \Generator; -} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/FilesystemStreams.php b/src/core/etl/src/Flow/ETL/Filesystem/FilesystemStreams.php index 6b13e0696..f5de49c0e 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/FilesystemStreams.php +++ b/src/core/etl/src/Flow/ETL/Filesystem/FilesystemStreams.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Filesystem; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Stream\{FileStream, Mode}; -use Flow\ETL\{Filesystem, Partition}; +use Flow\Filesystem\{DestinationStream, Path, Path\Filter, SourceStream, Stream\VoidStream}; +use Flow\Filesystem\{FilesystemTable, Partition}; /** * @psalm-suppress MissingTemplateParam @@ -18,11 +18,11 @@ final class FilesystemStreams implements \Countable, \IteratorAggregate private SaveMode $saveMode; /** - * @var array> + * @var array> */ private array $writingStreams; - public function __construct(private readonly Filesystem $filesystem) + public function __construct(private readonly FilesystemTable $fstab) { $this->writingStreams = []; $this->saveMode = SaveMode::ExceptionIfExists; @@ -32,6 +32,8 @@ public function closeWriters(Path $path) : void { $streams = []; + $fs = $this->fstab->for($path); + foreach ($this->writingStreams as $nextBasePath => $nextStreams) { if ($path->uri() === $nextBasePath) { foreach ($nextStreams as $fileStream) { @@ -44,16 +46,16 @@ public function closeWriters(Path $path) : void if ($fileStream->path()->partitions()->count()) { $partitionFilesPatter = new Path($fileStream->path()->parentDirectory()->path() . '/*', $fileStream->path()->options()); - foreach ($this->filesystem->scan($partitionFilesPatter) as $partitionFile) { - if (\str_ends_with($partitionFile->path(), self::FLOW_TMP_SUFFIX)) { + foreach ($fs->list($partitionFilesPatter) as $partitionFile) { + if (\str_ends_with($partitionFile->path->path(), self::FLOW_TMP_SUFFIX)) { continue; } - $this->filesystem->rm($partitionFile); + $fs->rm($partitionFile->path); } } - $this->filesystem->mv( + $fs->mv( $fileStream->path(), new Path( \str_replace(self::FLOW_TMP_SUFFIX, '', $fileStream->path()->uri()), @@ -84,11 +86,11 @@ public function exists(Path $path, array $partitions = []) : bool ? $path->addPartitions(...$partitions) : $path; - return $this->filesystem->exists($destination); + return $this->fstab->for($path)->status($destination) !== null; } /** - * @return \Traversable + * @return \Traversable */ public function getIterator() : \Traversable { @@ -111,7 +113,19 @@ public function isOpen(Path $path, array $partitions = []) : bool return \array_key_exists($destination->uri(), $this->writingStreams[$path->uri()]); } - public function read(Path $path, array $partitions = []) : FileStream + /** + * @return \Generator + */ + public function list(Path $path, Filter $pathFilter) : \Generator + { + $fs = $this->fstab->for($path); + + foreach ($fs->list($path, $pathFilter) as $file) { + yield $fs->readFrom($file->path); + } + } + + public function read(Path $path, array $partitions = []) : SourceStream { if ($path->isPattern()) { throw new RuntimeException("Path can't be pattern, given: " . $path->uri()); @@ -121,7 +135,7 @@ public function read(Path $path, array $partitions = []) : FileStream ? $path->addPartitions(...$partitions) : $path; - return $this->filesystem->open($destination, Mode::READ); + return $this->fstab->for($path)->readFrom($destination); } /** @@ -134,18 +148,10 @@ public function rm(Path $path, array $partitions = []) : void ? $path->addPartitions(...$partitions) : $path; - if ($this->filesystem->exists($destination)) { - $this->filesystem->rm($destination); - } - } + $fs = $this->fstab->for($path); - /** - * @return \Generator - */ - public function scan(Path $path, Partition\PartitionFilter $partitionFilter) : \Generator - { - foreach ($this->filesystem->scan($path, $partitionFilter) as $file) { - yield $this->filesystem->open($file, Mode::READ); + if ($fs->status($destination)) { + $fs->rm($destination); } } @@ -159,7 +165,7 @@ public function setSaveMode(SaveMode $saveMode) : self /** * @param array $partitions */ - public function writeTo(Path $path, array $partitions = []) : FileStream + public function writeTo(Path $path, array $partitions = []) : DestinationStream { if (!$path->extension()) { throw new RuntimeException('Stream path must have an extension, given: ' . $path->uri()); @@ -182,10 +188,12 @@ public function writeTo(Path $path, array $partitions = []) : FileStream $destinationPathUri = $destination->uri(); if (!\array_key_exists($destinationPathUri, $this->writingStreams[$pathUri])) { + $fs = $this->fstab->for($path); + $outputPath = $destination; if ($this->saveMode === SaveMode::Append) { - if ($this->filesystem->fileExists($outputPath)) { + if ($fs->status($outputPath) !== null) { $outputPath = $outputPath->randomize(); } } @@ -195,18 +203,18 @@ public function writeTo(Path $path, array $partitions = []) : FileStream } if ($this->saveMode === SaveMode::ExceptionIfExists) { - if ($this->filesystem->exists($destination)) { + if ($fs->status($destination)) { throw new RuntimeException('Destination path "' . $destinationPathUri . '" already exists, please change path to different or set different SaveMode'); } } if ($this->saveMode === SaveMode::Ignore) { - if ($this->filesystem->exists($destination)) { - return $this->writingStreams[$pathUri][$destinationPathUri] = FileStream::voidStream($outputPath); + if ($fs->status($destination)) { + return $this->writingStreams[$pathUri][$destinationPathUri] = new VoidStream($outputPath); } } - return $this->writingStreams[$pathUri][$destinationPathUri] = $this->filesystem->open($outputPath, Mode::WRITE_READ); + return $this->writingStreams[$pathUri][$destinationPathUri] = $fs->writeTo($outputPath); } return $this->writingStreams[$pathUri][$destinationPathUri]; diff --git a/src/core/etl/src/Flow/ETL/Filesystem/LocalBuffer.php b/src/core/etl/src/Flow/ETL/Filesystem/LocalBuffer.php deleted file mode 100644 index c44eb8bed..000000000 --- a/src/core/etl/src/Flow/ETL/Filesystem/LocalBuffer.php +++ /dev/null @@ -1,21 +0,0 @@ -isLocal()) { - return false; - } - - if ($path->isPattern()) { - return false; - } - - return \is_dir($path->path()); - } - - public function exists(Path $path) : bool - { - if (!$path->isLocal()) { - return false; - } - - if (!$path->isPattern()) { - return \file_exists($path->path()); - } - - foreach (Glob::glob($path->path()) as $filePath) { - if (\file_exists($filePath)) { - return true; - } - } - - return false; - } - - public function fileExists(Path $path) : bool - { - if (!$path->isLocal()) { - return false; - } - - if (!$path->isPattern()) { - return \file_exists($path->path()) && !\is_dir($path->path()); - } - - foreach (Glob::glob($path->path()) as $filePath) { - if (\is_dir($filePath)) { - continue; - } - - return true; - } - - return false; - } - - public function mv(Path $from, Path $to) : void - { - if (!$from->isLocal() || !$to->isLocal()) { - throw new InvalidArgumentException(\sprintf('Paths "%s" and "%s" are not local', $from->uri(), $to->uri())); - } - - if ($from->isPattern() || $to->isPattern()) { - throw new InvalidArgumentException('Pattern paths can\'t be moved'); - } - - if (\file_exists($to->path())) { - $this->rm($to); - } - - if (!\rename($from->path(), $to->path())) { - throw new InvalidArgumentException(\sprintf('Can\'t move "%s" to "%s"', $from->uri(), $to->uri())); - } - } - - public function open(Path $path, Mode $mode) : FileStream - { - if (!$path->isLocal()) { - throw new InvalidArgumentException(\sprintf('Path "%s" is not local', $path->uri())); - } - - if ($path->isPattern()) { - throw new InvalidArgumentException("Pattern paths can't be open: " . $path->uri()); - } - - if (!$this->directoryExists($path->parentDirectory())) { - if (!\mkdir($concurrentDirectory = $path->parentDirectory()->path(), recursive: true) && !\is_dir($concurrentDirectory)) { - throw new RuntimeException(\sprintf('Directory "%s" was not created', $concurrentDirectory)); - } - } - - return new FileStream($path, \fopen($path->path(), $mode->value, false, $path->context()->resource()) ?: null); - } - - public function rm(Path $path) : void - { - if (!$path->isLocal()) { - throw new InvalidArgumentException(\sprintf('Path "%s" is not local', $path->uri())); - } - - if (!$path->isPattern()) { - if (\is_dir($path->path())) { - $this->rmdir($path->path()); - } else { - if (\file_exists($path->path())) { - \unlink($path->path()); - } - } - - return; - } - - foreach (Glob::glob($path->path()) as $filePath) { - if (\is_dir($filePath)) { - $this->rmdir($filePath); - } else { - \unlink($filePath); - } - } - } - - public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilter()) : \Generator - { - if (!$path->isLocal()) { - throw new InvalidArgumentException(\sprintf('Path "%s" is not local', $path->uri())); - } - - if (!$path->isPattern()) { - if (!$this->fileExists($path)) { - throw new FileNotFoundException($path); - } - - yield $path; - - return; - - } - - foreach (Glob::glob($path->path()) as $filePath) { - if (\is_dir($filePath)) { - continue; - } - - if ($partitionFilter->keep(...(Path::realpath($filePath, $path->options()))->partitions()->toArray())) { - yield Path::realpath($filePath, $path->options()); - } - } - } - - private function rmdir(string $dirPath) : void - { - if (!\is_dir($dirPath)) { - throw new InvalidArgumentException("{$dirPath} must be a directory"); - } - - if (!\str_ends_with($dirPath, '/')) { - $dirPath .= '/'; - } - - $files = \scandir($dirPath); - - if (!$files) { - throw new RuntimeException("Can't read directory: {$dirPath}"); - } - - foreach ($files as $file) { - if (\in_array($file, ['.', '..'], true)) { - continue; - } - - $filePath = $dirPath . $file; - - if (\is_dir($filePath)) { - $this->rmdir($filePath); - } else { - \unlink($filePath); - } - } - - \rmdir($dirPath); - } -} diff --git a/src/core/etl/src/Flow/ETL/Partition/ScalarFunctionFilter.php b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php similarity index 70% rename from src/core/etl/src/Flow/ETL/Partition/ScalarFunctionFilter.php rename to src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php index 7b4dae62e..25914e284 100644 --- a/src/core/etl/src/Flow/ETL/Partition/ScalarFunctionFilter.php +++ b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php @@ -2,15 +2,16 @@ declare(strict_types=1); -namespace Flow\ETL\Partition; +namespace Flow\ETL\Filesystem; use function Flow\ETL\DSL\row; use Flow\ETL\Function\ScalarFunction; -use Flow\ETL\Partition; use Flow\ETL\PHP\Type\AutoCaster; use Flow\ETL\Row\EntryFactory; +use Flow\Filesystem\Path\Filter; +use Flow\Filesystem\{FileStatus, Partition}; -final class ScalarFunctionFilter implements PartitionFilter +final class ScalarFunctionFilter implements Filter { public function __construct( private readonly ScalarFunction $function, @@ -19,13 +20,13 @@ public function __construct( ) { } - public function keep(Partition ...$partitions) : bool + public function accept(FileStatus $status) : bool { return (bool) $this->function->eval( row( ...\array_map( fn (Partition $partition) => $this->entryFactory->create($partition->name, $this->caster->cast($partition->value)), - $partitions + $status->path->partitions()->toArray() ) ) ); diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Stream/FileStream.php b/src/core/etl/src/Flow/ETL/Filesystem/Stream/FileStream.php deleted file mode 100644 index d1b8f2615..000000000 --- a/src/core/etl/src/Flow/ETL/Filesystem/Stream/FileStream.php +++ /dev/null @@ -1,72 +0,0 @@ -resource)) { - throw new InvalidArgumentException('FileStream expects resource type, given: ' . \gettype($this->resource)); - } - } - - public static function voidStream(Path $path) : self - { - VoidStreamWrapper::register(); - - $resourcePath = 'void://' . $path->uri(); - - $resource = \fopen($resourcePath, Mode::WRITE->value); - - if ($resource === false) { - throw new RuntimeException("Cannot open void stream for {$resourcePath}"); - } - - return new self($path, $resource); - } - - /** - * @psalm-suppress InvalidPropertyAssignmentValue - */ - public function close() : void - { - if (!\is_resource($this->resource)) { - throw new RuntimeException('FileStream was closed'); - } - - \fclose($this->resource); - $this->resource = null; - } - - public function isOpen() : bool - { - return \is_resource($this->resource); - } - - public function path() : Path - { - return $this->path; - } - - /** - * @return resource - */ - public function resource() - { - if (!\is_resource($this->resource)) { - throw new RuntimeException('FileStream was closed'); - } - - return $this->resource; - } -} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/TmpfileBuffer.php b/src/core/etl/src/Flow/ETL/Filesystem/TmpfileBuffer.php deleted file mode 100644 index db944eeee..000000000 --- a/src/core/etl/src/Flow/ETL/Filesystem/TmpfileBuffer.php +++ /dev/null @@ -1,50 +0,0 @@ -stream)) { - /** - * @psalm-suppress InvalidPropertyAssignmentValue - */ - \fclose($this->stream); - } - } - - public function seek(int $offset, int $whence = SEEK_SET) : void - { - \fseek($this->stream(), $offset, $whence); - } - - /** - * @return resource - */ - public function stream() - { - if ($this->stream === null) { - $this->stream = \tmpfile(); - } - - return $this->stream; - } - - public function tell() : int|false - { - return \ftell($this->stream()); - } - - public function write(string $data) : void - { - \fwrite($this->stream(), $data); - } -} diff --git a/src/core/etl/src/Flow/ETL/Flow.php b/src/core/etl/src/Flow/ETL/Flow.php index 5fd08142f..99311a2fa 100644 --- a/src/core/etl/src/Flow/ETL/Flow.php +++ b/src/core/etl/src/Flow/ETL/Flow.php @@ -4,7 +4,7 @@ namespace Flow\ETL; -use Flow\ETL\Extractor\ProcessExtractor; +use Flow\ETL\Extractor\RowsExtractor; use Flow\ETL\Pipeline\SynchronousPipeline; final class Flow @@ -41,7 +41,7 @@ public function from(Extractor $extractor) : DataFrame public function process(Rows ...$rows) : DataFrame { return new DataFrame( - (new SynchronousPipeline(new ProcessExtractor(...$rows))), + (new SynchronousPipeline(new RowsExtractor(...$rows))), $this->config ); } diff --git a/src/core/etl/src/Flow/ETL/FlowContext.php b/src/core/etl/src/Flow/ETL/FlowContext.php index 10aafa936..e5f7d7b5b 100644 --- a/src/core/etl/src/Flow/ETL/FlowContext.php +++ b/src/core/etl/src/Flow/ETL/FlowContext.php @@ -7,6 +7,7 @@ use Flow\ETL\ErrorHandler\ThrowError; use Flow\ETL\Filesystem\FilesystemStreams; use Flow\ETL\Row\EntryFactory; +use Flow\Filesystem\{Filesystem, Path, Protocol}; /** * Mutable Flow execution context. @@ -36,6 +37,11 @@ public function errorHandler() : ErrorHandler return $this->errorHandler; } + public function filesystem(Path|Protocol $path) : Filesystem + { + return $this->config->fstab()->for($path); + } + public function setErrorHandler(ErrorHandler $handler) : self { $this->errorHandler = $handler; diff --git a/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php b/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php index 4f78ed59d..689c9fc33 100644 --- a/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php +++ b/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php @@ -5,7 +5,8 @@ namespace Flow\ETL\Formatter\ASCII; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\{Partition, Row, Rows}; +use Flow\ETL\{Row, Rows}; +use Flow\Filesystem\Partition; final class Body { diff --git a/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php b/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php index 4c9a5892e..1336a0877 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php +++ b/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php @@ -29,7 +29,6 @@ public function eval(Row $row) : mixed return null; } - /** @psalm-suppress ArgumentTypeCoercion */ $tz = match (\gettype($tz)) { 'string' => new \DateTimeZone($tz), 'object' => $tz instanceof \DateTimeZone ? $tz : null, diff --git a/src/core/etl/src/Flow/ETL/Loader/FileLoader.php b/src/core/etl/src/Flow/ETL/Loader/FileLoader.php index 97617efa6..9f9ce88f9 100644 --- a/src/core/etl/src/Flow/ETL/Loader/FileLoader.php +++ b/src/core/etl/src/Flow/ETL/Loader/FileLoader.php @@ -4,7 +4,7 @@ namespace Flow\ETL\Loader; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; interface FileLoader { diff --git a/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php b/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php index c71889fcb..6364f4192 100644 --- a/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php +++ b/src/core/etl/src/Flow/ETL/Loader/StreamLoader.php @@ -5,11 +5,11 @@ namespace Flow\ETL\Loader; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Stream\Mode; use Flow\ETL\Loader\StreamLoader\Output; use Flow\ETL\Row\Schema\Formatter\ASCIISchemaFormatter; use Flow\ETL\Row\Schema\SchemaFormatter; use Flow\ETL\{FlowContext, Formatter, Loader, Loader\StreamLoader\Type, Rows}; +use Flow\Filesystem\Stream\Mode; final class StreamLoader implements Closure, Loader { diff --git a/src/core/etl/src/Flow/ETL/Partition/FiltersCollection.php b/src/core/etl/src/Flow/ETL/Partition/FiltersCollection.php deleted file mode 100644 index 248be3519..000000000 --- a/src/core/etl/src/Flow/ETL/Partition/FiltersCollection.php +++ /dev/null @@ -1,29 +0,0 @@ - $filters - */ - public function __construct(public readonly array $filters) - { - - } - - public function keep(Partition ...$partitions) : bool - { - foreach ($this->filters as $filter) { - if (!$filter->keep(...$partitions)) { - return false; - } - } - - return true; - } -} diff --git a/src/core/etl/src/Flow/ETL/Partition/NoopFilter.php b/src/core/etl/src/Flow/ETL/Partition/NoopFilter.php deleted file mode 100644 index 0e1e583ca..000000000 --- a/src/core/etl/src/Flow/ETL/Partition/NoopFilter.php +++ /dev/null @@ -1,15 +0,0 @@ -type->isEqual($entry->type); } - /** - * @psalm-suppress ArgumentTypeCoercion - */ return $this->is($entry->name()) && $entry instanceof self && $this->type->isEqual($entry->type) diff --git a/src/core/etl/src/Flow/ETL/Rows.php b/src/core/etl/src/Flow/ETL/Rows.php index 02b1a7a68..91422f321 100644 --- a/src/core/etl/src/Flow/ETL/Rows.php +++ b/src/core/etl/src/Flow/ETL/Rows.php @@ -7,10 +7,11 @@ use function Flow\ETL\DSL\{array_to_rows, row}; use Flow\ETL\Exception\{DuplicatedEntriesException, InvalidArgumentException, RuntimeException}; use Flow\ETL\Join\Expression; -use Flow\ETL\Partition\CartesianProduct; +use Flow\ETL\Row\CartesianProduct; use Flow\ETL\Row\Comparator\NativeComparator; use Flow\ETL\Row\Factory\NativeEntryFactory; use Flow\ETL\Row\{Comparator, Entries, EntryFactory, Reference, References, Schema, SortOrder}; +use Flow\Filesystem\{Partition, Partitions}; /** * @implements \ArrayAccess diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleCacheTest.php index 1be1d52b5..11595f408 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleCacheTest.php @@ -17,7 +17,7 @@ public function test_saving_to_psr_simple_cache_implementation() : void { $cache = new PSRSimpleCache( new Psr16Cache( - new FilesystemAdapter(directory: \sys_get_temp_dir() . '/flow-etl-cache-' . bin2hex(random_bytes(16))) + new FilesystemAdapter(directory: __DIR__ . '/var/flow-etl-cache-' . bin2hex(random_bytes(16))) ), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php index 20780f6d6..c523ced3a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php @@ -13,14 +13,15 @@ int_entry, lit, overwrite, - partition, ref, row, rows, rows_partitioned, str_entry}; +use function Flow\Filesystem\DSL\partition; use Flow\ETL\Tests\Integration\IntegrationTestCase; -use Flow\ETL\{Partition, Rows}; +use Flow\ETL\{Rows}; +use Flow\Filesystem\Partition; final class PartitioningTest extends IntegrationTestCase { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php index efaf38113..3e9815678 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php @@ -4,16 +4,15 @@ namespace Flow\ETL\Tests\Integration\Extractor; -use function Flow\ETL\DSL\{flow_context, rows}; -use Flow\ETL\Extractor\PathPartitionsExtractor; -use Flow\ETL\Filesystem\Path; +use function Flow\ETL\DSL\{flow_context, from_path_partitions, rows}; use Flow\ETL\Tests\Integration\IntegrationTestCase; +use Flow\Filesystem\Path; final class PathPartitionsExtractorTest extends IntegrationTestCase { public function test_extracting_data_from_path_partitions() : void { - $extractor = new PathPartitionsExtractor(Path::realpath(__DIR__ . '/Fixtures/multi_partitioned/**/*')); + $extractor = from_path_partitions(Path::realpath(__DIR__ . '/Fixtures/multi_partitioned/**/*')); $extractedData = \iterator_to_array($extractor->extract(flow_context())); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/FilesystemStreamsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/FilesystemStreamsTest.php index 8a3226e06..b2d3b00a8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/FilesystemStreamsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/FilesystemStreamsTest.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams; use Flow\ETL\Filesystem\FilesystemStreams; -use Flow\ETL\Partition; -use Flow\ETL\Partition\NoopFilter; +use Flow\Filesystem\Partition; +use Flow\Filesystem\Path\Filter\KeepAll; final class FilesystemStreamsTest extends FilesystemStreamsTestCase { @@ -111,12 +111,12 @@ public function test_scan() : void $streams = $this->streams(); self::assertCount( 4, - \iterator_to_array($streams->scan($this->getPath(__FUNCTION__ . '/**/*.txt'), new NoopFilter())) + \iterator_to_array($streams->list($this->getPath(__FUNCTION__ . '/**/*.txt'), new KeepAll())) ); } protected function streams() : FilesystemStreams { - return new FilesystemStreams($this->fs); + return new FilesystemStreams($this->fstab()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/AppendModeTest.php index 65261fd26..115a70607 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/AppendModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/AppendModeTest.php @@ -5,8 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\NotPartitioned; use function Flow\ETL\DSL\append; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\Path; final class AppendModeTest extends FilesystemStreamsTestCase { @@ -28,16 +29,16 @@ public function test_open_stream_for_existing_file() : void self::assertFileExists($file->path()); $appendFileStream = $streams->writeTo($file); - \fwrite($appendFileStream->resource(), 'new content'); + $appendFileStream->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/*'))); self::assertCount(2, $files); foreach ($files as $streamFile) { - self::assertStringStartsWith('existing-file', $streamFile->basename()); - self::assertStringEndsWith('.txt', $streamFile->basename()); + self::assertStringStartsWith('existing-file', $streamFile->path->basename()); + self::assertStringEndsWith('.txt', $streamFile->path->basename()); } } @@ -51,18 +52,18 @@ public function test_open_stream_for_non_existing_file() : void self::assertFileDoesNotExist($file->path()); $appendFileStream = $streams->writeTo($file); - \fwrite($appendFileStream->resource(), 'new content'); + $appendFileStream->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/*'))); self::assertCount(1, $files); - self::assertSame('non-existing-file.txt', $files[0]->basename()); + self::assertSame('non-existing-file.txt', $files[0]->path->basename()); } protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(append()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/ExceptionIfExistsModeTest.php index 4cf4f0990..c5bd134c3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/ExceptionIfExistsModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/ExceptionIfExistsModeTest.php @@ -41,7 +41,7 @@ public function test_open_stream_for_non_existing_file() : void $file = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); $fileStream = $streams->writeTo($file); - \fwrite($fileStream->resource(), 'some content'); + $fileStream->append('some content'); $streams->closeWriters($file); self::assertFileExists($file->path()); @@ -50,7 +50,7 @@ public function test_open_stream_for_non_existing_file() : void protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(exception_if_exists()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/IgnoreModeTest.php index 21e513561..219536a21 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/IgnoreModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/IgnoreModeTest.php @@ -27,7 +27,7 @@ public function test_open_stream_for_existing_file() : void $path = $this->getPath(__FUNCTION__ . '/existing-file.txt'); $fileStream = $streams->writeTo($path); - \fwrite($fileStream->resource(), 'different content'); + $fileStream->append('different content'); $streams->closeWriters($path); self::assertFileExists($path->path()); @@ -43,7 +43,7 @@ public function test_open_stream_for_non_existing_file() : void $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); $fileStream = $streams->writeTo($path); - \fwrite($fileStream->resource(), 'some content'); + $fileStream->append('some content'); $streams->closeWriters($path); self::assertFileExists($path->path()); @@ -52,7 +52,7 @@ public function test_open_stream_for_non_existing_file() : void protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(ignore()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/OverwriteModeTest.php index 49dd17f1f..00d309fdb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/OverwriteModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/NotPartitioned/OverwriteModeTest.php @@ -5,8 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\NotPartitioned; use function Flow\ETL\DSL\overwrite; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\Path; final class OverwriteModeTest extends FilesystemStreamsTestCase { @@ -27,15 +28,15 @@ public function test_open_stream_for_existing_file() : void $fileStream = $streams->writeTo($path = $this->getPath(__FUNCTION__ . '/existing-file.txt')); self::assertStringEndsWith(FilesystemStreams::FLOW_TMP_SUFFIX, $fileStream->path()->path()); - \fwrite($fileStream->resource(), 'some other content'); + $fileStream->append('some other content'); self::assertSame('some content', \file_get_contents($path->path())); $streams->closeWriters($path); self::assertSame('some other content', \file_get_contents($path->path())); - self::assertCount(1, $files = \iterator_to_array($this->fs->scan(new Path($path->parentDirectory()->path() . '/*')))); - self::assertSame('existing-file.txt', $files[0]->basename()); + self::assertCount(1, $files = \iterator_to_array($this->fs()->list(new Path($path->parentDirectory()->path() . '/*')))); + self::assertSame('existing-file.txt', $files[0]->path->basename()); } public function test_open_stream_for_non_existing_file() : void @@ -45,7 +46,7 @@ public function test_open_stream_for_non_existing_file() : void $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); $fileStream = $streams->writeTo($path); - \fwrite($fileStream->resource(), 'some content'); + $fileStream->append('some content'); $streams->closeWriters($path); self::assertFileExists($path->path()); @@ -54,7 +55,7 @@ public function test_open_stream_for_non_existing_file() : void protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(overwrite()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/AppendModeTest.php index 0d83a3c41..dab86a1aa 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/AppendModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/AppendModeTest.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\Partitioned; use function Flow\ETL\DSL\append; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; -use Flow\ETL\Partition; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\{Partition, Path}; final class AppendModeTest extends FilesystemStreamsTestCase { @@ -32,16 +32,16 @@ public function test_open_stream_for_existing_partition_with_existing_file() : v $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'appended content'); + $fileStream->append('appended content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(2, $files); foreach ($files as $streamFile) { - self::assertStringStartsWith('file', $streamFile->basename()); - self::assertStringEndsWith('.txt', $streamFile->basename()); + self::assertStringStartsWith('file', $streamFile->path->basename()); + self::assertStringEndsWith('.txt', $streamFile->path->basename()); } } @@ -57,15 +57,15 @@ public function test_open_stream_for_existing_partition_without_existing_file() $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'appended content'); + $fileStream->append('appended content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('appended content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('appended content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_non_existing_partition() : void @@ -78,19 +78,19 @@ public function test_open_stream_for_non_existing_partition() : void $file = $this->getPath(__FUNCTION__ . '/file.txt'); $appendedFile = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($appendedFile->resource(), 'appended content'); + $appendedFile->append('appended content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/partition=value/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/partition=value/*'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('appended content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('appended content', \file_get_contents($files[0]->path->path())); } protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(append()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/ExceptionIfExistsModeTest.php index 32a620e97..83cc5b670 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/ExceptionIfExistsModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/ExceptionIfExistsModeTest.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\Partitioned; use function Flow\ETL\DSL\exception_if_exists; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; -use Flow\ETL\Partition; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\{Partition, Path}; final class ExceptionIfExistsModeTest extends FilesystemStreamsTestCase { @@ -46,15 +46,15 @@ public function test_open_stream_for_existing_partition_without_existing_file() $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'file content'); + $fileStream->append('file content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('file content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('file content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_non_existing_partition() : void @@ -67,19 +67,19 @@ public function test_open_stream_for_non_existing_partition() : void $file = $this->getPath(__FUNCTION__ . '/file.txt'); $appendedFile = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($appendedFile->resource(), 'file content'); + $appendedFile->append('file content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/partition=value/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/partition=value/*'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('file content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('file content', \file_get_contents($files[0]->path->path())); } protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(exception_if_exists()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/IgnoreModeTest.php index 07fb5d8d2..af26b27a0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/IgnoreModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/IgnoreModeTest.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\Partitioned; use function Flow\ETL\DSL\ignore; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; -use Flow\ETL\Partition; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\{Partition, Path}; final class IgnoreModeTest extends FilesystemStreamsTestCase { @@ -31,15 +31,15 @@ public function test_open_stream_for_existing_partition_with_existing_file() : v $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'new content'); + $fileStream->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertStringStartsWith('file.txt', $files[0]->basename()); - self::assertSame('file content', \file_get_contents($files[0]->path())); + self::assertStringStartsWith('file.txt', $files[0]->path->basename()); + self::assertSame('file content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_existing_partition_without_existing_file() : void @@ -54,15 +54,15 @@ public function test_open_stream_for_existing_partition_without_existing_file() $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'appended content'); + $fileStream->append('appended content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('appended content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('appended content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_non_existing_partition() : void @@ -75,19 +75,19 @@ public function test_open_stream_for_non_existing_partition() : void $file = $this->getPath(__FUNCTION__ . '/file.txt'); $appendedFile = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($appendedFile->resource(), 'appended content'); + $appendedFile->append('appended content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/partition=value/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/partition=value/*'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('appended content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('appended content', \file_get_contents($files[0]->path->path())); } protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(ignore()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/OverwriteModeTest.php index 5d3d4d4a8..33ba6d473 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/OverwriteModeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesystemStreams/Partitioned/OverwriteModeTest.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\Partitioned; use function Flow\ETL\DSL\overwrite; -use Flow\ETL\Filesystem\{FilesystemStreams, Path}; -use Flow\ETL\Partition; +use Flow\ETL\Filesystem\{FilesystemStreams}; use Flow\ETL\Tests\Integration\Filesystem\FilesystemStreams\FilesystemStreamsTestCase; +use Flow\Filesystem\{Partition, Path}; final class OverwriteModeTest extends FilesystemStreamsTestCase { @@ -31,15 +31,15 @@ public function test_open_stream_for_existing_partition_with_existing_file() : v $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'new content'); + $fileStream->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertStringStartsWith('file.txt', $files[0]->basename()); - self::assertSame('new content', \file_get_contents($files[0]->path())); + self::assertStringStartsWith('file.txt', $files[0]->path->basename()); + self::assertSame('new content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_existing_partition_without_existing_file() : void @@ -54,15 +54,15 @@ public function test_open_stream_for_existing_partition_without_existing_file() $file = $this->getPath(__FUNCTION__ . '/file.txt'); $fileStream = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($fileStream->resource(), 'new content'); + $fileStream->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/**/*.txt'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/**/*.txt'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('new content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('new content', \file_get_contents($files[0]->path->path())); } public function test_open_stream_for_non_existing_partition() : void @@ -75,19 +75,19 @@ public function test_open_stream_for_non_existing_partition() : void $file = $this->getPath(__FUNCTION__ . '/file.txt'); $appendedFile = $streams->writeTo($file, partitions: [new Partition('partition', 'value')]); - \fwrite($appendedFile->resource(), 'new content'); + $appendedFile->append('new content'); $streams->closeWriters($file); - $files = \iterator_to_array($this->fs->scan(new Path($file->parentDirectory()->path() . '/partition=value/*'))); + $files = \iterator_to_array($this->fs()->list(new Path($file->parentDirectory()->path() . '/partition=value/*'))); self::assertCount(1, $files); - self::assertSame('file.txt', $files[0]->basename()); - self::assertSame('new content', \file_get_contents($files[0]->path())); + self::assertSame('file.txt', $files[0]->path->basename()); + self::assertSame('new content', \file_get_contents($files[0]->path->path())); } protected function streams() : FilesystemStreams { - $streams = new FilesystemStreams($this->fs); + $streams = new FilesystemStreams($this->fstab()); $streams->setSaveMode(overwrite()); return $streams; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/partitioned/partition_01=a/file_01.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/partitioned/partition_01=a/file_01.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/partitioned/partition_01=b/file_02.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/partitioned/partition_01=b/file_02.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/LocalFilesystemTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/LocalFilesystemTest.php deleted file mode 100644 index 843a6d84e..000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/LocalFilesystemTest.php +++ /dev/null @@ -1,226 +0,0 @@ -open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/append.txt'), Mode::APPEND_WRITE); - \fwrite($stream->resource(), "some data to make file not empty\n"); - $stream->close(); - - $appendStream = $fs->open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/append.txt'), Mode::APPEND_WRITE); - \fwrite($appendStream->resource(), "some more data to make file not empty\n"); - $appendStream->close(); - - self::assertStringContainsString( - <<<'STRING' -some data to make file not empty -some more data to make file not empty -STRING, - \file_get_contents($appendStream->path()->path()) - ); - - $fs->rm($stream->path()); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_dir_exists() : void - { - self::assertTrue((new LocalFilesystem())->exists(new Path(__DIR__))); - self::assertFalse((new LocalFilesystem())->exists(new Path(__DIR__ . '/not_existing_directory'))); - } - - public function test_fie_exists() : void - { - self::assertTrue((new LocalFilesystem())->exists(new Path(__FILE__))); - self::assertFalse((new LocalFilesystem())->exists(new Path(__DIR__ . '/not_existing_file.php'))); - } - - public function test_file_pattern_exists() : void - { - self::assertTrue((new LocalFilesystem())->exists(new Path(__DIR__ . '/**/*.txt'))); - self::assertFalse((new LocalFilesystem())->exists(new Path(__DIR__ . '/**/*.pdf'))); - } - - public function test_open_file_stream_for_existing_file() : void - { - $stream = (new LocalFilesystem())->open(new Path(__FILE__), Mode::READ); - - self::assertIsResource($stream->resource()); - self::assertSame( - \file_get_contents(__FILE__), - \stream_get_contents($stream->resource()) - ); - } - - public function test_open_file_stream_for_non_existing_file() : void - { - $path = \sys_get_temp_dir() . '/flow_php_test_file_' . bin2hex(random_bytes(16)) . '.txt'; - - $stream = (new LocalFilesystem())->open(new Path($path), Mode::WRITE); - - self::assertIsResource($stream->resource()); - } - - public function test_reading_multi_partitioned_path() : void - { - $paths = \iterator_to_array( - (new LocalFilesystem()) - ->scan( - new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), - new ScalarFunctionFilter( - all( - ref('country')->equals(lit('pl')), - all( - ref('date')->cast('date')->greaterThanEqual(lit(new \DateTimeImmutable('2022-01-02'))), - ref('date')->cast('date')->lessThan(lit(new \DateTimeImmutable('2022-01-04'))) - ) - ), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ) - ) - ); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'), - ], - $paths - ); - } - - public function test_reading_partitioned_folder() : void - { - $paths = \iterator_to_array((new LocalFilesystem())->scan(new Path(__DIR__ . '/Fixtures/partitioned/**/*.txt'), new NoopFilter())); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - $paths - ); - } - - public function test_reading_partitioned_folder_with_partitions_filtering() : void - { - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - \iterator_to_array( - (new LocalFilesystem()) - ->scan( - new Path(__DIR__ . '/Fixtures/partitioned/**/*.txt'), - new ScalarFunctionFilter(ref('partition_01')->equals(lit('b')), new NativeEntryFactory(), new AutoCaster(Caster::default())) - ) - ) - ); - } - - public function test_reading_partitioned_folder_with_pattern() : void - { - $paths = \iterator_to_array((new LocalFilesystem())->scan(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=*/*.txt'), new NoopFilter())); - \sort($paths); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), - new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), - ], - $paths - ); - } - - public function test_remove_directory_with_content_when_exists() : void - { - $fs = new LocalFilesystem(); - - $dirPath = Path::realpath(\sys_get_temp_dir() . '/flow-fs-test-directory/'); - - $stream = $fs->open(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($dirPath)); - self::assertTrue($fs->exists($stream->path())); - $fs->rm($dirPath); - self::assertFalse($fs->exists($dirPath)); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_remove_file_when_exists() : void - { - $fs = new LocalFilesystem(); - - $stream = $fs->open(Path::realpath(\sys_get_temp_dir() . '/flow-fs-test/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($stream->path())); - $fs->rm($stream->path()); - self::assertFalse($fs->exists($stream->path())); - } - - public function test_remove_pattern() : void - { - $fs = new LocalFilesystem(); - - $dirPath = Path::realpath(\sys_get_temp_dir() . '/flow-fs-test-directory/'); - - $stream = $fs->open(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt'), Mode::WRITE); - \fwrite($stream->resource(), 'some data to make file not empty'); - $stream->close(); - - self::assertTrue($fs->exists($dirPath)); - self::assertTrue($fs->exists($stream->path())); - $fs->rm(Path::realpath($dirPath->path() . '/*.txt')); - self::assertTrue($fs->exists($dirPath)); - self::assertFalse($fs->exists($stream->path())); - $fs->rm($dirPath); - } - - public function test_that_scan_sort_files_by_path_names() : void - { - $paths = \iterator_to_array( - (new LocalFilesystem()) - ->scan( - new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), - ) - ); - - self::assertEquals( - [ - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt'), - new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt'), - ], - $paths - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/IntegrationTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/IntegrationTestCase.php index 6099c487e..6499ee020 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/IntegrationTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/IntegrationTestCase.php @@ -4,28 +4,29 @@ namespace Flow\ETL\Tests\Integration; -use Flow\ETL\Filesystem\{LocalFilesystem, Path}; -use Flow\ETL\{Config, Filesystem}; +use Flow\ETL\{Config}; +use Flow\Filesystem\{Filesystem, Path}; +use Flow\Filesystem\{FilesystemTable, Local\NativeLocalFilesystem}; use PHPUnit\Framework\TestCase; abstract class IntegrationTestCase extends TestCase { - protected string $cacheDir; + protected string $cacheDir = __DIR__ . '/var/'; - protected Filesystem $fs; + protected ?Filesystem $fs = null; - private string $baseMemoryLimit; + protected ?FilesystemTable $fstab = null; + + private string $baseMemoryLimit = '-1'; protected function setUp() : void { $this->baseMemoryLimit = \ini_get('memory_limit'); $this->cacheDir = Path::realpath(\getenv(Config::CACHE_DIR_ENV))->path(); - $this->fs = new LocalFilesystem(); - $this->cleanupCacheDir($this->cacheDir); - if (!$this->fs->directoryExists(Path::realpath($this->cacheDir))) { + if (!$this->fs()->status(Path::realpath($this->cacheDir))?->isDirectory()) { \mkdir($this->cacheDir, recursive: true); } } @@ -46,7 +47,7 @@ protected function cleanFiles() : void continue; } - $this->fs->rm(Path::realpath($this->filesDirectory() . DIRECTORY_SEPARATOR . $file)); + $this->fs()->rm(Path::realpath($this->filesDirectory() . DIRECTORY_SEPARATOR . $file)); } } @@ -55,6 +56,24 @@ protected function filesDirectory() : string throw new \RuntimeException('You need to implement filesDirectory method to point to your test files directory.'); } + protected function fs() : Filesystem + { + if ($this->fs === null) { + $this->fs = new NativeLocalFilesystem(); + } + + return $this->fs; + } + + protected function fstab() : FilesystemTable + { + if ($this->fstab === null) { + $this->fstab = new FilesystemTable($this->fs()); + } + + return $this->fstab; + } + protected function getPath(string $relativePath) : Path { return new Path($this->filesDirectory() . DIRECTORY_SEPARATOR . $relativePath); @@ -87,8 +106,8 @@ protected function setupFiles(array $datasets, $path = '') : void private function cleanupCacheDir(string $directory) : void { - if ($this->fs->directoryExists($path = Path::realpath($directory))) { - $this->fs->rm($path); + if ($this->fs()->status($path = Path::realpath($directory))?->isDirectory()) { + $this->fs()->rm($path); } } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SynchronousPipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SynchronousPipelineTest.php index fbf1b41e6..3bd2a287d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SynchronousPipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/SynchronousPipelineTest.php @@ -11,9 +11,16 @@ final class SynchronousPipelineTest extends IntegrationTestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_limit() : void { - $path = \sys_get_temp_dir() . '/synchronous_pipeline_' . __FUNCTION__ . '.csv'; + $path = __DIR__ . '/var/synchronous_pipeline_' . __FUNCTION__ . '.csv'; if (\file_exists($path)) { \unlink($path); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php new file mode 100644 index 000000000..1bd1b651b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php @@ -0,0 +1,70 @@ +extract(flow_context()) as $rows) { + self::assertCount(1, $rows); + $totalRows += $rows->count(); + } + + self::assertEquals(3, $totalRows); + } + + public function test_extracting_files_from_directory_after_getting_stop_signal() : void + { + $extractor = files(__DIR__ . '/Fixtures/FileListExtractor/*'); + $generator = $extractor->extract(flow_context()); + $totalRows = 0; + + foreach ($generator as $rows) { + self::assertCount(1, $rows); + $totalRows += $rows->count(); + $generator->send(Signal::STOP); + } + + self::assertEquals(1, $totalRows); + } + + public function test_extracting_files_from_directory_recursive() : void + { + $extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*'); + + $totalRows = 0; + + foreach ($extractor->extract(flow_context()) as $rows) { + self::assertCount(1, $rows); + $totalRows += $rows->count(); + } + + self::assertEquals(6, $totalRows); + } + + public function test_extracting_files_from_directory_with_limit() : void + { + $extractor = files(__DIR__ . '/Fixtures/FileListExtractor/**/*'); + $extractor->changeLimit(2); + + $totalRows = 0; + + foreach ($extractor->extract(flow_context()) as $rows) { + self::assertCount(1, $rows); + $totalRows += $rows->count(); + } + + self::assertEquals(2, $totalRows); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/LocalFileListExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/LocalFileListExtractorTest.php deleted file mode 100644 index 0746e6605..000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/LocalFileListExtractorTest.php +++ /dev/null @@ -1,99 +0,0 @@ -extract(flow_context()) as $rows) { - self::assertCount(1, $rows); - $totalRows += $rows->count(); - } - - self::assertEquals(6, $totalRows); - } - - public function test_extracting_files_from_directory_after_getting_stop_signal() : void - { - $extractor = local_files(__DIR__ . '/Fixtures/FileListExtractor', true); - $generator = $extractor->extract(flow_context()); - $totalRows = 0; - - foreach ($generator as $rows) { - self::assertCount(1, $rows); - $totalRows += $rows->count(); - $generator->send(Signal::STOP); - } - - self::assertEquals(1, $totalRows); - } - - public function test_extracting_files_from_directory_recursive() : void - { - $extractor = local_files(__DIR__ . '/Fixtures/FileListExtractor', true); - - $totalRows = 0; - - foreach ($extractor->extract(flow_context()) as $rows) { - self::assertCount(1, $rows); - $totalRows += $rows->count(); - } - - self::assertEquals(7, $totalRows); - } - - public function test_extracting_files_from_directory_with_limit() : void - { - $extractor = local_files(__DIR__ . '/Fixtures/FileListExtractor', true); - $extractor->changeLimit(2); - - $totalRows = 0; - - foreach ($extractor->extract(flow_context()) as $rows) { - self::assertCount(1, $rows); - $totalRows += $rows->count(); - } - - self::assertEquals(2, $totalRows); - } - - public function test_extracting_from_directory_with_pattern() : void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('LocalFileListExtractor does not support glob paths'); - local_files(__DIR__ . '/Fixtures/FileListExtractor/*'); - } - - public function test_extracting_from_remote_directory() : void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Path must point to a local directory'); - local_files('flow-remote-file://bucket-name/path/to/file'); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PipelineExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PipelineExtractorTest.php index a202d0bc0..ec3e59f44 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PipelineExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PipelineExtractorTest.php @@ -5,7 +5,7 @@ namespace Flow\ETL\Tests\Unit\Extractor; use function Flow\ETL\DSL\int_entry; -use Flow\ETL\Extractor\{PipelineExtractor, ProcessExtractor}; +use Flow\ETL\Extractor\{PipelineExtractor, RowsExtractor}; use Flow\ETL\Pipeline\SynchronousPipeline; use Flow\ETL\{Config, FlowContext, Row, Rows}; use PHPUnit\Framework\TestCase; @@ -14,7 +14,7 @@ final class PipelineExtractorTest extends TestCase { public function test_pipeline_extractor() : void { - $pipeline = new SynchronousPipeline(new ProcessExtractor( + $pipeline = new SynchronousPipeline(new RowsExtractor( new Rows(Row::create(int_entry('id', 1)), Row::create(int_entry('id', 2))), new Rows(Row::create(int_entry('id', 3)), Row::create(int_entry('id', 4))), new Rows(Row::create(int_entry('id', 5)), Row::create(int_entry('id', 6))), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ProcessExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php similarity index 96% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ProcessExtractorTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php index b27125cca..4f27bd006 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ProcessExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php @@ -8,7 +8,7 @@ use Flow\ETL\{Config, FlowContext, Row, Rows}; use PHPUnit\Framework\TestCase; -final class ProcessExtractorTest extends TestCase +final class RowsExtractorTest extends TestCase { public function test_process_extractor() : void { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php index f7e4e00dc..879b3490f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php @@ -6,9 +6,9 @@ use function Flow\ETL\DSL\{int_entry, ref, row, rows, str_entry, to_output, to_stream}; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\Stream\Mode; use Flow\ETL\Loader\StreamLoader; use Flow\ETL\{Config, FlowContext}; +use Flow\Filesystem\Stream\Mode; use PHPUnit\Framework\TestCase; final class StreamLoaderTest extends TestCase diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/ScalarFunctionFilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/ScalarFunctionFilterTest.php deleted file mode 100644 index 221cafa9b..000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/ScalarFunctionFilterTest.php +++ /dev/null @@ -1,97 +0,0 @@ -greaterThan(lit(10)), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - self::assertTrue($filter->keep(new Partition('foo', '100'))); - self::assertFalse($filter->keep(new Partition('foo', '5'))); - } - - public function test_filtering_datetime_partitions() : void - { - $filter = new ScalarFunctionFilter( - ref('foo')->greaterThan(lit(new \DateTimeImmutable('2021-01-01'))), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - self::assertTrue($filter->keep(new Partition('foo', '2021-01-02'))); - self::assertFalse($filter->keep(new Partition('foo', '2020-12-31'))); - } - - public function test_filtering_datetime_partitions_by_string_value() : void - { - $filter = new ScalarFunctionFilter( - ref('foo')->greaterThan(lit('2021-01-01')), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - $this->expectExceptionMessage("Can't compare '(datetime > string)' due to data type mismatch."); - self::assertTrue($filter->keep(new Partition('foo', '2021-01-02'))); - } - - public function test_filtering_when_partition_is_not_covered_by_any_filter() : void - { - $filter = new ScalarFunctionFilter( - ref('foo')->greaterThan(lit(10)), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - $this->expectExceptionMessage('Entry "foo" does not exist. Did you mean one of the following? ["bar"]'); - self::assertFalse($filter->keep(new Partition('bar', '100'))); - } - - public function test_filtering_with_multiple_partitions_and_condition() : void - { - $filter = new ScalarFunctionFilter( - all( - ref('foo')->greaterThanEqual(lit(100)), - ref('bar')->greaterThanEqual(lit(100)) - ), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - self::assertTrue($filter->keep(new Partition('foo', '100'), new Partition('bar', '100'))); - self::assertFalse($filter->keep(new Partition('foo', '100'), new Partition('bar', '5'))); - self::assertFalse($filter->keep(new Partition('foo', '5'), new Partition('bar', '100'))); - self::assertFalse($filter->keep(new Partition('foo', '5'), new Partition('bar', '5'))); - } - - public function test_filtering_with_multiple_partitions_or_condition() : void - { - $filter = new ScalarFunctionFilter( - any( - ref('foo')->greaterThanEqual(lit(100)), - ref('bar')->greaterThanEqual(lit(100)) - ), - new NativeEntryFactory(), - new AutoCaster(Caster::default()) - ); - - self::assertTrue($filter->keep(new Partition('foo', '100'), new Partition('bar', '100'))); - self::assertTrue($filter->keep(new Partition('foo', '100'), new Partition('bar', '5'))); - self::assertTrue($filter->keep(new Partition('foo', '5'), new Partition('bar', '100'))); - self::assertFalse($filter->keep(new Partition('foo', '5'), new Partition('bar', '5'))); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/LinkedPipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/LinkedPipelineTest.php index c3a658cb6..f60f4b236 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/LinkedPipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/LinkedPipelineTest.php @@ -5,7 +5,7 @@ namespace Flow\ETL\Tests\Unit\Pipeline; use function Flow\ETL\DSL\{bool_entry, int_entry, lit}; -use Flow\ETL\Extractor\ProcessExtractor; +use Flow\ETL\Extractor\RowsExtractor; use Flow\ETL\Pipeline\{LinkedPipeline, SynchronousPipeline}; use Flow\ETL\Transformer\ScalarFunctionTransformer; use Flow\ETL\{Config, FlowContext, Row, Rows}; @@ -16,7 +16,7 @@ final class LinkedPipelineTest extends TestCase public function test_linked_pipelines() : void { $pipeline = new LinkedPipeline( - (new SynchronousPipeline(new ProcessExtractor( + (new SynchronousPipeline(new RowsExtractor( new Rows( Row::create(int_entry('id', 1)), Row::create(int_entry('id', 2)) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/Optimizer/LimitOptimizationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/Optimizer/LimitOptimizationTest.php index 01fbd4330..09a06c0b0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/Optimizer/LimitOptimizationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Pipeline/Optimizer/LimitOptimizationTest.php @@ -6,11 +6,11 @@ use function Flow\ETL\DSL\ref; use Flow\ETL\Adapter\CSV\CSVExtractor; -use Flow\ETL\Filesystem\Path; use Flow\ETL\GroupBy; use Flow\ETL\Pipeline\Optimizer\LimitOptimization; use Flow\ETL\Pipeline\{GroupByPipeline, Optimizer, PartitioningPipeline, SynchronousPipeline}; use Flow\ETL\Transformer\{DropDuplicatesTransformer, LimitTransformer, RenameEntryTransformer, ScalarFunctionTransformer, SelectEntriesTransformer}; +use Flow\Filesystem\Path; use PHPUnit\Framework\TestCase; final class LimitOptimizationTest extends TestCase diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/CartesianProductTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/CartesianProductTest.php similarity index 95% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/CartesianProductTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Row/CartesianProductTest.php index cc2775770..a35ceaef4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Partition/CartesianProductTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/CartesianProductTest.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Partition; +namespace Flow\ETL\Tests\Unit\Row; -use Flow\ETL\Partition\CartesianProduct; +use Flow\ETL\Row\CartesianProduct; use PHPUnit\Framework\TestCase; final class CartesianProductTest extends TestCase diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php index 3a8fafa24..918012937 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php @@ -4,7 +4,8 @@ namespace Flow\ETL\Tests\Unit; -use function Flow\ETL\DSL\{array_entry, array_to_rows, bool_entry, bool_schema, datetime_entry, int_entry, int_schema, list_entry, partition, partitions, ref, row, rows, rows_partitioned, str_entry, str_schema, type_int, type_list, type_string}; +use function Flow\ETL\DSL\{array_entry, array_to_rows, bool_entry, bool_schema, datetime_entry, int_entry, int_schema, list_entry, ref, row, rows, rows_partitioned, str_entry, str_schema, type_int, type_list, type_string}; +use function Flow\Filesystem\DSL\{partition, partitions}; use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException}; use Flow\ETL\PHP\Type\Logical\List\ListElement; use Flow\ETL\PHP\Type\Logical\ListType; diff --git a/src/lib/array-dot/README.md b/src/lib/array-dot/README.md index 5fe1a47d8..8c2bc94c7 100644 --- a/src/lib/array-dot/README.md +++ b/src/lib/array-dot/README.md @@ -10,5 +10,5 @@ it a valuable asset for developers aiming to streamline their array handling tas configuration data, nested JSON objects, or any other complex array structures, the Array Dot library is a reliable companion for achieving cleaner and more efficient array operations. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/array-dot.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/azure-sdk/.gitattributes b/src/lib/azure-sdk/.gitattributes new file mode 100644 index 000000000..e02097205 --- /dev/null +++ b/src/lib/azure-sdk/.gitattributes @@ -0,0 +1,9 @@ +*.php text eol=lf + +/.github export-ignore +/tests export-ignore + +/README.md export-ignore + +/.gitattributes export-ignore +/.gitignore export-ignore diff --git a/src/lib/azure-sdk/.github/workflows/readonly.yaml b/src/lib/azure-sdk/.github/workflows/readonly.yaml new file mode 100644 index 000000000..da596bcdd --- /dev/null +++ b/src/lib/azure-sdk/.github/workflows/readonly.yaml @@ -0,0 +1,17 @@ +name: Readonly + +on: + pull_request_target: + types: [opened] + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: superbrothers/close-pull-request@v3 + with: + comment: | + Hi, thank you for your contribution. + Unfortunately, this repository is read-only. It's a split from our main monorepo repository. + In order to proceed with this PR please open it against https://github.com/flow-php/flow repository. + Thank you. \ No newline at end of file diff --git a/src/lib/azure-sdk/CONTRIBUTING.md b/src/lib/azure-sdk/CONTRIBUTING.md new file mode 100644 index 000000000..a2d0671c7 --- /dev/null +++ b/src/lib/azure-sdk/CONTRIBUTING.md @@ -0,0 +1,6 @@ +## Contributing + +This repo is **READ ONLY**, in order to contribute to Flow PHP project, please +open PR against [flow](https://github.com/flow-php/flow) monorepo. + +Changes merged to monorepo are automatically propagated into sub repositories. \ No newline at end of file diff --git a/src/lib/azure-sdk/LICENSE b/src/lib/azure-sdk/LICENSE new file mode 100644 index 000000000..bc3cc4d08 --- /dev/null +++ b/src/lib/azure-sdk/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020-present Flow PHP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/lib/azure-sdk/README.md b/src/lib/azure-sdk/README.md new file mode 100644 index 000000000..afb67d0f3 --- /dev/null +++ b/src/lib/azure-sdk/README.md @@ -0,0 +1,6 @@ +# Dremel + +## Installation + +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/azure-sdk.md) +- 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/adapter/etl-adapter-filesystem/composer.json b/src/lib/azure-sdk/composer.json similarity index 58% rename from src/adapter/etl-adapter-filesystem/composer.json rename to src/lib/azure-sdk/composer.json index 2c25adead..daa43f5df 100644 --- a/src/adapter/etl-adapter-filesystem/composer.json +++ b/src/lib/azure-sdk/composer.json @@ -1,24 +1,23 @@ { - "name": "flow-php/etl-adapter-filesystem", + "name": "flow-php/azure-sdk", "type": "library", - "description": "PHP ETL - Adapter - Filesystem", + "description": "PHP ETL - Filesystem abstraction", "keywords": [ "etl", "extract", "transform", "load", - "parquet", - "adapter" + "filesystem", + "remote", + "azure", + "aws", + "gcp" ], "require": { "php": "~8.1.0 || ~8.2.0 || ~8.3.0", - "ext-json": "*", - "flow-php/etl": "^0.7 || 1.x-dev", - "league/flysystem": "^3.0" - }, - "require-dev": { - "league/flysystem-aws-s3-v3": "^3.0", - "league/flysystem-azure-blob-storage": "^3.0" + "psr/log": "^2.0 || ^3.0", + "psr/http-client": "^1.0", + "php-http/discovery": "^1.0" }, "config": { "optimize-autoloader": true, @@ -32,7 +31,7 @@ ] }, "files": [ - "src/Flow/ETL/Adapter/Filesystem/functions.php" + "src/Flow/Azure/SDK/DSL/functions.php" ] }, "autoload-dev": { diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/AuthorizationFactory.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/AuthorizationFactory.php new file mode 100644 index 000000000..3357f64c4 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/AuthorizationFactory.php @@ -0,0 +1,12 @@ +computeSignature( + $this->normalizeHeaders($request), + (string) $request->getUri(), + $this->parseQueryPart($request->getUri()->getQuery()), + $request->getMethod() + ); + + return 'SharedKey ' . $this->account . ':' . base64_encode( + hash_hmac('sha256', $signature, (string) base64_decode($this->accountKey, true), true) + ); + } + + private function computeCanonicalizedHeaders(array $headers) : array + { + $canonicalizedHeaders = []; + $normalizedHeaders = []; + + foreach ($headers as $header => $value) { + $header = \strtolower($header); + + if (\str_starts_with($header, 'x-ms-')) { + $value = \str_replace("\r\n", ' ', $value); + + /** + * @psalm-suppress PossiblyInvalidArgument + */ + $value = \ltrim($value); + $header = \rtrim($header); + + $normalizedHeaders[$header] = $value; + } + } + + \ksort($normalizedHeaders); + + foreach ($normalizedHeaders as $key => $value) { + $canonicalizedHeaders[] = $key . ':' . $value; + } + + return $canonicalizedHeaders; + } + + private function computeCanonicalizedResource(string $url, array $queryParams) : string + { + $queryParams = array_change_key_case($queryParams); + + $canonicalizedResource = '/' . $this->account; + + $canonicalizedResource .= parse_url($url, PHP_URL_PATH); + + if (\count($queryParams) > 0) { + \ksort($queryParams); + } + + foreach ($queryParams as $key => $value) { + $canonicalizedResource .= "\n" . $key . ':' . $value; + } + + return $canonicalizedResource; + } + + private function computeSignature(array $headers, string $url, array $queryParams, string $httpMethod) : string + { + $canonicalizedHeaders = $this->computeCanonicalizedHeaders($headers); + $canonicalizedResource = $this->computeCanonicalizedResource($url, $queryParams); + + $stringToSign = []; + $stringToSign[] = \strtoupper($httpMethod); + + $includedHeaders = ['content-encoding', 'content-language', 'content-length', 'content-md5', 'content-type', 'date', 'if-modified-since', 'if-match', 'if-none-match', 'if-unmodified-since', 'range']; + + $lowercaseHeaders = array_change_key_case($headers); + + foreach ($includedHeaders as $header) { + $stringToSign[] = \array_key_exists($header, $lowercaseHeaders) ? $lowercaseHeaders[$header] : null; + } + + if (count($canonicalizedHeaders) > 0) { + $stringToSign[] = \implode("\n", $canonicalizedHeaders); + } + + $stringToSign[] = $canonicalizedResource; + + return \implode("\n", $stringToSign); + } + + private function normalizeHeaders(RequestInterface $request) : array + { + $headers = []; + + foreach ($request->getHeaders() as $key => $value) { + if (is_array($value) && count($value) == 1) { + $headers[strtolower($key)] = $value[0]; + } else { + $headers[strtolower($key)] = $value; + } + } + + return $headers; + } + + private function parseQueryPart(string $queryPart, bool $urlEncoding = true) : array + { + $result = []; + + if ($queryPart === '') { + return $result; + } + + if ($urlEncoding === true) { + $decoder = static fn (string $value) : string => rawurldecode(str_replace('+', ' ', $value)); + } else { + $decoder = static fn (string $str) : string => $str; + } + + foreach (explode('&', $queryPart) as $kvp) { + $parts = explode('=', $kvp, 2); + $key = $decoder($parts[0]); + $value = isset($parts[1]) ? $decoder($parts[1]) : null; + + if (!array_key_exists($key, $result)) { + $result[$key] = $value; + } else { + if (!is_array($result[$key])) { + $result[$key] = [$result[$key]]; + } + $result[$key][] = $value; + } + } + + return $result; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService.php new file mode 100644 index 000000000..fe7a6cc73 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService.php @@ -0,0 +1,586 @@ +httpFactory->put( + $this->urlFactory->create( + $this->configuration, + $toBlob, + $options->toURIParameters(), + ) + ); + + $request = $request + ->withHeader('date', \gmdate('D, d M Y H:i:s T', time())) + ->withHeader('x-ms-copy-source', $this->urlFactory->create( + $this->configuration, + $fromBlob, + )); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Copy Blob', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + $request = $request->withHeader('content-length', '0'); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Copy Blob', ['response' => $response]); + + if ($response->getStatusCode() !== 202) { + $this->logger->critical('Azure - Blob Service - Copy Blob', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function deleteBlob(string $blob, DeleteBlobOptions $options = new DeleteBlobOptions()) : void + { + $request = $this->httpFactory->delete( + $this->urlFactory->create( + $this->configuration, + $blob, + $options->toURIParameters(), + ) + ); + + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Delete Blob', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Delete Blob', ['response' => $response]); + + if ($response->getStatusCode() !== 202) { + $this->logger->critical('Azure - Blob Service - Delete Blob', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function deleteContainer(DeleteContainerOptions $options = new DeleteContainerOptions()) : void + { + $request = $this->httpFactory->delete( + $this->urlFactory->create( + $this->configuration, + null, + $options->toURIParameters(), + ) + ); + + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Delete Container', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Delete Container', ['response' => $response]); + + if ($response->getStatusCode() !== 202) { + $this->logger->critical('Azure - Blob Service - Delete Container', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function getBlob(string $blob, GetBlobOptions $options = new GetBlobOptions()) : BlobContent + { + $request = $this->httpFactory->get( + $this->urlFactory->create( + $this->configuration, + $blob, + $options->toURIParameters(), + ) + ); + + $request = $request->withHeader('content-type', 'application/x-www-form-urlencoded'); + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Get Blob', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Get Blob', ['response' => $response]); + + if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { + $this->logger->critical('Azure - Blob Service - Get Blob', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + + return new BlobContent($response); + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function getBlobProperties(string $blob, GetBlobPropertiesOptions $options = new GetBlobPropertiesOptions()) : ?BlobProperties + { + $request = $this->httpFactory->get( + $this->urlFactory->create( + $this->configuration, + $blob, + $options->toURIParameters(), + ) + ); + + $request = $request->withHeader('content-type', 'application/x-www-form-urlencoded'); + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Get Blob Properties', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Get Blob Properties', ['response' => $response]); + + if ($response->getStatusCode() === 404) { + return null; + } + + if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { + $this->logger->critical('Azure - Blob Service - Get Blob Properties', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + + return new BlobProperties($response); + } + + public function getBlockBlobBlockList(string $blob, GetBlockBlobBlockListOptions $options = new GetBlockBlobBlockListOptions()) : BlockList + { + $request = $this->httpFactory->get( + $this->urlFactory->create( + $this->configuration, + $blob, + \array_merge( + $options->toURIParameters(), + ['comp' => 'blocklist'] + ) + ) + ); + + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Get Block Blob Block List', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Get Block Blob Block List', ['response' => $response]); + + if ($response->getStatusCode() !== 200) { + $this->logger->critical('Azure - Blob Service - Get Block Blob Block List', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + + $normalized = (new SimpleXMLNormalizer())->toArray($response->getBody()->getContents()); + + $blocks = []; + + if (\array_key_exists('CommittedBlocks', $normalized) && \is_array($normalized['CommittedBlocks'])) { + if (isset($normalized['CommittedBlocks']['Block']['Name'])) { + $blocks[] = new Block($normalized['CommittedBlocks']['Block']['Name'], BlockState::COMMITTED, (int) $normalized['CommittedBlocks']['Block']['Size']); + } else { + foreach ($normalized['CommittedBlocks']['Block'] as $block) { + $blocks[] = new Block($block['Name'], BlockState::COMMITTED, (int) $block['Size']); + } + } + } + + if (\array_key_exists('UncommittedBlocks', $normalized) && \is_array($normalized['UncommittedBlocks'])) { + if (isset($normalized['UncommittedBlocks']['Block']['Name'])) { + $blocks[] = new Block($normalized['UncommittedBlocks']['Block']['Name'], BlockState::UNCOMMITTED, (int) $normalized['UncommittedBlocks']['Block']['Size']); + } else { + foreach ($normalized['UncommittedBlocks']['Block'] as $block) { + $blocks[] = new Block($block['Name'], BlockState::UNCOMMITTED, (int) $block['Size']); + } + } + } + + return new BlockList(...$blocks); + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function getContainerProperties(GetContainerPropertiesOptions $options = new GetContainerPropertiesOptions()) : ?ContainerProperties + { + $request = $this->httpFactory->get( + $this->urlFactory->create( + $this->configuration, + null, + array_merge( + $options->toURIParameters(), + ['restype' => 'container'] + ) + ) + ); + + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Get Container Properties', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Get Container Properties', ['response' => $response]); + + if ($response->getStatusCode() === 404) { + return null; + } + + if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { + $this->logger->critical('Azure - Blob Service - Get Container Properties', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + + return new ContainerProperties($response); + } + + /** + * @throws AzureException + * + * @return \Generator + */ + public function listBlobs(ListBlobOptions $options = new ListBlobOptions()) : \Generator + { + $request = $this->httpFactory->get( + $this->urlFactory->create( + $this->configuration, + queryParameters: \array_merge( + $options->toURIParameters(), + ['restype' => 'container', 'comp' => 'list'] + ) + ) + ); + + $request = $request->withHeader('content-type', 'application/x-www-form-urlencoded'); + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - List Blobs', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - List Blobs', ['response' => $response]); + + if ($response->getStatusCode() !== 200) { + $this->logger->critical('Azure - Blob Service - List Blobs', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + + $normalized = (new SimpleXMLNormalizer())->toArray($response->getBody()->getContents()); + + if ($normalized['Blobs'] === null) { + return; + } + + if (isset($normalized['Blobs']['Blob']['Name'])) { + yield new Blob($normalized['Blobs']['Blob']); + + return; + } + + foreach ($normalized['Blobs']['Blob'] as $blobData) { + yield new Blob($blobData); + } + + if ($normalized['NextMarker'] !== null) { + yield from $this->listBlobs($options->withMarker($normalized['NextMarker'])); + } + } + + /** + * @param null|resource|string $content + * + * @throws AzureException + */ + public function putBlockBlob(string $path, $content = null, ?int $size = null, PutBlockBlobOptions $options = new PutBlockBlobOptions()) : void + { + if ($content !== null) { + if (!\is_resource($content) && !\is_string($content)) { + throw new InvalidArgumentException('Content must be a resource or a string'); + } + + if ($size === null) { + throw new InvalidArgumentException('Size must be provided when content is provided'); + } + } + + $request = $this->httpFactory->put( + $this->urlFactory->create( + $this->configuration, + $path, + $options->toURIParameters(), + ) + ); + + $request = $request + ->withHeader('content-type', 'application/octet-stream') + ->withHeader('x-ms-blob-content-type', 'application/octet-stream') + ->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + if ($content) { + $request = $request + ->withHeader('content-length', (string) $size) + ->withBody($this->httpFactory->stream($content)); + } + + $this->logger->info('Azure - Blob Service - Put Blob Block', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + if (!$content) { + $request = $request->withHeader('content-length', '0'); + } + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Put Blob Block', ['response' => $response]); + + if ($response->getStatusCode() !== 201) { + $this->logger->critical('Azure - Blob Service - Put Blob Block', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @param resource|string $content + * + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function putBlockBlobBlock(string $path, string $blockId, $content, int $size, PutBlockBlobBlockOptions $options = new PutBlockBlobBlockOptions()) : void + { + $request = $this->httpFactory->put( + $this->urlFactory->create( + $this->configuration, + $path, + \array_merge( + $options->toURIParameters(), + ['comp' => 'block', 'blockid' => $blockId] + ) + ) + ); + + $request = $request + ->withHeader('content-type', 'application/x-www-form-urlencoded') + ->withHeader('date', \gmdate('D, d M Y H:i:s T', time())) + ->withHeader('content-length', (string) $size); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $request = $request + ->withBody($this->httpFactory->stream($content)) + ->withHeader('authorization', $this->authorizationFactory->for($request)); + + $this->logger->info('Azure - Blob Service - Put Block Blob Block', ['request' => $request]); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Put Block Blob Block', ['response' => $response]); + + if ($response->getStatusCode() !== 201) { + $this->logger->critical('Azure - Blob Service - Put Block Blob Block', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function putBlockBlobBlockList(string $path, BlockList $blockList, PutBlockBlobBlockListOptions $options = new PutBlockBlobBlockListOptions(), Serializer $serializer = new SimpleXMLSerializer()) : void + { + $request = $this->httpFactory->put( + $this->urlFactory->create( + $this->configuration, + $path, + queryParameters: \array_merge( + $options->toURIParameters(), + ['comp' => 'blocklist'] + ) + ) + ); + + $request = $request + ->withHeader('content-type', 'application/x-www-form-urlencoded') + ->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $request = $request + ->withBody($this->httpFactory->stream($blockListString = $serializer->serialize($blockList))) + ->withHeader('content-length', (string) \strlen($blockListString)); + + $this->logger->info('Azure - Blob Service - Put Block Blob Block List', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Put Block Blob Block List', ['response' => $response]); + + if ($response->getStatusCode() !== 201) { + $this->logger->critical('Azure - Blob Service - Put Block Blob Block List', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } + + /** + * @throws AzureException + * @throws ClientExceptionInterface + */ + public function putContainer(CreateContainerOptions $options = new CreateContainerOptions()) : void + { + $request = $this->httpFactory->put( + $this->urlFactory->create( + $this->configuration, + null, + \array_merge( + $options->toURIParameters(), + ['restype' => 'container'] + ) + ) + ); + + $request = $request->withHeader('date', \gmdate('D, d M Y H:i:s T', time())); + + foreach ($options->toHeaders() as $header => $value) { + $request = $request->withHeader($header, $value); + } + + $this->logger->info('Azure - Blob Service - Put Container', ['request' => $request]); + + $request = $request->withHeader('authorization', $this->authorizationFactory->for($request)); + + $response = $this->httpClient->sendRequest($request); + + $this->logger->info('Azure - Blob Service - Put Container', ['response' => $response]); + + if ($response->getStatusCode() !== 201) { + $this->logger->critical('Azure - Blob Service - Put Container', ['response' => $response]); + + throw new AzureException(__METHOD__, $request, $response); + } + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/Block.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/Block.php new file mode 100644 index 000000000..b157c2341 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/Block.php @@ -0,0 +1,28 @@ + + */ + private array $blocks; + + public function __construct(Block ...$blocks) + { + $this->blocks = $blocks; + } + + public function all() : array + { + return $this->blocks; + } + + public function append(Block $block) : self + { + $this->blocks[] = $block; + + return $this; + } + + public function last() : ?Block + { + if (!\count($this->blocks)) { + return null; + } + + $blocks = $this->blocks; + + return \array_pop($blocks); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/BlockState.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/BlockState.php new file mode 100644 index 000000000..3d3de5579 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/BlockBlob/BlockState.php @@ -0,0 +1,12 @@ +version !== null) { + $headers['x-ms-version'] = $this->version; + } + + $headers['User-Agent'] = $this->userAgentHeader(); + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + if ($this->snapshot === null) { + if ($this->deleteSnapshots !== null) { + $headers['x-ms-delete-snapshots'] = $this->deleteSnapshots->value; + } + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + if ($this->snapshot !== null) { + $uriParameters['snapshot'] = $this->snapshot; + } + + if ($this->deleteType !== null) { + $uriParameters['deleteType'] = $this->deleteType->value; + } + + return $uriParameters; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withSnapshot(string $snapshot) : self + { + $this->snapshot = $snapshot; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } + + public function withVersionId(string $versionId) : self + { + $this->versionId = $versionId; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/CreateContainerOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/CreateContainerOptions.php new file mode 100644 index 000000000..429e36492 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/CreateContainerOptions.php @@ -0,0 +1,80 @@ +userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->publicAccess !== null) { + $headers['x-ms-blob-public-access'] = $this->publicAccess->value; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withPublicAccess(PublicAccess $publicAccess) : self + { + $this->publicAccess = $publicAccess; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/PublicAccess.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/PublicAccess.php new file mode 100644 index 000000000..5c32207d9 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/CreateContainer/PublicAccess.php @@ -0,0 +1,11 @@ +userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + if ($this->snapshot === null) { + if ($this->deleteSnapshots !== null) { + $headers['x-ms-delete-snapshots'] = $this->deleteSnapshots->value; + } + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + if ($this->snapshot !== null) { + $uriParameters['snapshot'] = $this->snapshot; + } + + if ($this->deleteType !== null) { + $uriParameters['deleteType'] = $this->deleteType->value; + } + + return $uriParameters; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withSnapshot(string $snapshot) : self + { + $this->snapshot = $snapshot; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } + + public function withVersionId(string $versionId) : self + { + $this->versionId = $versionId; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/DeleteBlob/DeleteSnapshots.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/DeleteBlob/DeleteSnapshots.php new file mode 100644 index 000000000..16d0a4e93 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/DeleteBlob/DeleteSnapshots.php @@ -0,0 +1,11 @@ +userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/BlobContent.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/BlobContent.php new file mode 100644 index 000000000..0871dda72 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/BlobContent.php @@ -0,0 +1,42 @@ +response->getStatusCode() < 200 || $this->response->getStatusCode() >= 300) { + throw new \RuntimeException('Blob content could not be fetched'); + } + } + + public function content() : string + { + return $this->response->getBody()->getContents(); + } + + public function length() : int + { + return (int) $this->response->getHeaderLine('Content-Length'); + } + + /** + * @return resource + */ + public function stream() + { + $stream = $this->response->getBody()->detach(); + + if (!\is_resource($stream)) { + throw new Exception('Blob content stream could not be accessed'); + } + + return $stream; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/GetBlobOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/GetBlobOptions.php new file mode 100644 index 000000000..63b114cc5 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/GetBlobOptions.php @@ -0,0 +1,185 @@ +userAgentHeader(); + + if ($this->range !== null) { + $headers['x-ms-range'] = $this->range->toString(); + } + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + if ($this->rangeGetContentMd5) { + $headers['x-ms-range-get-content-md5'] = 'true'; + } + + if ($this->rangeGetContentCrc64) { + $headers['x-ms-range-get-content-crc64'] = 'true'; + } + + if ($this->origin !== null) { + $headers['Origin'] = $this->origin; + } + + if ($this->encryptionKey !== null) { + $headers['x-ms-encryption-key'] = $this->encryptionKey; + } + + if ($this->encryptionKeySha256 !== null) { + $headers['x-ms-encryption-key-sha256'] = $this->encryptionKeySha256; + } + + if ($this->encryptionAlgorithm !== null) { + $headers['x-ms-encryption-algorithm'] = $this->encryptionAlgorithm; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + if ($this->snapshot !== null) { + $uriParameters['snapshot'] = $this->snapshot; + } + + return $uriParameters; + } + + public function withEncryption(string $encryptionKey, string $encryptionAlgorithm, ?string $encryptionKeySha256 = null) : self + { + $this->encryptionKey = $encryptionKey; + $this->encryptionKeySha256 = $encryptionKeySha256; + $this->encryptionAlgorithm = $encryptionAlgorithm; + + return $this; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withOrigin(string $origin) : self + { + $this->origin = $origin; + + return $this; + } + + public function withRange(Range $range) : self + { + $this->range = $range; + + return $this; + } + + public function withRangeGetContentCrc64(bool $rangeGetContentCrc64) : self + { + $this->rangeGetContentCrc64 = $rangeGetContentCrc64; + + return $this; + } + + public function withRangeGetContentMd5(bool $rangeGetContentMd5) : self + { + $this->rangeGetContentMd5 = $rangeGetContentMd5; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withSnapshot(string $snapshot) : self + { + $this->snapshot = $snapshot; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } + + public function withVersionId(string $versionId) : self + { + $this->versionId = $versionId; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/Range.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/Range.php new file mode 100644 index 000000000..029a898b9 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlob/Range.php @@ -0,0 +1,27 @@ +start = $start; + $this->end = $end; + } + + public function toString() : string + { + if (!isset($this->end)) { + return \sprintf('bytes=%d-', $this->start); + } + + return \sprintf('bytes=%d-%d', $this->start, $this->end); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/BlobProperties.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/BlobProperties.php new file mode 100644 index 000000000..8a61d092a --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/BlobProperties.php @@ -0,0 +1,20 @@ +response->getHeaderLine('Content-Length'); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/GetBlobPropertiesOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/GetBlobPropertiesOptions.php new file mode 100644 index 000000000..cad0e4583 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlobProperties/GetBlobPropertiesOptions.php @@ -0,0 +1,133 @@ +userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + if ($this->encryptionKey !== null) { + $headers['x-ms-encryption-key'] = $this->encryptionKey; + } + + if ($this->encryptionKeySha256 !== null) { + $headers['x-ms-encryption-key-sha256'] = $this->encryptionKeySha256; + } + + if ($this->encryptionAlgorithm !== null) { + $headers['x-ms-encryption-algorithm'] = $this->encryptionAlgorithm; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + if ($this->snapshot !== null) { + $uriParameters['snapshot'] = $this->snapshot; + } + + return $uriParameters; + } + + public function withEncryption(string $encryptionKey, string $encryptionAlgorithm, ?string $encryptionKeySha256 = null) : self + { + $this->encryptionKey = $encryptionKey; + $this->encryptionKeySha256 = $encryptionKeySha256; + $this->encryptionAlgorithm = $encryptionAlgorithm; + + return $this; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withSnapshot(string $snapshot) : self + { + $this->snapshot = $snapshot; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } + + public function withVersionId(string $versionId) : self + { + $this->versionId = $versionId; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlockBlobBlockList/BlockListType.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlockBlobBlockList/BlockListType.php new file mode 100644 index 000000000..55f3837fb --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetBlockBlobBlockList/BlockListType.php @@ -0,0 +1,12 @@ +version; + $headers['user-agent'] = $this->userAgentHeader(); + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + $uriParameters['blocklisttype'] = $this->blockListType->value; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + if ($this->snapshot !== null) { + $uriParameters['snapshot'] = $this->snapshot; + } + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + return $uriParameters; + } + + public function withBlockListType(BlockListType $blockListType) : self + { + $this->blockListType = $blockListType; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withSnapshot(string $snapshot) : self + { + $this->snapshot = $snapshot; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/ContainerProperties.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/ContainerProperties.php new file mode 100644 index 000000000..06ab6ee45 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/ContainerProperties.php @@ -0,0 +1,18 @@ +response->getStatusCode() < 200 || $this->response->getStatusCode() >= 300) { + throw new InvalidArgumentException('Container properties could not be fetched'); + } + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/GetContainerPropertiesOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/GetContainerPropertiesOptions.php new file mode 100644 index 000000000..f77e6da19 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/GetContainerProperties/GetContainerPropertiesOptions.php @@ -0,0 +1,93 @@ +userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->leaseId !== null) { + $headers['x-ms-lease-id'] = $this->leaseId; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->versionId !== null) { + $uriParameters['versionId'] = $this->versionId; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withLeaseId(string $leaseId) : self + { + $this->leaseId = $leaseId; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } + + public function withVersionId(string $versionId) : self + { + $this->versionId = $versionId; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/Blob.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/Blob.php new file mode 100644 index 000000000..72396dcd2 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/Blob.php @@ -0,0 +1,22 @@ +data['Name']; + } + + public function size() : int + { + return (int) $this->data['Properties']['Content-Length']; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/ListBlobOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/ListBlobOptions.php new file mode 100644 index 000000000..c0cd354ed --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/ListBlobOptions.php @@ -0,0 +1,152 @@ + + */ + private ?array $include = null; + + private ?string $marker = null; + + private ?int $maxResults = null; + + private ?string $prefix = null; + + private ?string $requestId = null; + + private ?OptionShowOnly $showOnly = null; + + private ?int $timeoutSeconds = null; + + private ?string $version = BlobService::VERSION; + + public function __construct() + { + } + + public function toHeaders() : array + { + $headers = []; + + $headers['user-agent'] = $this->userAgentHeader(); + + if ($this->version !== null) { + $headers['x-ms-version'] = $this->version; + } + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->prefix !== null) { + $uriParameters['prefix'] = $this->prefix; + } + + if ($this->delimiter !== null) { + $uriParameters['delimiter'] = $this->delimiter; + } + + if ($this->maxResults !== null) { + $uriParameters['maxresults'] = $this->maxResults; + } + + if ($this->marker !== null) { + $uriParameters['marker'] = $this->marker; + } + + if ($this->include !== null) { + $uriParameters['include'] = \array_map(static fn (OptionInclude $include) => $include->value, $this->include); + } + + if ($this->showOnly !== null) { + $uriParameters['showonly'] = $this->showOnly->value; + } + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withDelimiter(string $delimiter) : self + { + $this->delimiter = $delimiter; + + return $this; + } + + public function withInclude(OptionInclude ...$include) : self + { + $this->include = $include; + + return $this; + } + + public function withMarker(string $marker) : self + { + $this->marker = $marker; + + return $this; + } + + public function withMaxResults(int $maxResults) : self + { + $this->maxResults = $maxResults; + + return $this; + } + + public function withPrefix(string $prefix) : self + { + $this->prefix = $prefix; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withShowOnly(OptionShowOnly $showOnly) : self + { + $this->showOnly = $showOnly; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/OptionInclude.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/OptionInclude.php new file mode 100644 index 000000000..98d0cc825 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/ListBlobs/OptionInclude.php @@ -0,0 +1,20 @@ +version; + + $headers['user-agent'] = $this->userAgentHeader(); + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->encryptionAlgorithm !== null) { + $headers['x-ms-encryption-algorithm'] = $this->encryptionAlgorithm; + } + + if ($this->encryptionKey !== null) { + $headers['x-ms-encryption-key'] = $this->encryptionKey; + } + + if ($this->encryptionKeySha256 !== null) { + $headers['x-ms-encryption-key-sha256'] = $this->encryptionKeySha256; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withEncryption(string $encryptionKey, string $encryptionAlgorithm, ?string $encryptionKeySha256 = null) : self + { + $this->encryptionKey = $encryptionKey; + $this->encryptionKeySha256 = $encryptionKeySha256; + $this->encryptionAlgorithm = $encryptionAlgorithm; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlock/PutBlockBlobBlockOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlock/PutBlockBlobBlockOptions.php new file mode 100644 index 000000000..1350ee9ad --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlock/PutBlockBlobBlockOptions.php @@ -0,0 +1,92 @@ +version; + + $headers['user-agent'] = $this->userAgentHeader(); + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->encryptionAlgorithm !== null) { + $headers['x-ms-encryption-algorithm'] = $this->encryptionAlgorithm; + } + + if ($this->encryptionKey !== null) { + $headers['x-ms-encryption-key'] = $this->encryptionKey; + } + + if ($this->encryptionKeySha256 !== null) { + $headers['x-ms-encryption-key-sha256'] = $this->encryptionKeySha256; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withEncryption(string $encryptionKey, string $encryptionAlgorithm, ?string $encryptionKeySha256 = null) : self + { + $this->encryptionKey = $encryptionKey; + $this->encryptionKeySha256 = $encryptionKeySha256; + $this->encryptionAlgorithm = $encryptionAlgorithm; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/PutBlockBlobBlockListOptions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/PutBlockBlobBlockListOptions.php new file mode 100644 index 000000000..d950d0325 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/PutBlockBlobBlockListOptions.php @@ -0,0 +1,105 @@ +version; + + $headers['user-agent'] = $this->userAgentHeader(); + + if ($this->requestId !== null) { + $headers['x-ms-client-request-id'] = $this->requestId; + } + + if ($this->encryptionAlgorithm !== null) { + $headers['x-ms-encryption-algorithm'] = $this->encryptionAlgorithm; + } + + if ($this->encryptionKey !== null) { + $headers['x-ms-encryption-key'] = $this->encryptionKey; + } + + if ($this->encryptionKeySha256 !== null) { + $headers['x-ms-encryption-key-sha256'] = $this->encryptionKeySha256; + } + + if ($this->contentLength !== null) { + $headers['Content-Length'] = $this->contentLength; + } + + return $headers; + } + + public function toURIParameters() : array + { + $uriParameters = []; + + if ($this->timeoutSeconds !== null) { + $uriParameters['timeout'] = $this->timeoutSeconds; + } + + return $uriParameters; + } + + public function withContentLength(int $contentLength) : self + { + $this->contentLength = $contentLength; + + return $this; + } + + public function withEncryption(string $encryptionKey, string $encryptionAlgorithm, ?string $encryptionKeySha256 = null) : self + { + $this->encryptionKey = $encryptionKey; + $this->encryptionKeySha256 = $encryptionKeySha256; + $this->encryptionAlgorithm = $encryptionAlgorithm; + + return $this; + } + + public function withRequestId(string $requestId) : self + { + $this->requestId = $requestId; + + return $this; + } + + public function withTimeoutSeconds(int $timeoutSeconds) : self + { + $this->timeoutSeconds = $timeoutSeconds; + + return $this; + } + + public function withVersion(string $version) : self + { + $this->version = $version; + + return $this; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/SimpleXMLSerializer.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/SimpleXMLSerializer.php new file mode 100644 index 000000000..1bfabb0eb --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/PutBlockBlobBlockList/SimpleXMLSerializer.php @@ -0,0 +1,40 @@ +'); + + foreach ($data->all() as $block) { + $xml->addChild($block->state->value, $block->id); + } + + $xmlString = $xml->asXML(); + + if (\is_bool($xmlString)) { + throw new Exception('Failed to serialize data'); + } + + return $xmlString; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzureURLFactory.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzureURLFactory.php new file mode 100644 index 000000000..bda8b0f08 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzureURLFactory.php @@ -0,0 +1,28 @@ +account, + $this->host, + $configuration->container, + $path ? ('/' . \trim($path, '/')) : '', + $queryParameters ? ('?' . \http_build_query($queryParameters)) : '' + ); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzuriteURLFactory.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzuriteURLFactory.php new file mode 100644 index 000000000..d034fe8ce --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobService/URLFactory/AzuriteURLFactory.php @@ -0,0 +1,29 @@ +secure ? 'https' : 'http', + $this->host, + $this->port, + $configuration->account, + $configuration->container, + $path ? ('/' . \trim($path, '/')) : '', + $queryParameters ? ('?' . \http_build_query($queryParameters)) : '' + ); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobServiceInterface.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobServiceInterface.php new file mode 100644 index 000000000..aff81006a --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/BlobServiceInterface.php @@ -0,0 +1,55 @@ + + */ + public function listBlobs(ListBlobOptions $options = new ListBlobOptions()) : \Generator; + + /** + * @param null|resource|string $content + */ + public function putBlockBlob(string $path, $content = null, ?int $size = null, PutBlockBlobOptions $options = new PutBlockBlobOptions()) : void; + + /** + * @param resource|string $content + */ + public function putBlockBlobBlock(string $path, string $blockId, $content, int $size, PutBlockBlobBlockOptions $options = new PutBlockBlobBlockOptions()) : void; + + public function putBlockBlobBlockList(string $path, BlockList $blockList, PutBlockBlobBlockListOptions $options = new PutBlockBlobBlockListOptions(), Serializer $serializer = new SimpleXMLSerializer()) : void; + + public function putContainer(CreateContainerOptions $options = new CreateContainerOptions()) : void; +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php new file mode 100644 index 000000000..21cf9cea4 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php @@ -0,0 +1,57 @@ +userAgentHeader; + } + + public function withUserAgent(string $userAgentHeader) : void + { + $this->userAgentHeader = $userAgentHeader; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/Exception/AzureException.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/Exception/AzureException.php new file mode 100644 index 000000000..96c1ff684 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/Exception/AzureException.php @@ -0,0 +1,15 @@ +requestFactory->createRequest('DELETE', $url); + } + + public function get(string $url) : RequestInterface + { + return $this->requestFactory->createRequest('GET', $url); + } + + public function post(string $url) : RequestInterface + { + return $this->requestFactory->createRequest('POST', $url); + } + + public function put(string $url) : RequestInterface + { + return $this->requestFactory->createRequest('PUT', $url); + } + + /** + * @param resource|string $content + */ + public function stream($content) : StreamInterface + { + if (!\is_string($content) && !\is_resource($content)) { + throw new InvalidArgumentException('Content must be a string or a resource'); + } + + if (\is_string($content)) { + return $this->streamFactory->createStream($content); + } + + return $this->streamFactory->createStreamFromResource($content); + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/Normalizer.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/Normalizer.php new file mode 100644 index 000000000..afdd95fd5 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/Normalizer.php @@ -0,0 +1,10 @@ +normalize(new \SimpleXMLElement($data)); + } + + private function normalize(\SimpleXMLElement|array $xml) : array + { + $normalized = []; + + foreach ((array) $xml as $key => $value) { + $normalizedValue = ($value instanceof \SimpleXMLElement) || is_array($value) ? $this->normalize($value) : $value; + + $normalized[$key] = \is_array($normalizedValue) && !\count($normalizedValue) ? null : $normalizedValue; + } + + return $normalized; + } +} diff --git a/src/lib/azure-sdk/src/Flow/Azure/SDK/RequestIdFactory.php b/src/lib/azure-sdk/src/Flow/Azure/SDK/RequestIdFactory.php new file mode 100644 index 000000000..dc96c7151 --- /dev/null +++ b/src/lib/azure-sdk/src/Flow/Azure/SDK/RequestIdFactory.php @@ -0,0 +1,12 @@ + new AzureURLFactory()], + ['factory' => new AzuriteURLFactory()], + ]; + } + + #[DataProvider('factoryProvider')] + public function test_creating_get_url_with_array_query_parameters(URLFactory $factory) : void + { + $configuration = new Configuration('account', 'container'); + + $url = $factory->create($configuration, null, ['foo' => ['biz', 'bar']]); + + self::assertStringEndsWith('?foo%5B0%5D=biz&foo%5B1%5D=bar', $url); + } + + #[DataProvider('factoryProvider')] + public function test_creating_get_url_with_query_parameters(URLFactory $factory) : void + { + $configuration = new Configuration('account', 'container'); + + $url = $factory->create($configuration, null, ['foo' => 'bar']); + + self::assertStringEndsWith('?foo=bar', $url); + } + + #[DataProvider('factoryProvider')] + public function test_creating_get_url_without_query_parameters(URLFactory $factory) : void + { + $configuration = new Configuration('account', 'container'); + + $url = $factory->create($configuration); + + self::assertStringNotContainsString('?', $url); + } +} diff --git a/src/lib/doctrine-dbal-bulk/README.md b/src/lib/doctrine-dbal-bulk/README.md index efd4ce9b7..32d98cf03 100644 --- a/src/lib/doctrine-dbal-bulk/README.md +++ b/src/lib/doctrine-dbal-bulk/README.md @@ -9,5 +9,5 @@ reliability even in demanding data-intensive environments. This aligns perfectly processing, making the Doctrine DBAL Bulk library an invaluable addition to your data transformation and processing toolkit. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/doctrine-dbal-bulk.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/dremel/README.md b/src/lib/dremel/README.md index a89999a6f..3ac0b5bad 100644 --- a/src/lib/dremel/README.md +++ b/src/lib/dremel/README.md @@ -2,5 +2,5 @@ ## Installation -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/dremel.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/filesystem/.gitattributes b/src/lib/filesystem/.gitattributes new file mode 100644 index 000000000..e02097205 --- /dev/null +++ b/src/lib/filesystem/.gitattributes @@ -0,0 +1,9 @@ +*.php text eol=lf + +/.github export-ignore +/tests export-ignore + +/README.md export-ignore + +/.gitattributes export-ignore +/.gitignore export-ignore diff --git a/src/lib/filesystem/.github/workflows/readonly.yaml b/src/lib/filesystem/.github/workflows/readonly.yaml new file mode 100644 index 000000000..da596bcdd --- /dev/null +++ b/src/lib/filesystem/.github/workflows/readonly.yaml @@ -0,0 +1,17 @@ +name: Readonly + +on: + pull_request_target: + types: [opened] + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: superbrothers/close-pull-request@v3 + with: + comment: | + Hi, thank you for your contribution. + Unfortunately, this repository is read-only. It's a split from our main monorepo repository. + In order to proceed with this PR please open it against https://github.com/flow-php/flow repository. + Thank you. \ No newline at end of file diff --git a/src/lib/filesystem/CONTRIBUTING.md b/src/lib/filesystem/CONTRIBUTING.md new file mode 100644 index 000000000..a2d0671c7 --- /dev/null +++ b/src/lib/filesystem/CONTRIBUTING.md @@ -0,0 +1,6 @@ +## Contributing + +This repo is **READ ONLY**, in order to contribute to Flow PHP project, please +open PR against [flow](https://github.com/flow-php/flow) monorepo. + +Changes merged to monorepo are automatically propagated into sub repositories. \ No newline at end of file diff --git a/src/lib/filesystem/LICENSE b/src/lib/filesystem/LICENSE new file mode 100644 index 000000000..bc3cc4d08 --- /dev/null +++ b/src/lib/filesystem/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020-present Flow PHP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/lib/filesystem/README.md b/src/lib/filesystem/README.md new file mode 100644 index 000000000..245aea7ef --- /dev/null +++ b/src/lib/filesystem/README.md @@ -0,0 +1,6 @@ +# Dremel + +## Installation + +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/filesystem.md) +- 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/filesystem/composer.json b/src/lib/filesystem/composer.json new file mode 100644 index 000000000..21720047c --- /dev/null +++ b/src/lib/filesystem/composer.json @@ -0,0 +1,41 @@ +{ + "name": "flow-php/filesystem", + "type": "library", + "description": "PHP ETL - Filesystem abstraction", + "keywords": [ + "etl", + "extract", + "transform", + "load", + "filesystem", + "remote", + "azure", + "aws", + "gcp" + ], + "require": { + "php": "~8.1.0 || ~8.2.0 || ~8.3.0" + }, + "config": { + "optimize-autoloader": true, + "sort-packages": true + }, + "license": "MIT", + "autoload": { + "psr-4": { + "Flow\\": [ + "src/Flow" + ] + }, + "files": [ + "src/Flow/Filesystem/DSL/functions.php" + ] + }, + "autoload-dev": { + "psr-4": { + "Flow\\": "tests/Flow" + } + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php b/src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php new file mode 100644 index 000000000..eb76a323b --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php @@ -0,0 +1,50 @@ + $options + */ +function path(string $path, array $options = []) : Path +{ + return new Path($path, $options); +} + +function path_real(string $path, array $options = []) : Path +{ + return Path::realpath($path, $options); +} + +function native_local_filesystem() : NativeLocalFilesystem +{ + return new NativeLocalFilesystem(); +} + +function fstab(Filesystem ...$filesystems) : FilesystemTable +{ + if (!\count($filesystems)) { + $filesystems[] = native_local_filesystem(); + } + + return new FilesystemTable(...$filesystems); +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/DestinationStream.php b/src/lib/filesystem/src/Flow/Filesystem/DestinationStream.php new file mode 100644 index 000000000..528f6bb5c --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/DestinationStream.php @@ -0,0 +1,15 @@ +isFile; + } + + public function isFile() : bool + { + return $this->isFile; + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Filesystem.php b/src/lib/filesystem/src/Flow/Filesystem/Filesystem.php new file mode 100644 index 000000000..cf0790c88 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Filesystem.php @@ -0,0 +1,28 @@ + + */ + public function list(Path $path, Filter $pathFilter = new KeepAll()) : \Generator; + + public function mv(Path $from, Path $to) : bool; + + public function protocol() : Protocol; + + public function readFrom(Path $path) : SourceStream; + + public function rm(Path $path) : bool; + + public function status(Path $path) : ?FileStatus; + + public function writeTo(Path $path) : DestinationStream; +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/FilesystemTable.php b/src/lib/filesystem/src/Flow/Filesystem/FilesystemTable.php new file mode 100644 index 000000000..40bc12c58 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/FilesystemTable.php @@ -0,0 +1,55 @@ + + */ + private array $fstab; + + public function __construct(Filesystem ...$filesystems) + { + $fstab = []; + + foreach ($filesystems as $filesystem) { + $fstab[$filesystem->protocol()->name] = $filesystem; + } + + $this->fstab = $fstab; + } + + public function for(Path|Protocol $path) : Filesystem + { + $protocol = $path instanceof Path ? $path->protocol() : $path; + + if (!\array_key_exists($protocol->name, $this->fstab)) { + throw new InvalidArgumentException("Filesystem with protocol {$protocol->name} is not mounted."); + } + + return $this->fstab[$protocol->name]; + } + + public function mount(Filesystem $filesystem) : void + { + if (isset($this->fstab[$filesystem->protocol()->name])) { + throw new InvalidArgumentException("Filesystem with protocol {$filesystem->protocol()->name} is already mounted."); + } + + $this->fstab[$filesystem->protocol()->name] = $filesystem; + } + + public function unmount(Filesystem $filesystem) : void + { + if (!isset($this->fstab[$filesystem->protocol()->name])) { + throw new InvalidArgumentException("Filesystem with protocol {$filesystem->protocol()->name} is not mounted."); + } + + unset($this->fstab[$filesystem->protocol()->name]); + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Local/NativeLocalFilesystem.php b/src/lib/filesystem/src/Flow/Filesystem/Local/NativeLocalFilesystem.php new file mode 100644 index 000000000..2ccc65792 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Local/NativeLocalFilesystem.php @@ -0,0 +1,180 @@ +protocol()->validateScheme($path); + + if (!$path->isPattern()) { + if ($pathFilter->accept($status = new FileStatus($path, \is_file($path->path())))) { + yield $status; + } + + return; + + } + + foreach (Glob::glob($path->path()) as $filePath) { + $status = new FileStatus(Path::realpath($filePath, $path->options()), \is_file($filePath)); + + if ($pathFilter->accept($status)) { + yield $status; + } + } + } + + public function mv(Path $from, Path $to) : bool + { + $this->protocol()->validateScheme($from); + $this->protocol()->validateScheme($to); + + if (\file_exists($to->path())) { + $this->rm($to); + } + + if (!\rename($from->path(), $to->path())) { + return false; + } + + return true; + } + + public function protocol() : Protocol + { + return new Protocol('file'); + } + + public function readFrom(Path $path) : SourceStream + { + $this->protocol()->validateScheme($path); + + if ($path->isPattern()) { + throw new InvalidArgumentException("Pattern paths can't be open: " . $path->uri()); + } + + if (!$this->status($path->parentDirectory())) { + if (!\mkdir($concurrentDirectory = $path->parentDirectory()->path(), recursive: true) && !\is_dir($concurrentDirectory)) { + throw new RuntimeException(\sprintf('Directory "%s" was not created', $concurrentDirectory)); + } + } + + return NativeLocalSourceStream::open($path); + } + + public function rm(Path $path) : bool + { + $this->protocol()->validateScheme($path); + + if (!$path->isPattern()) { + if (!\file_exists($path->path())) { + return false; + } + + if (\is_dir($path->path())) { + $this->rmdir($path->path()); + } else { + \unlink($path->path()); + } + + return true; + } + + $deletedCount = 0; + + foreach (Glob::glob($path->path()) as $filePath) { + if (\is_dir($filePath)) { + $this->rmdir($filePath); + } else { + \unlink($filePath); + } + + $deletedCount++; + } + + return (bool) $deletedCount; + } + + public function status(Path $path) : ?FileStatus + { + $this->protocol()->validateScheme($path); + + if (!$path->isPattern() && \file_exists($path->path())) { + return new FileStatus( + $path, + \is_file($path->path()) + ); + } + + foreach (Glob::glob($path->path()) as $filePath) { + if (\file_exists($filePath)) { + return new FileStatus(new Path($filePath, $path->options()), true); + } + } + + return null; + } + + public function writeTo(Path $path) : DestinationStream + { + $this->protocol()->validateScheme($path); + + if ($path->isPattern()) { + throw new InvalidArgumentException("Pattern paths can't be written: " . $path->uri()); + } + + if (!$this->status($path->parentDirectory())) { + if (!\mkdir($concurrentDirectory = $path->parentDirectory()->path(), recursive: true) && !\is_dir($concurrentDirectory)) { + throw new RuntimeException(\sprintf('Directory "%s" was not created', $concurrentDirectory)); + } + } + + return NativeLocalDestinationStream::openBlank($path); + } + + private function rmdir(string $dirPath) : void + { + if (!\is_dir($dirPath)) { + throw new InvalidArgumentException("{$dirPath} must be a directory"); + } + + if (!\str_ends_with($dirPath, '/')) { + $dirPath .= '/'; + } + + $files = \scandir($dirPath); + + if (!$files) { + throw new RuntimeException("Can't read directory: {$dirPath}"); + } + + foreach ($files as $file) { + if (\in_array($file, ['.', '..'], true)) { + continue; + } + + $filePath = $dirPath . $file; + + if (\is_dir($filePath)) { + $this->rmdir($filePath); + } else { + \unlink($filePath); + } + } + + \rmdir($dirPath); + } +} diff --git a/src/core/etl/src/Flow/ETL/Partition.php b/src/lib/filesystem/src/Flow/Filesystem/Partition.php similarity index 98% rename from src/core/etl/src/Flow/ETL/Partition.php rename to src/lib/filesystem/src/Flow/Filesystem/Partition.php index 0d69e1683..117e0a7d5 100644 --- a/src/core/etl/src/Flow/ETL/Partition.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Partition.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL; +namespace Flow\Filesystem; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Row; use Flow\ETL\Row\Entry\{ArrayEntry, DateTimeEntry, JsonEntry, diff --git a/src/core/etl/src/Flow/ETL/Partitions.php b/src/lib/filesystem/src/Flow/Filesystem/Partitions.php similarity index 98% rename from src/core/etl/src/Flow/ETL/Partitions.php rename to src/lib/filesystem/src/Flow/Filesystem/Partitions.php index 150e7c449..f44b813ae 100644 --- a/src/core/etl/src/Flow/ETL/Partitions.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Partitions.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL; +namespace Flow\Filesystem; use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException}; diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Path.php b/src/lib/filesystem/src/Flow/Filesystem/Path.php similarity index 80% rename from src/core/etl/src/Flow/ETL/Filesystem/Path.php rename to src/lib/filesystem/src/Flow/Filesystem/Path.php index 2ffee0ce8..94dce4c65 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Path.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Path.php @@ -2,11 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem; +namespace Flow\Filesystem; -use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException}; -use Flow\ETL\Filesystem\Stream\ResourceContext; -use Flow\ETL\{Partition, Partitions}; +use Flow\Filesystem\Exception\{InvalidArgumentException, RuntimeException}; +use Flow\Filesystem\Stream\ResourceContext; final class Path { @@ -22,44 +21,25 @@ final class Path private string $path; - private string $scheme; + private Protocol $protocol; /** * @param array $options - * - * @throws InvalidArgumentException */ public function __construct(string $uri, private readonly array $options = []) { - $urlParts = \parse_url($uri); + $scheme = \preg_match('/^([a-zA-Z0-9+-]+):\/\//', $uri, $matches) ? $matches[1] : 'file'; - if (!\is_array($urlParts)) { - throw new InvalidArgumentException("Invalid uri: {$uri}"); - } + $path = \str_replace($scheme . '://', '', $uri); - if (\array_key_exists('scheme', $urlParts) && !\in_array($urlParts['scheme'], \stream_get_wrappers(), true)) { - throw new InvalidArgumentException("Unknown scheme \"{$urlParts['scheme']}\""); + if (!\str_starts_with($path, DIRECTORY_SEPARATOR)) { + $path = DIRECTORY_SEPARATOR . $path; } - $path = \array_key_exists('scheme', $urlParts) - ? \str_replace($urlParts['scheme'] . '://', '', $uri) - : $uri; - - if (\array_key_exists('scheme', $urlParts)) { - $path = !\str_starts_with($path, DIRECTORY_SEPARATOR) ? (DIRECTORY_SEPARATOR . $path) : $path; - } else { - if (!\str_starts_with($path, DIRECTORY_SEPARATOR)) { - throw new InvalidArgumentException("Relative paths are not supported, consider using instead Path::realpath: {$uri}"); - } - - if (\str_contains($path, '..' . DIRECTORY_SEPARATOR)) { - throw new InvalidArgumentException("Relative paths are not supported, consider using instead Path::realpath: {$uri}"); - } - } + $pathInfo = \pathinfo($path); $this->path = $path; - $pathInfo = \pathinfo($this->path); - $this->scheme = \array_key_exists('scheme', $urlParts) ? $urlParts['scheme'] : 'file'; + $this->protocol = new Protocol($scheme); $this->extension = \array_key_exists('extension', $pathInfo) ? $pathInfo['extension'] : false; $this->filename = $pathInfo['filename']; $this->basename = $pathInfo['basename']; @@ -141,13 +121,6 @@ public static function realpath(string $path, array $options = []) : self return new self(DIRECTORY_SEPARATOR . \implode(DIRECTORY_SEPARATOR, $absoluteParts), $options); } - public static function tmpFile(string $extension, ?string $name = null) : self - { - $name = \ltrim($name ?? bin2hex(random_bytes(16)), '/'); - - return new self(\sys_get_temp_dir() . DIRECTORY_SEPARATOR . $name . '.' . $extension); - } - public function addPartitions(Partition $partition, Partition ...$partitions) : self { if ($this->isPattern()) { @@ -169,7 +142,7 @@ public function addPartitions(Partition $partition, Partition ...$partitions) : */ $base = \trim(\mb_substr($this->path(), 0, \mb_strrpos($this->path(), $this->basename())), DIRECTORY_SEPARATOR); - return new self($this->scheme . '://' . $base . $partitionsPath . DIRECTORY_SEPARATOR . $this->basename(), $this->options); + return new self($this->protocol->scheme() . $base . $partitionsPath . DIRECTORY_SEPARATOR . $this->basename(), $this->options); } public function basename() : string @@ -203,7 +176,7 @@ public function isEqual(self $path) : bool public function isLocal() : bool { - return $this->scheme === 'file'; + return $this->protocol->is('file'); } public function isPattern() : bool @@ -250,7 +223,7 @@ public function parentDirectory() : self $dirname = $dirname === '' ? '/' : $dirname; return new self( - $this->scheme . '://' . $dirname, + $this->protocol->scheme() . $dirname, $this->options ); } @@ -291,6 +264,11 @@ public function path() : string return $this->path; } + public function protocol() : Protocol + { + return $this->protocol; + } + /** * @psalm-suppress PossiblyFalseArgument */ @@ -302,16 +280,11 @@ public function randomize() : self $base = \trim(\mb_substr($this->path(), 0, \mb_strrpos($this->path(), $this->basename())), DIRECTORY_SEPARATOR); return new self( - $this->scheme . '://' . $base . DIRECTORY_SEPARATOR . $this->filename . '_' . bin2hex(random_bytes(16)) . $extension, + $this->protocol->scheme() . $base . DIRECTORY_SEPARATOR . $this->filename . '_' . \bin2hex(\random_bytes(16)) . $extension, $this->options ); } - public function scheme() : string - { - return $this->scheme; - } - public function setExtension(string $extension) : self { if ($this->extension) { @@ -322,7 +295,7 @@ public function setExtension(string $extension) : self $pathinfo = \pathinfo($this->path); $path = ($pathinfo['dirname'] ?? '') . DIRECTORY_SEPARATOR . $pathinfo['filename'] . '.' . $extension; - return new self($this->scheme . '://' . \ltrim($path, DIRECTORY_SEPARATOR), $this->options); + return new self($this->protocol->scheme() . \ltrim($path, DIRECTORY_SEPARATOR), $this->options); } return new self($this->uri() . '.' . $extension, $this->options); @@ -342,6 +315,10 @@ public function staticPart() : self $pathInfo = \pathinfo($this->path); if (!\array_key_exists('dirname', $pathInfo) || $pathInfo['dirname'] === DIRECTORY_SEPARATOR) { + if ($this->isPathPattern($pathInfo['basename'])) { + return new self($this->protocol->scheme() . DIRECTORY_SEPARATOR, $this->options); + } + return $this; } @@ -355,7 +332,7 @@ public function staticPart() : self $staticPath[] = $folder; } - return new self($this->scheme() . '://' . \implode(DIRECTORY_SEPARATOR, $staticPath), $this->options); + return new self($this->protocol->scheme() . \implode(DIRECTORY_SEPARATOR, $staticPath), $this->options); } /** @@ -363,7 +340,7 @@ public function staticPart() : self */ public function uri() : string { - return $this->scheme . '://' . \ltrim($this->path, DIRECTORY_SEPARATOR); + return $this->protocol->scheme() . \ltrim($this->path, DIRECTORY_SEPARATOR); } /** diff --git a/src/lib/filesystem/src/Flow/Filesystem/Path/Filter.php b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter.php new file mode 100644 index 000000000..5283dd47f --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter.php @@ -0,0 +1,12 @@ + $filters + */ + private array $filters; + + public function __construct(Filter ...$filters) + { + $this->filters = $filters; + } + + public function accept(FileStatus $status) : bool + { + foreach ($this->filters as $filter) { + if (!$filter->accept($status)) { + return false; + } + } + + return true; + } + + public function add(Filter $filter) : self + { + return new self(...\array_merge($this->filters, [$filter])); + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/KeepAll.php b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/KeepAll.php new file mode 100644 index 000000000..e9aa46884 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Path/Filter/KeepAll.php @@ -0,0 +1,15 @@ +isFile(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Paths.php b/src/lib/filesystem/src/Flow/Filesystem/Paths.php similarity index 92% rename from src/core/etl/src/Flow/ETL/Filesystem/Paths.php rename to src/lib/filesystem/src/Flow/Filesystem/Paths.php index 45646b5d9..8451c32f5 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Paths.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Paths.php @@ -2,9 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem; - -use Flow\ETL\Partitions; +namespace Flow\Filesystem; final class Paths { diff --git a/src/lib/filesystem/src/Flow/Filesystem/Protocol.php b/src/lib/filesystem/src/Flow/Filesystem/Protocol.php new file mode 100644 index 000000000..2f7da22a1 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Protocol.php @@ -0,0 +1,35 @@ +name)) === \str_replace('://', '', \mb_strtolower($name)); + } + + public function scheme() : string + { + return $this->name . '://'; + } + + public function validateScheme(string|Path $scheme) : void + { + if ($scheme instanceof Path) { + $scheme = $scheme->protocol()->scheme(); + } + + if (!$this->is($scheme)) { + throw new InvalidSchemeException($scheme, $this->scheme()); + } + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/SizeUnits.php b/src/lib/filesystem/src/Flow/Filesystem/SizeUnits.php new file mode 100644 index 000000000..5cd8645cc --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/SizeUnits.php @@ -0,0 +1,29 @@ + $length number of bytes to read from the stream + * + * @return \Generator + */ + public function iterate(int $length = 1) : \Generator; + + /** + * @param int<1, max> $length number of bytes to read from the stream + * @param int $offset The offset where to start reading from the stream. If negative, reading will start from the end of the stream. + */ + public function read(int $length, int $offset) : string; + + /** + * @param string $separator The line separator, content will be read until the first occurrence of the separator + * @param null|int<1, max> $length Number of bytes to read in one step. If the end of the stream or separator is reached before the specified number of bytes are read, the remaining bytes are returned. + * Otherwise we are reading until the separator is found or end of file is reached. When working with remote streams it might be a good idea to set length to few mb in orders to reduce number of network requests. + * When no value is provided, filesystems will use a default value, for example NativeLocalFilesystem is going to use 8192 length. + * + * @return \Generator + */ + public function readLines(string $separator = "\n", ?int $length = null) : \Generator; + + /** + * @return null|int The size of the stream in bytes + */ + public function size() : ?int; +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream.php new file mode 100644 index 000000000..89886031c --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream.php @@ -0,0 +1,14 @@ +id === '') { + throw new InvalidArgumentException('Block id cannot be empty.'); + } + + if ($totalSize < 0) { + throw new InvalidArgumentException('Block size must be greater than 0, got: ' . $totalSize); + } + + if (\file_exists($path->path())) { + throw new InvalidArgumentException('Block file already exists: ' . $path->path()); + } + + $handle = \fopen($path->path(), 'w+b'); + + if ($handle === false) { + throw new RuntimeException('Could not open block file: ' . $path->path()); + } + + $this->handle = $handle; + } + + /** + * @throws RuntimeException when we try to append more data than block can handle + */ + public function append(string $data) : void + { + if ($this->spaceLeft() < strlen($data)) { + throw new RuntimeException('Block is full, space left: ' . $this->spaceLeft() . ' bytes, trying to append: ' . strlen($data) . ' bytes.'); + } + + \fwrite($this->handle, $data); + $this->size += strlen($data); + } + + /** + * @param resource $resource + */ + public function fromResource($resource, int $offset = 0) : int + { + if (!\is_resource($resource)) { + throw new InvalidArgumentException('Block::fromResource expects resource type, given: ' . \gettype($resource)); + } + + if ($offset < 0) { + throw new InvalidArgumentException('Block::fromResource expects offset to be greater or equal to 0, given: ' . $offset); + } + + $result = \stream_copy_to_stream($resource, $this->handle, $this->spaceLeft(), $offset); + + if ($result === false) { + throw new RuntimeException('Could not copy stream to block.'); + } + + $this->size += $result; + + return $result; + } + + public function id() : string + { + return $this->id; + } + + public function path() : Path + { + return $this->path; + } + + /** + * Current block size in bytes. + */ + public function size() : int + { + return $this->size; + } + + public function spaceLeft() : int + { + return $this->totalSize - $this->size; + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/BlockVoidLifecycle.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/BlockVoidLifecycle.php new file mode 100644 index 000000000..c0f145ace --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/BlockVoidLifecycle.php @@ -0,0 +1,18 @@ +blockLocation = $blockLocation ?: \sys_get_temp_dir(); + } + + public function create(int $size) : Block + { + $id = \bin2hex(\random_bytes(16)); + + return new Block($id, $size, new Path($this->blockLocation . DIRECTORY_SEPARATOR . $id)); + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/NativeLocalStreamBlock.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/NativeLocalStreamBlock.php new file mode 100644 index 000000000..7578d3431 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/Block/NativeLocalStreamBlock.php @@ -0,0 +1,49 @@ +path()->protocol()->is('file')) { + throw new RuntimeException('FileBlock can be used only with file:// protocol, got: ' . $stream->path()->protocol()->scheme()); + } + } + + public function append(string $data) : void + { + if ($this->spaceLeft() < strlen($data)) { + throw new RuntimeException('Block is full, space left: ' . $this->spaceLeft() . ' bytes, trying to append: ' . strlen($data) . ' bytes.'); + } + + $this->currentSize += strlen($data); + } + + public function id() : string + { + return $this->id; + } + + public function read() : NativeLocalSourceStream + { + return NativeLocalSourceStream::open($this->stream->path()); + } + + public function size() : int + { + return $this->currentSize; + } + + public function spaceLeft() : int + { + return $this->size - $this->currentSize; + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/BlockFactory.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/BlockFactory.php new file mode 100644 index 000000000..cd002e1c3 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/BlockFactory.php @@ -0,0 +1,10 @@ +currentBlock = $this->blockFactory->create($this->blockSize); + } + + public function all() : array + { + return \array_merge($this->blocks, [$this->currentBlock]); + } + + public function append(string $data) : void + { + /** + * @phpstan-ignore-next-line + */ + foreach (\str_split($data, $this->blockSize) as $chunk) { + if ($this->block()->spaceLeft() < \strlen($chunk)) { + // cut the chunk to fit into the block, store it in the block and move remaining part to next block + $spaceLeft = $this->block()->spaceLeft(); + $this->block()->append(\substr($chunk, 0, $spaceLeft)); + $this->block()->append(\substr($chunk, $spaceLeft)); + } else { + $this->block()->append($chunk); + } + } + + $this->size += \strlen($data); + } + + /** + * @return Block - current block that might not be filled yet + */ + public function block() : Block + { + if ($this->currentBlock->spaceLeft() === 0) { + $this->blocks[] = $this->currentBlock; + $this->blockLifecycle->filled($this->currentBlock); + $this->currentBlock = $this->blockFactory->create($this->blockSize); + } + + return $this->currentBlock; + } + + public function count() : int + { + return \count($this->blocks) + 1; + } + + public function done() : void + { + if ($this->currentBlock->size() === 0) { + return; + } + + $this->blocks[] = $this->currentBlock; + $this->blockLifecycle->filled($this->currentBlock); + } + + /** + * @param resource $resource + */ + public function fromResource($resource) : void + { + if (!\is_resource($resource)) { + throw new InvalidArgumentException('DestinationStream::fromResource expects resource type, given: ' . \gettype($resource)); + } + + // use Block::fromStream and simply move offset after each block + $offset = \ftell($resource); + + if ($offset === false) { + throw new InvalidArgumentException('Cannot determine current position in the stream'); + } + + while (!\feof($resource)) { + $bytesCopied = $this->block()->fromResource($resource, $offset); + $offset += $bytesCopied; + $this->size += $bytesCopied; + } + } + + public function size() : int + { + return $this->size; + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/MemorySourceStream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/MemorySourceStream.php new file mode 100644 index 000000000..d12d20438 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/MemorySourceStream.php @@ -0,0 +1,68 @@ +content)) { + throw new InvalidArgumentException('MemorySourceStream expects non-empty content'); + } + } + + public function close() : void + { + } + + public function content() : string + { + return $this->content; + } + + public function isOpen() : bool + { + return true; + } + + public function iterate(int $length = 1) : \Generator + { + foreach (\str_split($this->content, $length) as $chunk) { + yield $chunk; + } + } + + public function path() : Path + { + return new Path('memory://'); + } + + public function read(int $length, int $offset) : string + { + return \substr($this->content, $offset, $length); + } + + public function readLines(string $separator = "\n", ?int $length = null) : \Generator + { + /** @phpstan-ignore-next-line */ + foreach (\explode($separator, $this->content) as $line) { + if (\strlen($line)) { + yield $line; + } + } + } + + public function size() : int + { + return \strlen($this->content); + } +} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Stream/Mode.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/Mode.php similarity index 88% rename from src/core/etl/src/Flow/ETL/Filesystem/Stream/Mode.php rename to src/lib/filesystem/src/Flow/Filesystem/Stream/Mode.php index c5ba55d83..c9a8cd456 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Stream/Mode.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/Mode.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem\Stream; +namespace Flow\Filesystem\Stream; enum Mode : string { diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalDestinationStream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalDestinationStream.php new file mode 100644 index 000000000..d009cf376 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalDestinationStream.php @@ -0,0 +1,114 @@ +handle = $handle; + } + + public static function openBlank(Path $path) : self + { + $resource = \fopen($path->path(), 'wb', false, $path->context()->resource()); + + if ($resource === false) { + throw new RuntimeException("Cannot open file: {$path->uri()}"); + } + + return new self($path, $resource); + } + + public function append(string $data) : self + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot write to closed stream'); + } + + \fseek($this->handle, 0, \SEEK_END); + \fwrite($this->handle, $data); + + return $this; + } + + public function close() : void + { + if (!\is_resource($this->handle)) { + $this->handle = null; + + return; + } + + \fclose($this->handle); + $this->handle = null; + } + + /** + * @param resource $resource + */ + public function fromResource($resource) : self + { + if (!\is_resource($resource)) { + throw new InvalidArgumentException('DestinationStream::fromResource expects resource type, given: ' . \gettype($resource)); + } + + if (!$this->isOpen()) { + throw new RuntimeException('Cannot write to closed stream'); + } + + $meta = \stream_get_meta_data($resource); + + if ($meta['seekable']) { + \rewind($resource); + } + + \stream_copy_to_stream($resource, $this->handle); + + return $this; + } + + /** + * @psalm-assert-if-true resource $this->handle + */ + public function isOpen() : bool + { + return \is_resource($this->handle); + } + + public function path() : Path + { + return $this->path; + } + + public function write(string $data) : void + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot write to closed stream'); + } + + \fseek($this->handle, 0); + \fwrite($this->handle, $data); + } +} diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php new file mode 100644 index 000000000..40387b888 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/NativeLocalSourceStream.php @@ -0,0 +1,153 @@ +handle = $handle; + } + + public static function open(Path $path) : self + { + $resource = \fopen($path->path(), 'rb', false, $path->context()->resource()); + + if ($resource === false) { + throw new RuntimeException("Cannot open file: {$path->uri()}"); + } + + return new self($path, $resource); + } + + public function close() : void + { + if (!\is_resource($this->handle)) { + $this->handle = null; + + return; + } + + \fclose($this->handle); + $this->handle = null; + } + + public function content() : string + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } + + \fseek($this->handle, 0); + + $content = \stream_get_contents($this->handle); + + if ($content === false) { + throw new RuntimeException("Cannot read file content: {$this->path->uri()}"); + } + + return $content; + } + + /** + * @psalm-assert-if-true resource $this->handle + */ + public function isOpen() : bool + { + return \is_resource($this->handle); + } + + /** + * @param int<1, max> $length + * + * @return \Generator + */ + public function iterate(int $length = 1) : \Generator + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } + + \fseek($this->handle, 0); + + while (!\feof($this->handle)) { + $string = \fread($this->handle, $length); + + if ($string === false) { + break; + } + + yield $string; + } + } + + public function path() : Path + { + return $this->path; + } + + public function read(int $length, int $offset) : string + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } + + \fseek($this->handle, $offset, $offset < 0 ? \SEEK_END : \SEEK_SET); + + $result = \fread($this->handle, $length); + + return $result === false ? '' : $result; + } + + /** + * @param ?int<1, max> $length + * + * @return \Generator + */ + public function readLines(string $separator = "\n", ?int $length = null) : \Generator + { + if (!$this->isOpen()) { + throw new RuntimeException('Cannot read from closed stream'); + } + + \fseek($this->handle, 0); + + while (!\feof($this->handle)) { + $line = \stream_get_line($this->handle, 0, $separator); + + if ($line === false) { + break; + } + + yield $line; + } + } + + public function size() : int + { + $size = \filesize($this->path->path()); + + if ($size === false) { + throw new RuntimeException("Cannot get file size: {$this->path->uri()}"); + } + + return $size; + } +} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Stream/ResourceContext.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/ResourceContext.php similarity index 79% rename from src/core/etl/src/Flow/ETL/Filesystem/Stream/ResourceContext.php rename to src/lib/filesystem/src/Flow/Filesystem/Stream/ResourceContext.php index 3356abbc5..a4665a5f9 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Stream/ResourceContext.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/ResourceContext.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem\Stream; +namespace Flow\Filesystem\Stream; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; final class ResourceContext { @@ -17,7 +17,7 @@ private function __construct(private readonly string $scheme, private readonly a public static function from(Path $path) : self { - return new self($path->scheme(), $path->options()); + return new self($path->protocol()->scheme(), $path->options()); } /** diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Stream/StreamWrapper.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/StreamWrapper.php similarity index 96% rename from src/core/etl/src/Flow/ETL/Filesystem/Stream/StreamWrapper.php rename to src/lib/filesystem/src/Flow/Filesystem/Stream/StreamWrapper.php index d53cbfb14..78dd069f5 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Stream/StreamWrapper.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/StreamWrapper.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem\Stream; +namespace Flow\Filesystem\Stream; /** * @property null|resource $context diff --git a/src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStream.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStream.php new file mode 100644 index 000000000..1f68cca11 --- /dev/null +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStream.php @@ -0,0 +1,74 @@ +path; + } + + public function read(int $length, int $offset) : string + { + return ''; + } + + public function readLines(string $separator = "\n", ?int $length = null) : \Generator + { + /** @phpstan-ignore-next-line */ + foreach ([] as $char) { + yield $char; + } + } + + public function size() : int + { + return 0; + } + + public function write(string $data) : void + { + } +} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/Stream/VoidStreamWrapper.php b/src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStreamWrapper.php similarity index 97% rename from src/core/etl/src/Flow/ETL/Filesystem/Stream/VoidStreamWrapper.php rename to src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStreamWrapper.php index 89645c05c..1839f30ef 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/Stream/VoidStreamWrapper.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Stream/VoidStreamWrapper.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem\Stream; +namespace Flow\Filesystem\Stream; final class VoidStreamWrapper implements StreamWrapper { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/file.txt similarity index 100% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/Fixtures/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/orders.csv b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/orders.csv new file mode 100644 index 000000000..78f967e1c --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/orders.csv @@ -0,0 +1,44 @@ +order_id,created_at,updated_at,discount,address,notes,items +e13d7098-5a78-3389-9289-022812e9ffa9,2024-06-17T19:24:49+00:00,2024-06-17T19:24:49+00:00,12.45,"{""street"":""9742 Jaskolski Forks Suite 585"",""city"":""South Lucianoside"",""zip"":""90339-5731"",""country"":""Saint Vincent and the Grenadines""}","[""Doloremque cum et adipisci sunt maiores qui."",""Distinctio fuga neque ut est et velit."",""Laborum consequuntur dolores quia quos eveniet."",""Voluptatem ipsam et quaerat atque hic quia maxime hic."",""Modi omnis non dolorem illo.""]","[{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +947df050-3abb-3f5a-92c5-5b6ceb49f0d6,2024-02-23T19:18:53+00:00,2024-02-23T19:18:53+00:00,,"{""street"":""37051 Alejandrin Orchard"",""city"":""Lebsackhaven"",""zip"":""85928-9879"",""country"":""Taiwan""}","[""Neque dolor et minima nulla aliquid."",""Est quas quod exercitationem ducimus nulla ut."",""Aut tempora quia quod aperiam vitae veritatis placeat.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55}]" +6315f9e2-86bf-3321-afa7-dfed4a31f10d,2024-04-02T11:30:25+00:00,2024-04-02T11:30:25+00:00,47.1,"{""street"":""792 Golda Estates Suite 028"",""city"":""Rubymouth"",""zip"":""50549"",""country"":""Turkey""}","[""Et porro fugiat fugiat culpa vitae dolores."",""Sit quibusdam minus consequuntur quo id distinctio aut tempora."",""Incidunt vel consequuntur beatae delectus."",""Cupiditate rerum minus iste ea illo et.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75}]" +4cccb632-fade-34e2-890c-ba557e227b5a,2024-05-06T00:17:57+00:00,2024-05-06T00:17:57+00:00,19.76,"{""street"":""30203 Wallace Plain"",""city"":""North Gennaro"",""zip"":""95802-1226"",""country"":""Zambia""}","[""Aliquam saepe iste suscipit officiis fuga."",""Blanditiis eaque cupiditate pariatur amet iure."",""Est repellat distinctio nesciunt dolorum ipsum recusandae."",""Qui vero nisi qui blanditiis consequatur magnam."",""Eius odio eveniet dolor est rem quia.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":9,""price"":397.55},{""sku"":""SKU_0003"",""quantity"":3,""price"":203.16}]" +82384f8c-9adb-38be-92b5-3c362a5f733e,2024-05-10T11:17:41+00:00,2024-05-10T11:17:41+00:00,,"{""street"":""757 Tobin Ports Suite 557"",""city"":""North Edison"",""zip"":""88755"",""country"":""Benin""}","[""Beatae nesciunt aut quia a.""]","[{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":1,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55}]" +e3fcf736-0f8c-3d97-bc5a-ffde26c0e2c1,2024-01-25T20:14:24+00:00,2024-01-25T20:14:24+00:00,,"{""street"":""9088 Tracey Ville Suite 686"",""city"":""Rosschester"",""zip"":""45566"",""country"":""Spain""}","[""Provident quam cum reprehenderit eius.""]","[{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0003"",""quantity"":7,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":6,""price"":246.3}]" +b987a49a-b4c5-37de-b5d9-2ec119cebedb,2024-06-03T23:22:13+00:00,2024-06-03T23:22:13+00:00,,"{""street"":""6867 Tyrell Flats"",""city"":""Hudsonmouth"",""zip"":""86961-2954"",""country"":""Bangladesh""}","[""Quibusdam maiores ex earum vel accusamus necessitatibus."",""Ex corporis laboriosam id optio qui beatae."",""Officia in at eveniet est architecto."",""Ullam porro ipsa eos dignissimos nihil recusandae dignissimos.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":10,""price"":247.75}]" +663523a9-713b-3354-8f0e-8f7142816aaf,2024-03-22T23:31:26+00:00,2024-03-22T23:31:26+00:00,25.88,"{""street"":""1577 Terence Tunnel"",""city"":""Berniecebury"",""zip"":""12143"",""country"":""Singapore""}","[""In rem maxime iure natus ipsam dolores."",""Sit hic voluptatibus facilis quibusdam consequuntur omnis aut est.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +6259fa2c-ec68-36a9-8476-dbd5d916465f,2024-05-10T10:12:52+00:00,2024-05-10T10:12:52+00:00,21.67,"{""street"":""987 Lloyd Radial"",""city"":""Zulaufberg"",""zip"":""18542-5386"",""country"":""Greece""}","[""Voluptatem non atque ad ipsum provident ut quia quas."",""Aspernatur maxime est quas debitis mollitia."",""Voluptatem et expedita ducimus inventore."",""Nihil dolorum distinctio qui facilis illo autem occaecati provident.""]","[{""sku"":""SKU_0003"",""quantity"":10,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":5,""price"":247.75},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55}]" +f7153c83-34b6-3769-95de-2109d932d925,2024-02-26T09:20:45+00:00,2024-02-26T09:20:45+00:00,18.93,"{""street"":""2039 Nova Summit Apt. 838"",""city"":""West Florian"",""zip"":""67943-5978"",""country"":""Turkmenistan""}","[""Culpa error recusandae fugit consequatur nam."",""Natus aspernatur aut dolor beatae ut.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":10,""price"":397.55}]" +966b91b5-e252-3787-96f9-c487a1ba3067,2024-05-10T11:34:49+00:00,2024-05-10T11:34:49+00:00,8.97,"{""street"":""518 Leannon Dam"",""city"":""Ziemannshire"",""zip"":""23503"",""country"":""Gambia""}","[""Ipsum adipisci veniam eaque quas."",""Vitae ad possimus sed dignissimos est occaecati.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":8,""price"":247.75}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" \ No newline at end of file diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=a/file_01.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=a/file_01.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=a/file_01.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=a/file_01.txt diff --git a/src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=b/file_02.txt b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=b/file_02.txt similarity index 100% rename from src/adapter/etl-adapter-filesystem/tests/Flow/ETL/Adapter/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=b/file_02.txt rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Fixtures/partitioned/partition_01=b/file_02.txt diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalDestinationStreamTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalDestinationStreamTest.php new file mode 100644 index 000000000..3a4d506a2 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalDestinationStreamTest.php @@ -0,0 +1,50 @@ +writeTo(new Path(__DIR__ . '/var/file.txt')); + self::assertTrue($stream->isOpen()); + $stream->close(); + self::assertFalse($stream->isOpen()); + } + + public function test_writing_content_from_resource() : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(new Path(__DIR__ . '/var/orders.csv')); + $stream->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $stream->close(); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/orders.csv'))->isFile()); + self::assertFalse($fs->status(new Path(__DIR__ . '/var/orders.csv'))->isDirectory()); + self::assertSame(\file_get_contents(__DIR__ . '/Fixtures/orders.csv'), $fs->readFrom(new Path(__DIR__ . '/var/orders.csv'))->content()); + + $fs->rm(new Path(__DIR__ . '/var/orders.csv')); + } + + public function test_writing_contente() : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(new Path(__DIR__ . '/var/file.txt')); + $stream->append('Hello, World!'); + $stream->close(); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/file.txt'))->isFile()); + self::assertFalse($fs->status(new Path(__DIR__ . '/var/file.txt'))->isDirectory()); + self::assertSame('Hello, World!', $fs->readFrom(new Path(__DIR__ . '/var/file.txt'))->content()); + + $fs->rm(new Path(__DIR__ . '/var/file.txt')); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTest.php new file mode 100644 index 000000000..4c0d53637 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTest.php @@ -0,0 +1,368 @@ +status(new Path(__DIR__))->isFile()); + self::assertTrue((new NativeLocalFilesystem())->status(new Path(__DIR__))->isDirectory()); + self::assertNull((new NativeLocalFilesystem())->status(new Path(__DIR__ . '/not_existing_directory'))); + } + + public function test_fie_exists() : void + { + self::assertTrue((new NativeLocalFilesystem())->status(new Path(__FILE__))->isFile()); + self::assertFalse((new NativeLocalFilesystem())->status(new Path(__FILE__))->isDirectory()); + self::assertNull((new NativeLocalFilesystem())->status(new Path(__DIR__ . '/not_existing_file.php'))); + } + + public function test_file_pattern_exists() : void + { + self::assertTrue((new NativeLocalFilesystem())->status(new Path(__DIR__ . '/**/*.txt'))->isFile()); + self::assertNull((new NativeLocalFilesystem())->status(new Path(__DIR__ . '/**/*.pdf'))); + } + + public function test_file_status_on_existing_file() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/file.txt'))->isFile()); + } + + public function test_file_status_on_existing_folder() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders'))->isDirectory()); + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/'))->isDirectory()); + } + + public function test_file_status_on_non_existing_file() : void + { + $fs = new NativeLocalFilesystem(); + + self::assertNull($fs->status(new Path(__DIR__ . '/var/non-existing-file.txt'))); + } + + public function test_file_status_on_non_existing_folder() : void + { + $fs = new NativeLocalFilesystem(); + + self::assertNull($fs->status(new Path(__DIR__ . '/var/non-existing-folder/'))); + } + + public function test_file_status_on_non_existing_pattern() : void + { + $fs = new NativeLocalFilesystem(); + + self::assertNull($fs->status(new Path(__DIR__ . '/var/non-existing-folder/*'))); + } + + public function test_file_status_on_partial_path() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/some_path_to/file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertNull($fs->status(new Path(__DIR__ . '/var/some_path'))); + } + + public function test_file_status_on_pattern() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/some_path_to/file.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/some_path_to/*.txt'))->isFile()); + self::assertSame( + 'file:/' . __DIR__ . '/var/some_path_to/file.txt', + $fs->status(new Path(__DIR__ . '/var/some_path_to/*.txt'))->path->uri() + ); + } + + public function test_file_status_on_root_folder() : void + { + $fs = new NativeLocalFilesystem(); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/'))->isDirectory()); + } + + public function test_move_blob() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/file.txt'))->append('Hello, World!'); + + $fs->mv(new Path(__DIR__ . '/var/file.txt'), new Path(__DIR__ . '/var/file_mv.txt')); + + self::assertNull($fs->status(new Path(__DIR__ . '/var/file.txt'))); + self::assertSame('Hello, World!', $fs->readFrom(new Path(__DIR__ . '/var/file_mv.txt'))->content()); + } + + public function test_not_removing_a_content_when_its_not_a_full_folder_path_pattern() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->isFile()); + + self::assertFalse($fs->rm(new Path(__DIR__ . '/var/nested/orders/ord'))); + } + + public function test_open_file_stream_for_existing_file() : void + { + $stream = (new NativeLocalFilesystem())->readFrom(new Path(__FILE__)); + + self::assertIsString($stream->read(100, 0)); + self::assertSame( + \mb_substr(\file_get_contents(__FILE__), 0, 100), + $stream->read(100, 0) + ); + } + + public function test_open_file_stream_for_non_existing_file() : void + { + $path = __DIR__ . '/var/file.txt'; + + $stream = (new NativeLocalFilesystem())->writeTo(new Path($path)); + + self::assertInstanceOf(NativeLocalDestinationStream::class, $stream); + } + + public function test_reading_multi_partitioned_path() : void + { + $paths = \iterator_to_array( + (new NativeLocalFilesystem()) + ->list( + new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), + new ScalarFunctionFilter( + all( + ref('country')->equals(lit('pl')), + all( + ref('date')->cast('date')->greaterThanEqual(lit(new \DateTimeImmutable('2022-01-02'))), + ref('date')->cast('date')->lessThan(lit(new \DateTimeImmutable('2022-01-04'))) + ) + ), + new NativeEntryFactory(), + new AutoCaster(Caster::default()) + ) + ) + ); + \sort($paths); + + $path1 = new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'); + $path1->partitions(); + $path2 = new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'); + $path2->partitions(); + + self::assertEquals( + [ + new FileStatus($path1, true), + new FileStatus($path2, true), + ], + $paths + ); + } + + public function test_reading_partitioned_folder() : void + { + $paths = \iterator_to_array((new NativeLocalFilesystem())->list(new Path(__DIR__ . '/Fixtures/partitioned/**/*.txt'), new KeepAll())); + \sort($paths); + + self::assertEquals( + [ + new FileStatus(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), true), + ], + $paths + ); + } + + public function test_reading_partitioned_folder_with_partitions_filtering() : void + { + $path = new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'); + $path->partitions(); + + self::assertEquals( + [ + new FileStatus($path, true), + ], + \iterator_to_array( + (new NativeLocalFilesystem()) + ->list( + new Path(__DIR__ . '/Fixtures/partitioned/**/*.txt'), + new ScalarFunctionFilter(ref('partition_01')->equals(lit('b')), new NativeEntryFactory(), new AutoCaster(Caster::default())) + ) + ) + ); + } + + public function test_reading_partitioned_folder_with_pattern() : void + { + $paths = \iterator_to_array((new NativeLocalFilesystem())->list(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=*/*.txt'), new KeepAll())); + \sort($paths); + + self::assertEquals( + [ + new FileStatus(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=a/file_01.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/partitioned/partition_01=b/file_02.txt'), true), + ], + $paths + ); + } + + public function test_remove_directory_with_content_when_exists() : void + { + $fs = new NativeLocalFilesystem(); + + $dirPath = Path::realpath(__DIR__ . '/var/flow-fs-test-directory/'); + + $stream = $fs->writeTo(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt')); + $stream->append('some data to make file not empty'); + $stream->close(); + + self::assertTrue($fs->status($dirPath)->isDirectory()); + self::assertTrue($fs->status($stream->path())->isFile()); + } + + public function test_remove_file_when_exists() : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(Path::realpath(__DIR__ . '/var/flow-fs-test/remove_file_when_exists.txt')); + $stream->append('some data to make file not empty'); + $stream->close(); + + self::assertTrue($fs->status($stream->path())->isFile()); + } + + public function test_remove_pattern() : void + { + $fs = new NativeLocalFilesystem(); + + $dirPath = Path::realpath(__DIR__ . '/var/flow-fs-test-directory/'); + + $stream = $fs->writeTo(Path::realpath($dirPath->path() . '/remove_file_when_exists.txt')); + $stream->append('some data to make file not empty'); + $stream->close(); + + self::assertTrue($fs->status($dirPath)->isDirectory()); + self::assertTrue($fs->status($stream->path())->isFile()); + $fs->rm(Path::realpath($dirPath->path() . '/*.txt')); + self::assertTrue($fs->status($dirPath)->isDirectory()); + self::assertNull($fs->status($stream->path())); + $fs->rm($dirPath); + } + + public function test_removing_folder() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->isFile()); + + $fs->rm(new Path(__DIR__ . '/var/nested/orders')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/orders.csv'))->isFile()); + self::assertNull($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.csv'))); + self::assertNull($fs->status(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))); + } + + public function test_removing_folder_pattern() : void + { + $fs = new NativeLocalFilesystem(); + + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.txt'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $fs->writeTo(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.csv'))->isFile()); + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))->isFile()); + + $fs->rm(new Path(__DIR__ . '/var/nested/orders/*.csv')); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.txt'))->isFile()); + self::assertNull($fs->status(new Path(__DIR__ . '/var/nested/orders/orders.csv'))); + self::assertNull($fs->status(new Path(__DIR__ . '/var/nested/orders/orders_01.csv'))); + } + + public function test_that_scan_sort_files_by_path_names() : void + { + $paths = \iterator_to_array( + (new NativeLocalFilesystem()) + ->list( + new Path(__DIR__ . '/Fixtures/multi_partitions/**/*.txt'), + ) + ); + + self::assertEquals( + [ + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=de/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-01/country=pl/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=de/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-02/country=pl/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=de/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-03/country=pl/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=de/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-04/country=pl/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=de/file.txt'), true), + new FileStatus(new Path(__DIR__ . '/Fixtures/multi_partitions/date=2022-01-05/country=pl/file.txt'), true), + ], + $paths + ); + } + + public function test_writing_to_azure_blob_storage() : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(new Path(__DIR__ . '/var/file.txt')); + $stream->append('Hello, World!'); + $stream->close(); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/file.txt'))->isFile()); + self::assertFalse($fs->status(new Path(__DIR__ . '/var/file.txt'))->isDirectory()); + self::assertSame('Hello, World!', $fs->readFrom(new Path(__DIR__ . '/var/file.txt'))->content()); + + $fs->rm(new Path(__DIR__ . '/var/file.txt')); + } + + public function test_writing_to_to_azure_from_resources() : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(new Path(__DIR__ . '/var/orders.csv')); + $stream->fromResource(\fopen(__DIR__ . '/Fixtures/orders.csv', 'rb')); + $stream->close(); + + self::assertTrue($fs->status(new Path(__DIR__ . '/var/orders.csv'))->isFile()); + self::assertFalse($fs->status(new Path(__DIR__ . '/var/orders.csv'))->isDirectory()); + self::assertSame(\file_get_contents(__DIR__ . '/Fixtures/orders.csv'), $fs->readFrom(new Path(__DIR__ . '/var/orders.csv'))->content()); + + $fs->rm(new Path(__DIR__ . '/var/orders.csv')); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTestCase.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTestCase.php new file mode 100644 index 000000000..8dc8715aa --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalFilesystemTestCase.php @@ -0,0 +1,28 @@ +rm(new Path(__DIR__ . '/var/*')); + } + + protected function givenFileExists(string $path, string $content) : void + { + $fs = new NativeLocalFilesystem(); + + $stream = $fs->writeTo(new Path($path)); + $stream->append($content); + $stream->close(); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php new file mode 100644 index 000000000..08428820c --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/NativeLocalSourceStreamTest.php @@ -0,0 +1,88 @@ +givenFileExists(__DIR__ . '/var/file.txt', $content); + + $stream = (new NativeLocalFilesystem())->readFrom(new Path(__DIR__ . '/var/file.txt')); + + self::assertSame($content, \implode('', \iterator_to_array($stream->iterate()))); + + $stream->close(); + } + + public function test_reading_from_blob_by_limit_and_offset() : void + { + $content = <<<'TEXT' +This is some +multi line file +that we are storing on azure blob +TEXT; + $this->givenFileExists(__DIR__ . '/var/file.txt', $content); + + $stream = (new NativeLocalFilesystem())->readFrom(new Path(__DIR__ . '/var/file.txt')); + + self::assertSame($content, $stream->content()); + + self::assertSame('This is some', $stream->read(12, 0)); + self::assertSame(12, \strlen($stream->read(12, 0))); + self::assertSame('multi line file', $stream->read(15, 13)); + self::assertSame(15, \strlen($stream->read(15, 13))); + self::assertSame('that we are storing on azure blob', $stream->read(33, 29)); + self::assertSame(33, \strlen($stream->read(33, 29))); + + $stream->close(); + } + + #[DataProvider('line_lengths')] + public function test_reading_lines_from_file(int $lineLength) : void + { + $content = <<<'TEXT' +This is some +multi line file +that we are storing on azure blob +TEXT; + $this->givenFileExists(__DIR__ . '/var/file.txt', $content); + + $stream = (new NativeLocalFilesystem())->readFrom(new Path(__DIR__ . '/var/file.txt')); + + self::assertSame($content, $stream->content()); + + $lines = $stream->readLines(length: $lineLength); + self::assertSame('This is some', $lines->current()); + $lines->next(); + self::assertSame('multi line file', $lines->current()); + $lines->next(); + self::assertSame('that we are storing on azure blob', $lines->current()); + $lines->next(); + self::assertNull($lines->current()); + + $stream->close(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/RealpathTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/RealpathTest.php similarity index 89% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/RealpathTest.php rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/RealpathTest.php index edfca9943..d1d7090a2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/RealpathTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/RealpathTest.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Integration\Filesystem; +namespace Flow\Filesystem\Tests\Integration; -use Flow\ETL\Filesystem\Path; +use Flow\Filesystem\Path; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/BlocksTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/BlocksTest.php new file mode 100644 index 000000000..aee302892 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/BlocksTest.php @@ -0,0 +1,39 @@ +fromResource($file); + + self::assertSame($fileSize, $blocks->size()); + self::assertSame((int) ceil($fileSize / $blockSize), \count($blocks->all())); + } + + public function test_moving_resource_to_existing_blocks() : void + { + $blocks = new Blocks($blockSize = SizeUnits::kbToBytes(10)); + + $file = \fopen(__DIR__ . '/Fixtures/orders.csv', 'rb'); + $fileSize = \filesize(__DIR__ . '/Fixtures/orders.csv'); + + $blocks->append(\str_repeat('a', 100)); + $blocks->fromResource($file); + + self::assertSame($fileSize + 100, $blocks->size()); + self::assertCount((int) ceil($fileSize / $blockSize), $blocks->all()); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/Fixtures/orders.csv b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/Fixtures/orders.csv new file mode 100644 index 000000000..78f967e1c --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/Stream/Fixtures/orders.csv @@ -0,0 +1,44 @@ +order_id,created_at,updated_at,discount,address,notes,items +e13d7098-5a78-3389-9289-022812e9ffa9,2024-06-17T19:24:49+00:00,2024-06-17T19:24:49+00:00,12.45,"{""street"":""9742 Jaskolski Forks Suite 585"",""city"":""South Lucianoside"",""zip"":""90339-5731"",""country"":""Saint Vincent and the Grenadines""}","[""Doloremque cum et adipisci sunt maiores qui."",""Distinctio fuga neque ut est et velit."",""Laborum consequuntur dolores quia quos eveniet."",""Voluptatem ipsam et quaerat atque hic quia maxime hic."",""Modi omnis non dolorem illo.""]","[{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +947df050-3abb-3f5a-92c5-5b6ceb49f0d6,2024-02-23T19:18:53+00:00,2024-02-23T19:18:53+00:00,,"{""street"":""37051 Alejandrin Orchard"",""city"":""Lebsackhaven"",""zip"":""85928-9879"",""country"":""Taiwan""}","[""Neque dolor et minima nulla aliquid."",""Est quas quod exercitationem ducimus nulla ut."",""Aut tempora quia quod aperiam vitae veritatis placeat.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55}]" +6315f9e2-86bf-3321-afa7-dfed4a31f10d,2024-04-02T11:30:25+00:00,2024-04-02T11:30:25+00:00,47.1,"{""street"":""792 Golda Estates Suite 028"",""city"":""Rubymouth"",""zip"":""50549"",""country"":""Turkey""}","[""Et porro fugiat fugiat culpa vitae dolores."",""Sit quibusdam minus consequuntur quo id distinctio aut tempora."",""Incidunt vel consequuntur beatae delectus."",""Cupiditate rerum minus iste ea illo et.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75}]" +4cccb632-fade-34e2-890c-ba557e227b5a,2024-05-06T00:17:57+00:00,2024-05-06T00:17:57+00:00,19.76,"{""street"":""30203 Wallace Plain"",""city"":""North Gennaro"",""zip"":""95802-1226"",""country"":""Zambia""}","[""Aliquam saepe iste suscipit officiis fuga."",""Blanditiis eaque cupiditate pariatur amet iure."",""Est repellat distinctio nesciunt dolorum ipsum recusandae."",""Qui vero nisi qui blanditiis consequatur magnam."",""Eius odio eveniet dolor est rem quia.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":9,""price"":397.55},{""sku"":""SKU_0003"",""quantity"":3,""price"":203.16}]" +82384f8c-9adb-38be-92b5-3c362a5f733e,2024-05-10T11:17:41+00:00,2024-05-10T11:17:41+00:00,,"{""street"":""757 Tobin Ports Suite 557"",""city"":""North Edison"",""zip"":""88755"",""country"":""Benin""}","[""Beatae nesciunt aut quia a.""]","[{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":1,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55}]" +e3fcf736-0f8c-3d97-bc5a-ffde26c0e2c1,2024-01-25T20:14:24+00:00,2024-01-25T20:14:24+00:00,,"{""street"":""9088 Tracey Ville Suite 686"",""city"":""Rosschester"",""zip"":""45566"",""country"":""Spain""}","[""Provident quam cum reprehenderit eius.""]","[{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0003"",""quantity"":7,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":6,""price"":246.3}]" +b987a49a-b4c5-37de-b5d9-2ec119cebedb,2024-06-03T23:22:13+00:00,2024-06-03T23:22:13+00:00,,"{""street"":""6867 Tyrell Flats"",""city"":""Hudsonmouth"",""zip"":""86961-2954"",""country"":""Bangladesh""}","[""Quibusdam maiores ex earum vel accusamus necessitatibus."",""Ex corporis laboriosam id optio qui beatae."",""Officia in at eveniet est architecto."",""Ullam porro ipsa eos dignissimos nihil recusandae dignissimos.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":10,""price"":247.75}]" +663523a9-713b-3354-8f0e-8f7142816aaf,2024-03-22T23:31:26+00:00,2024-03-22T23:31:26+00:00,25.88,"{""street"":""1577 Terence Tunnel"",""city"":""Berniecebury"",""zip"":""12143"",""country"":""Singapore""}","[""In rem maxime iure natus ipsam dolores."",""Sit hic voluptatibus facilis quibusdam consequuntur omnis aut est.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +6259fa2c-ec68-36a9-8476-dbd5d916465f,2024-05-10T10:12:52+00:00,2024-05-10T10:12:52+00:00,21.67,"{""street"":""987 Lloyd Radial"",""city"":""Zulaufberg"",""zip"":""18542-5386"",""country"":""Greece""}","[""Voluptatem non atque ad ipsum provident ut quia quas."",""Aspernatur maxime est quas debitis mollitia."",""Voluptatem et expedita ducimus inventore."",""Nihil dolorum distinctio qui facilis illo autem occaecati provident.""]","[{""sku"":""SKU_0003"",""quantity"":10,""price"":203.16},{""sku"":""SKU_0005"",""quantity"":5,""price"":247.75},{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55}]" +f7153c83-34b6-3769-95de-2109d932d925,2024-02-26T09:20:45+00:00,2024-02-26T09:20:45+00:00,18.93,"{""street"":""2039 Nova Summit Apt. 838"",""city"":""West Florian"",""zip"":""67943-5978"",""country"":""Turkmenistan""}","[""Culpa error recusandae fugit consequatur nam."",""Natus aspernatur aut dolor beatae ut.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":10,""price"":397.55}]" +966b91b5-e252-3787-96f9-c487a1ba3067,2024-05-10T11:34:49+00:00,2024-05-10T11:34:49+00:00,8.97,"{""street"":""518 Leannon Dam"",""city"":""Ziemannshire"",""zip"":""23503"",""country"":""Gambia""}","[""Ipsum adipisci veniam eaque quas."",""Vitae ad possimus sed dignissimos est occaecati.""]","[{""sku"":""SKU_0004"",""quantity"":8,""price"":246.3},{""sku"":""SKU_0005"",""quantity"":8,""price"":247.75}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +607e5afa-3783-39ce-8e38-6d76884f97c9,2024-05-14T13:13:46+00:00,2024-05-14T13:13:46+00:00,,"{""street"":""992 Mraz Alley"",""city"":""Robertsborough"",""zip"":""78402-8952"",""country"":""Reunion""}","[""Quisquam sed rerum explicabo nam autem incidunt qui.""]","[{""sku"":""SKU_0005"",""quantity"":2,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":5,""price"":203.16},{""sku"":""SKU_0004"",""quantity"":1,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":10,""price"":246.3}]" +05476f35-2efc-3708-af80-10b10f2ce581,2024-02-24T16:33:00+00:00,2024-02-24T16:33:00+00:00,,"{""street"":""6172 Tad Summit Suite 243"",""city"":""Dakotashire"",""zip"":""97070"",""country"":""Cuba""}","[""Numquam quo et eos vel."",""Beatae commodi ut natus sed molestiae aliquid molestias."",""Sit id quia sed expedita ea qui qui."",""Dignissimos molestiae nemo enim ut officiis cumque.""]","[{""sku"":""SKU_0005"",""quantity"":6,""price"":247.75},{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16},{""sku"":""SKU_0002"",""quantity"":5,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":1,""price"":247.75}]" +71f9884e-3d07-3f69-8f19-0b137a01b98d,2024-01-20T00:34:39+00:00,2024-01-20T00:34:39+00:00,,"{""street"":""9371 Rice Ridges"",""city"":""Collinsbury"",""zip"":""08802-8220"",""country"":""Tuvalu""}","[""Aliquam omnis aliquid quaerat consequatur expedita quo."",""Exercitationem quibusdam et ut vel natus quos."",""Dolor quod amet sapiente asperiores dolor."",""Aperiam mollitia non consequatur repellendus.""]","[{""sku"":""SKU_0002"",""quantity"":1,""price"":397.55},{""sku"":""SKU_0004"",""quantity"":4,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":9,""price"":246.3},{""sku"":""SKU_0004"",""quantity"":2,""price"":246.3}]" +de23917a-15bd-33cc-9996-2a9e1a43ee85,2024-05-20T00:52:43+00:00,2024-05-20T00:52:43+00:00,,"{""street"":""4354 Hailie Parks"",""city"":""Greenfeldermouth"",""zip"":""08199"",""country"":""Central African Republic""}","[""Asperiores officiis ad eius vel ut quia aut."",""Est et non quae sapiente sunt autem consequatur.""]","[{""sku"":""SKU_0002"",""quantity"":2,""price"":397.55}]" +10a8b132-11ca-3288-9d1d-5ef2428ad942,2024-03-07T04:04:38+00:00,2024-03-07T04:04:38+00:00,,"{""street"":""121 Brown Rue Suite 026"",""city"":""North Gilberto"",""zip"":""42296-5028"",""country"":""Ecuador""}","[""Aspernatur autem optio sequi doloremque consequatur aut."",""Veniam esse non vel necessitatibus sed."",""Consectetur error et molestiae eum."",""Perferendis accusantium qui fugit minima vitae odit.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55},{""sku"":""SKU_0005"",""quantity"":4,""price"":247.75}]" +384f222d-fb15-33ba-b384-5e091c147960,2024-01-31T16:09:59+00:00,2024-01-31T16:09:59+00:00,,"{""street"":""772 Stroman Points"",""city"":""Corkeryhaven"",""zip"":""89395-4907"",""country"":""Cameroon""}","[""Dolor doloribus accusantium rem nihil."",""Quisquam dicta nulla delectus possimus eos."",""Vitae non est autem nisi molestiae unde."",""Rerum est quos repudiandae qui."",""Rerum alias ea quae.""]","[{""sku"":""SKU_0003"",""quantity"":9,""price"":203.16}]" +c36018e3-4368-3660-8d0d-71bef80b5a46,2024-01-17T12:30:55+00:00,2024-01-17T12:30:55+00:00,,"{""street"":""797 Hammes Ramp"",""city"":""Pollichstad"",""zip"":""37308-2344"",""country"":""Aruba""}","[""Nihil ullam sed culpa alias.""]","[{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" +371fa03b-970e-3ad3-90c2-2314a55acd06,2024-04-05T10:09:51+00:00,2024-04-05T10:09:51+00:00,16.26,"{""street"":""98873 Kirlin Dam Apt. 755"",""city"":""Bergnaumview"",""zip"":""94010-1544"",""country"":""Cote d'Ivoire""}","[""Laborum amet rem nobis consequuntur voluptatem."",""Et tempore eligendi ullam necessitatibus ut."",""Amet veritatis similique et facere.""]","[{""sku"":""SKU_0004"",""quantity"":7,""price"":246.3},{""sku"":""SKU_0002"",""quantity"":8,""price"":397.55},{""sku"":""SKU_0002"",""quantity"":3,""price"":397.55}]" \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionTest.php similarity index 97% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionTest.php rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionTest.php index 5fc20d925..82ea4752d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit; +namespace Flow\Filesystem\Tests\Unit; use function Flow\ETL\DSL\{datetime_entry, ref, row, xml_entry}; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Partition; use Flow\ETL\Row\Entry\XMLEntry; +use Flow\Filesystem\Partition; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionsTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionsTest.php similarity index 88% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionsTest.php rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionsTest.php index a422f3f6b..1bfe6417d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/PartitionsTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PartitionsTest.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit; +namespace Flow\Filesystem\Tests\Unit; -use Flow\ETL\{Partition, Partitions}; +use Flow\Filesystem\Partition; +use Flow\Filesystem\{Partitions}; use PHPUnit\Framework\TestCase; final class PartitionsTest extends TestCase diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Stream/PathTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php similarity index 84% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Stream/PathTest.php rename to src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php index 3815dce2b..b3380650f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Stream/PathTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php @@ -2,11 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Stream; +namespace Flow\Filesystem\Tests\Unit; -use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Filesystem\Path; -use Flow\ETL\{Partition, Partitions}; +use Flow\Filesystem\{Partition, Path}; +use Flow\Filesystem\{Partitions}; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -28,7 +27,12 @@ public static function paths() : \Generator yield 'file://file.csv' => ['file://file.csv', 'file', 'file://file.csv']; yield 'file:///' => ['file:///', 'file', 'file://']; yield '/' => ['/', 'file', 'file://']; - yield 'flow-file://file.csv' => ['flow-file://folder/file.csv', 'flow-file', 'flow-file://folder/file.csv']; + yield '/absolute/path/to/file.txt' => ['/absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; + yield 'file://absolute/path/to/file.txt' => ['file://absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; + yield 'file:///absolute/path/to/file.txt' => ['file:///absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; + yield 'flow-file://' => ['flow-file://', 'flow-file', 'flow-file://']; + yield 'flow-file:///' => ['flow-file:///', 'flow-file', 'flow-file://']; + yield 'flow-file://folder/file.csv' => ['flow-file://folder/file.csv', 'flow-file', 'flow-file://folder/file.csv']; } public static function paths_pattern_matching() : \Generator @@ -55,7 +59,9 @@ public static function paths_with_static_parts() : \Generator yield '/file.csv' => ['/file.csv', '/file.csv']; yield '/nested/folder/*/file.csv' => ['/nested/folder', '/nested/folder/*/file.csv']; yield '/nested/folder/path/{one|two}/file.csv' => ['/nested/folder/path', '/nested/folder/path/{one|two}/file.csv']; - yield '/file*.csv' => ['/file*.csv', '/file*.csv']; + yield '/file*.csv' => ['/', '/file*.csv']; + yield '/{one|two|tree}.csv' => ['/', '/{one|two|tree}.csv']; + yield '/file.{parquet|csv}' => ['/', '/file.{parquet|csv}']; yield 'flow-file://nested/partition={one,two}/*.csv' => ['flow-file://nested', 'flow-file://nested/partition={one,two}/*.csv']; yield 'flow-file://nested/partition=[one]/*.csv' => ['flow-file://nested', 'flow-file://nested/partition=[one]/*.csv']; yield '/nested/partition=[one]/*.csv' => ['file://nested', '/nested/partition=[one]/*.csv']; @@ -63,15 +69,8 @@ public static function paths_with_static_parts() : \Generator protected function setUp() : void { - if (!\in_array('flow-file', \stream_get_wrappers(), true)) { - \stream_wrapper_register('flow-file', self::class); - } - } - - protected function tearDown() : void - { - if (\in_array('flow-file', \stream_get_wrappers(), true)) { - \stream_wrapper_unregister('flow-file'); + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); } } @@ -123,8 +122,8 @@ public function test_directories(string $uri, string $dirPath) : void public function test_equal_paths_starts_with() : void { self::assertTrue( - Path::realpath(\sys_get_temp_dir() . '/some/path/file.json') - ->startsWith(Path::realpath(\sys_get_temp_dir() . '/some/path/file.json')) + Path::realpath(__DIR__ . '/var/some/path/file.json') + ->startsWith(Path::realpath(__DIR__ . '/var/some/path/file.json')) ); } @@ -162,7 +161,7 @@ public function test_not_matching_items_under_directory_that_matches_pattern() : #[DataProvider('paths')] public function test_parsing_path(string $uri, string $schema, string $parsedUri) : void { - self::assertEquals($schema, (new Path($uri))->scheme()); + self::assertEquals($schema, (new Path($uri))->protocol()->name); self::assertEquals($parsedUri, (new Path($uri))->uri()); } @@ -188,16 +187,16 @@ public function test_partitions_paths() : void public function test_path_starting_with_other_path() : void { self::assertTrue( - Path::realpath(\sys_get_temp_dir() . '/some/path/file.json') - ->startsWith(Path::realpath(\sys_get_temp_dir() . '/some/path')) + Path::realpath(__DIR__ . '/var/some/path/file.json') + ->startsWith(Path::realpath(__DIR__ . '/var/some/path')) ); } public function test_pattern_path_starting_with_realpath_path() : void { self::assertTrue( - Path::realpath(\sys_get_temp_dir() . '/some/path/*.json') - ->startsWith(Path::realpath(\sys_get_temp_dir() . '/some/path')) + Path::realpath(__DIR__ . '/var/some/path/*.json') + ->startsWith(Path::realpath(__DIR__ . '/var/some/path')) ); } @@ -228,7 +227,7 @@ public function test_randomization_folder_path() : void public function test_realpath_starting_with_non_realpath_path() : void { self::assertFalse( - Path::realpath(\sys_get_temp_dir() . '/some/path/file.json') + Path::realpath(__DIR__ . '/var/some/path/file.json') ->startsWith(new Path('/some/path')) ); } @@ -262,12 +261,4 @@ public function test_set_same_extension() : void $path->setExtension('csv') ); } - - public function test_unknown_stream_scheme() : void - { - $this->expectExceptionMessage('Unknown scheme "flow-invalid"'); - $this->expectException(InvalidArgumentException::class); - - new Path('flow-invalid://some_file.txt'); - } } diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/BlocksTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/BlocksTest.php new file mode 100644 index 000000000..f98781804 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Stream/BlocksTest.php @@ -0,0 +1,35 @@ +createMock(BlockLifecycle::class); + $blockLifecycle->expects(self::exactly(4)) + ->method('filled'); + + $blocks = new Blocks( + 100, + new NativeLocalFileBlocksFactory(), + $blockLifecycle + ); + + $blocks->append(\str_repeat('a', 100)); // block 1 + $blocks->append(\str_repeat('a', 150)); // block 2 and 3 + $blocks->append(\str_repeat('a', 70)); // block 3 and 4 + $blocks->append(\str_repeat('a', 90)); // block 5 + + self::assertSame(410, $blocks->size()); + self::assertSame(90, $blocks->block()->spaceLeft()); + self::assertSame(10, $blocks->block()->size()); + self::assertCount(5, $blocks->all()); + } +} diff --git a/src/lib/parquet-viewer/README.md b/src/lib/parquet-viewer/README.md index 5fd32c7be..1f58726b2 100644 --- a/src/lib/parquet-viewer/README.md +++ b/src/lib/parquet-viewer/README.md @@ -2,5 +2,5 @@ ## Installation -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/parquet-viewer.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php index 54caf5059..cbc8393d3 100644 --- a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php +++ b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php @@ -86,7 +86,7 @@ protected function execute(InputInterface $input, OutputInterface $output) : int $column->flatPath(), $column->type() ? $column->type()->name : 'group', $column->logicalType() ? $column->logicalType()->name() : '-', - $column->repetition()->name, + $column->repetition()?->name ?? 'N/A', $column->maxRepetitionsLevel(), $column->maxDefinitionsLevel(), ]); diff --git a/src/lib/parquet/src/Flow/Parquet/Consts.php b/src/lib/parquet/src/Flow/Parquet/Consts.php index 0d6032ec7..ed84505ba 100644 --- a/src/lib/parquet/src/Flow/Parquet/Consts.php +++ b/src/lib/parquet/src/Flow/Parquet/Consts.php @@ -6,12 +6,6 @@ final class Consts { - public const GB_SIZE = 1073741824; - - public const KB_SIZE = 1024; - - public const MB_SIZE = 1048576; - public const PHP_INT32_MAX = 2147483647; public const PHP_INT64_MAX = 9223372036854775807; diff --git a/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int32DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int32DateTimeConverter.php index ffeec54dd..10af90c87 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int32DateTimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int32DateTimeConverter.php @@ -30,9 +30,6 @@ public function toParquetType(mixed $data) : int return $this->dateTimeToMicroseconds($data); } - /** - * @psalm-suppress ArgumentTypeCoercion - */ private function dateTimeToMicroseconds(\DateTimeInterface $dateTime) : int { $microseconds = \number_format((((int) $dateTime->format('u')) / 1000), 0, '', '') . '000'; diff --git a/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int64DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int64DateTimeConverter.php index 9aea210d7..cec3372e1 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int64DateTimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/Converter/Int64DateTimeConverter.php @@ -30,9 +30,6 @@ public function toParquetType(mixed $data) : int return $this->dateTimeToMicroseconds($data); } - /** - * @psalm-suppress ArgumentTypeCoercion - */ private function dateTimeToMicroseconds(\DateTimeInterface $dateTime) : int { return (int) \bcadd(\bcmul($dateTime->format('U'), '1000000'), $dateTime->format('u')); diff --git a/src/lib/parquet/src/Flow/Parquet/Options.php b/src/lib/parquet/src/Flow/Parquet/Options.php index ce7d60cde..f5835f28e 100644 --- a/src/lib/parquet/src/Flow/Parquet/Options.php +++ b/src/lib/parquet/src/Flow/Parquet/Options.php @@ -4,6 +4,7 @@ namespace Flow\Parquet; +use Flow\Filesystem\SizeUnits; use Flow\Parquet\Exception\InvalidArgumentException; final class Options @@ -19,10 +20,10 @@ public function __construct() Option::BYTE_ARRAY_TO_STRING->name => true, Option::ROUND_NANOSECONDS->name => false, Option::INT_96_AS_DATETIME->name => true, - Option::PAGE_SIZE_BYTES->name => Consts::KB_SIZE * 8, - Option::ROW_GROUP_SIZE_BYTES->name => Consts::MB_SIZE * 4, + Option::PAGE_SIZE_BYTES->name => SizeUnits::KiB_SIZE * 8, + Option::ROW_GROUP_SIZE_BYTES->name => SizeUnits::MiB_SIZE * 4, Option::ROW_GROUP_SIZE_CHECK_INTERVAL->name => 1000, - Option::DICTIONARY_PAGE_SIZE->name => Consts::MB_SIZE, + Option::DICTIONARY_PAGE_SIZE->name => SizeUnits::MiB_SIZE, Option::DICTIONARY_PAGE_MIN_CARDINALITY_RATION->name => 0.4, Option::GZIP_COMPRESSION_LEVEL->name => 9, Option::WRITER_VERSION->name => 1, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile.php index af82f408c..6fab991f8 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile.php @@ -4,6 +4,7 @@ namespace Flow\Parquet; +use Flow\Filesystem\SourceStream; use Flow\Parquet\Data\DataConverter; use Flow\Parquet\Exception\{InvalidArgumentException, RuntimeException}; use Flow\Parquet\ParquetFile\ColumnChunkReader\WholeChunkReader; @@ -13,8 +14,8 @@ use Flow\Parquet\ParquetFile\Schema\{Column, FlatColumn, NestedColumn}; use Flow\Parquet\ParquetFile\{ColumnPageHeader, Metadata, PageReader, Schema}; use Flow\Parquet\Thrift\FileMetaData; -use Flow\Parquet\ThriftStream\TPhpFileStream; use Thrift\Protocol\TCompactProtocol; +use Thrift\Transport\TMemoryBuffer; final class ParquetFile { @@ -22,11 +23,8 @@ final class ParquetFile private ?Metadata $metadata = null; - /** - * @param resource $stream - */ public function __construct( - private $stream, + private SourceStream $stream, private readonly ByteOrder $byteOrder, private readonly DataConverter $dataConverter, private readonly Options $options @@ -35,10 +33,7 @@ public function __construct( public function __destruct() { - /** @psalm-suppress RedundantConditionGivenDocblockType */ - if (\is_resource($this->stream)) { - \fclose($this->stream); - } + $this->stream->close(); } public function metadata() : Metadata @@ -47,22 +42,19 @@ public function metadata() : Metadata return $this->metadata; } - \fseek($this->stream, -4, SEEK_END); - - if (\fread($this->stream, 4) !== self::PARQUET_MAGIC_NUMBER) { + if ($this->stream->read(4, -4) !== self::PARQUET_MAGIC_NUMBER) { throw new InvalidArgumentException('Given file is not valid Parquet file'); } - \fseek($this->stream, -8, SEEK_END); - /** * @phpstan-ignore-next-line */ - $metadataLength = \unpack($this->byteOrder->value, \fread($this->stream, 4))[1]; - \fseek($this->stream, -($metadataLength + 8), SEEK_END); + $metadataLength = \unpack($this->byteOrder->value, $this->stream->read(4, -8))[1]; + + $metadata = $this->stream->read($metadataLength, -($metadataLength + 8)); $thriftMetadata = new FileMetaData(); - $thriftMetadata->read(new TCompactProtocol(new TPhpFileStream($this->stream))); + $thriftMetadata->read(new TCompactProtocol(new TMemoryBuffer($metadata))); $this->metadata = Metadata::fromThrift($thriftMetadata); diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Codec.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Codec.php index 09d44519d..eb9d296e4 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Codec.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Codec.php @@ -30,7 +30,7 @@ public function compress(string $data, Compressions $compression) : string }; if ($result === false) { - throw new RuntimeException('Failed to decompress data'); + throw new RuntimeException('Failed to compress data'); } return $result; diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader.php index 80cc3215f..1597ae284 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader.php @@ -4,15 +4,14 @@ namespace Flow\Parquet\ParquetFile; +use Flow\Filesystem\SourceStream; use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema\FlatColumn; interface ColumnChunkReader { /** - * @param resource $stream - * * @return \Generator> */ - public function read(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \Generator; + public function read(ColumnChunk $columnChunk, FlatColumn $column, SourceStream $stream) : \Generator; } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader/WholeChunkReader.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader/WholeChunkReader.php index 41f7fbf1f..cf6436655 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader/WholeChunkReader.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkReader/WholeChunkReader.php @@ -4,15 +4,16 @@ namespace Flow\Parquet\ParquetFile\ColumnChunkReader; +use Flow\Filesystem\SourceStream; use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\ParquetFile\Data\DataBuilder; use Flow\Parquet\ParquetFile\Page\{ColumnData, PageHeader}; use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\{ColumnChunkReader, PageReader}; -use Flow\Parquet\ThriftStream\TPhpFileStream; +use Flow\Parquet\ThriftStream\{TPhpFileStream}; use Thrift\Protocol\TCompactProtocol; -use Thrift\Transport\TBufferedTransport; +use Thrift\Transport\{TBufferedTransport}; final class WholeChunkReader implements ColumnChunkReader { @@ -23,33 +24,35 @@ public function __construct( } /** - * @param resource $stream - * * @return \Generator> */ - public function read(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \Generator + public function read(ColumnChunk $columnChunk, FlatColumn $column, SourceStream $stream) : \Generator { - $offset = $columnChunk->pageOffset(); + $pageStream = fopen('php://temp', 'rb+'); + + if ($pageStream === false) { + throw new RuntimeException('Cannot open temporary stream'); + } - \fseek($stream, $offset); + /** @phpstan-ignore-next-line */ + \fwrite($pageStream, $stream->read($columnChunk->totalCompressedSize(), $columnChunk->pageOffset())); + \rewind($pageStream); - $firstHeader = $this->readHeader($stream, $offset); + $header = $this->readHeader($pageStream); - if ($firstHeader === null) { + if ($header === null) { throw new RuntimeException('Cannot read first page header'); } - if ($firstHeader->type()->isDictionaryPage()) { + if ($header->type()->isDictionaryPage()) { $dictionary = $this->pageReader->readDictionary( $column, - $firstHeader, + $header, $columnChunk->codec(), - $stream + $pageStream ); - $offset = \ftell($stream); } else { $dictionary = null; - \fseek($stream, $offset); } $columnData = ColumnData::initialize($column); @@ -57,8 +60,7 @@ public function read(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \G $rowsToRead = $columnChunk->valuesCount(); while (true) { - /** @phpstan-ignore-next-line */ - $dataHeader = $this->readHeader($stream, $offset); + $dataHeader = $dictionary ? $this->readHeader($pageStream) : $header; /** There are no more pages in given column chunk */ if ($dataHeader === null || $columnData->size() >= $rowsToRead || $dataHeader->type()->isDataPage() === false) { @@ -70,6 +72,8 @@ public function read(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \G $yieldedRows++; if ($yieldedRows >= $rowsToRead) { + \fclose($pageStream); + return; } } @@ -82,22 +86,21 @@ public function read(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \G $dataHeader, $columnChunk->codec(), $dictionary, - $stream + $pageStream )); - - $offset = \ftell($stream); } + + \fclose($pageStream); } /** * @param resource $stream */ - private function readHeader($stream, int $pageOffset) : ?PageHeader + private function readHeader($stream) : ?PageHeader { $currentOffset = \ftell($stream); try { - \fseek($stream, $pageOffset); $thriftHeader = new \Flow\Parquet\Thrift\PageHeader(); @$thriftHeader->read(new TCompactProtocol(new TBufferedTransport(new TPhpFileStream($stream)))); diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer.php index 441191ed1..567092f72 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer.php @@ -4,13 +4,11 @@ namespace Flow\Parquet\ParquetFile; +use Flow\Filesystem\SourceStream; use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema\FlatColumn; interface ColumnChunkViewer { - /** - * @param resource $stream - */ - public function view(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \Generator; + public function view(ColumnChunk $columnChunk, FlatColumn $column, SourceStream $stream) : \Generator; } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer/WholeChunkViewer.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer/WholeChunkViewer.php index 61975617c..48e7d4169 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer/WholeChunkViewer.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/ColumnChunkViewer/WholeChunkViewer.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\ParquetFile\ColumnChunkViewer; +use Flow\Filesystem\SourceStream; use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\ParquetFile\ColumnChunkViewer; use Flow\Parquet\ParquetFile\Page\PageHeader; @@ -15,29 +16,30 @@ final class WholeChunkViewer implements ColumnChunkViewer { - /** - * @param resource $stream - */ - public function view(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \Generator + public function view(ColumnChunk $columnChunk, FlatColumn $column, SourceStream $stream) : \Generator { - $offset = $columnChunk->pageOffset(); + $pageStream = fopen('php://temp', 'rb+'); + + if ($pageStream === false) { + throw new RuntimeException('Cannot open temporary stream'); + } - \fseek($stream, $offset); + /** @phpstan-ignore-next-line */ + \fwrite($pageStream, $stream->read($columnChunk->totalCompressedSize(), $columnChunk->pageOffset())); + \rewind($pageStream); if ($columnChunk->dictionaryPageOffset()) { - $dictionaryHeader = $this->readHeader($stream, $offset); + $dictionaryHeader = $this->readHeader($pageStream); if ($dictionaryHeader === null) { - throw new RuntimeException('Dictionary page header not found in column chunk under offset: ' . $offset); + throw new RuntimeException('Dictionary page header not found in column chunk under offset: ' . $columnChunk->pageOffset()); } yield $dictionaryHeader; - - $offset += $dictionaryHeader->compressedPageSize(); } while (true) { - $dataHeader = $this->readHeader($stream, $offset); + $dataHeader = $this->readHeader($pageStream); /** There are no more pages in given column chunk */ if ($dataHeader === null || $dataHeader->type()->isDataPage() === false) { @@ -45,20 +47,19 @@ public function view(ColumnChunk $columnChunk, FlatColumn $column, $stream) : \G } yield $dataHeader; - - $offset += $dataHeader->compressedPageSize(); } + + \fclose($pageStream); } /** * @param resource $stream */ - private function readHeader($stream, int $pageOffset) : ?PageHeader + private function readHeader($stream) : ?PageHeader { $currentOffset = \ftell($stream); try { - \fseek($stream, $pageOffset); $thriftHeader = new \Flow\Parquet\Thrift\PageHeader(); @$thriftHeader->read(new TCompactProtocol(new TBufferedTransport(new TPhpFileStream($stream)))); diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/ColumnData.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/ColumnData.php index aafc1693f..9bad6fdb9 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/ColumnData.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/ColumnData.php @@ -64,8 +64,6 @@ public function size() : int } /** - * @psalm-suppress ArgumentTypeCoercion - * * @return array{0: self, 1: self} */ public function splitLastRow() : array diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/ColumnChunk.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/ColumnChunk.php index 4e610cb16..849aee4f1 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/ColumnChunk.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroup/ColumnChunk.php @@ -39,9 +39,6 @@ public function __construct( ) { } - /** - * @psalm-suppress RedundantConditionGivenDocblockType - */ public static function fromThrift(\Flow\Parquet\Thrift\ColumnChunk $thrift) : self { return new self( @@ -93,9 +90,6 @@ public function flatPath() : string return \implode('.', $this->path); } - /** - * @psalm-suppress ArgumentTypeCoercion - */ public function pageOffset() : int { $offset = \min( diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php index c871c065d..a01f6d48e 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php @@ -64,9 +64,6 @@ public static function enum() : self return new self(self::ENUM); } - /** - * @psalm-suppress RedundantConditionGivenDocblockType - */ public static function fromThrift(\Flow\Parquet\Thrift\LogicalType $logicalType) : self { $name = null; diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php index cf31fedb4..64b3f1c50 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/NestedColumn.php @@ -38,8 +38,6 @@ public static function create(string $name, array $columns) : self } /** - * @psalm-suppress RedundantConditionGivenDocblockType - * * @param array $children */ public static function fromThrift(SchemaElement $schemaElement, array $children) : self diff --git a/src/lib/parquet/src/Flow/Parquet/Reader.php b/src/lib/parquet/src/Flow/Parquet/Reader.php index f73cb8adb..127315480 100644 --- a/src/lib/parquet/src/Flow/Parquet/Reader.php +++ b/src/lib/parquet/src/Flow/Parquet/Reader.php @@ -4,8 +4,8 @@ namespace Flow\Parquet; +use Flow\Filesystem\{Path, SourceStream, Stream\NativeLocalSourceStream}; use Flow\Parquet\Data\DataConverter; -use Flow\Parquet\Exception\InvalidArgumentException; final class Reader { @@ -17,40 +17,16 @@ public function __construct( public function read(string $path) : ParquetFile { - if (!\file_exists($path)) { - throw new InvalidArgumentException("File {$path} does not exist"); - } - - $stream = \fopen($path, 'rb'); - - if (!\is_resource($stream)) { - throw new InvalidArgumentException("File {$path} is not a valid resource"); - } - - $streamMetadata = \stream_get_meta_data($stream); - - if (!$streamMetadata['seekable']) { - throw new InvalidArgumentException("File {$path} is not seekable"); - } - - return new ParquetFile($stream, $this->byteOrder, DataConverter::initialize($this->options), $this->options); + return new ParquetFile( + NativeLocalSourceStream::open(Path::realpath($path)), + $this->byteOrder, + DataConverter::initialize($this->options), + $this->options + ); } - /** - * @param resource $stream - */ - public function readStream($stream) : ParquetFile + public function readStream(SourceStream $stream) : ParquetFile { - if (!\is_resource($stream)) { - throw new InvalidArgumentException('Given argument is not a valid resource'); - } - - $streamMetadata = \stream_get_meta_data($stream); - - if (!$streamMetadata['seekable']) { - throw new InvalidArgumentException('Given stream is not seekable'); - } - return new ParquetFile($stream, $this->byteOrder, DataConverter::initialize($this->options), $this->options); } diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftStream/TPhpFileStream.php b/src/lib/parquet/src/Flow/Parquet/ThriftStream/TPhpFileStream.php index 6314f7343..7b3568843 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftStream/TPhpFileStream.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftStream/TPhpFileStream.php @@ -47,7 +47,7 @@ public function isOpen() : bool public function open() : void { if (!\is_resource($this->stream)) { - throw new TException('TPhpStream: Could not open php://input'); + throw new TException('TPhpStream: Could not open stream'); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer.php b/src/lib/parquet/src/Flow/Parquet/Writer.php index 6478bd087..4ca67d783 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer.php @@ -5,11 +5,12 @@ namespace Flow\Parquet; use Composer\InstalledVersions; +use Flow\Filesystem\Stream\{NativeLocalDestinationStream}; +use Flow\Filesystem\{DestinationStream, Path}; use Flow\Parquet\Data\DataConverter; use Flow\Parquet\Exception\{InvalidArgumentException, RuntimeException}; use Flow\Parquet\ParquetFile\RowGroupBuilder\PageSizeCalculator; use Flow\Parquet\ParquetFile\{Compressions, Metadata, RowGroupBuilder, RowGroups, Schema}; -use Flow\Parquet\Thrift\FileMetaData; use Flow\Parquet\ThriftStream\TPhpFileStream; use Thrift\Protocol\TCompactProtocol; @@ -21,15 +22,11 @@ final class Writer private ?RowGroupBuilder $rowGroupBuilder = null; - /** - * @var null|resource - */ - private $stream; + private ?DestinationStream $stream = null; public function __construct( - private Compressions $compression = Compressions::SNAPPY, - private Options $options = new Options(), - private ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN + private readonly Compressions $compression = Compressions::SNAPPY, + private readonly Options $options = new Options(), ) { switch ($this->compression) { case Compressions::UNCOMPRESSED: @@ -49,21 +46,6 @@ public function __destruct() } } - /** - * Reopen existing parquet file, read metadata and append new rows. - * Once all rows are appended, the file is automatically closed. - * - * @param iterable> $rows - */ - public function append(string $path, iterable $rows) : void - { - $this->reopen($path); - - $this->writeBatch($rows); - - $this->close(); - } - public function close() : void { if (!$this->isOpen()) { @@ -72,22 +54,35 @@ public function close() : void if (!$this->rowGroupBuilder()->isEmpty()) { $rowGroupContainer = $this->rowGroupBuilder()->flush($this->fileOffset); - \fwrite($this->stream(), $rowGroupContainer->binaryBuffer); + $this->stream()->append($rowGroupContainer->binaryBuffer); $this->metadata()->rowGroups()->add($rowGroupContainer->rowGroup); $this->fileOffset += \strlen($rowGroupContainer->binaryBuffer); } $this->rowGroupBuilder = null; - $start = \ftell($this->stream()); - $this->metadata()->toThrift()->write(new TCompactProtocol(new TPhpFileStream($this->stream()))); - $end = \ftell($this->stream()); - $size = $end - $start; - \fwrite($this->stream(), \pack('l', $size)); - \fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER); + $metadataHandle = \fopen('php://temp/maxmemory:' . (5 * 1024 * 1024), 'rb+'); + + if ($metadataHandle === false) { + throw new RuntimeException('Cannot open temporary stream'); + } + + $this->metadata()->toThrift()->write(new TCompactProtocol(new TPhpFileStream($metadataHandle))); + $metadata = \stream_get_contents($metadataHandle, offset: 0); + + if ($metadata === false) { + throw new RuntimeException('Cannot read metadata from temporary stream'); + } + + $this->stream()->append($metadata); + \fclose($metadataHandle); + + $size = \strlen($metadata); + + $this->stream()->append(\pack('l', $size)); + $this->stream()->append(ParquetFile::PARQUET_MAGIC_NUMBER); - /** @psalm-suppress InvalidPassByReference */ - \fclose($this->stream()); + $this->stream()->close(); $this->stream = null; $this->fileOffset = 0; @@ -109,14 +104,10 @@ public function open(string $path, Schema $schema) : void throw new InvalidArgumentException("File {$path} already exists"); } - $stream = \fopen($path, 'wb'); - - if ($stream === false) { - throw new RuntimeException("Can't open {$path} for writing"); - } + $stream = NativeLocalDestinationStream::openBlank(new Path($path)); $this->stream = $stream; - \fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER); + $this->stream()->append(ParquetFile::PARQUET_MAGIC_NUMBER); $this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER); $this->initMetadata($schema); @@ -125,32 +116,12 @@ public function open(string $path, Schema $schema) : void /** * Opens a writer for an existing stream. - * - * @param resource $resource */ - public function openForStream($resource, Schema $schema) : void + public function openForStream(DestinationStream $stream, Schema $schema) : void { - if (!\is_resource($resource)) { - throw new InvalidArgumentException('Given argument is not a valid resource'); - } - - $streamMetadata = \stream_get_meta_data($resource); - - if (!$streamMetadata['seekable']) { - throw new InvalidArgumentException('Given stream is not seekable'); - } - - if (!\str_starts_with($streamMetadata['mode'], 'w')) { - throw new InvalidArgumentException('Given stream is not opened in write mode, expected wb, got: ' . $streamMetadata['mode']); - } - - $this->stream = $resource; - - if (\ftell($this->stream()) !== 0) { - \fseek($this->stream(), 0); - } + $this->stream = $stream; - \fwrite($this->stream(), ParquetFile::PARQUET_MAGIC_NUMBER); + $this->stream()->append(ParquetFile::PARQUET_MAGIC_NUMBER); $this->fileOffset = \strlen(ParquetFile::PARQUET_MAGIC_NUMBER); $this->initMetadata($schema); @@ -158,123 +129,6 @@ public function openForStream($resource, Schema $schema) : void $this->initGroupBuilder($schema); } - /** - * Reopen existing Parquet file for appending new rows. - * This method will read the metadata from the end of the file and truncate the file. - */ - public function reopen(string $path) : void - { - if ($this->isOpen()) { - throw new RuntimeException('Writer is already open'); - } - - if (!\file_exists($path)) { - throw new InvalidArgumentException("File {$path} don't exists"); - } - - $stream = \fopen($path, 'ab+'); - - if ($stream === false) { - throw new RuntimeException("Can't open {$path} for writing"); - } - - $this->stream = $stream; - - \fseek($this->stream(), -4, SEEK_END); - - if (\fread($this->stream(), 4) !== ParquetFile::PARQUET_MAGIC_NUMBER) { - throw new InvalidArgumentException('Given file is not valid Parquet file'); - } - - \fseek($this->stream(), -8, SEEK_END); - - /** - * @phpstan-ignore-next-line - */ - $metadataLength = \unpack($this->byteOrder->value, \fread($this->stream(), 4))[1]; - \fseek($this->stream(), -($metadataLength + 8), SEEK_END); - - $thriftMetadata = new FileMetaData(); - $thriftMetadata->read(new TCompactProtocol(new TPhpFileStream($this->stream()))); - - $this->metadata = Metadata::fromThrift($thriftMetadata); - - $this->initGroupBuilder($this->metadata()->schema()); - - \fseek($this->stream(), -($metadataLength + 8), SEEK_END); - - $fileSizeWithoutMetadata = \ftell($this->stream()); - - if ($fileSizeWithoutMetadata === false || $fileSizeWithoutMetadata <= 0) { - throw new RuntimeException('File is empty'); - } - - // Truncate previous metadata - \ftruncate($this->stream(), $fileSizeWithoutMetadata); - - $this->fileOffset = $fileSizeWithoutMetadata; - } - - /** - * Opens a writer for an existing stream. - * - * @param resource $resource - */ - public function reopenForStream($resource) : void - { - if ($this->isOpen()) { - throw new RuntimeException('Writer is already open'); - } - - if (!\is_resource($resource)) { - throw new InvalidArgumentException('Given argument is not a valid resource'); - } - - $streamMetadata = \stream_get_meta_data($resource); - - if (!$streamMetadata['seekable']) { - throw new InvalidArgumentException('Given stream is not seekable'); - } - - \fseek($resource, 0); - - $this->stream = $resource; - - \fseek($this->stream(), -4, SEEK_END); - - if (\fread($this->stream(), 4) !== ParquetFile::PARQUET_MAGIC_NUMBER) { - throw new InvalidArgumentException('Given file is not valid Parquet file'); - } - - \fseek($this->stream(), -8, SEEK_END); - - /** - * @phpstan-ignore-next-line - */ - $metadataLength = \unpack($this->byteOrder->value, \fread($this->stream(), 4))[1]; - \fseek($this->stream(), -($metadataLength + 8), SEEK_END); - - $thriftMetadata = new FileMetaData(); - $thriftMetadata->read(new TCompactProtocol(new TPhpFileStream($this->stream()))); - - $this->metadata = Metadata::fromThrift($thriftMetadata); - - $this->initGroupBuilder($this->metadata()->schema()); - - \fseek($this->stream(), -($metadataLength + 8), SEEK_END); - - $fileSizeWithoutMetadata = \ftell($this->stream()); - - if ($fileSizeWithoutMetadata === false || $fileSizeWithoutMetadata <= 0) { - throw new RuntimeException('File is empty'); - } - - // Truncate previous metadata - \ftruncate($this->stream(), $fileSizeWithoutMetadata); - - $this->fileOffset = $fileSizeWithoutMetadata; - } - /** * Create new parquet file, write rows, write metadata and close the file. * @@ -317,7 +171,7 @@ public function writeRow(array $row) : void if (($this->rowGroupBuilder()->statistics()->rowsCount() % $interval === 0) && $this->rowGroupBuilder()->isFull()) { $rowGroupContainer = $this->rowGroupBuilder()->flush($this->fileOffset); - \fwrite($this->stream(), $rowGroupContainer->binaryBuffer); + $this->stream()->append($rowGroupContainer->binaryBuffer); $this->metadata()->rowGroups()->add($rowGroupContainer->rowGroup); $this->fileOffset += \strlen($rowGroupContainer->binaryBuffer); } @@ -326,10 +180,9 @@ public function writeRow(array $row) : void /** * Create new parquet file directly in stream, write rows, write metadata and close the file. * - * @param resource $resource * @param iterable> $rows */ - public function writeStream($resource, Schema $schema, iterable $rows) : void + public function writeStream(DestinationStream $resource, Schema $schema, iterable $rows) : void { $this->openForStream($resource, $schema); @@ -376,10 +229,7 @@ private function rowGroupBuilder() : RowGroupBuilder return $this->rowGroupBuilder; } - /** - * @return resource - */ - private function stream() + private function stream() : DestinationStream { if ($this->stream === null) { throw new RuntimeException('Writer is not open'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php index d4de8ee4b..22511ce90 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php @@ -12,9 +12,16 @@ final class CompressionTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_writing_and_reading_file_with_gzip_compression() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(Compressions::GZIP); @@ -61,7 +68,7 @@ public function test_writing_and_reading_file_with_gzip_compression() : void public function test_writing_and_reading_file_with_snappy_compression() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(Compressions::SNAPPY); @@ -108,7 +115,7 @@ public function test_writing_and_reading_file_with_snappy_compression() : void public function test_writing_and_reading_file_with_uncompressed_compression() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(Compressions::UNCOMPRESSED); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php index 2ea96e22d..40b73da56 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php @@ -12,9 +12,16 @@ final class ListsWritingTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_writing_list_of_ints() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::list('list_of_ints', ListElement::int32())); @@ -38,7 +45,7 @@ public function test_writing_list_of_ints() : void public function test_writing_list_of_strings() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::list('list_of_strings', ListElement::string())); @@ -62,7 +69,7 @@ public function test_writing_list_of_strings() : void public function test_writing_list_of_structures() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with( @@ -96,7 +103,7 @@ public function test_writing_list_of_structures() : void public function test_writing_list_with_nullable_elements() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::list('list_of_ints', ListElement::int32())); @@ -122,7 +129,7 @@ public function test_writing_list_with_nullable_elements() : void public function test_writing_list_with_nullable_list_values() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::list('list_of_ints', ListElement::int32())); @@ -148,7 +155,7 @@ public function test_writing_list_with_nullable_list_values() : void public function test_writing_nullable_list_of_ints() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::list('list_of_ints', ListElement::int32())); @@ -174,7 +181,7 @@ public function test_writing_nullable_list_of_ints() : void public function test_writing_nullable_list_of_structures() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with( diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php index b2d1d3908..db8a0caf8 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php @@ -12,9 +12,16 @@ final class MapsWritingTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_writing_map_of_int_int() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::map('map_int_int', MapKey::int32(), MapValue::int32())); @@ -43,7 +50,7 @@ public function test_writing_map_of_int_int() : void public function test_writing_map_of_int_string() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::map('map_int_string', MapKey::int32(), MapValue::string())); @@ -72,7 +79,7 @@ public function test_writing_map_of_int_string() : void public function test_writing_nullable_map_of_int_int() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::map('map_int_int', MapKey::int32(), MapValue::int32())); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php index 426a7160a..0d083bddd 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php @@ -12,9 +12,16 @@ final class SimpleTypesWritingTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_writing_bool_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::boolean('boolean')); @@ -40,7 +47,7 @@ public function test_writing_bool_column() : void public function test_writing_bool_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::boolean('boolean')); @@ -66,7 +73,7 @@ public function test_writing_bool_nullable_column() : void public function test_writing_date_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::date('date')); @@ -94,7 +101,7 @@ public function test_writing_date_column() : void public function test_writing_date_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::date('date')); @@ -122,7 +129,7 @@ public function test_writing_date_nullable_column() : void public function test_writing_decimal_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::decimal('decimal')); @@ -150,7 +157,7 @@ public function test_writing_decimal_column() : void public function test_writing_decimal_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::decimal('decimal')); @@ -178,7 +185,7 @@ public function test_writing_decimal_nullable_column() : void public function test_writing_double_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::double('double')); @@ -206,7 +213,7 @@ public function test_writing_double_column() : void public function test_writing_double_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::double('double')); @@ -234,7 +241,7 @@ public function test_writing_double_nullable_column() : void public function test_writing_enum_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::enum('enum')); @@ -262,7 +269,7 @@ public function test_writing_enum_column() : void public function test_writing_float_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::float('float')); @@ -288,7 +295,7 @@ public function test_writing_float_column() : void public function test_writing_float_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::float('float')); @@ -314,7 +321,7 @@ public function test_writing_float_nullable_column() : void public function test_writing_int32_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::int32('int32')); @@ -342,7 +349,7 @@ public function test_writing_int32_column() : void public function test_writing_int32_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::int32('int32')); @@ -370,7 +377,7 @@ public function test_writing_int32_nullable_column() : void public function test_writing_int64() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::int64('int64')); @@ -397,7 +404,7 @@ public function test_writing_int64() : void public function test_writing_int64_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::int64('int64')); @@ -424,7 +431,7 @@ public function test_writing_int64_nullable_column() : void public function test_writing_json_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::json('json')); @@ -451,7 +458,7 @@ public function test_writing_json_column() : void public function test_writing_json_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::json('json')); @@ -480,7 +487,7 @@ public function test_writing_json_nullable_column() : void public function test_writing_string_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::string('string')); @@ -507,7 +514,7 @@ public function test_writing_string_column() : void public function test_writing_string_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::string('string')); @@ -534,7 +541,7 @@ public function test_writing_string_nullable_column() : void public function test_writing_time_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::time('time')); @@ -559,7 +566,7 @@ public function test_writing_time_column() : void public function test_writing_time_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::time('time')); @@ -584,7 +591,7 @@ public function test_writing_time_nullable_column() : void public function test_writing_timestamp_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::dateTime('dateTime')); @@ -611,7 +618,7 @@ public function test_writing_timestamp_column() : void public function test_writing_timestamp_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::dateTime('dateTime')); @@ -638,7 +645,7 @@ public function test_writing_timestamp_nullable_column() : void public function test_writing_uuid_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::uuid('uuid')); @@ -665,7 +672,7 @@ public function test_writing_uuid_column() : void public function test_writing_uuid_nullable_column() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(FlatColumn::uuid('uuid')); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php index 11ea0a116..5262f55cc 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php @@ -12,9 +12,16 @@ final class StructsWritingTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_writing_flat_nullable_structure() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::struct('struct', [ @@ -60,7 +67,7 @@ public function test_writing_flat_nullable_structure() : void public function test_writing_flat_structure() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::struct('struct', [ @@ -104,7 +111,7 @@ public function test_writing_flat_structure() : void public function test_writing_flat_structure_with_nullable_elements() : void { - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $writer = new Writer(); $schema = Schema::with(NestedColumn::struct('struct', [ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php index edccf722d..2e0724e0a 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php @@ -6,7 +6,8 @@ use Composer\InstalledVersions; use Faker\Factory; -use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Filesystem\Stream\NativeLocalDestinationStream; +use Flow\Filesystem\{Path}; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\{FlatColumn, ListElement, NestedColumn}; use Flow\Parquet\{Consts, Option, Options, Reader, Writer}; @@ -14,77 +15,11 @@ final class WriterTest extends TestCase { - public function test_appending_to_file() : void + protected function setUp() : void { - $writer = new Writer(); - - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; - - $schema = $this->createSchema(); - $row = $this->createRow(); - - $writer->write($path, $schema, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); - - $writer->append($path, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); - - self::assertSame( - [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row], - \iterator_to_array((new Reader())->read($path)->values()) - ); - self::assertFileExists($path); - \unlink($path); - } - - public function test_appending_to_in_batches_file() : void - { - $writer = new Writer(); - - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; - - $schema = $this->createSchema(); - $row = $this->createRow(); - - $writer->write($path, $schema, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); - - $writer->reopen($path); - $writer->writeBatch([$row, $row]); - $writer->writeBatch([$row, $row]); - $writer->writeBatch([$row, $row]); - $writer->writeBatch([$row, $row]); - $writer->writeBatch([$row, $row]); - $writer->close(); - - self::assertSame( - [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row], - \iterator_to_array((new Reader())->read($path)->values()) - ); - self::assertFileExists($path); - \unlink($path); - } - - public function test_appending_to_stream() : void - { - $writer = new Writer(); - - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; - - $schema = $this->createSchema(); - $row = $this->createRow(); - - $stream = \fopen($path, 'wb+'); - $writer->writeStream($stream, $schema, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); - - $stream = \fopen($path, 'ab+'); - $writer->reopenForStream($stream); - $writer->writeBatch([$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); - $writer->close(); - - self::assertSame( - [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row, $row], - \iterator_to_array((new Reader())->read($path)->values()) - ); - self::assertFileExists($path); - \unlink($path); + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } } public function test_closing_not_open_writer() : void @@ -101,7 +36,7 @@ public function test_created_by_metadata() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); $writer->open($path, $schema); @@ -116,7 +51,7 @@ public function test_opening_already_open_writer() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); @@ -138,22 +73,6 @@ public function test_writing_batch_to_not_open_stream() : void $writer->writeBatch([$this->createRow()]); } - public function test_writing_batch_to_not_writable_stream() : void - { - $writer = new Writer(); - - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Given stream is not opened in write mode, expected wb, got: rb+'); - - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; - \file_put_contents($path, 'test'); - $stream = \fopen($path, 'rb+'); - - $writer->openForStream($stream, $this->createSchema()); - $writer->writeBatch([$this->createRow()]); - \unlink($path); - } - public function test_writing_column_statistics() : void { $writer = new Writer( @@ -161,7 +80,7 @@ public function test_writing_column_statistics() : void ->set(Option::WRITER_VERSION, 1) ); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with($column = FlatColumn::int32('int32')); @@ -190,7 +109,7 @@ public function test_writing_data_page_v2_statistics() : void ->set(Option::WRITER_VERSION, 2) ); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with($column = FlatColumn::int32('int32')); @@ -217,7 +136,7 @@ public function test_writing_in_batches_to_file() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); @@ -244,7 +163,7 @@ public function test_writing_in_batches_to_file_without_explicit_close() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); @@ -270,14 +189,14 @@ public function test_writing_in_batches_to_stream() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); $row = $this->createRow(); $stream = \fopen($path, 'wb+'); - $writer->openForStream($stream, $schema); + $writer->openForStream(new NativeLocalDestinationStream(new Path($path), $stream), $schema); $writer->writeBatch([$row, $row]); $writer->writeBatch([$row, $row]); $writer->writeBatch([$row, $row]); @@ -308,7 +227,7 @@ public function test_writing_to_file() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); $row = $this->createRow(); @@ -330,7 +249,7 @@ public function test_writing_to_file_v2() : void ->set(Option::WRITER_VERSION, 2) ); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-v2-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); $row = $this->createRow(); @@ -350,14 +269,14 @@ public function test_writing_to_stream() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = $this->createSchema(); $row = $this->createRow(); $stream = \fopen($path, 'wb+'); - $writer->writeStream($stream, $schema, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); + $writer->writeStream(new NativeLocalDestinationStream(new Path($path), $stream), $schema, [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row]); self::assertSame( [$row, $row, $row, $row, $row, $row, $row, $row, $row, $row], diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterValidatorTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterValidatorTest.php index 215e6495e..449b77ed9 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterValidatorTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterValidatorTest.php @@ -10,12 +10,19 @@ final class WriterValidatorTest extends TestCase { + protected function setUp() : void + { + if (!\file_exists(__DIR__ . '/var')) { + \mkdir(__DIR__ . '/var'); + } + } + public function test_skipping_required_row() : void { $this->expectExceptionMessage('Column "string" is required'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with( Schema\FlatColumn::int32('id'), @@ -30,7 +37,7 @@ public function test_writing_int_value_to_string_column() : void $this->expectExceptionMessage('Column "string" is not string, got "integer" instead'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\FlatColumn::string('string')); @@ -42,7 +49,7 @@ public function test_writing_null_to_list_that_is_required() : void $this->expectExceptionMessage('Column "list" is required'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\NestedColumn::list('list', Schema\ListElement::string())->makeRequired()); @@ -54,7 +61,7 @@ public function test_writing_null_to_list_with_element_is_required() : void $this->expectExceptionMessage('Column "list.list.element" is not string, got "NULL" instead'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\NestedColumn::list('list', Schema\ListElement::string(required: true))); @@ -66,7 +73,7 @@ public function test_writing_null_to_map_with_value_required() : void $this->expectExceptionMessage('Column "map.key_value.value" is not string, got "NULL" instead'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\NestedColumn::map('map', Schema\MapKey::string(), Schema\MapValue::string(required: true))); @@ -78,7 +85,7 @@ public function test_writing_null_to_required_map() : void $this->expectExceptionMessage('Column "map" is required'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\NestedColumn::map('map', Schema\MapKey::string(), Schema\MapValue::string())->makeRequired()); @@ -90,7 +97,7 @@ public function test_writing_null_value_to_required_column() : void $this->expectExceptionMessage('Column "string" is required'); $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with(Schema\FlatColumn::string('string')->makeRequired()); @@ -100,7 +107,7 @@ public function test_writing_null_value_to_required_column() : void public function test_writing_row_with_missing_optional_columns() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with( Schema\FlatColumn::int32('id'), @@ -134,7 +141,7 @@ public function test_writing_row_with_missing_optional_columns() : void public function test_writing_row_with_missing_optional_columns_in_different_columns() : void { $writer = new Writer(); - $path = \sys_get_temp_dir() . '/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; + $path = __DIR__ . '/var/test-writer-validator-parquet-test-' . bin2hex(random_bytes(16)) . '.parquet'; $schema = Schema::with( Schema\FlatColumn::int32('id'), diff --git a/src/lib/rdsl/README.md b/src/lib/rdsl/README.md index c0896f6cc..1bad22248 100644 --- a/src/lib/rdsl/README.md +++ b/src/lib/rdsl/README.md @@ -4,5 +4,5 @@ Remote Domain Specific Language This library allows to execute a DSL functions defined in JSON (and possibly more) format on a remote server. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/rdsl.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/src/lib/snappy/README.md b/src/lib/snappy/README.md index 0a4cf583f..394943322 100644 --- a/src/lib/snappy/README.md +++ b/src/lib/snappy/README.md @@ -6,5 +6,5 @@ This library is a port of javascript [snappyjs](https://github.com/zhipeng-jia/s Whenever it's possible, it's recommended to install [PHP Extension.](https://github.com/kjdev/php-ext-snappy) Otherwise, this lib will register polyfill functions. -- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/README.md) +- 📜 [Documentation](https://github.com/flow-php/flow/blob/1.x/docs/components/libs/snappy.md) - 🛠️ [Contributing](https://github.com/flow-php/flow/blob/1.x/CONTRIBUTING.md) \ No newline at end of file diff --git a/tools/box/composer.lock b/tools/box/composer.lock index f56d13d29..cce6b0266 100644 --- a/tools/box/composer.lock +++ b/tools/box/composer.lock @@ -2028,16 +2028,16 @@ }, { "name": "symfony/console", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91" + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/be5854cee0e8c7b110f00d695d11debdfa1a2a91", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91", + "url": "https://api.github.com/repos/symfony/console/zipball/6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", "shasum": "" }, "require": { @@ -2102,7 +2102,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.8" + "source": "https://github.com/symfony/console/tree/v6.4.9" }, "funding": [ { @@ -2118,7 +2118,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2265,16 +2265,16 @@ }, { "name": "symfony/filesystem", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3" + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/4d37529150e7081c51b3c5d5718c55a04a9503f3", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/b51ef8059159330b74a4d52f68e671033c0fe463", + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463", "shasum": "" }, "require": { @@ -2311,7 +2311,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.8" + "source": "https://github.com/symfony/filesystem/tree/v6.4.9" }, "funding": [ { @@ -2327,7 +2327,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/finder", @@ -2937,16 +2937,16 @@ }, { "name": "symfony/string", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d" + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/a147c0f826c4a1f3afb763ab8e009e37c877a44d", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d", + "url": "https://api.github.com/repos/symfony/string/zipball/76792dbd99690a5ebef8050d9206c60c59e681d7", + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7", "shasum": "" }, "require": { @@ -3003,7 +3003,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.8" + "source": "https://github.com/symfony/string/tree/v6.4.9" }, "funding": [ { @@ -3019,20 +3019,20 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:25:38+00:00" }, { "name": "symfony/var-dumper", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/var-dumper.git", - "reference": "ad23ca4312395f0a8a8633c831ef4c4ee542ed25" + "reference": "c31566e4ca944271cc8d8ac6887cbf31b8c6a172" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/var-dumper/zipball/ad23ca4312395f0a8a8633c831ef4c4ee542ed25", - "reference": "ad23ca4312395f0a8a8633c831ef4c4ee542ed25", + "url": "https://api.github.com/repos/symfony/var-dumper/zipball/c31566e4ca944271cc8d8ac6887cbf31b8c6a172", + "reference": "c31566e4ca944271cc8d8ac6887cbf31b8c6a172", "shasum": "" }, "require": { @@ -3088,7 +3088,7 @@ "dump" ], "support": { - "source": "https://github.com/symfony/var-dumper/tree/v6.4.8" + "source": "https://github.com/symfony/var-dumper/tree/v6.4.9" }, "funding": [ { @@ -3104,7 +3104,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-27T13:23:14+00:00" }, { "name": "thecodingmachine/safe", diff --git a/tools/cs-fixer/composer.lock b/tools/cs-fixer/composer.lock index 47f20da6d..0191e3c2f 100644 --- a/tools/cs-fixer/composer.lock +++ b/tools/cs-fixer/composer.lock @@ -1252,16 +1252,16 @@ }, { "name": "symfony/console", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91" + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/be5854cee0e8c7b110f00d695d11debdfa1a2a91", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91", + "url": "https://api.github.com/repos/symfony/console/zipball/6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", "shasum": "" }, "require": { @@ -1326,7 +1326,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.8" + "source": "https://github.com/symfony/console/tree/v6.4.9" }, "funding": [ { @@ -1342,7 +1342,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1569,16 +1569,16 @@ }, { "name": "symfony/filesystem", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3" + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/4d37529150e7081c51b3c5d5718c55a04a9503f3", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/b51ef8059159330b74a4d52f68e671033c0fe463", + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463", "shasum": "" }, "require": { @@ -1615,7 +1615,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.8" + "source": "https://github.com/symfony/filesystem/tree/v6.4.9" }, "funding": [ { @@ -1631,7 +1631,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/finder", @@ -2446,16 +2446,16 @@ }, { "name": "symfony/string", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d" + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/a147c0f826c4a1f3afb763ab8e009e37c877a44d", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d", + "url": "https://api.github.com/repos/symfony/string/zipball/76792dbd99690a5ebef8050d9206c60c59e681d7", + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7", "shasum": "" }, "require": { @@ -2512,7 +2512,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.8" + "source": "https://github.com/symfony/string/tree/v6.4.9" }, "funding": [ { @@ -2528,7 +2528,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:25:38+00:00" } ], "aliases": [], diff --git a/tools/infection/composer.lock b/tools/infection/composer.lock index 57b3526f2..c59a55a3e 100644 --- a/tools/infection/composer.lock +++ b/tools/infection/composer.lock @@ -1108,16 +1108,16 @@ }, { "name": "symfony/console", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91" + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/be5854cee0e8c7b110f00d695d11debdfa1a2a91", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91", + "url": "https://api.github.com/repos/symfony/console/zipball/6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", "shasum": "" }, "require": { @@ -1182,7 +1182,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.8" + "source": "https://github.com/symfony/console/tree/v6.4.9" }, "funding": [ { @@ -1198,7 +1198,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1269,16 +1269,16 @@ }, { "name": "symfony/filesystem", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3" + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/4d37529150e7081c51b3c5d5718c55a04a9503f3", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/b51ef8059159330b74a4d52f68e671033c0fe463", + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463", "shasum": "" }, "require": { @@ -1315,7 +1315,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.8" + "source": "https://github.com/symfony/filesystem/tree/v6.4.9" }, "funding": [ { @@ -1331,7 +1331,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/finder", @@ -1861,16 +1861,16 @@ }, { "name": "symfony/string", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d" + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/a147c0f826c4a1f3afb763ab8e009e37c877a44d", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d", + "url": "https://api.github.com/repos/symfony/string/zipball/76792dbd99690a5ebef8050d9206c60c59e681d7", + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7", "shasum": "" }, "require": { @@ -1927,7 +1927,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.8" + "source": "https://github.com/symfony/string/tree/v6.4.9" }, "funding": [ { @@ -1943,7 +1943,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:25:38+00:00" }, { "name": "thecodingmachine/safe", diff --git a/tools/phpbench/composer.lock b/tools/phpbench/composer.lock index 950d55d94..1ad7c4097 100644 --- a/tools/phpbench/composer.lock +++ b/tools/phpbench/composer.lock @@ -264,16 +264,16 @@ }, { "name": "phpbench/phpbench", - "version": "1.2.15", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/phpbench/phpbench.git", - "reference": "f7000319695cfad04a57fc64bf7ef7abdf4c437c" + "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/f7000319695cfad04a57fc64bf7ef7abdf4c437c", - "reference": "f7000319695cfad04a57fc64bf7ef7abdf4c437c", + "url": "https://api.github.com/repos/phpbench/phpbench/zipball/a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0", + "reference": "a3e1ef08d9d7736d43a7fbd444893d6a073c0ca0", "shasum": "" }, "require": { @@ -285,29 +285,30 @@ "ext-spl": "*", "ext-tokenizer": "*", "php": "^8.1", - "phpbench/container": "^2.1", + "phpbench/container": "^2.2", "phpbench/dom": "~0.3.3", "psr/log": "^1.1 || ^2.0 || ^3.0", "seld/jsonlint": "^1.1", - "symfony/console": "^4.2 || ^5.0 || ^6.0 || ^7.0", - "symfony/filesystem": "^4.2 || ^5.0 || ^6.0 || ^7.0", - "symfony/finder": "^4.2 || ^5.0 || ^6.0 || ^7.0", - "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0", - "symfony/process": "^4.2 || ^5.0 || ^6.0 || ^7.0", + "symfony/console": "^6.1 || ^7.0", + "symfony/filesystem": "^6.1 || ^7.0", + "symfony/finder": "^6.1 || ^7.0", + "symfony/options-resolver": "^6.1 || ^7.0", + "symfony/process": "^6.1 || ^7.0", "webmozart/glob": "^4.6" }, "require-dev": { "dantleech/invoke": "^2.0", + "ergebnis/composer-normalize": "^2.39", "friendsofphp/php-cs-fixer": "^3.0", "jangregor/phpstan-prophecy": "^1.0", "phpspec/prophecy": "dev-master", "phpstan/extension-installer": "^1.1", "phpstan/phpstan": "^1.0", "phpstan/phpstan-phpunit": "^1.0", - "phpunit/phpunit": "^10.0", - "rector/rector": "^0.18.10", - "symfony/error-handler": "^5.2 || ^6.0 || ^7.0", - "symfony/var-dumper": "^4.0 || ^5.0 || ^6.0 || ^7.0" + "phpunit/phpunit": "^10.4", + "rector/rector": "^0.18.11 || ^1.0.0", + "symfony/error-handler": "^6.1 || ^7.0", + "symfony/var-dumper": "^6.1 || ^7.0" }, "suggest": { "ext-xdebug": "For Xdebug profiling extension." @@ -350,7 +351,7 @@ ], "support": { "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.2.15" + "source": "https://github.com/phpbench/phpbench/tree/1.3.1" }, "funding": [ { @@ -358,7 +359,7 @@ "type": "github" } ], - "time": "2023-11-29T12:21:11+00:00" + "time": "2024-06-30T11:04:37+00:00" }, { "name": "psr/cache", @@ -578,16 +579,16 @@ }, { "name": "symfony/console", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91" + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/be5854cee0e8c7b110f00d695d11debdfa1a2a91", - "reference": "be5854cee0e8c7b110f00d695d11debdfa1a2a91", + "url": "https://api.github.com/repos/symfony/console/zipball/6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", + "reference": "6edb5363ec0c78ad4d48c5128ebf4d083d89d3a9", "shasum": "" }, "require": { @@ -652,7 +653,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.8" + "source": "https://github.com/symfony/console/tree/v6.4.9" }, "funding": [ { @@ -668,7 +669,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/deprecation-contracts", @@ -739,16 +740,16 @@ }, { "name": "symfony/filesystem", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3" + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/4d37529150e7081c51b3c5d5718c55a04a9503f3", - "reference": "4d37529150e7081c51b3c5d5718c55a04a9503f3", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/b51ef8059159330b74a4d52f68e671033c0fe463", + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463", "shasum": "" }, "require": { @@ -785,7 +786,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.8" + "source": "https://github.com/symfony/filesystem/tree/v6.4.9" }, "funding": [ { @@ -801,7 +802,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/finder", @@ -1398,16 +1399,16 @@ }, { "name": "symfony/string", - "version": "v6.4.8", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d" + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/a147c0f826c4a1f3afb763ab8e009e37c877a44d", - "reference": "a147c0f826c4a1f3afb763ab8e009e37c877a44d", + "url": "https://api.github.com/repos/symfony/string/zipball/76792dbd99690a5ebef8050d9206c60c59e681d7", + "reference": "76792dbd99690a5ebef8050d9206c60c59e681d7", "shasum": "" }, "require": { @@ -1464,7 +1465,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.8" + "source": "https://github.com/symfony/string/tree/v6.4.9" }, "funding": [ { @@ -1480,7 +1481,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-06-28T09:25:38+00:00" }, { "name": "webmozart/glob", diff --git a/tools/phpunit/composer.lock b/tools/phpunit/composer.lock index 6cf4409d0..7a6f3cf59 100644 --- a/tools/phpunit/composer.lock +++ b/tools/phpunit/composer.lock @@ -69,16 +69,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.0.2", + "version": "v5.1.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13" + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/683130c2ff8c2739f4822ff7ac5c873ec529abd1", + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1", "shasum": "" }, "require": { @@ -89,7 +89,7 @@ }, "require-dev": { "ircmaxell/php-yacc": "^0.0.7", - "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0" + "phpunit/phpunit": "^9.0" }, "bin": [ "bin/php-parse" @@ -121,9 +121,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.1.0" }, - "time": "2024-03-05T20:51:40+00:00" + "time": "2024-07-01T20:03:41+00:00" }, { "name": "phar-io/manifest", @@ -245,16 +245,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "10.1.14", + "version": "10.1.15", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b" + "reference": "5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", - "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae", + "reference": "5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae", "shasum": "" }, "require": { @@ -311,7 +311,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.14" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.15" }, "funding": [ { @@ -319,7 +319,7 @@ "type": "github" } ], - "time": "2024-03-12T15:33:41+00:00" + "time": "2024-06-29T08:25:15+00:00" }, { "name": "phpunit/php-file-iterator", @@ -566,16 +566,16 @@ }, { "name": "phpunit/phpunit", - "version": "10.5.24", + "version": "10.5.25", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "5f124e3e3e561006047b532fd0431bf5bb6b9015" + "reference": "831bf82312be6037e811833ddbea0b8de60ea314" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/5f124e3e3e561006047b532fd0431bf5bb6b9015", - "reference": "5f124e3e3e561006047b532fd0431bf5bb6b9015", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/831bf82312be6037e811833ddbea0b8de60ea314", + "reference": "831bf82312be6037e811833ddbea0b8de60ea314", "shasum": "" }, "require": { @@ -647,7 +647,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.24" + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.25" }, "funding": [ { @@ -663,7 +663,7 @@ "type": "tidelift" } ], - "time": "2024-06-20T13:09:54+00:00" + "time": "2024-07-03T05:49:17+00:00" }, { "name": "sebastian/cli-parser", diff --git a/web/landing/templates/blog/2024-04-04/building-custom-extractor-google-analytics/extractor-04.php b/web/landing/templates/blog/2024-04-04/building-custom-extractor-google-analytics/extractor-04.php index 3331b0012..7e40e3cec 100644 --- a/web/landing/templates/blog/2024-04-04/building-custom-extractor-google-analytics/extractor-04.php +++ b/web/landing/templates/blog/2024-04-04/building-custom-extractor-google-analytics/extractor-04.php @@ -32,7 +32,7 @@ public function extract(FlowContext $context): \Generator /** @var AccountSummary $account */ foreach ($list->iterateAllElements() as $accountSummary) { $signal = yield rows(ga_account_summary_to_row($accountSummary)); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { return; @@ -44,7 +44,7 @@ public function extract(FlowContext $context): \Generator foreach ($list->iterateAllElements() as $accountSummary) { $signal = yield rows(ga_account_summary_to_row($accountSummary)); - $this->countRow(); + $this->incrementReturnedRows(); if ($signal === Signal::STOP || $this->reachedLimit()) { return;