Flow filesystem introduction (#1104)

* Proof of concept of reading remote streams with seeking * First draft of filesystem lib with related bridges * Adjusted FilesExtractor to latest filesystem lib changes * Cleanups and docs updates * Final cleanups * Updated monorepo split workflow configuration * Try to run azurite at github actions through nodejs * Fixed failing tests * Static analysis fixes * Use __DIR__ '/var/' in tests instead of system tmp path * Fixed failing tests * Fixed accessing not initialized values in tests
flow-php · Jul 3, 2024 · 811f214 · 811f214
1 parent 8ae179f
commit 811f214
Show file tree

Hide file tree

Showing 351 changed files with 8,273 additions and 4,205 deletions.
diff --git a/.github/workflows/monorepo-split.yml b/.github/workflows/monorepo-split.yml
@@ -26,6 +26,8 @@ jobs:
 
           - local_path: 'src/lib/array-dot'
             split_repository: 'array-dot'
+          - local_path: 'src/lib/azure-sdk'
+            split_repository: 'azure-sdk'
           - local_path: 'src/lib/doctrine-dbal-bulk'
             split_repository: 'doctrine-dbal-bulk'
           - local_path: 'src/lib/parquet'
@@ -34,6 +36,8 @@ jobs:
             split_repository: 'parquet-viewer'
           - local_path: 'src/lib/dremel'
             split_repository: 'dremel'
+          - local_path: 'src/lib/filesystem'
+            split_repository: 'filesystem'
           - local_path: 'src/lib/rdsl'
             split_repository: 'rdsl'
           - local_path: 'src/lib/snappy'
@@ -49,8 +53,6 @@ jobs:
             split_repository: 'etl-adapter-doctrine'
           - local_path: 'src/adapter/etl-adapter-elasticsearch'
             split_repository: 'etl-adapter-elasticsearch'
-          - local_path: 'src/adapter/etl-adapter-filesystem'
-            split_repository: 'etl-adapter-filesystem'
           - local_path: 'src/adapter/etl-adapter-meilisearch'
             split_repository: 'etl-adapter-meilisearch'
           - local_path: 'src/adapter/etl-adapter-google-sheet'
@@ -68,6 +70,11 @@ jobs:
           - local_path: 'src/adapter/etl-adapter-xml'
             split_repository: 'etl-adapter-xml'
 
+          - local_path: 'src/bridge/filesystem/azure'
+            split_repository: 'filesystem-azure-bridge'
+          - local_path: 'src/bridge/monolog/http'
+            split_repository: 'monolog-http-bridge'
+
           - local_path: 'src/tools/homebrew'
             split_repository: 'homebrew-flow'
 

diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
@@ -96,6 +96,20 @@ jobs:
       - name: "List PHP configuration"
         run: php -i
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: '14'
+
+      - name: Install Azurite storage emulator
+        run: npm install -g azurite
+
+      - name: Start Azurite blob endpoint
+        shell: bash
+        run: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 &
+        env:
+          AZURITE_ACCOUNTS: flowphpaccount01:flowphpkey01
+
       - name: "Get Composer Cache Directory"
         id: composer-cache
         run: |

diff --git a/.php-cs-fixer.php b/.php-cs-fixer.php
@@ -12,6 +12,7 @@
         __DIR__ . '/src/core/**/src',
         __DIR__ . '/src/core/**/tests',
         __DIR__ . '/src/adapter/**/src',
+        __DIR__ . '/src/bridge/**/**/src',
         __DIR__ . '/src/adapter/**/tests',
         __DIR__ . '/src/lib/**/src',
         __DIR__ . '/src/lib/**/tests',

diff --git a/compose.yml.dist b/compose.yml.dist
@@ -36,3 +36,15 @@ services:
         environment:
             - MEILI_MASTER_KEY=masterKey
             - MEILI_NO_ANALYTICS=true
+    azurite:
+        image: mcr.microsoft.com/azure-storage/azurite
+        container_name: flow-php-azurite
+        hostname: azurite
+        restart: always
+        command: "azurite --loose --blobHost 0.0.0.0 --blobPort 10000 --location /workspace --debug /workspace/debug.log"
+        environment:
+            - AZURITE_ACCOUNTS=flowphpaccount01:flowphpkey01
+        ports:
+            - 10000:10000
+        volumes:
+            - ./var/azurite:/workspace
diff --git a/composer.json b/composer.json
@@ -24,11 +24,12 @@
         "elasticsearch/elasticsearch": "^7.6|^8.0",
         "google/apiclient": "^2.13",
         "halaxa/json-machine": "^1.0",
-        "league/flysystem": "^3.0",
         "meilisearch/meilisearch-php": "^1.1",
-        "monolog/monolog": "^3.0",
+        "monolog/monolog": "^2.0||^3.0",
         "packaged/thrift": "^0.15.0",
+        "php-http/discovery": "^1.0",
         "psr/http-client": "^1.0",
+        "psr/http-message": "^1.0",
         "psr/log": "^2.0 || ^3.0",
         "psr/simple-cache": "^1.0 || ^2.0 || ^3.0",
         "symfony/console": "^6.3 || ^7.0",
@@ -39,8 +40,6 @@
         "fakerphp/faker": "^1.23",
         "fig/log-test": "^1.1",
         "jawira/case-converter": "^3.4",
-        "league/flysystem-aws-s3-v3": "^3.0",
-        "league/flysystem-azure-blob-storage": "^3.0",
         "moneyphp/money": "^4",
         "nyholm/psr7": "^1.8",
         "php-http/curl-client": "^2.2",
@@ -53,21 +52,23 @@
     },
     "autoload": {
         "files": [
-            "src/functions.php",
             "src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php",
             "src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php",
             "src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php",
             "src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php",
             "src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php",
-            "src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/functions.php",
             "src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php",
             "src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php",
             "src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php",
             "src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php",
             "src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php",
             "src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php",
+            "src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php",
             "src/core/etl/src/Flow/ETL/DSL/functions.php",
+            "src/functions.php",
             "src/lib/array-dot/src/Flow/ArrayDot/array_dot.php",
+            "src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php",
+            "src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php",
             "src/lib/parquet/src/Flow/Parquet/functions.php",
             "src/lib/snappy/polyfill.php"
         ],
@@ -78,7 +79,6 @@
                 "src/adapter/etl-adapter-csv/src/Flow",
                 "src/adapter/etl-adapter-doctrine/src/Flow",
                 "src/adapter/etl-adapter-elasticsearch/src/Flow",
-                "src/adapter/etl-adapter-filesystem/src/Flow",
                 "src/adapter/etl-adapter-google-sheet/src/Flow",
                 "src/adapter/etl-adapter-http/src/Flow",
                 "src/adapter/etl-adapter-json/src/Flow",
@@ -87,10 +87,14 @@
                 "src/adapter/etl-adapter-parquet/src/Flow",
                 "src/adapter/etl-adapter-text/src/Flow",
                 "src/adapter/etl-adapter-xml/src/Flow",
+                "src/bridge/filesystem/azure/src/Flow",
+                "src/bridge/monolog/http/src/Flow",
                 "src/core/etl/src/Flow",
                 "src/lib/array-dot/src/Flow",
+                "src/lib/azure-sdk/src/Flow",
                 "src/lib/doctrine-dbal-bulk/src/Flow",
                 "src/lib/dremel/src/Flow",
+                "src/lib/filesystem/src/Flow",
                 "src/lib/parquet-viewer/src/Flow",
                 "src/lib/parquet/src/Flow",
                 "src/lib/rdsl/src/Flow",
@@ -112,7 +116,6 @@
                 "src/adapter/etl-adapter-csv/tests/Flow",
                 "src/adapter/etl-adapter-doctrine/tests/Flow",
                 "src/adapter/etl-adapter-elasticsearch/tests/Flow",
-                "src/adapter/etl-adapter-filesystem/tests/Flow",
                 "src/adapter/etl-adapter-google-sheet/tests/Flow",
                 "src/adapter/etl-adapter-http/tests/Flow",
                 "src/adapter/etl-adapter-json/tests/Flow",
@@ -121,10 +124,14 @@
                 "src/adapter/etl-adapter-parquet/tests/Flow",
                 "src/adapter/etl-adapter-text/tests/Flow",
                 "src/adapter/etl-adapter-xml/tests/Flow",
+                "src/bridge/filesystem/azure/tests/Flow",
+                "src/bridge/monolog/http/tests/Flow",
                 "src/core/etl/tests/Flow",
                 "src/lib/array-dot/tests/Flow",
+                "src/lib/azure-sdk/tests/Flow",
                 "src/lib/doctrine-dbal-bulk/tests/Flow",
                 "src/lib/dremel/tests/Flow",
+                "src/lib/filesystem/tests/Flow",
                 "src/lib/parquet-viewer/tests/Flow",
                 "src/lib/parquet/tests/Flow",
                 "src/lib/rdsl/tests/Flow",
@@ -145,6 +152,7 @@
     },
     "replace": {
         "flow-php/array-dot": "self.version",
+        "flow-php/azure-sdk": "self.version",
         "flow-php/doctrine-dbal-bulk": "self.version",
         "flow-php/doctrine-dbal-bulk-tools": "self.version",
         "flow-php/dremel": "self.version",
@@ -165,6 +173,9 @@
         "flow-php/etl-adapter-parquet": "self.version",
         "flow-php/etl-adapter-text": "self.version",
         "flow-php/etl-adapter-xml": "self.version",
+        "flow-php/filesystem": "self.version",
+        "flow-php/filesytem-azure-bridge": "self.version",
+        "flow-php/monolog-http-bridge": "self.version",
         "flow-php/parquet": "self.version",
         "flow-php/parquet-viewer": "self.version",
         "flow-php/rdsl": "self.version",
@@ -204,7 +215,7 @@
             "tools/phpbench/vendor/bin/phpbench run --report=flow-report --group=transformer"
         ],
         "test:mutation": [
-            "tools/infection/vendor/bin/infection -j2"
+            "tools/infection/vendor/bin/infection --threads=max"
         ],
         "test:monorepo": "tools/monorepo/vendor/bin/monorepo-builder validate",
         "static:analyze": [