Added a DSL documentation to landing page (#1188)

* Added a DSL documentation to landing page

* CS fixes

* Lowered version of nikic parser due to conflicts with infection

* Removed function extractor tests from test suite due to conflicts with phpunit

* Updated parser version

.github/workflows/publish-website.yml

@@ -45,7 +45,13 @@ jobs:
           restore-keys: |
             php-${{ matrix.php-version }}-composer-website-
 
-      - name: "Install dependencies"
+      - name: "Install project dependencies"
+        run: "composer install --no-interaction --no-progress --no-suggest"
+
+      - name: "Generate documentation"
+        run: "composer build:docs"
+
+      - name: "Install Landing dependencies"
         run: "composer install --no-interaction --no-progress --no-suggest"
         working-directory: "web/landing"
 

bin/docs.php

@@ -0,0 +1,64 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+use Flow\Documentation\{FunctionCollector, FunctionsExtractor};
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\{InputArgument, InputInterface};
+use Symfony\Component\Console\Output\OutputInterface;
+
+require __DIR__ . '/../vendor/autoload.php';
+
+if (false === in_array(PHP_SAPI, ['cli', 'phpdbg', 'embed'], true)) {
+    print PHP_EOL . 'This app may only be invoked from a command line, got "' . PHP_SAPI . '"' . PHP_EOL;
+
+    exit(1);
+}
+
+ini_set('memory_limit', -1);
+
+$application = new Application('Flow-PHP - Documentation');
+
+$application->add(new class extends Command {
+    public function configure() : void
+    {
+        $this
+            ->setName('dsl:dump')
+            ->setDescription('Dump DSL into json file.')
+            ->addArgument('output', InputArgument::REQUIRED, 'Where to dump dsl.');
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output) : int
+    {
+
+        $paths = [
+            __DIR__ . '/../src/core/etl/src/Flow/ETL/DSL/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php',
+        ];
+
+        $extractor = new FunctionsExtractor(new FunctionCollector());
+
+        $normalizedFunctions = [];
+
+        foreach ($extractor->extract($paths) as $function) {
+            $normalizedFunctions[] = $function->normalize();
+        }
+
+        \file_put_contents(__DIR__ . '/../' . \ltrim($input->getArgument('output'), '/'), \json_encode($normalizedFunctions));
+
+        return Command::SUCCESS;
+    }
+});
+
+$application->run();

composer.json

@@ -42,6 +42,7 @@
         "aeon-php/calendar": "^1.0",
         "fakerphp/faker": "^1.23",
         "fig/log-test": "^1.1",
+        "nikic/php-parser": "^4.18",
         "nyholm/psr7": "^1.8",
         "php-http/curl-client": "^2.2",
         "php-http/mock-client": "^1.5",
@@ -100,7 +101,8 @@
                 "src/lib/parquet-viewer/src/Flow",
                 "src/lib/parquet/src/Flow",
                 "src/lib/rdsl/src/Flow",
-                "src/lib/snappy/src/Flow"
+                "src/lib/snappy/src/Flow",
+                "src/tools/documentation/src/Flow"
             ],
             "Flow\\Doctrine\\Bulk\\": [
                 "src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk"
@@ -137,7 +139,8 @@
                 "src/lib/parquet-viewer/tests/Flow",
                 "src/lib/parquet/tests/Flow",
                 "src/lib/rdsl/tests/Flow",
-                "src/lib/snappy/tests/Flow"
+                "src/lib/snappy/tests/Flow",
+                "src/tools/documentation/tests/Flow"
             ],
             "Flow\\Doctrine\\Bulk\\Tests\\": [
                 "src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests"
@@ -240,6 +243,9 @@
         "build:phar": [
             "tools/box/vendor/bin/box compile"
         ],
+        "build:docs": [
+            "bin/docs.php dsl:dump web/landing/resources/dsl.json"
+        ],
         "pre-autoload-dump": [
             "Google\\Task\\Composer::cleanup"
         ],

examples/topics/data_frame/data_frame/code.php

@@ -5,9 +5,13 @@
 use function Flow\ETL\DSL\{array_entry, array_expand, data_frame, from_rows, int_entry, ref, row, rows, to_output, to_stream};
 
 data_frame()
-    ->read(from_rows(rows(
-        row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])),
-    )))
+    ->read(
+        from_rows(
+            rows(
+                row(int_entry('id', 1), array_entry('array', ['a' => 1, 'b' => 2, 'c' => 3])),
+            )
+        )
+    )
     ->write(to_output(false))
     ->withEntry('expanded', array_expand(ref('array')))
     ->write(to_stream(__DIR__ . '/output.txt', truncate: false))

examples/topics/data_frame/data_frame/description.md

@@ -0,0 +1 @@
+Simple example of reading from rows and writing to stdout.

src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php

@@ -7,12 +7,13 @@
 use function Flow\ETL\DSL\from_all;
 use Flow\ETL\Adapter\CSV\Detector\{Option, Options};
 use Flow\ETL\Row\Schema;
-use Flow\ETL\{Extractor, Loader};
+use Flow\ETL\{Attribute\DSL, Attribute\Module, Attribute\Type as DSLType, Extractor, Loader};
 use Flow\Filesystem\{Path, SourceStream};
 
 /**
  * @param int<1, max> $characters_read_in_line
  */
+#[DSL(module: Module::CSV, type: DSLType::EXTRACTOR)]
 function from_csv(
     string|Path|array $path,
     bool $with_header = true,
@@ -54,6 +55,7 @@ function from_csv(
     );
 }
 
+#[DSL(module: Module::CSV, type: DSLType::LOADER)]
 function to_csv(
     string|Path $uri,
     bool $with_header = true,
@@ -80,6 +82,7 @@ function to_csv(
  * @param null|Option $fallback - fallback option to use when no best option can be detected, default is Option(',', '"', '\\')
  * @param null|Options $options - options to use for detection, default is Options::all()
  */
+#[DSL(module: Module::CSV, type: DSLType::HELPER)]
 function csv_detect_separator(SourceStream $stream, int $lines = 5, ?Option $fallback = new Option(',', '"', '\\'), ?Options $options = null) : Option
 {
     return (new CSVDetector($stream, $fallback, $options))->detect($lines);