Dremel & Parquet performance improvements (#862)

flow-php · Dec 3, 2023 · 085d8b3 · 085d8b3
1 parent 587ad5b
commit 085d8b3
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 12 deletions.
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/ParquetLoader.php
@@ -72,7 +72,7 @@ public function destination() : Path
 
     public function load(Rows $rows, FlowContext $context) : void
     {
-        if ($this->schema === null) {
+        if ($this->schema === null && $this->inferredSchema === null) {
             $this->inferSchema($rows);
         }
 

diff --git a/src/lib/dremel/src/Flow/Dremel/Dremel.php b/src/lib/dremel/src/Flow/Dremel/Dremel.php
@@ -75,7 +75,12 @@ public function shred(array $data, int $maxDefinitionLevel) : DataShredded
     {
         $definitions = [];
         $this->buildDefinitions($data, $definitions, $maxDefinitionLevel);
-        $repetitions = $this->buildRepetitions($data);
+        $repetitions = [];
+        $this->buildRepetitions($data, 0, 0, $repetitions);
+
+        if (!\count($repetitions) || \max($repetitions) === 0) {
+            $repetitions = [];
+        }
 
         return new DataShredded(
             $repetitions,
@@ -126,10 +131,8 @@ private function buildDefinitions(array $data, array &$definitions, int $maxDefi
         }
     }
 
-    private function buildRepetitions(array $data, int $currentLevel = 0, int $topIndex = 0) : array
+    private function buildRepetitions(array $data, int $currentLevel, int $topIndex, array &$output) : void
     {
-        $output = [];
-
         foreach ($data as $index => $item) {
             if (\is_array($item)) {
 
@@ -139,7 +142,12 @@ private function buildRepetitions(array $data, int $currentLevel = 0, int $topIn
                     continue;
                 }
 
-                $output = \array_merge($output, $this->buildRepetitions($item, $currentLevel + 1, $index));
+                $childRepetitions = [];
+                $this->buildRepetitions($item, $currentLevel + 1, $index, $childRepetitions);
+
+                foreach ($childRepetitions as $repetition) {
+                    $output[] = $repetition;
+                }
             } else {
                 if (!\count($output)) {
                     $output[] = $topIndex === 0 ? 0 : $currentLevel - 1;
@@ -148,12 +156,6 @@ private function buildRepetitions(array $data, int $currentLevel = 0, int $topIn
                 }
             }
         }
-
-        if (!\count($output) || \max($output) === 0) {
-            return [];
-        }
-
-        return $output;
     }
 
     private function value(int $definition, int $maxDefinitionLevel, array $values, int &$valueIndex) : mixed