Preallocate list capacity when serializing (to prevent later resizing)

aloneguid · Jan 5, 2024 · 643307d · 643307d
1 parent 50ccd64
commit 643307d
Showing 1 changed file with 21 additions and 6 deletions.
diff --git a/src/Parquet/Serialization/ParquetSerializer.cs b/src/Parquet/Serialization/ParquetSerializer.cs
@@ -177,8 +177,9 @@ public static async Task<IList<T>> DeserializeAsync<T>(Stream source,
 
             Assembler<T> asm = GetAssembler<T>();
 
-            var result = new List<T>();
             using ParquetReader reader = await ParquetReader.CreateAsync(source, options, cancellationToken: cancellationToken);
+
+            List<T> result = GetList<T>(reader.Metadata?.RowGroups[rowGroupIndex].NumRows);
             await DeserializeRowGroupAsync(reader, rowGroupIndex, asm, result, cancellationToken);
 
             return result;
@@ -200,9 +201,11 @@ public static async Task<IList<T>> DeserializeAsync<T>(Stream source,
 
             Assembler<T> asm = GetAssembler<T>();
 
-            var result = new List<T>();
-
             using ParquetReader reader = await ParquetReader.CreateAsync(source, options, cancellationToken: cancellationToken);
+
+            long? requestedCapacity = reader.Metadata?.RowGroups.Sum(x => x.NumRows);
+            List<T> result = GetList<T>(requestedCapacity);
+
             for(int rgi = 0; rgi < reader.RowGroupCount; rgi++) {
 
                 await DeserializeRowGroupAsync(reader, rgi, asm, result, cancellationToken);
@@ -253,9 +256,11 @@ public static async IAsyncEnumerable<T> DeserializeAllAsync<T>(Stream source,
 
             Assembler<T> asm = GetAssembler<T>();
 
-            var result = new List<T>();
-
             using ParquetReader reader = await ParquetReader.CreateAsync(source, options, cancellationToken: cancellationToken);
+
+            long? requestedCapacity = reader.Metadata?.RowGroups.Max(x => x.NumRows);
+            List<T> result = GetList<T>(requestedCapacity);
+
             for(int rgi = 0; rgi < reader.RowGroupCount; rgi++) {
 
                 await DeserializeRowGroupAsync(reader, rgi, asm, result, cancellationToken);
@@ -284,7 +289,7 @@ public static async Task<IList<T>> DeserializeAsync<T>(ParquetRowGroupReader row
 
             Assembler<T> asm = GetAssembler<T>();
 
-            var result = new List<T>();
+            List<T> result = GetList<T>(rowGroupReader.RowGroup.NumRows);
 
             await DeserializeRowGroupAsync(rowGroupReader, schema, asm, result, cancellationToken);
 
@@ -354,5 +359,15 @@ private static async Task DeserializeRowGroupAsync<T>(ParquetRowGroupReader rg,
                 }
             }
         }
+
+        private static List<T> GetList<T>(long? requestedCapacity) {
+            if(requestedCapacity == null)
+                return new List<T>();
+
+            if(requestedCapacity >= int.MaxValue)
+                return new List<T>(int.MaxValue);
+
+            return new List<T>((int)requestedCapacity);
+        }
     }
 }