Merge pull request apache#57 from Shopify/do_not_shrink_shuffle_batch…

…_size Do not shrink shuffle batch size
JasonMWhite · Jun 9, 2015 · 044b478 · 044b478
2 parents b68a83b + 85db5cc
commit 044b478
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py
@@ -513,7 +513,6 @@ def load(f):
                 chunks.append(load(open(path, 'rb')))
                 current_chunk = []
                 gc.collect()
-                batch //= 2
                 limit = self._next_limit()
                 MemoryBytesSpilled += max(used_memory - get_used_memory(), 0) << 20
                 DiskBytesSpilled += os.path.getsize(path)