diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py index 81c420ce16541..a046a22d070a4 100644 --- a/python/pyspark/shuffle.py +++ b/python/pyspark/shuffle.py @@ -513,7 +513,6 @@ def load(f): chunks.append(load(open(path, 'rb'))) current_chunk = [] gc.collect() - batch //= 2 limit = self._next_limit() MemoryBytesSpilled += max(used_memory - get_used_memory(), 0) << 20 DiskBytesSpilled += os.path.getsize(path)