From 4ef5dbcf0afc4ed594a65bdebe89ca0e75a2a991 Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Thu, 20 Dec 2018 13:17:56 +0000
Subject: [PATCH 1/3] perf: do not create new buffers

---
 src/builder/builder.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/builder/builder.js b/src/builder/builder.js
index b06987e..f50993a 100644
--- a/src/builder/builder.js
+++ b/src/builder/builder.js
@@ -110,7 +110,7 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
         if (options.progress && typeof options.progress === 'function') {
           options.progress(chunk.byteLength)
         }
-        return Buffer.from(chunk)
+        return chunk
       }),
       asyncMap((buffer, callback) => {
         if (options.rawLeaves) {

From 6a86d5511a5613df3e94c08d66e346f2266d9ee9 Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Thu, 20 Dec 2018 14:32:32 +0000
Subject: [PATCH 2/3] perf: write files in parallel chunks, use a through
 instead of a map

---
 package.json           |  1 +
 src/builder/builder.js | 82 +++++++++++++++++++++---------------------
 2 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/package.json b/package.json
index c6c3c85..9d6cf27 100644
--- a/package.json
+++ b/package.json
@@ -61,6 +61,7 @@
     "pull-batch": "^1.0.0",
     "pull-block": "^1.4.0",
     "pull-pair": "^1.1.0",
+    "pull-paramap": "^1.2.2",
     "pull-pause": "0.0.2",
     "pull-pushable": "^2.2.0",
     "pull-stream": "^3.6.9",
diff --git a/src/builder/builder.js b/src/builder/builder.js
index f50993a..9fe6667 100644
--- a/src/builder/builder.js
+++ b/src/builder/builder.js
@@ -4,12 +4,11 @@ const extend = require('deep-extend')
 const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream/pull')
 const values = require('pull-stream/sources/values')
-const asyncMap = require('pull-stream/throughs/async-map')
-const map = require('pull-stream/throughs/map')
 const collect = require('pull-stream/sinks/collect')
 const through = require('pull-through')
 const parallel = require('async/parallel')
 const waterfall = require('async/waterfall')
+const paraMap = require('pull-paramap')
 const persist = require('../utils/persist')
 const reduce = require('./reduce')
 const {
@@ -106,50 +105,53 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
     pull(
       file.content,
       chunker,
-      map(chunk => {
+      through(chunk => {
         if (options.progress && typeof options.progress === 'function') {
           options.progress(chunk.byteLength)
         }
-        return chunk
       }),
-      asyncMap((buffer, callback) => {
-        if (options.rawLeaves) {
-          return callback(null, {
-            size: buffer.length,
-            leafSize: buffer.length,
-            data: buffer
-          })
-        }
-
-        const file = new UnixFS(options.leafType, buffer)
+      paraMap((buffer, callback) => {
+        waterfall([
+          (cb) => {
+            if (options.rawLeaves) {
+              return cb(null, {
+                size: buffer.length,
+                leafSize: buffer.length,
+                data: buffer
+              })
+            }
 
-        DAGNode.create(file.marshal(), [], (err, node) => {
-          if (err) {
-            return callback(err)
+            const file = new UnixFS(options.leafType, buffer)
+
+            DAGNode.create(file.marshal(), [], (err, node) => {
+              if (err) {
+                return cb(err)
+              }
+
+              cb(null, {
+                size: node.size,
+                leafSize: file.fileSize(),
+                data: node
+              })
+            })
+          },
+          (leaf, cb) => {
+            persist(leaf.data, ipld, options, (error, results) => {
+              if (error) {
+                return cb(error)
+              }
+
+              cb(null, {
+                size: leaf.size,
+                leafSize: leaf.leafSize,
+                data: results.node,
+                multihash: results.cid.buffer,
+                path: leaf.path,
+                name: ''
+              })
+            })
           }
-
-          callback(null, {
-            size: node.size,
-            leafSize: file.fileSize(),
-            data: node
-          })
-        })
-      }),
-      asyncMap((leaf, callback) => {
-        persist(leaf.data, ipld, options, (error, results) => {
-          if (error) {
-            return callback(error)
-          }
-
-          callback(null, {
-            size: leaf.size,
-            leafSize: leaf.leafSize,
-            data: results.node,
-            multihash: results.cid.buffer,
-            path: leaf.path,
-            name: ''
-          })
-        })
+        ], callback)
       }),
       through( // mark as single node if only one single node
         function onData (data) {

From df0abfac9f8ec6b97f666d15114e2c58deb5752e Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Thu, 20 Dec 2018 15:36:07 +0000
Subject: [PATCH 3/3] fix: pull-stream/throughs/through is not pull-through

---
 src/builder/builder.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/builder/builder.js b/src/builder/builder.js
index 9fe6667..f4f9ade 100644
--- a/src/builder/builder.js
+++ b/src/builder/builder.js
@@ -5,7 +5,8 @@ const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream/pull')
 const values = require('pull-stream/sources/values')
 const collect = require('pull-stream/sinks/collect')
-const through = require('pull-through')
+const through = require('pull-stream/throughs/through')
+const pullThrough = require('pull-through')
 const parallel = require('async/parallel')
 const waterfall = require('async/waterfall')
 const paraMap = require('pull-paramap')
@@ -153,7 +154,7 @@ module.exports = function builder (createChunker, ipld, createReducer, _options)
           }
         ], callback)
       }),
-      through( // mark as single node if only one single node
+      pullThrough( // mark as single node if only one single node
         function onData (data) {
           count++
           if (previous) {