feat: add support for sharded CAR uploads (#36)

Adds a `--car` flag to `w3 up` that allows the file to be a CAR file. If
flagged, we call `uploadCAR` on the client, which skips the unixfs
encoding but still shards and registers an upload.

Also adds a couple of parameters to control the shard size and request
concurrency: `--shard-size` and `--concurrent-requests`.

Finally, this also includes a fix to `filesize` which ensures it avoids
output like `0.0MB` and also calculates correctly the value for the
given units (we were dividing by 1024 not 1000 and should have been
outputting `MiB` if that were the case).

README.md

@@ -68,8 +68,11 @@ w3 up recipies.txt
 
 Upload file(s) to web3.storage. The IPFS Content ID (CID) for your files is calculated on your machine, and sent up along with your files. web3.storage makes your content available on the IPFS network
 
-- `--no-wrap` Don't wrap input files with a directory
-- `-H, --hidden` Include paths that start with "."
+- `--no-wrap` Don't wrap input files with a directory.
+- `-H, --hidden` Include paths that start with ".".
+- `-c, --car` File is a CAR file.
+- `--shard-size` Shard uploads into CAR files of approximately this size in bytes.
+- `--concurrent-requests` Send up to this many CAR shards concurrently.
 
 ### `w3 ls`
 

bin.js

@@ -34,6 +34,9 @@ cli.command('up <file>')
   .describe('Store a file(s) to the service and register an upload.')
   .option('--no-wrap', 'Don\'t wrap input files with a directory.', false)
   .option('-H, --hidden', 'Include paths that start with ".".')
+  .option('-c, --car', 'File is a CAR file.', false)
+  .option('--shard-size', 'Shard uploads into CAR files of approximately this size in bytes.')
+  .option('--concurrent-requests', 'Send up to this many CAR shards concurrently.')
   .action(upload)
 
 cli.command('open <cid>')

index.js

@@ -22,17 +22,28 @@ export async function upload (firstPath, opts) {
   const files = await filesFromPaths(paths, { hidden })
   const totalSize = files.reduce((total, f) => total + f.size, 0)
   spinner.stopAndPersist({ text: `${files.length} file${files.length === 1 ? '' : 's'} (${filesize(totalSize)})` })
+
+  if (opts.car && files.length > 1) {
+    console.error('Error: multiple CAR files not supported')
+    process.exit(1)
+  }
+
   spinner.start('Storing')
   /** @type {(o?: import('@web3-storage/w3up-client/src/types').UploadOptions) => Promise<import('@web3-storage/w3up-client/src/types').AnyLink>} */
-  const uploadFn = files.length === 1 && opts['no-wrap']
-    ? client.uploadFile.bind(client, files[0])
-    : client.uploadDirectory.bind(client, files)
+  const uploadFn = opts.car
+    ? client.uploadCAR.bind(client, files[0])
+    : files.length === 1 && opts['no-wrap']
+      ? client.uploadFile.bind(client, files[0])
+      : client.uploadDirectory.bind(client, files)
+
   const root = await uploadFn({
     onShardStored: ({ cid, size }) => {
       totalSent += size
       spinner.stopAndPersist({ text: cid.toString() })
       spinner.start(`Storing ${Math.round((totalSent / totalSize) * 100)}%`)
-    }
+    },
+    shardSize: opts['shard-size'] && parseInt(opts['shard-size']),
+    concurrentRequests: opts['concurrent-requests'] && parseInt(opts['concurrent-requests'])
   })
   spinner.stopAndPersist({ symbol: '⁂', text: `Stored ${files.length} file${files.length === 1 ? '' : 's'}` })
   console.log(`⁂ https://w3s.link/ipfs/${root}`)

lib.js

@@ -32,8 +32,10 @@ export function checkPathsExist (paths) {
 }
 
 export function filesize (bytes) {
-  const size = bytes / 1024 / 1024
-  return `${size.toFixed(1)}MB`
+  if (bytes < 50) return `${bytes}B` // avoid 0.0KB
+  if (bytes < 50000) return `${(bytes / 1000).toFixed(1)}KB` // avoid 0.0MB
+  if (bytes < 50000000) return `${(bytes / 1000 / 1000).toFixed(1)}MB` // avoid 0.0GB
+  return `${(bytes / 1000 / 1000 / 1000).toFixed(1)}GB`
 }
 
 /**

package.json

@@ -47,7 +47,7 @@
     "@ucanto/core": "^4.0.3",
     "@ucanto/transport": "^4.0.3",
     "@web3-storage/access": "^9.1.1",
-    "@web3-storage/w3up-client": "^4.0.1",
+    "@web3-storage/w3up-client": "^4.1.0",
     "open": "^8.4.0",
     "ora": "^6.1.2",
     "pretty-tree": "^1.0.0",

test/bin.spec.js

@@ -128,6 +128,42 @@ test('w3 up', async (t) => {
   t.regex(stderr, /Stored 1 file/)
 })
 
+test('w3 up --car', async (t) => {
+  const env = t.context.env.alice
+
+  await execa('./bin.js', ['space', 'create'], { env })
+
+  const service = mockService({
+    store: {
+      add: provide(StoreCapabilities.add, () => ({
+        status: 'upload',
+        headers: { 'x-test': 'true' },
+        url: 'http://localhost:9200'
+      }))
+    },
+    upload: {
+      add: provide(UploadCapabilities.add, ({ invocation }) => {
+        const { nb } = invocation.capabilities[0]
+        if (!nb) throw new Error('missing nb')
+        t.assert(nb.shards)
+        t.is(nb.shards[0]?.toString(), 'bagbaieracyt3l5gpf3ovcmedm6ktgvxzi6gpp7x42ffu43zrqh2qwm6q7peq')
+        return nb
+      })
+    }
+  })
+
+  t.context.setService(service)
+
+  const { stderr } = await execa('./bin.js', ['up', '--car', 'test/fixtures/pinpie.car'], { env })
+
+  t.true(service.store.add.called)
+  t.is(service.store.add.callCount, 1)
+  t.true(service.upload.add.called)
+  t.is(service.upload.add.callCount, 1)
+
+  t.regex(stderr, /Stored 1 file/)
+})
+
 test('w3 ls', async (t) => {
   const env = t.context.env.alice
 

test/lib.spec.js

@@ -1,5 +1,5 @@
 import test from 'ava'
-import { filesFromPaths } from '../lib.js'
+import { filesFromPaths, filesize } from '../lib.js'
 
 test('filesFromPaths', async (t) => {
   const files = await filesFromPaths(['node_modules'])
@@ -26,3 +26,18 @@ test('filesFromPaths single file has name', async (t) => {
   t.is(files.length, 1)
   t.is(files[0].name, 'empty.car')
 })
+
+test('filesize', t => {
+  [
+    [5, '5B'],
+    [50, '0.1KB'],
+    [500, '0.5KB'],
+    [5_000, '5.0KB'],
+    [50_000, '0.1MB'],
+    [500_000, '0.5MB'],
+    [5_000_000, '5.0MB'],
+    [50_000_000, '0.1GB'],
+    [500_000_000, '0.5GB'],
+    [5_000_000_000, '5.0GB']
+  ].forEach(([size, str]) => t.is(filesize(size), str))
+})