feat: add file

web3-storage · Jul 1, 2021 · 3e5bb9d · 3e5bb9d
1 parent 9eae4a4
commit 3e5bb9d
Show file tree

Hide file tree

Showing 19 changed files with 1,159 additions and 738 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/client/examples/browser/.gitignore b/packages/client/examples/browser/.gitignore
@@ -0,0 +1,5 @@
+node_modules
+.DS_Store
+dist
+dist-ssr
+*.local
diff --git a/packages/client/examples/browser/README.md b/packages/client/examples/browser/README.md
@@ -0,0 +1,18 @@
+# Files demo browser - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
+
+A demo using web3.storage client in the browser to pre-calculate the CID for an asset then storing it on tbd.storage and confirming that it uses the exact same CID for the asset.
+
+## Getting started
+
+```console
+npm install
+npm run dev
+
+# or
+yarn
+yarn dev
+```
+
+Then visit `http://localhost:3000?key=<your web3.storage API KEY here>`
diff --git a/packages/client/examples/browser/index.html b/packages/client/examples/browser/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>CAR upload - nft.storage</title>
+  </head>
+  <body>
+    <pre id="out"></pre>
+    <script type="module" src="/main.js"></script>
+  </body>
+</html>
diff --git a/packages/client/examples/browser/main.js b/packages/client/examples/browser/main.js
@@ -0,0 +1,51 @@
+import { Web3Storage } from 'web3.storage'
+import { Web3File } from 'web3-file'
+
+const endpoint = 'https://api.web3.storage' // the default
+const token =
+  new URLSearchParams(window.location.search).get('key') || 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token })
+
+  const files = prepareFiles()
+
+  // send the files to tbd.storage
+  const cid = await storage.put(files)
+
+  // TODO
+  console.log('added', cid)
+  // check that the CID is pinned
+  // const status = await store.status(cid)
+  // log(status)
+}
+
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/data2.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/dir/data2.zip' }
+    )
+  ]
+}
+
+main()
diff --git a/packages/client/examples/browser/package.json b/packages/client/examples/browser/package.json
@@ -0,0 +1,14 @@
+{
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "serve": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.3.7"
+  },
+  "dependencies": {
+    "web3.storage": "../../"
+  }
+}
diff --git a/packages/client/examples/node.js/README.md b/packages/client/examples/node.js/README.md
@@ -0,0 +1,3 @@
+# Files demo Node.js - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
diff --git a/packages/client/examples/node.js/files.js b/packages/client/examples/node.js/files.js
@@ -0,0 +1,46 @@
+import { Web3Storage } from '../../src/lib.js'
+import { Web3File } from 'web3-file'
+
+// TODO
+const endpoint = 'https://api.web3.storage' // the default
+const token = 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token }) 
+
+  const files = prepareFiles()
+  const cid = await storage.put(files)
+
+  console.log('added', cid)
+}
+
+// TODO: Read a fixstures folder instead
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/data2.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/dir/data2.zip' }
+    )
+  ]
+}
+
+main()
diff --git a/packages/client/examples/node.js/package.json b/packages/client/examples/node.js/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "filecoin-storage-examples",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Examples of using filecoin.storage in Node.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Vasco Santos",
+  "license": "(Apache-2.0 AND MIT)"
+}
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -32,8 +32,13 @@
     "@ipld/car": "^3.1.2",
     "@web-std/blob": "^2.1.0",
     "@web-std/fetch": "^2.0.1",
+    "@web-std/file": "^1.1.0",
     "browser-readablestream-to-it": "^1.0.2",
-    "ipfs-car": "^0.3.5"
+    "carbites": "^1.0.6",
+    "ipfs-car": "^0.3.5",
+    "p-retry": "^4.5.0",
+    "streaming-iterables": "^6.0.0",
+    "web3-file": "^0.1.0"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^19.0.0",

diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js
@@ -13,12 +13,24 @@
  * ```
  * @module
  */
+import { transform } from 'streaming-iterables'
+import pRetry from 'p-retry'
+import { pack } from 'ipfs-car/pack'
+import { TreewalkCarSplitter } from 'carbites/treewalk'
 import * as API from './lib/interface.js'
-import { fetch, Blob } from './platform.js'
+import {
+  fetch,
+  Blob,
+  Blockstore
+} from './platform.js'
 import { CarReader } from '@ipld/car/reader'
 import { unpack } from 'ipfs-car/unpack'
 import toIterable from 'browser-readablestream-to-it'
 
+const MAX_ADD_RETRIES = 5
+const MAX_CONCURRENT_UPLOADS = 3
+const MAX_CHUNK_SIZE = 1024 * 1024 * 10 // chunk to ~10MB CARs
+
 /**
  * @implements API.Service
  */
@@ -75,33 +87,66 @@ class Web3Storage {
 
   /**
    * @param {API.Service} service
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {{onStoredChunk?: (size: number) => void}} [options]
    * @returns {Promise<API.CIDString>}
    */
-  static async store({ endpoint, token }, blob) {
+  static async put({ endpoint, token }, files, { onStoredChunk } = {}) {
     const url = new URL(`/car`, endpoint)
+    const headers = Web3Storage.headers(token)
+    const targetSize = MAX_CHUNK_SIZE
 
-    if (blob.size === 0) {
-      throw new Error('Content size is 0, make sure to provide some content')
-    }
+    const blockstore = new Blockstore()
+    const { out } = await pack({
+      input: files,
+      blockstore
+    })
+    const splitter = await TreewalkCarSplitter.fromIterable(out, targetSize)
 
-    const car =
-      blob.type !== 'application/car'
-        ? blob.slice(0, blob.size, 'application/car')
-        : blob
+    const upload = transform(
+      MAX_CONCURRENT_UPLOADS,
+      async (/** @type {AsyncIterable<Uint8Array>} */ car) => {
+        const carParts = []
+        for await (const part of car) {
+          carParts.push(part)
+        }
 
-    const request = await fetch(url.toString(), {
-      method: 'POST',
-      headers: Web3Storage.headers(token),
-      body: car,
-    })
-    const result = await request.json()
+        const carFile = new Blob(carParts, {
+          type: 'application/car',
+        })
 
-    if (result.ok) {
-      return result.value.cid
-    } else {
-      throw new Error(result.error.message)
+        const res = await pRetry(
+          async () => {
+            const request = await fetch(url.toString(), {
+              method: 'POST',
+              headers,
+              body: carFile,
+            })
+            const result = await request.json()
+
+            if (result.ok) {
+              return result.value.cid
+            } else {
+              throw new Error(result.error.message)
+            }
+          },
+          { retries: MAX_ADD_RETRIES }
+        )
+        onStoredChunk && onStoredChunk(carFile.size)
+        return res
+      }
+    )
+
+    let root
+    for await (const cid of upload(splitter.cars())) {
+      root = cid
     }
+
+    // Destroy Blockstore
+    await blockstore.destroy()
+
+    // @ts-ignore there will always be a root, or carbites will fail
+    return root
   }
 
   /**
@@ -130,9 +175,7 @@ class Web3Storage {
   // Just a sugar so you don't have to pass around endpoint and token around.
 
   /**
-   * Stores files encoded as a single [Content Addressed Archive
-   * (CAR)](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md).
-   *
+   * Puts files.
    * Takes a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob)
    *
    * Returns the corresponding Content Identifier (CID).
@@ -144,10 +187,11 @@ class Web3Storage {
    * const cid = await client.store(car)
    * console.assert(cid === root)
    * ```
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {{onStoredChunk?: (size: number) => void}} [options]
    */
-  store(blob) {
-    return Web3Storage.store(this, blob)
+  put(files, options) {
+    return Web3Storage.put(this, files, options)
   }
 
   /**

diff --git a/packages/client/src/lib/interface.ts b/packages/client/src/lib/interface.ts
@@ -1,6 +1,8 @@
 import type { UnixFSEntry } from 'ipfs-car/unpack'
 import type { CID } from 'multiformats'
-export type { CID , UnixFSEntry }
+import type { Web3File } from 'web3-file'
+export type { CID , UnixFSEntry, Web3File }
+
 
 /**
  * Define nominal type of U based on type of T. Similar to Opaque types in Flow
@@ -23,9 +25,14 @@ export type CIDString = Tagged<string, CID>
 
 export interface API {
   /**
-   * Stores a single file and returns a corresponding CID.
+   * Stores files and returns a corresponding CID.
    */
-  store(service: Service, content: Blob | File): Promise<CIDString>
+  put(
+    service: Service,
+    files: Iterable<Web3File>,
+    options?: { onStoredChunk?: (size: number) => void }
+  ): Promise<CIDString>
+
   /**
    * Get files for a root CID packed as a CAR file
    */

diff --git a/packages/client/src/platform.js b/packages/client/src/platform.js
@@ -1,4 +1,14 @@
 import fetch, { Request, Response, Headers } from '@web-std/fetch'
 import { Blob } from '@web-std/blob'
+import { File } from '@web-std/file'
+import { FsBlockStore as Blockstore } from 'ipfs-car/blockstore/fs'
 
-export { fetch, Request, Response, Headers, Blob }
+export {
+  fetch,
+  Request,
+  Response,
+  Headers,
+  Blob,
+  File,
+  Blockstore
+}
diff --git a/packages/client/src/platform.ts b/packages/client/src/platform.ts
@@ -1,2 +1,6 @@
+import { MemoryBlockStore } from 'ipfs-car/blockstore/memory'
+
 export const fetch = globalThis.fetch
 export const Blob = globalThis.Blob
+export const File = globalThis.File
+export const Blockstore = MemoryBlockStore
diff --git a/packages/client/src/platform.web.js b/packages/client/src/platform.web.js
@@ -1,5 +1,9 @@
+// TODO: Use indexedDb
+import { MemoryBlockStore } from 'ipfs-car/blockstore/memory'
+
 export const fetch = globalThis.fetch
 export const Request = globalThis.Request
 export const Response = globalThis.Response
 export const Blob = globalThis.Blob
 export const File = globalThis.File
+export const Blockstore = MemoryBlockStore
diff --git a/packages/client/test/interface.spec.js b/packages/client/test/interface.spec.js
@@ -6,6 +6,6 @@ describe('interface', () => {
     assert.equal(typeof Web3Storage, 'function')
     const client = new Web3Storage({ token: 'secret' })
     assert.ok(client instanceof Web3Storage)
-    assert.equal(typeof client.store, 'function')
+    assert.equal(typeof client.put, 'function')
   })
 })