web3-storage · olizilla · Jul 2, 2021 · Jun 30, 2021 · Jul 1, 2021 · Jul 1, 2021
@@ -0,0 +1,5 @@
+node_modules
+.DS_Store
+dist
+dist-ssr
+*.local
@@ -0,0 +1,18 @@
+# Files demo browser - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
+
+A demo using web3.storage client in the browser to pre-calculate the CID for an asset then storing it on tbd.storage and confirming that it uses the exact same CID for the asset.
+
+## Getting started
+
+```console
+npm install
+npm run dev
+
+# or
+yarn
+yarn dev
+```
+
+Then visit `http://localhost:3000?key=<your web3.storage API KEY here>`
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>CAR upload - web3.storage</title>
+  </head>
+  <body>
+    <pre id="out"></pre>
+    <script type="module" src="/main.js"></script>
+  </body>
+</html>
@@ -0,0 +1,50 @@
+import { Web3Storage, Web3File } from 'web3.storage'
+
+const endpoint = 'https://api.web3.storage' // the default
+const token =
+  new URLSearchParams(window.location.search).get('key') || 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token })
+
+  const files = prepareFiles()
+
+  // send the files to web3.storage
+  const cid = await storage.put(files)
+
+  // TODO
+  console.log('added', cid)
+  // check that the CID is pinned
+  // const status = await store.status(cid)
+  // log(status)
+}
+
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromText(
+      data,
+      'data.txt',
+      { path: '/dir/data.txt' }
+    ),
+    Web3File.fromText(
+      data2,
+      'data2.txt',
+      { path: '/dir/data2.txt' }
+    ),
+    Web3File.fromText(
+      data,
+      'data.txt',
+      { path: '/dir/otherdir/data.txt' }
+    ),
+    Web3File.fromText(
+      data2,
+      'data2.txt',
+      { path: '/dir/otherdir/data2.txt' }
+    )
+  ]
+}
+
+main()
@@ -0,0 +1,14 @@
+{
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "serve": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.3.7"
+  },
+  "dependencies": {
+    "web3.storage": "../../"
+  }
+}
@@ -0,0 +1,3 @@
+# Files demo Node.js - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
@@ -0,0 +1,45 @@
+import { Web3Storage, Web3File } from '../../src/lib.js'
+
+// TODO
+const endpoint = 'https://api.web3.storage' // the default
+const token = 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token }) 
+
+  const files = prepareFiles()
+  const cid = await storage.put(files)
+
+  console.log('added', cid)
+}
+
+// TODO: Read a fixtures folder instead
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromText(
+      data,
+      'data.txt',
+      { path: '/dir/data.txt' }
+    ),
+    Web3File.fromText(
+      data2,
+      'data2.txt',
+      { path: '/dir/data2.txt' }
+    ),
+    Web3File.fromText(
+      data,
+      'data.txt',
+      { path: '/dir/otherdir/data.txt' }
+    ),
+    Web3File.fromText(
+      data2,
+      'data2.txt',
+      { path: '/dir/otherdir/data2.txt' }
+    )
+  ]
+}
+
+main()
@@ -0,0 +1,12 @@
+{
+  "name": "web3-storage-examples",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Examples of using web3.storage in Node.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Vasco Santos",
+  "license": "(Apache-2.0 AND MIT)"
+}
@@ -32,8 +32,13 @@
     "@ipld/car": "^3.1.2",
     "@web-std/blob": "^2.1.0",
     "@web-std/fetch": "^2.0.1",
+    "@web-std/file": "^1.1.0",
     "browser-readablestream-to-it": "^1.0.2",
-    "ipfs-car": "^0.3.5"
+    "carbites": "^1.0.6",
+    "ipfs-car": "^0.4.1",
+    "p-retry": "^4.5.0",
+    "streaming-iterables": "^6.0.0",
+    "web3-file": "^0.2.0"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^19.0.0",

@@ -13,12 +13,25 @@
  * ```
  * @module
  */
+import { transform } from 'streaming-iterables'
+import pRetry from 'p-retry'
+import { pack } from 'ipfs-car/pack'
+import { TreewalkCarSplitter } from 'carbites/treewalk'
+import { Web3File } from 'web3-file'
 import * as API from './lib/interface.js'
-import { fetch, Blob } from './platform.js'
+import {
+  fetch,
+  Blob,
+  Blockstore
+} from './platform.js'
 import { CarReader } from '@ipld/car/reader'
 import { unpack } from 'ipfs-car/unpack'
 import toIterable from 'browser-readablestream-to-it'
 
+const MAX_PUT_RETRIES = 5
+const MAX_CONCURRENT_UPLOADS = 3
+const MAX_CHUNK_SIZE = 1024 * 1024 * 10 // chunk to ~10MB CARs
+
 /**
  * @implements API.Service
  */
@@ -75,33 +88,69 @@ class Web3Storage {
 
   /**
    * @param {API.Service} service
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {API.PutOptions} [options]
    * @returns {Promise<API.CIDString>}
    */
-  static async store({ endpoint, token }, blob) {
+  static async put({ endpoint, token }, files, { onStoredChunk, maxRetries = MAX_PUT_RETRIES } = {}) {
     const url = new URL(`/car`, endpoint)
+    const headers = Web3Storage.headers(token)
+    const targetSize = MAX_CHUNK_SIZE
 
-    if (blob.size === 0) {
-      throw new Error('Content size is 0, make sure to provide some content')
-    }
+    let root
+    const blockstore = new Blockstore()
 
-    const car =
-      blob.type !== 'application/car'
-        ? blob.slice(0, blob.size, 'application/car')
-        : blob
+    try {
+      const { out } = await pack({
+        input: files,
+        blockstore
+      })
+      const splitter = await TreewalkCarSplitter.fromIterable(out, targetSize)
 
-    const request = await fetch(url.toString(), {
-      method: 'POST',
-      headers: Web3Storage.headers(token),
-      body: car,
-    })
-    const result = await request.json()
+      const upload = transform(
+        MAX_CONCURRENT_UPLOADS,
+        async (/** @type {AsyncIterable<Uint8Array>} */ car) => {
+          const carParts = []
+          for await (const part of car) {
+            carParts.push(part)
+          }
+
+          const carFile = new Blob(carParts, {
+            type: 'application/car',
+          })
+
+          const res = await pRetry(
+            async () => {
+              const request = await fetch(url.toString(), {
+                method: 'POST',
+                headers,
+                body: carFile,
+              })
+              const result = await request.json()
 
-    if (result.ok) {
-      return result.value.cid
-    } else {
-      throw new Error(result.error.message)
+              if (result.ok) {
+                return result.value.cid
+              } else {
+                throw new Error(result.error.message)
+              }
+            },
+            { retries: maxRetries }
+          )
+          onStoredChunk && onStoredChunk(carFile.size)
+          return res
+        }
+      )
+
+      for await (const cid of upload(splitter.cars())) {
+        root = cid
+      }
+    } finally {
+      // Destroy Blockstore
+      await blockstore.destroy()
     }
+
+    // @ts-ignore there will always be a root, or carbites will fail
+    return root
   }
 
   /**
@@ -130,9 +179,8 @@ class Web3Storage {
   // Just a sugar so you don't have to pass around endpoint and token around.
 
   /**
-   * Stores files encoded as a single [Content Addressed Archive
-   * (CAR)](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md).
-   *
+   * Uploads files to web3.storage. Files are hashed in the client and uploaded as a single 
+   * [Content Addressed Archive(CAR)](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md).
    * Takes a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob)
    *
    * Returns the corresponding Content Identifier (CID).
@@ -144,10 +192,11 @@ class Web3Storage {
    * const cid = await client.store(car)
    * console.assert(cid === root)
    * ```
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {API.PutOptions} [options]
    */
-  store(blob) {
-    return Web3Storage.store(this, blob)
+  put(files, options) {
+    return Web3Storage.put(this, files, options)
   }
 
   /**
@@ -232,7 +281,7 @@ function toCarResponse(res) {
   return response
 }
 
-export { Web3Storage, Blob }
+export { Web3Storage, Blob, Web3File }
 
 /**
  * Just to verify API compatibility.

@@ -1,6 +1,8 @@
 import type { UnixFSEntry } from 'ipfs-car/unpack'
 import type { CID } from 'multiformats'
-export type { CID , UnixFSEntry }
+import type { Web3File } from 'web3-file'
+export type { CID, UnixFSEntry, Web3File }
+
 
 /**
  * Define nominal type of U based on type of T. Similar to Opaque types in Flow
@@ -23,15 +25,25 @@ export type CIDString = Tagged<string, CID>
 
 export interface API {
   /**
-   * Stores a single file and returns a corresponding CID.
+   * Stores files and returns a corresponding CID.
    */
-  store(service: Service, content: Blob | File): Promise<CIDString>
+  put(
+    service: Service,
+    files: Iterable<Web3File>,
+    options?: PutOptions
+  ): Promise<CIDString>
+
   /**
    * Get files for a root CID packed as a CAR file
    */
   get(service: Service, cid: CIDString): Promise<CarResponse | null>
 }
 
+export type PutOptions = {
+  onStoredChunk?: (size: number) => void,
+  maxRetries?: number
+}
+
 export interface IpfsFile extends File {
   cid: CIDString,  
 }

@@ -1,4 +1,14 @@
 import fetch, { Request, Response, Headers } from '@web-std/fetch'
 import { Blob } from '@web-std/blob'
+import { File } from '@web-std/file'
+import { FsBlockStore as Blockstore } from 'ipfs-car/blockstore/fs'
 
-export { fetch, Request, Response, Headers, Blob }
+export {
+  fetch,
+  Request,
+  Response,
+  Headers,
+  Blob,
+  File,
+  Blockstore
+}
@@ -1,2 +1,6 @@
+import { MemoryBlockStore } from 'ipfs-car/blockstore/memory'
+
 export const fetch = globalThis.fetch
 export const Blob = globalThis.Blob
+export const File = globalThis.File
+export const Blockstore = MemoryBlockStore