Skip to content

Commit

Permalink
fix: tests in CI (#1034)
Browse files Browse the repository at this point in the history 
Node.js v16.14.0 included [a fix](nodejs/node#40706) that meant that `@web-std/blob` started using the Node.js native version.

Bad news, Node.js currently copies the buffer on every iteration when obtaining a stream from `File.stream()`. It also has a fixed and small chunk size of `65536` bytes. This makes reading the stream VERY slow and this test fails because it times out.

I opened an issue about this here: nodejs/node#42108

Once the test was fixed, the cloudflare build for the website started failing so I had to update next.js to v12. WTF!?!

After that, the client build started failing with:

```
Error: Build failed with 2 errors:
../../node_modules/parse-link-header/index.js:3:17: error: Could not resolve "querystring" (use "platform: 'node'" when building for node)
../../node_modules/parse-link-header/index.js:4:18: error: Could not resolve "url" (use "platform: 'node'" when building for node)
    at failureErrorWithLog (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1493:15)
    at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1151:28
    at runOnEndCallbacks (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:941:63)
    at buildResponseToResult (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1149:7)
    at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:1258:14
    at /Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:629:9
    at handleIncomingPacket (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:726:9)
    at Socket.readFromStdout (/Users/alan/Code/web3-storage/web3.storage/node_modules/esbuild/lib/main.js:596:7)
    at Socket.emit (node:events:520:28)
    at addChunk (node:internal/streams/readable:315:12) {
  errors: [
    {
      detail: undefined,
      location: [Object],
      notes: [],
      pluginName: '',
      text: `Could not resolve "querystring" (use "platform: 'node'" when building for node)`
    },
    {
      detail: undefined,
      location: [Object],
      notes: [],
      pluginName: '',
      text: `Could not resolve "url" (use "platform: 'node'" when building for node)`
    }
  ],
  warnings: []
}
```
So I had to roll the update to `parse-link-header` from #1032 into here as well.
  • Loading branch information
Alan Shaw authored Feb 24, 2022
1 parent 0e5d0f8 commit ad26c27
Show file tree
Hide file tree
Showing 7 changed files with 13,473 additions and 8,042 deletions.
21,447 changes: 13,423 additions & 8,024 deletions package-lock.json
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions packages/client/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,18 @@
},
"dependencies": {
"@ipld/car": "^3.1.4",
"@web-std/blob": "^3.0.1",
"@web-std/fetch": "^3.0.0",
"@web-std/file": "^3.0.0",
"@web-std/blob": "^3.0.3",
"@web-std/fetch": "^3.0.3",
"@web-std/file": "^3.0.2",
"@web3-storage/parse-link-header": "^3.1.0",
"browser-readablestream-to-it": "^1.0.3",
"carbites": "^1.0.6",
"cborg": "^1.6.0",
"files-from-path": "^0.2.3",
"ipfs-car": "^0.6.1",
"ipfs-car": "^0.6.2",
"ipns": "^0.16.0",
"libp2p-crypto": "^0.21.0",
"p-retry": "^4.5.0",
"parse-link-header": "^2.0.0",
"streaming-iterables": "^6.0.0",
"uint8arrays": "^3.0.0"
},
Expand All @@ -52,7 +52,6 @@
"@rollup/plugin-json": "^4.1.0",
"@rollup/plugin-node-resolve": "^13.1.3",
"@types/mocha": "9.0.0",
"@types/parse-link-header": "^1.0.1",
"bundlesize": "^0.18.1",
"cors": "^2.8.5",
"del-cli": "^4.0.0",
Expand All @@ -62,6 +61,7 @@
"npm-run-all": "^4.1.5",
"nyc": "15.1.0",
"playwright-test": "^7.2.2",
"randombytes": "^2.1.0",
"rollup": "2.63.0",
"rollup-plugin-multi-input": "1.3.1",
"rollup-plugin-terser": "^7.0.2",
Expand Down
6 changes: 3 additions & 3 deletions packages/client/src/lib.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import { transform } from 'streaming-iterables'
import pRetry from 'p-retry'
import { pack } from 'ipfs-car/pack'
import parseLink from 'parse-link-header'
import { parseLinkHeader } from '@web3-storage/parse-link-header'
import { unpackStream } from 'ipfs-car/unpack'
import { TreewalkCarSplitter } from 'carbites/treewalk'
import { CarReader } from '@ipld/car'
Expand Down Expand Up @@ -469,13 +469,13 @@ function toWeb3Response (res) {
async function * paginator (fn, service, opts) {
let res = await fn(service, opts)
yield res
let link = parseLink(res.headers.get('Link') || '')
let link = parseLinkHeader(res.headers.get('Link') || '')
// @ts-ignore
while (link && link.next) {
// @ts-ignore
res = await fn(service, link.next)
yield res
link = parseLink(res.headers.get('Link') || '')
link = parseLinkHeader(res.headers.get('Link') || '')
}
}

Expand Down
24 changes: 22 additions & 2 deletions packages/client/test/put.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { CID } from 'multiformats/cid'
import { encode } from 'multiformats/block'
import * as json from '@ipld/dag-json'
import { sha256 } from 'multiformats/hashes/sha2'
import { ReadableStream } from '@web-std/blob'

describe('put', () => {
const { AUTH_TOKEN, API_PORT } = process.env
Expand Down Expand Up @@ -85,12 +86,31 @@ describe('put', () => {
})

it('adds big files', async function () {
this.timeout(30e3)
this.timeout(60e3)
const client = new Web3Storage({ token, endpoint })
let uploadedChunks = 0

const files = [
new File([randomBytes(1024e6)], '102mb.txt')
// Previously: new File([randomBytes(1024e6)], '102mb.txt')
//
// Node.js currently copies the buffer on every iteration when obtaining a
// stream from File.stream(). It also has a fixed and small chunk size of
// 65536 bytes. This makes reading the stream VERY slow and this test
// fails because it times out.
//
// TODO: revert to using File if this issue gets resolved:
// https://github.com/nodejs/node/issues/42108
{
name: '102mb.txt',
stream () {
return new ReadableStream({
pull (controller) {
controller.enqueue(randomBytes(1024e6))
controller.close()
}
})
}
}
]

await client.put(files, {
Expand Down
1 change: 0 additions & 1 deletion packages/website/next-env.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
/// <reference types="next" />
/// <reference types="next/types/global" />
/// <reference types="next/image-types/global" />

// NOTE: This file should not be edited
Expand Down
2 changes: 1 addition & 1 deletion packages/website/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"filesize": "^6.1.0",
"gray-matter": "^4.0.3",
"magic-sdk": "^4.2.1",
"next": "^11.0.1",
"next": "^12.1.0",
"p-map": "^5.1.0",
"pretty-bytes": "^5.6.0",
"rc-tooltip": "^5.1.1",
Expand Down
23 changes: 18 additions & 5 deletions packages/website/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
"allowJs": true,
"checkJs": true,
"target": "esnext",
"lib": ["ESNext", "DOM"],
"lib": [
"ESNext",
"DOM"
],
"strict": true,
"moduleResolution": "node",
"sourceMap": true,
Expand All @@ -17,11 +20,21 @@
"resolveJsonModule": true,
"isolatedModules": true,
"paths": {
"web3.storage": ["../client"]
}
"web3.storage": [
"../client"
]
},
"incremental": true
},
"include": ["lib", "pages", "components"],
"exclude": ["node_modules/", ".next/"],
"include": [
"lib",
"pages",
"components"
],
"exclude": [
"node_modules/",
".next/"
],
"references": [
{
"path": "../client"
Expand Down

0 comments on commit ad26c27

Please sign in to comment.