Upload Directory that is large returns Error: Request body larger than maxBodyLength limit

[ltfschoen]

I tried using the script node ./scripts/siaSkynet.js in this repo https://github.com/ltfschoen/ethquad that I created to upload a build directory that contained a p5.js game that was ~16MB to Skynet. I basically built the game from my other repo https://github.com/ltfschoen/flappytips with yarn; yarn run build, and then copied the contents of the build/ directory across and into the client/buid/ directory of https://github.com/ltfschoen/ethquad so my script would deploy it.

But it generated error Error: Request body larger than maxBodyLength limit. See below for full log.

I found this potential solution https://stackoverflow.com/questions/56868023/error-request-body-larger-than-maxbodylength-limit-when-sending-base64-post-req and when I implemented it, it successfully uploaded the large folder. It simply involve modifying this axios.post request in this repo here https://github.com/NebulousLabs/nodejs-skynet/blob/master/src/upload.js#L75 by adding 'maxContentLength': Infinity, 'maxBodyLength': Infinity, so it became:

    axios
      .post(url, formData, { headers: formData.getHeaders(), params: params,
        'maxContentLength': Infinity,
        'maxBodyLength': Infinity
      })

Uploading directory to Skynet:  /Users/ls/code/src/ltfschoen/ethquad/client/build
Error: Request body larger than maxBodyLength limit
    at RedirectableRequest.write (/Users/ls/code/src/ltfschoen/ethquad/node_modules/follow-redirects/index.js:97:24)
    at FormData.ondata (internal/streams/legacy.js:19:31)
    at FormData.emit (events.js:314:20)
    at FormData.CombinedStream.write (/Users/ls/code/src/ltfschoen/ethquad/node_modules/combined-stream/lib/combined_stream.js:138:8)
    at DelayedStream.ondata (internal/streams/legacy.js:19:31)
    at DelayedStream.emit (events.js:314:20)
    at DelayedStream._handleEmit (/Users/ls/code/src/ltfschoen/ethquad/node_modules/delayed-stream/lib/delayed_stream.js:82:15)
    at ReadStream.source.emit (/Users/ls/code/src/ltfschoen/ethquad/node_modules/delayed-stream/lib/delayed_stream.js:29:19)
    at addChunk (_stream_readable.js:303:12)
    at readableAddChunk (_stream_readable.js:279:9) {
  config: {
    url: 'https://siasky.net/skynet/skyfile',
    method: 'post',
    params: { filename: 'EthQuad-Website' },
    data: FormData {
      _overheadLength: 4218,
      _valueLength: 0,
      _valuesToMeasure: [Array],
      writable: true,
      readable: true,
      dataSize: 0,
      maxDataSize: 2097152,
      pauseStreams: true,
      _released: true,
      _streams: [Array],
      _currentStream: [DelayedStream],
      _insideLoop: false,
      _pendingNext: false,
      _boundary: '--------------------------030930603990912989043150',
      _events: [Object: null prototype],
      _eventsCount: 3
    },
    headers: {
      Accept: 'application/json, text/plain, */*',
      'Content-Type': 'multipart/form-data; boundary=--------------------------030930603990912989043150',
      'User-Agent': 'axios/0.19.2'
    },
    transformRequest: [ [Function: transformRequest] ],
    transformResponse: [ [Function: transformResponse] ],
    timeout: 0,
    adapter: [Function: httpAdapter],
    xsrfCookieName: 'XSRF-TOKEN',
    xsrfHeaderName: 'X-XSRF-TOKEN',
    maxContentLength: -1,
    validateStatus: [Function: validateStatus]
  },
  request: <ref *1> Writable {
    _writableState: WritableState {
      objectMode: false,
      highWaterMark: 16384,
      finalCalled: false,
      needDrain: false,
      ending: false,
      ended: false,
      finished: false,
      destroyed: false,
      decodeStrings: true,
      defaultEncoding: 'utf8',
      length: 0,
      writing: false,
      corked: 0,
      sync: true,
      bufferProcessing: false,
      onwrite: [Function: bound onwrite],
      writecb: null,
      writelen: 0,
      afterWriteTickInfo: null,
      buffered: [],
      bufferedIndex: 0,
      allBuffers: true,
      allNoop: true,
      pendingcb: 0,
      prefinished: false,
      errorEmitted: false,
      emitClose: true,
      autoDestroy: true,
      errored: false,
      closed: false
    },
    _events: [Object: null prototype] {
      response: [Function: handleResponse],
      error: [Function: handleRequestError]
    },
    _eventsCount: 2,
    _maxListeners: undefined,
    _options: {
      protocol: 'https:',
      maxRedirects: 21,
      maxBodyLength: 10485760,
      path: '/skynet/skyfile?filename=EthQuad-Website',
      method: 'POST',
      headers: [Object],
      agent: undefined,
      agents: [Object],
      auth: undefined,
      hostname: 'siasky.net',
      port: null,
      nativeProtocols: [Object],
      pathname: '/skynet/skyfile',
      search: '?filename=EthQuad-Website'
    },
    _redirectCount: 0,
    _redirects: [],
    _requestBodyLength: 10464525,
    _requestBodyBuffers: [
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object], [Object], [Object],
      [Object], [Object], [Object], [Object],
      ... 119 more items
    ],
    _onNativeResponse: [Function (anonymous)],
    _currentRequest: ClientRequest {
      _events: [Object: null prototype],
      _eventsCount: 6,
      _maxListeners: undefined,
      outputData: [],
      outputSize: 0,
      writable: true,
      destroyed: true,
      _last: true,
      chunkedEncoding: true,
      shouldKeepAlive: false,
      useChunkedEncodingByDefault: true,
      sendDate: false,
      _removedConnection: false,
      _removedContLen: false,
      _removedTE: false,
      _contentLength: null,
      _hasBody: true,
      _trailer: '',
      finished: false,
      _headerSent: true,
      socket: [TLSSocket],
      _header: 'POST /skynet/skyfile?filename=EthQuad-Website HTTP/1.1\r\n' +
        'Accept: application/json, text/plain, */*\r\n' +
        'Content-Type: multipart/form-data; boundary=--------------------------030930603990912989043150\r\n' +
        'User-Agent: axios/0.19.2\r\n' +
        'Host: siasky.net\r\n' +
        'Connection: close\r\n' +
        'Transfer-Encoding: chunked\r\n' +
        '\r\n',
      _onPendingData: [Function: noopPendingOutput],
      agent: [Agent],
      socketPath: undefined,
      method: 'POST',
      maxHeaderSize: undefined,
      insecureHTTPParser: undefined,
      path: '/skynet/skyfile?filename=EthQuad-Website',
      _ended: false,
      res: null,
      aborted: true,
      timeoutCb: null,
      upgradeOrConnect: false,
      parser: [HTTPParser],
      maxHeadersCount: null,
      reusedSocket: false,
      host: 'siasky.net',
      protocol: 'https:',
      _redirectable: [Circular *1],
      [Symbol(kCapture)]: false,
      [Symbol(kNeedDrain)]: true,
      [Symbol(corked)]: 0,
      [Symbol(kOutHeaders)]: [Object: null prototype]
    },
    _currentUrl: 'https://siasky.net/skynet/skyfile?filename=EthQuad-Website',
    [Symbol(kCapture)]: false
  },
  response: undefined,
  isAxiosError: true,
  toJSON: [Function (anonymous)]
}

In that repo I modified ./scripts/siaSkynet.js

node ./scripts/siaSkynet.js

[mrcnski]

Thanks @ltfschoen, great contributions 🙂

I'll push a fix soon but Karol is on vacation so it may take some time to get reviewed. Glad you got it working though 👍