Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(benchmark): make it fair #2929

Merged
merged 2 commits into from
Mar 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions benchmarks/_util/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
'use strict'

const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest':
relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

module.exports = { makeParallelRequests, printResults }
104 changes: 37 additions & 67 deletions benchmarks/benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@ const { isMainThread } = require('node:worker_threads')

const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')

const { makeParallelRequests, printResults } = require('./_util')

let nodeFetch
const axios = require('axios')
let superagent
let got

const util = require('node:util')
const _request = require('request')
const request = util.promisify(_request)
const { promisify } = require('node:util')
const request = promisify(require('request'))

const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
const dest = {}
Expand Down Expand Up @@ -136,53 +136,6 @@ class SimpleRequest {
}
}

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

const experiments = {
'http - no keepalive' () {
return makeParallelRequests(resolve => {
Expand Down Expand Up @@ -217,8 +170,8 @@ const experiments = {
'undici - pipeline' () {
return makeParallelRequests(resolve => {
dispatcher
.pipeline(undiciOptions, data => {
return data.body
.pipeline(undiciOptions, ({ body }) => {
return body
})
.end()
.pipe(
Expand Down Expand Up @@ -288,9 +241,15 @@ if (process.env.PORT) {
})
}

const axiosOptions = {
url: dest.url,
method: 'GET',
responseType: 'stream',
httpAgent: axiosAgent
}
experiments.axios = () => {
return makeParallelRequests(resolve => {
axios.get(dest.url, { responseType: 'stream', httpAgent: axiosAgent }).then(res => {
axios.request(axiosOptions).then(res => {
res.data.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
Expand All @@ -300,26 +259,37 @@ if (process.env.PORT) {
})
}

const gotOptions = {
url: dest.url,
method: 'GET',
agent: {
http: gotAgent
},
// avoid body processing
isStream: true
}
experiments.got = () => {
return makeParallelRequests(resolve => {
got.get(dest.url, { agent: { http: gotAgent } }).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
}).catch(console.log)
got(gotOptions).pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
})
}

const requestOptions = {
url: dest.url,
method: 'GET',
agent: requestAgent,
// avoid body toString
encoding: null
}
experiments.request = () => {
return makeParallelRequests(resolve => {
request(dest.url, { agent: requestAgent }).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
request(requestOptions).then(() => {
// already body consumed
resolve()
}).catch(console.log)
})
}
Expand Down
88 changes: 21 additions & 67 deletions benchmarks/post-benchmark.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,23 @@ const os = require('node:os')
const path = require('node:path')
const { Writable, Readable, pipeline } = require('node:stream')
const { isMainThread } = require('node:worker_threads')

const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')

const { makeParallelRequests, printResults } = require('./_util')

let nodeFetch
const axios = require('axios')
let superagent
let got

const util = require('node:util')
const _request = require('request')
const request = util.promisify(_request)
const { promisify } = require('node:util')
const request = promisify(require('request'))

const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
const dest = {}
Expand Down Expand Up @@ -147,53 +148,6 @@ class SimpleRequest {
}
}

function makeParallelRequests (cb) {
const promises = new Array(parallelRequests)
for (let i = 0; i < parallelRequests; ++i) {
promises[i] = new Promise(cb)
}
return Promise.all(promises)
}

function printResults (results) {
// Sort results by least performant first, then compare relative performances and also printing padding
let last

const rows = Object.entries(results)
// If any failed, put on the top of the list, otherwise order by mean, ascending
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
.map(([name, result]) => {
if (!result.success) {
return {
Tests: name,
Samples: result.size,
Result: 'Errored',
Tolerance: 'N/A',
'Difference with Slowest': 'N/A'
}
}

// Calculate throughput and relative performance
const { size, mean, standardError } = result
const relative = last !== 0 ? (last / mean - 1) * 100 : 0

// Save the slowest for relative comparison
if (typeof last === 'undefined') {
last = mean
}

return {
Tests: name,
Samples: size,
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
}
})

return console.table(rows)
}

const experiments = {
'http - no keepalive' () {
return makeParallelRequests(resolve => {
Expand Down Expand Up @@ -236,8 +190,8 @@ const experiments = {
this.push(null)
}
}),
dispatcher.pipeline(undiciOptions, data => {
return data.body
dispatcher.pipeline(undiciOptions, ({ body }) => {
return body
}),
new Writable({
write (chunk, encoding, callback) {
Expand Down Expand Up @@ -341,22 +295,23 @@ if (process.env.PORT) {
}

const gotOptions = {
url: dest.url,
method: 'POST',
headers,
agent: {
http: gotAgent
},
// avoid body processing
isStream: true,
body: data
}
experiments.got = () => {
return makeParallelRequests(resolve => {
got(dest.url, gotOptions).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
}).catch(console.log)
got(gotOptions).pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
})
}

Expand All @@ -365,16 +320,15 @@ if (process.env.PORT) {
method: 'POST',
headers,
agent: requestAgent,
body: data
body: data,
// avoid body toString
encoding: null
}
experiments.request = () => {
return makeParallelRequests(resolve => {
request(requestOptions).then(res => {
res.pipe(new Writable({
write (chunk, encoding, callback) {
callback()
}
})).on('finish', resolve)
request(requestOptions).then(() => {
// already body consumed
resolve()
}).catch(console.log)
})
}
Expand Down
10 changes: 8 additions & 2 deletions benchmarks/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,16 @@ if (cluster.isPrimary) {
}
} else {
const buf = Buffer.alloc(64 * 1024, '_')

const headers = {
'Content-Length': `${buf.byteLength}`,
'Content-Type': 'text/plain; charset=UTF-8'
}
let i = 0
const server = createServer((req, res) => {
const server = createServer((_req, res) => {
i++
setTimeout(function () {
setTimeout(() => {
res.writeHead(200, headers)
res.end(buf)
}, timeout)
}).listen(port)
Expand Down
Loading
Loading