From 1a8700ccd79b9619f0ed271a88c5982c9a6bf1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zihua=20=E5=AD=90=E9=AA=85?= Date: Sat, 7 May 2016 10:49:11 +0800 Subject: [PATCH] feat: add dropBufferSupport option to improve the performance (#293) --- API.md | 2 + README.md | 7 +- benchmarks/single_node.js | 217 ++++++------------------- lib/commander.js | 27 ++- lib/redis.js | 6 + lib/redis/parser.js | 8 +- test/functional/drop_buffer_support.js | 91 +++++++++++ 7 files changed, 181 insertions(+), 177 deletions(-) create mode 100644 test/functional/drop_buffer_support.js diff --git a/API.md b/API.md index ab0898e2..de42c188 100644 --- a/API.md +++ b/API.md @@ -59,6 +59,8 @@ Creates a Redis instance | [options.connectionName] | string | null | Connection name. | | [options.db] | number | 0 | Database index to use. | | [options.password] | string | null | If set, client will send AUTH command with the value of this option when connected. | +| [options.parser] | string | null | Either "hiredis" or "javascript". If not set, "hiredis" parser will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. | +| [options.dropBufferSupport] | boolean | false | Drop the buffer support for better performance. This option is recommanded to be enabled when "hiredis" parser is used. Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details. | | [options.enableReadyCheck] | boolean | true | When a connection is established to the Redis server, the server might still be loading the database from disk. While loading, the server not respond to any commands. To work around this, when this option is `true`, ioredis will check the status of the Redis server, and when the Redis server is able to process commands, a `ready` event will be emitted. | | [options.enableOfflineQueue] | boolean | true | By default, if there is no active connection to the Redis server, commands are added to a queue and are executed once the connection is "ready" (when `enableReadyCheck` is `true`, "ready" means the Redis server has loaded the database from disk, otherwise means the connection to the Redis server has been established). If this option is false, when execute the command when the connection isn't ready, an error will be returned. | | [options.connectTimeout] | number | 10000 | The milliseconds before a timeout occurs during the initial connection to the Redis server. | diff --git a/README.md b/README.md index f777ad52..f0a8ecd6 100644 --- a/README.md +++ b/README.md @@ -807,10 +807,9 @@ var cluster = new Redis.Cluster([ }); ``` -## Native Parser -If [hiredis](https://github.com/redis/hiredis-node) is installed (by `npm install hiredis`), -ioredis will use it by default. Otherwise, a pure JavaScript parser will be used. -Typically, there's not much difference between them in terms of performance. +## Improve Performance +ioredis supports two parsers, "hiredis" and "javascript". Refer to https://github.com/luin/ioredis/wiki/Improve-Performance +for details about the differences between them in terms of performance.
diff --git a/benchmarks/single_node.js b/benchmarks/single_node.js index baf8b8c6..fe1c4f8e 100644 --- a/benchmarks/single_node.js +++ b/benchmarks/single_node.js @@ -1,13 +1,10 @@ 'use strict'; var childProcess = require('child_process'); -var nodeRedis = require('redis'); -var IORedis = require('../'); -var ndredis, ioredis; +var Redis = require('../'); console.log('=========================='); -console.log('ioredis: ' + require('../package.json').version); -console.log('node_redis: ' + require('redis/package.json').version); +console.log('redis: ' + require('../package.json').version); var os = require('os'); console.log('CPU: ' + os.cpus().length); console.log('OS: ' + os.platform() + ' ' + os.arch()); @@ -15,207 +12,87 @@ console.log('node version: ' + process.version); console.log('current commit: ' + childProcess.execSync('git rev-parse --short HEAD')); console.log('=========================='); +var redisJD, redisJ, redisBD, redisB; var waitReady = function (next) { - var pending = 2; - ndredis.on('ready', function () { + var pending = 4; + function check() { if (!--pending) { next(); } - }); + } + redisJD = new Redis({ parser: 'javascript', dropBufferSupport: true }); + redisJ = new Redis({ parser: 'javascript', dropBufferSupport: false }); + redisBD = new Redis({ parser: 'hiredis', dropBufferSupport: true }); + redisB = new Redis({ parser: 'hiredis', dropBufferSupport: false }); + redisJD.on('ready', check); + redisJ.on('ready', check); + redisBD.on('ready', check); + redisB.on('ready', check); +}; - ioredis.on('ready', function () { - if (!--pending) { - next(); - } - }); +var quit = function () { + redisJD.quit(); + redisJ.quit(); + redisBD.quit(); + redisB.quit(); }; -suite('simple set', function () { +suite('SET foo bar', function () { set('mintime', 5000); set('concurrency', 300); before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); waitReady(start); }); - bench('ioredis', function (next) { - ioredis.set('foo', 'bar', next); - }); - - bench('node_redis', function (next) { - ndredis.set('foo', 'bar', next); + bench('javascript parser + dropBufferSupport: true', function (next) { + redisJD.set('foo', 'bar', next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('simple get', function () { - set('mintime', 5000); - set('concurrency', 300); - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndredis.set('foo', 'bar', start); - }); + bench('javascript parser', function (next) { + redisJ.setBuffer('foo', 'bar', next); }); - bench('ioredis', function (next) { - ioredis.get('foo', next); + bench('hiredis parser + dropBufferSupport: true', function (next) { + redisBD.set('foo', 'bar', next); }); - bench('node_redis', function (next) { - ndredis.get('foo', next); + bench('hiredis parser', function (next) { + redisB.setBuffer('foo', 'bar', next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); + after(quit); }); -suite('simple get with pipeline', function () { +suite('LRANGE foo 0 99', function () { set('mintime', 5000); set('concurrency', 300); before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndredis.set('foo', 'bar', start); - }); - }); - - bench('ioredis', function (next) { - var pipeline = ioredis.pipeline(); - for (var i = 0; i < 10; ++i) { - pipeline.get('foo'); - } - pipeline.exec(next); - }); - - bench('node_redis', function (next) { - var pending = 0; - for (var i = 0; i < 10; ++i) { - pending += 1; - ndredis.get('foo', check); + var redis = new Redis(); + var item = []; + for (var i = 0; i < 100; ++i) { + item.push((Math.random() * 100000 | 0) + 'str'); } - function check() { - if (!--pending) { - next(); - } - } - }); - - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('lrange 100', function () { - set('mintime', 5000); - set('concurrency', 300); - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - var item = []; - for (var i = 0; i < 100; ++i) { - item.push((Math.random() * 100000 | 0) + 'str'); - } - ndredis.del('foo'); - ndredis.lpush('foo', item, start); - }); - }); - - bench('ioredis', function (next) { - ioredis.lrange('foo', 0, 99, next); - }); - - bench('node_redis', function (next) { - ndredis.lrange('foo', 0, 99, next); - }); - - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('publish', function () { - set('mintime', 5000); - set('concurrency', 300); - - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - start(); + redis.del('foo'); + redis.lpush('foo', item, function () { + waitReady(start); }); }); - bench('ioredis', function (next) { - ioredis.publish('foo', 'bar', next); - }); - - bench('node_redis', function (next) { - ndredis.publish('foo', 'bar', next); + bench('javascript parser + dropBufferSupport: true', function (next) { + redisJD.lrange('foo', 0, 99, next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('subscribe', function () { - set('mintime', 5000); - set('concurrency', 300); - - var ndpublisher = null; - var iopublisher = null; - var ndsubscriber = null; - var iosubscriber = null; - - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndsubscriber = ndredis; - ndsubscriber.subscribe('foo'); - iosubscriber = ioredis; - iosubscriber.subscribe('foo'); - - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndpublisher = ndredis; - iopublisher = ioredis; - start(); - }); - }); + bench('javascript parser', function (next) { + redisJ.lrangeBuffer('foo', 0, 99, next); }); - bench('ioredis', function (next) { - iosubscriber.removeAllListeners('message'); - ndsubscriber.removeAllListeners('message'); - iosubscriber.on('message', next); - iopublisher.publish('foo', 'bar'); + bench('hiredis parser + dropBufferSupport: true', function (next) { + redisBD.lrange('foo', 0, 99, next); }); - bench('node_redis', function (next) { - iosubscriber.removeAllListeners('message'); - ndsubscriber.removeAllListeners('message'); - ndsubscriber.on('message', next); - ndpublisher.publish('foo', 'bar'); + bench('hiredis parser', function (next) { + redisB.lrangeBuffer('foo', 0, 99, next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); + after(quit); }); diff --git a/lib/commander.js b/lib/commander.js index e5f9c1c5..701ab843 100644 --- a/lib/commander.js +++ b/lib/commander.js @@ -3,6 +3,11 @@ var _ = require('lodash'); var Command = require('./command'); var Script = require('./script'); +var Promise = require('bluebird'); + +var DROP_BUFFER_SUPPORT_ERROR = '*Buffer methods are not available ' + + 'because "dropBufferSupport" option is enabled.' + + 'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.'; /** * Commander @@ -106,7 +111,16 @@ function generateFunction(_commandName, _encoding) { args[i - firstArgIndex] = arguments[i]; } - var options = { replyEncoding: _encoding }; + var options; + if (this.options.dropBufferSupport) { + if (!_encoding) { + return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback); + } + options = { replyEncoding: null }; + } else { + options = { replyEncoding: _encoding }; + } + if (this.options.showFriendlyErrorStack) { options.errorStack = new Error().stack; } @@ -133,7 +147,16 @@ function generateScriptingFunction(_script, _encoding) { args[i] = arguments[i]; } - var options = { replyEncoding: _encoding }; + var options; + if (this.options.dropBufferSupport) { + if (!_encoding) { + return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback); + } + options = { replyEncoding: null }; + } else { + options = { replyEncoding: _encoding }; + } + if (this.options.showFriendlyErrorStack) { options.errorStack = new Error().stack; } diff --git a/lib/redis.js b/lib/redis.js index b810807f..d1197603 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -36,6 +36,11 @@ var ScanStream = require('./scan_stream'); * @param {number} [options.db=0] - Database index to use. * @param {string} [options.password=null] - If set, client will send AUTH command * with the value of this option when connected. + * @param {string} [options.parser=null] - Either "hiredis" or "javascript". If not set, "hiredis" parser + * will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. + * @param {boolean} [options.dropBufferSupport=false] - Drop the buffer support for better performance. + * This option is recommanded to be enabled when "hiredis" parser is used. + * Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details. * @param {boolean} [options.enableReadyCheck=true] - When a connection is established to * the Redis server, the server might still be loading the database from disk. * While loading, the server not respond to any commands. @@ -166,6 +171,7 @@ Redis.defaultOptions = { db: 0, // Others parser: null, + dropBufferSupport: false, enableOfflineQueue: true, enableReadyCheck: true, autoResubscribe: true, diff --git a/lib/redis/parser.js b/lib/redis/parser.js index 6137cc4c..2068c42f 100644 --- a/lib/redis/parser.js +++ b/lib/redis/parser.js @@ -20,7 +20,7 @@ exports.initParser = function () { this.replyParser = new Parser({ name: this.options.parser, stringNumbers: this.options.stringNumbers, - returnBuffers: true, + returnBuffers: !this.options.dropBufferSupport, returnError: function (err) { _this.returnError(new ReplyError(err.message)); }, @@ -33,6 +33,12 @@ exports.initParser = function () { _this.disconnect(true); } }); + + if (this.replyParser.name === 'hiredis' && !this.options.dropBufferSupport) { + console.warn('[WARN] ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' + + 'It\'s highly recommanded to enable this option. ' + + 'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.'); + } }; exports.returnError = function (err) { diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js new file mode 100644 index 00000000..9f16fd0f --- /dev/null +++ b/test/functional/drop_buffer_support.js @@ -0,0 +1,91 @@ +'use strict'; + +describe('dropBufferSupport', function () { + it('should be disabled by default', function () { + var redis = new Redis({ lazyConnect: true }); + expect(redis.options).to.have.property('dropBufferSupport', false); + }); + + it('should return strings correctly', function (done) { + var redis = new Redis({ dropBufferSupport: false }); + redis.set('foo', new Buffer('bar'), function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('OK'); + redis.get('foo', function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('bar'); + redis.disconnect(); + done(); + }); + }); + }); + + context('enabled', function () { + it('should reject the buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.getBuffer('foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + + redis.callBuffer('get', 'foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); + }); + }); + + it('should reject the custom buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.getevalBuffer(function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); + }); + + it('should return strings correctly', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.set('foo', new Buffer('bar'), function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('OK'); + redis.get('foo', function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('bar'); + redis.disconnect(); + done(); + }); + }); + }); + + it('should return strings for custom commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.geteval(function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('string'); + redis.disconnect(); + done(); + }); + }); + + it('should work with pipeline', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + var pipeline = redis.pipeline(); + pipeline.set('foo', 'bar'); + pipeline.get(new Buffer('foo')); + pipeline.exec(function (err, res) { + expect(err).to.eql(null); + expect(res[0][1]).to.eql('OK'); + expect(res[1][1]).to.eql('bar'); + done(); + }); + }); + }); +});