From 6129721ad14ba48f458ff393814ecb78fc9d8989 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 29 Dec 2014 12:59:50 +0100 Subject: [PATCH 1/3] lib: make linklist functions return useful values Make a number of functions from require('_linklist') return the list or the item just inserted. Makes the API a little less unwieldy to use. --- lib/_linklist.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/_linklist.js b/lib/_linklist.js index 048e933ee3a725..c94c431bb61551 100644 --- a/lib/_linklist.js +++ b/lib/_linklist.js @@ -24,6 +24,7 @@ function init(list) { list._idleNext = list; list._idlePrev = list; + return list; } exports.init = init; @@ -57,6 +58,7 @@ function remove(item) { item._idleNext = null; item._idlePrev = null; + return item; } exports.remove = remove; @@ -68,6 +70,7 @@ function append(list, item) { list._idleNext._idlePrev = item; item._idlePrev = list; list._idleNext = item; + return item; } exports.append = append; From dffe15b9a8b6369c96509e48c341a6d59cc51f2e Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 29 Dec 2014 13:05:05 +0100 Subject: [PATCH 2/3] benchmark: add util.format() benchmark Add a simple (too simple!) benchmark for util.format(). --- benchmark/util/util-format.js | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 benchmark/util/util-format.js diff --git a/benchmark/util/util-format.js b/benchmark/util/util-format.js new file mode 100644 index 00000000000000..8368d5abcf52f6 --- /dev/null +++ b/benchmark/util/util-format.js @@ -0,0 +1,41 @@ +// Copyright (c) 2014, Ben Noordhuis +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +'use strict'; + +var common = require('../common.js'); +var format = require('util').format; + +var bench = common.createBenchmark(main, {}); + +function main(conf) { + var fmts = []; + + for (var i = 0; i < 26; i += 1) { + var k = i, fmt = ''; + do { + fmt += '%' + String.fromCharCode(97 + k); + k = (k + 1) % 26; + } while (k != i); + fmts.push(fmt); + } + + bench.start(); + + for (var i = 0; i < 1e5; i += 1) + for (var k = 0; k < fmts.length; k += 1) + format(fmts[k]); + + bench.end(1e5); +} From 5a74f84192e3f0489302f73a281089d30c5d5a38 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 29 Dec 2014 13:06:26 +0100 Subject: [PATCH 3/3] lib: optimize util.format() Optimize util.format() by parsing the format string and generating specialized code on the fly, then caching the result in a LRU list. This change is based on the observation that most applications that log extensively, often have very skewed distributions of log patterns. It's common to see top 10 or top 25 of popular patterns, followed by a long (sometimes very long) tail of less popular patterns. This is a work in progress: the common case is currently 2-25x faster but the worst case - every pattern unique - is 4-5x slower. --- lib/sprintf.js | 187 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/util.js | 44 ++---------- node.gyp | 1 + 3 files changed, 194 insertions(+), 38 deletions(-) create mode 100644 lib/sprintf.js diff --git a/lib/sprintf.js b/lib/sprintf.js new file mode 100644 index 00000000000000..c49cdb5e99e5c3 --- /dev/null +++ b/lib/sprintf.js @@ -0,0 +1,187 @@ +// Copyright (c) 2014, Ben Noordhuis +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +'use strict'; + +var L = require('_linklist'); +var inspect = require('util').inspect; +var quickeval = process.binding('contextify').quickeval; + +function LRU(maxsize) { + this.dict = Object.create(null); + this.list = L.init({}); + this.size = 0; + this.maxsize = maxsize; +} + +LRU.prototype.find = function(key) { + var val = this.dict[key]; + + // Move to front of the list unless it's already the first node. + if (val && val !== L.peek(this.list)) + L.append(this.list, L.remove(val)); + + return val; +}; + +LRU.prototype.insert = function(key, val) { + while (this.size >= this.maxsize) { + delete this.dict[L.shift(this.list).key]; + this.size -= 1; + } + + var val = L.append(this.list, {key: key}); + this.dict[key] = val; + this.size += 1; + + return val; +}; + +function format(fmt /*, ... */) { + return vsprintf(fmt, arguments, compat); +} + +function sprintf(fmt /*, ... */) { + return vsprintf(fmt, arguments, extended); +} + +function vsprintf(fmt, argv, compile) { + if (typeof(fmt) !== 'string') { + if (compile === compat) + return concat(argv); + fmt = '' + fmt; + } + + var lru = compile.lru; + var cached = lru.find(fmt); + + if (cached === undefined) { + cached = lru.insert(fmt); + cached.fun = compile(fmt); + } + + return cached.fun.apply(null, argv); +} + +function concat(argv) { + var argc = argv.length; + var objects = new Array(argc); + + for (var i = 0; i < argc; i += 1) + objects[i] = inspect(argv[i]); + + return objects.join(' '); +} + +// util.format() compatibility mode. +function compat(fmt) { + var index = 0; + var argc = 0; + var body = ''; + + for (;;) { + var start = index; + index = fmt.indexOf('%', index); + var end = index === -1 ? fmt.length : index; + + if (start !== end) { + body += ' s += "' + untaint(fmt.slice(start, end)) + '";'; + } + + if (index === -1) + break; + + index += 1; + + var c = fmt[index]; + switch (c) { + case 'd': + // %d is really %f; it coerces the argument to a number + // before turning it into a string. + body += ' s += argc > ' + (argc + 1) + ' ? +a' + argc + ' : "%d";'; + argc += 1; + index += 1; + continue; + + case 'j': + body += ' if (argc > ' + (argc + 1) + ')'; + body += ' try { s += JSON.stringify(a' + argc + '); }'; + body += ' catch (e) { s += "[Circular]"; }'; + body += ' else s += "%j";'; + argc += 1; + index += 1; + continue; + + case 's': + body += ' s += argc > ' + (argc + 1) + ' ? a' + argc + ' : "%s";'; + argc += 1; + index += 1; + continue; + + case '"': + c = '\\"'; + // Fall through. + + default: + body += ' s += "' + c + '";'; + index += 1; + continue; + } + } + + // The dummy argument lets format() and sprintf() call the formatter + // with .apply() without having to unshift the format argument first. + var source = 'return function(dummy'; + + for (var i = 0; i < argc; i += 1) + source += ', a' + i; + + source += ') { var argc = arguments.length, s = "";' + body; + source += ' for (var i = ' + argc + ' + 1; i < arguments.length; i += 1) {'; + source += ' var arg = arguments[i];'; + source += ' if (arg === null || typeof(arg) !== "object") s += " " + arg;'; + source += ' else s += " " + inspect(arg);'; + source += ' }'; + source += ' return s;'; + source += ' };'; + + return Function('inspect', source)(inspect); +} + +function untaint(s) { + return s.replace(/[\\"]|[^\x20-\x7F]/g, function(c) { + switch (c) { + case '\t': return '\\t'; + case '\n': return '\\n'; + case '\f': return '\\f'; + case '\r': return '\\r'; + case '"': return '\\"'; + case '\\': return '\\\\'; + } + return '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4); + }); +} + +compat.lru = new LRU(64); + +function extended(fmt) { + throw Error('unimplemented'); +} + +extended.lru = new LRU(64); + +// Hidden export for lib/util.js +Object.defineProperty(sprintf, '_format', {value: format}); + +module.exports = sprintf; diff --git a/lib/util.js b/lib/util.js index 4fed8311a0902a..8a08b855b6f550 100644 --- a/lib/util.js +++ b/lib/util.js @@ -21,45 +21,13 @@ 'use strict'; -var formatRegExp = /%[sdj%]/g; -exports.format = function(f) { - if (!isString(f)) { - var objects = []; - for (var i = 0; i < arguments.length; i++) { - objects.push(inspect(arguments[i])); - } - return objects.join(' '); - } - - var i = 1; - var args = arguments; - var len = args.length; - var str = String(f).replace(formatRegExp, function(x) { - if (x === '%%') return '%'; - if (i >= len) return x; - switch (x) { - case '%s': return String(args[i++]); - case '%d': return Number(args[i++]); - case '%j': - try { - return JSON.stringify(args[i++]); - } catch (_) { - return '[Circular]'; - } - default: - return x; - } - }); - for (var x = args[i]; i < len; x = args[++i]) { - if (isNull(x) || !isObject(x)) { - str += ' ' + x; - } else { - str += ' ' + inspect(x); - } - } - return str; -}; +var formatter; +exports.format = function() { + if (!formatter) + formatter = require('sprintf')._format; + return formatter.apply(null, arguments); +}; // Mark that a method should not be used. // Returns a modified function which warns once by default. diff --git a/node.gyp b/node.gyp index be266825bf7293..05b6808dc4a722 100644 --- a/node.gyp +++ b/node.gyp @@ -54,6 +54,7 @@ 'lib/_stream_duplex.js', 'lib/_stream_transform.js', 'lib/_stream_passthrough.js', + 'lib/sprintf.js', 'lib/string_decoder.js', 'lib/sys.js', 'lib/timers.js',