readline: use icu based string width calculation

Rather than the pseudo-wcwidth impl used currently, use the ICU
character properties database to calculate string width and
determine if a character is full width or not. This allows the
algorithm to correctly identify emoji's as full width, ensures
the algorithm will continue to fucntion properly as new unicode
codepoints are added, and it's faster.

This was originally part of a proposal to add a new unicode module,
but has been split out.

Refs: #8075
PR-URL: #9040
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: Steven R Loomis <srloomis@us.ibm.com>

lib/internal/readline.js

@@ -1,103 +1,117 @@
 'use strict';
 
-// Regexes used for ansi escape code splitting
+// Regex used for ansi escape code splitting
 // eslint-disable-next-line no-control-regex
-const metaKeyCodeReAnywhere = /(?:\x1b)([a-zA-Z0-9])/;
-const functionKeyCodeReAnywhere = new RegExp('(?:\x1b+)(O|N|\\[|\\[\\[)(?:' + [
-  '(\\d+)(?:;(\\d+))?([~^$])',
-  '(?:M([@ #!a`])(.)(.))', // mouse
-  '(?:1;)?(\\d+)?([a-zA-Z])'
-].join('|') + ')');
+// Adopted from https://github.com/chalk/ansi-regex/blob/master/index.js
+// License: MIT, authors: @sindresorhus, Qix-, and arjunmehta
+// Matches all ansi escape code sequences in a string
+const ansi =
+  /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g;
 
 
 module.exports = {
   emitKeys,
-  getStringWidth,
-  isFullWidthCodePoint,
   stripVTControlCharacters
 };
 
+if (process.binding('config').hasIntl) {
+  const icu = process.binding('icu');
+  module.exports.getStringWidth = function getStringWidth(str, options) {
+    options = options || {};
+    if (!Number.isInteger(str))
+      str = stripVTControlCharacters(String(str));
+    return icu.getStringWidth(str,
+                              Boolean(options.ambiguousAsFullWidth),
+                              Boolean(options.expandEmojiSequence));
+  };
+  module.exports.isFullWidthCodePoint =
+    function isFullWidthCodePoint(code, options) {
+      if (typeof code !== 'number')
+        return false;
+      return icu.getStringWidth(code, options) === 2;
+    };
+} else {
+  /**
+   * Returns the number of columns required to display the given string.
+   */
+  module.exports.getStringWidth = function getStringWidth(str) {
+    if (Number.isInteger(str))
+      return module.exports.isFullWidthCodePoint(str) ? 2 : 1;
 
-/**
- * Returns the number of columns required to display the given string.
- */
-function getStringWidth(str) {
-  let width = 0;
+    let width = 0;
 
-  str = stripVTControlCharacters(str);
+    str = stripVTControlCharacters(String(str));
 
-  for (var i = 0; i < str.length; i++) {
-    const code = str.codePointAt(i);
+    for (var i = 0; i < str.length; i++) {
+      const code = str.codePointAt(i);
 
-    if (code >= 0x10000) { // surrogates
-      i++;
-    }
+      if (code >= 0x10000) { // surrogates
+        i++;
+      }
 
-    if (isFullWidthCodePoint(code)) {
-      width += 2;
-    } else {
-      width++;
+      if (module.exports.isFullWidthCodePoint(code)) {
+        width += 2;
+      } else {
+        width++;
+      }
     }
-  }
-
-  return width;
-}
 
+    return width;
+  };
 
-/**
- * Returns true if the character represented by a given
- * Unicode code point is full-width. Otherwise returns false.
- */
-function isFullWidthCodePoint(code) {
-  if (isNaN(code)) {
-    return false;
-  }
+  /**
+   * Returns true if the character represented by a given
+   * Unicode code point is full-width. Otherwise returns false.
+   */
+  module.exports.isFullWidthCodePoint = function isFullWidthCodePoint(code) {
+    if (!Number.isInteger(code)) {
+      return false;
+    }
 
-  // Code points are derived from:
-  // http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt
-  if (code >= 0x1100 && (
-      code <= 0x115f ||  // Hangul Jamo
-      0x2329 === code || // LEFT-POINTING ANGLE BRACKET
-      0x232a === code || // RIGHT-POINTING ANGLE BRACKET
-      // CJK Radicals Supplement .. Enclosed CJK Letters and Months
-      (0x2e80 <= code && code <= 0x3247 && code !== 0x303f) ||
-      // Enclosed CJK Letters and Months .. CJK Unified Ideographs Extension A
-      0x3250 <= code && code <= 0x4dbf ||
-      // CJK Unified Ideographs .. Yi Radicals
-      0x4e00 <= code && code <= 0xa4c6 ||
-      // Hangul Jamo Extended-A
-      0xa960 <= code && code <= 0xa97c ||
-      // Hangul Syllables
-      0xac00 <= code && code <= 0xd7a3 ||
-      // CJK Compatibility Ideographs
-      0xf900 <= code && code <= 0xfaff ||
-      // Vertical Forms
-      0xfe10 <= code && code <= 0xfe19 ||
-      // CJK Compatibility Forms .. Small Form Variants
-      0xfe30 <= code && code <= 0xfe6b ||
-      // Halfwidth and Fullwidth Forms
-      0xff01 <= code && code <= 0xff60 ||
-      0xffe0 <= code && code <= 0xffe6 ||
-      // Kana Supplement
-      0x1b000 <= code && code <= 0x1b001 ||
-      // Enclosed Ideographic Supplement
-      0x1f200 <= code && code <= 0x1f251 ||
-      // CJK Unified Ideographs Extension B .. Tertiary Ideographic Plane
-      0x20000 <= code && code <= 0x3fffd)) {
-    return true;
-  }
+    // Code points are derived from:
+    // http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt
+    if (code >= 0x1100 && (
+        code <= 0x115f ||  // Hangul Jamo
+        0x2329 === code || // LEFT-POINTING ANGLE BRACKET
+        0x232a === code || // RIGHT-POINTING ANGLE BRACKET
+        // CJK Radicals Supplement .. Enclosed CJK Letters and Months
+        (0x2e80 <= code && code <= 0x3247 && code !== 0x303f) ||
+        // Enclosed CJK Letters and Months .. CJK Unified Ideographs Extension A
+        0x3250 <= code && code <= 0x4dbf ||
+        // CJK Unified Ideographs .. Yi Radicals
+        0x4e00 <= code && code <= 0xa4c6 ||
+        // Hangul Jamo Extended-A
+        0xa960 <= code && code <= 0xa97c ||
+        // Hangul Syllables
+        0xac00 <= code && code <= 0xd7a3 ||
+        // CJK Compatibility Ideographs
+        0xf900 <= code && code <= 0xfaff ||
+        // Vertical Forms
+        0xfe10 <= code && code <= 0xfe19 ||
+        // CJK Compatibility Forms .. Small Form Variants
+        0xfe30 <= code && code <= 0xfe6b ||
+        // Halfwidth and Fullwidth Forms
+        0xff01 <= code && code <= 0xff60 ||
+        0xffe0 <= code && code <= 0xffe6 ||
+        // Kana Supplement
+        0x1b000 <= code && code <= 0x1b001 ||
+        // Enclosed Ideographic Supplement
+        0x1f200 <= code && code <= 0x1f251 ||
+        // CJK Unified Ideographs Extension B .. Tertiary Ideographic Plane
+        0x20000 <= code && code <= 0x3fffd)) {
+      return true;
+    }
 
-  return false;
+    return false;
+  };
 }
 
-
 /**
  * Tries to remove all VT control characters. Use to estimate displayed
  * string width. May be buggy due to not running a real state machine
  */
 function stripVTControlCharacters(str) {
-  str = str.replace(new RegExp(functionKeyCodeReAnywhere.source, 'g'), '');
-  return str.replace(new RegExp(metaKeyCodeReAnywhere.source, 'g'), '');
+  return str.replace(ansi, '');
 }
 
 

lib/readline.js

@@ -124,6 +124,14 @@ function Interface(input, output, completer, terminal) {
 
   function onkeypress(s, key) {
     self._ttyWrite(s, key);
+    if (key && key.sequence) {
+      // if the key.sequence is half of a surrogate pair
+      // (>= 0xd800 and <= 0xdfff), refresh the line so
+      // the character is displayed appropriately.
+      const ch = key.sequence.codePointAt(0);
+      if (ch >= 0xd800 && ch <= 0xdfff)
+        self._refreshLine();
+    }
   }
 
   function onresize() {

src/node_i18n.cc

@@ -31,6 +31,7 @@
 #include "v8.h"
 
 #include <unicode/putil.h>
+#include <unicode/uchar.h>
 #include <unicode/udata.h>
 #include <unicode/uidna.h>
 
@@ -185,13 +186,102 @@ static void ToASCII(const FunctionCallbackInfo<Value>& args) {
                           len).ToLocalChecked());
 }
 
+// This is similar to wcwidth except that it takes the current unicode
+// character properties database into consideration, allowing it to
+// correctly calculate the column widths of things like emoji's and
+// newer wide characters. wcwidth, on the other hand, uses a fixed
+// algorithm that does not take things like emoji into proper
+// consideration.
+static int GetColumnWidth(UChar32 codepoint,
+                          bool ambiguous_as_full_width = false) {
+  if (!u_isdefined(codepoint) ||
+      u_iscntrl(codepoint) ||
+      u_getCombiningClass(codepoint) > 0 ||
+      u_hasBinaryProperty(codepoint, UCHAR_EMOJI_MODIFIER)) {
+    return 0;
+  }
+  // UCHAR_EAST_ASIAN_WIDTH is the Unicode property that identifies a
+  // codepoint as being full width, wide, ambiguous, neutral, narrow,
+  // or halfwidth.
+  const int eaw = u_getIntPropertyValue(codepoint, UCHAR_EAST_ASIAN_WIDTH);
+  switch (eaw) {
+    case U_EA_FULLWIDTH:
+    case U_EA_WIDE:
+      return 2;
+    case U_EA_AMBIGUOUS:
+      // See: http://www.unicode.org/reports/tr11/#Ambiguous for details
+      if (ambiguous_as_full_width) {
+        return 2;
+      }
+      // Fall through if ambiguous_as_full_width if false.
+    case U_EA_NEUTRAL:
+      if (u_hasBinaryProperty(codepoint, UCHAR_EMOJI_PRESENTATION)) {
+        return 2;
+      }
+      // Fall through
+    case U_EA_HALFWIDTH:
+    case U_EA_NARROW:
+    default:
+      return 1;
+  }
+}
+
+// Returns the column width for the given String.
+static void GetStringWidth(const FunctionCallbackInfo<Value>& args) {
+  Environment* env = Environment::GetCurrent(args);
+  if (args.Length() < 1)
+    return;
+
+  bool ambiguous_as_full_width = args[1]->BooleanValue();
+  bool expand_emoji_sequence = args[2]->BooleanValue();
+
+  if (args[0]->IsNumber()) {
+    args.GetReturnValue().Set(
+        GetColumnWidth(args[0]->Uint32Value(),
+                       ambiguous_as_full_width));
+    return;
+  }
+
+  TwoByteValue value(env->isolate(), args[0]);
+  // reinterpret_cast is required by windows to compile
+  UChar* str = reinterpret_cast<UChar*>(*value);
+  UChar32 c;
+  UChar32 p;
+  size_t n = 0;
+  uint32_t width = 0;
+
+  while (n < value.length()) {
+    p = c;
+    U16_NEXT(str, n, value.length(), c);
+    // Don't count individual emoji codepoints that occur within an
+    // emoji sequence. This is not necessarily foolproof. Some
+    // environments display emoji sequences in the appropriate
+    // condensed form (as a single emoji glyph), other environments
+    // may not understand an emoji sequence and will display each
+    // individual emoji separately. When this happens, the width
+    // calculated will be off, and there's no reliable way of knowing
+    // in advance if a particular sequence is going to be supported.
+    // The expand_emoji_sequence option allows the caller to skip this
+    // check and count each code within an emoji sequence separately.
+    if (!expand_emoji_sequence &&
+        n > 0 && p == 0x200d &&  // 0x200d == ZWJ (zero width joiner)
+        (u_hasBinaryProperty(c, UCHAR_EMOJI_PRESENTATION) ||
+         u_hasBinaryProperty(c, UCHAR_EMOJI_MODIFIER))) {
+      continue;
+    }
+    width += GetColumnWidth(c, ambiguous_as_full_width);
+  }
+  args.GetReturnValue().Set(width);
+}
+
 void Init(Local<Object> target,
           Local<Value> unused,
           Local<Context> context,
           void* priv) {
   Environment* env = Environment::GetCurrent(context);
   env->SetMethod(target, "toUnicode", ToUnicode);
   env->SetMethod(target, "toASCII", ToASCII);
+  env->SetMethod(target, "getStringWidth", GetStringWidth);
 }
 
 }  // namespace i18n

test/parallel/test-icu-stringwidth.js

@@ -0,0 +1,43 @@
+// Flags: --expose_internals
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const readline = require('internal/readline');
+
+if (!process.binding('config').hasIntl) {
+  common.skip('missing intl... skipping test');
+  return;
+}
+
+// Test column width
+assert.strictEqual(readline.getStringWidth('a'), 1);
+assert.strictEqual(readline.getStringWidth('丁'), 2);
+assert.strictEqual(readline.getStringWidth('\ud83d\udc78\ud83c\udfff'), 2);
+assert.strictEqual(readline.getStringWidth('👅'), 2);
+assert.strictEqual(readline.getStringWidth('\n'), 0);
+assert.strictEqual(readline.getStringWidth('\u200Ef\u200F'), 1);
+assert.strictEqual(readline.getStringWidth(97), 1);
+
+// The following is an emoji sequence. In some implementations, it is
+// represented as a single glyph, in other implementations as a sequence
+// of individual glyphs. By default, the algorithm will assume the single
+// glyph interpretation and return a value of 2. By passing the
+// expandEmojiSequence: true option, each component will be counted
+// individually.
+assert.strictEqual(readline.getStringWidth('👩‍👩‍👧‍👧'), 2);
+assert.strictEqual(
+    readline.getStringWidth('👩‍👩‍👧‍👧', {expandEmojiSequence: true}), 8);
+
+// By default, unicode characters whose width is considered ambiguous will
+// be considered half-width. For these characters, getStringWidth will return
+// 1. In some contexts, however, it is more appropriate to consider them full
+// width. By default, the algorithm will assume half width. By passing
+// the ambiguousAsFullWidth: true option, ambiguous characters will be counted
+// as 2 columns.
+assert.strictEqual(readline.getStringWidth('\u01d4'), 1);
+assert.strictEqual(
+    readline.getStringWidth('\u01d4', {ambiguousAsFullWidth: true}), 2);
+
+// Control chars and combining chars are zero
+assert.strictEqual(readline.getStringWidth('\u200E\n\u220A\u20D2'), 1);