diff --git a/.gitignore b/.gitignore index 6581dee9d1f..39775b1767d 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ ipch/ email.md deps/v8-* deps/icu +deps/icu*.zip +deps/icu*.tgz ./node_modules .svn/ diff --git a/README.md b/README.md index 0032c63c073..88588ecbf52 100644 --- a/README.md +++ b/README.md @@ -83,20 +83,76 @@ make doc man doc/node.1 ``` -### To build `Intl` (ECMA-402) support: +### `Intl` (ECMA-402) support: -*Note:* more docs, including how to reduce disk footprint, are on +[Intl](https://github.com/joyent/node/wiki/Intl) support is not +enabled by default. + +#### "small" (English only) support + +This option will build with "small" (English only) support, but +the full `Intl` (ECMA-402) APIs. It will download the ICU library +as needed. + +Unix/Macintosh: + +```sh +./configure --with-intl=small-icu +``` + +Windows: + +```sh +vcbuild small-icu +``` + +The `small-icu` mode builds +with English-only data. You can add full data at runtime. + +*Note:* more docs are on [the wiki](https://github.com/joyent/node/wiki/Intl). +#### Build with full ICU support (all locales supported by ICU): + +*Note*, this may download ICU if you don't have an ICU in `deps/icu` + +Unix/Macintosh: + +```sh +./configure --with-intl=full-icu +``` + +Windows: + +```sh +vcbuild full-icu +``` + +#### Build with no Intl support `:-(` + +The `Intl` object will not be available. + +Unix/Macintosh: + +```sh +./configure --with-intl=none +``` + +Windows: + +```sh +vcbuild intl-none +``` + #### Use existing installed ICU (Unix/Macintosh only): ```sh pkg-config --modversion icu-i18n && ./configure --with-intl=system-icu ``` -#### Build ICU from source: +#### Build with a specific ICU: -First: Unpack latest ICU +First: Unpack latest ICU to `deps/icu` [icu4c-**##.#**-src.tgz](http://icu-project.org/download) (or `.zip`) as `deps/icu` (You'll have: `deps/icu/source/...`) diff --git a/configure b/configure index c558f7f8dd7..37d8a4b6804 100755 --- a/configure +++ b/configure @@ -6,6 +6,7 @@ import re import shlex import subprocess import sys +import shutil CC = os.environ.get('CC', 'cc') @@ -13,6 +14,10 @@ root_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.join(root_dir, 'tools', 'gyp', 'pylib')) from gyp.common import GetFlavor +# imports in tools/configure.d +sys.path.insert(0, os.path.join(root_dir, 'tools', 'configure.d')) +import nodedownload + # parse our options parser = optparse.OptionParser() @@ -712,6 +717,34 @@ def glob_to_var(dir_base, dir_sub): return list def configure_intl(o): + icus = [ + { + 'url': 'http://download.icu-project.org/files/icu4c/54.1/icu4c-54_1-src.zip', + # from https://ssl.icu-project.org/files/icu4c/54.1/icu4c-src-54_1.md5: + 'md5': '6b89d60e2f0e140898ae4d7f72323bca', + }, + ] + def icu_download(path): + # download ICU, if needed + for icu in icus: + url = icu['url'] + md5 = icu['md5'] + local = url.split('/')[-1] + targetfile = os.path.join(root_dir, 'deps', local) + if not os.path.isfile(targetfile): + nodedownload.retrievefile(url, targetfile) + else: + print ' Re-using existing %s' % targetfile + if os.path.isfile(targetfile): + sys.stdout.write(' Checking file integrity with MD5:\r') + gotmd5 = nodedownload.md5sum(targetfile) + print ' MD5: %s %s' % (gotmd5, targetfile) + if (md5 == gotmd5): + return targetfile + else: + print ' Expected: %s *MISMATCH*' % md5 + print '\n ** Corrupted ZIP? Delete %s to retry download.\n' % targetfile + return None icu_config = { 'variables': {} } @@ -723,7 +756,6 @@ def configure_intl(o): write(icu_config_name, do_not_edit + pprint.pformat(icu_config, indent=2) + '\n') - # small ICU is off by default. # always set icu_small, node.gyp depends on it being defined. o['variables']['icu_small'] = b(False) @@ -739,6 +771,8 @@ def configure_intl(o): o['variables']['icu_gyp_path'] = options.with_icu_path return # --with-intl= + if with_intl is None: + with_intl = 'none' # The default mode of Intl if with_intl == 'none' or with_intl is None: o['variables']['v8_enable_i18n_support'] = 0 return # no Intl @@ -769,20 +803,45 @@ def configure_intl(o): # Note: non-ICU implementations could use other 'with_intl' # values. + icu_parent_path = os.path.join(root_dir, 'deps') + icu_full_path = os.path.join(icu_parent_path, 'icu') + icu_small_path = os.path.join(icu_parent_path, 'icu-small') + icu_small_tag = os.path.join(icu_full_path, 'is-small-icu.txt') + + ## Use (or not) an embedded small-icu. + if with_intl == 'small-icu': + if not os.path.isdir(icu_full_path) and os.path.isdir(icu_small_path): + # deps/small-icu -> deps/icu + print 'Copying small ICU %s to %s' % (icu_small_path, icu_full_path) + shutil.copytree(icu_small_path, icu_full_path) + #else: + # print 'Not copying %s to %s' % (icu_small_path, icu_full_path) + elif os.path.isfile(icu_small_tag): + print 'deleting small-icu %s for --with-intl=%s' % (icu_full_path, with_intl) + shutil.rmtree(icu_full_path) + # ICU mode. (icu-generic.gyp) byteorder = sys.byteorder o['variables']['icu_gyp_path'] = 'tools/icu/icu-generic.gyp' # ICU source dir relative to root - icu_full_path = os.path.join(root_dir, 'deps/icu') o['variables']['icu_path'] = icu_full_path if not os.path.isdir(icu_full_path): - print 'Error: ICU path is not a directory: %s' % (icu_full_path) + print '* ECMA-402 (Intl) support didn\'t find ICU in %s..' % (icu_full_path) + # can we download (or find) a zipfile? + localzip = icu_download(icu_full_path) + if localzip: + nodedownload.unpack(localzip, icu_parent_path) + if not os.path.isdir(icu_full_path): + print ' Cannot build Intl without ICU in %s.' % (icu_full_path) + print ' (Fix, or disable with "--with-intl=none" )' sys.exit(1) + else: + print '* Using ICU in %s' % (icu_full_path) # Now, what version of ICU is it? We just need the "major", such as 54. # uvernum.h contains it as a #define. uvernum_h = os.path.join(icu_full_path, 'source/common/unicode/uvernum.h') if not os.path.isfile(uvernum_h): - print 'Error: could not load %s - is ICU installed?' % uvernum_h + print ' Error: could not load %s - is ICU installed?' % uvernum_h sys.exit(1) icu_ver_major = None matchVerExp = r'^\s*#define\s+U_ICU_VERSION_SHORT\s+"([^"]*)".*' @@ -792,7 +851,7 @@ def configure_intl(o): if m: icu_ver_major = m.group(1) if not icu_ver_major: - print 'Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h + print ' Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h sys.exit(1) icu_endianness = sys.byteorder[0]; # TODO(srl295): EBCDIC should be 'e' o['variables']['icu_ver_major'] = icu_ver_major @@ -819,8 +878,8 @@ def configure_intl(o): # this is the icudt*.dat file which node will be using (platform endianness) o['variables']['icu_data_file'] = icu_data_file if not os.path.isfile(icu_data_path): - print 'Error: ICU prebuilt data file %s does not exist.' % icu_data_path - print 'See the README.md.' + print ' Error: ICU prebuilt data file %s does not exist.' % icu_data_path + print ' See the README.md.' # .. and we're not about to build it from .gyp! sys.exit(1) # map from variable name to subdirs diff --git a/test/simple/test-intl.js b/test/simple/test-intl.js new file mode 100644 index 00000000000..1215641d5eb --- /dev/null +++ b/test/simple/test-intl.js @@ -0,0 +1,62 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common'); +var assert = require('assert'); + +var enablei18n = process.config.variables.v8_enable_i18n_support; +if (enablei18n === undefined) { + enablei18n = false; +} + +var haveIntl = ( global.Intl != undefined ); + +if (!haveIntl) { + assert.equal(enablei18n, false, '"Intl" object is NOT present but v8_enable_i18n_support is ' + enablei18n); + console.log('Skipping Intl tests because Intl object not present.'); +} else { + assert.equal(enablei18n, true, '"Intl" object is present but v8_enable_i18n_support is ' + enablei18n + '. Is this test out of date?'); + + // Check with toLocaleString + var date0 = new Date(0); + var GMT = 'Etc/GMT'; + var optsGMT = {timeZone: GMT}; + var localeString0 = date0.toLocaleString(['en'], optsGMT); + var expectString0 = '1/1/1970, 12:00:00 AM'; // epoch + assert.equal(localeString0, expectString0); + + // check with a Formatter + var dtf = new Intl.DateTimeFormat(['en'], {timeZone: GMT, month: 'short', year: '2-digit'}); + var localeString1 = dtf.format(date0); + assert.equal(localeString1, 'Jan 70'); + + // number format + assert.equal(new Intl.NumberFormat(['en']).format(12345.67890), '12,345.679'); + + var coll = new Intl.Collator(['en'],{sensitivity:'base',ignorePunctuation:true}); + + assert.equal(coll.compare('blackbird', 'black-bird'), 0, 'ignore punctuation failed'); + + assert.equal(coll.compare('blackbird', 'red-bird'), -1, 'compare less failed'); + assert.equal(coll.compare('bluebird', 'blackbird'), 1, 'compare greater failed'); + assert.equal(coll.compare('Bluebird', 'bluebird'), 0, 'ignore case failed'); + assert.equal(coll.compare('\ufb03', 'ffi'), 0, 'ffi ligature (contraction) failed'); +} diff --git a/tools/configure.d/nodedownload.py b/tools/configure.d/nodedownload.py new file mode 100644 index 00000000000..18602c8455b --- /dev/null +++ b/tools/configure.d/nodedownload.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# Moved some utilities here from ../../configure + +import urllib +import hashlib +import sys +import zipfile + +def formatSize(amt): + """Format a size as a string in MB""" + return "{:.1f}".format(amt / 1024000.) + +def spin(c): + """print out a spinner based on 'c'""" +# spin = "\\|/-" + spin = ".:|'" + return (spin[c % len(spin)]) + +class ConfigOpener(urllib.FancyURLopener): + """fancy opener used by retrievefile. Set a UA""" + # append to existing version (UA) + version = '%s node.js/configure' % urllib.URLopener.version + +def reporthook(count, size, total): + """internal hook used by retrievefile""" + sys.stdout.write(' Fetch: %c %sMB total, %sMB downloaded \r' % + (spin(count), + formatSize(total), + formatSize(count*size))) + +def retrievefile(url, targetfile): + """fetch file 'url' as 'targetfile'. Return targetfile or throw.""" + try: + sys.stdout.write(' <%s>\nConnecting...\r' % url) + sys.stdout.flush() + msg = ConfigOpener().retrieve(url, targetfile, reporthook=reporthook) + print '' # clear the line + return targetfile + except: + print ' ** Error occurred while downloading\n <%s>' % url + raise + +def md5sum(targetfile): + """md5sum a file. Return the hex digest.""" + digest = hashlib.md5() + with open(targetfile, 'rb') as f: + chunk = f.read(1024) + while chunk != "": + digest.update(chunk) + chunk = f.read(1024) + return digest.hexdigest() + +def unpack(packedfile, parent_path): + """Unpack packedfile into parent_path. Assumes .zip.""" + with zipfile.ZipFile(packedfile, 'r') as icuzip: + print ' Extracting source zip: %s' % packedfile + icuzip.extractall(parent_path) diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index 220d2c16a66..419efb525fb 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -11,6 +11,17 @@ }, 'includes': [ '../../icu_config.gypi' ], 'targets': [ + { + # a target for additional uconfig defines, target only + 'target_name': 'icu_uconfig_target', + 'type': 'none', + 'toolsets': [ 'target' ], + 'direct_dependent_settings': { + 'defines': [ + 'UCONFIG_NO_CONVERSION=1', + ] + }, + }, { # a target to hold uconfig defines. # for now these are hard coded, but could be defined. @@ -97,19 +108,67 @@ 'sources': [ '<@(icu_src_i18n)' ], + 'conditions': [ + [ 'icu_ver_major == 54', { 'sources!': [ + ## Strip out the following for ICU 54 only. + ## add more conditions in the future? + ## if your compiler can dead-strip, this will + ## make ZERO difference to binary size. + ## Made ICU-specific for future-proofing. + + # alphabetic index + '../../deps/icu/source/i18n/alphaindex.cpp', + # BOCSU + # misc + '../../deps/icu/source/i18n/dtitvfmt.cpp', + '../../deps/icu/source/i18n/dtitvinf.cpp', + '../../deps/icu/source/i18n/dtitv_impl.h', + '../../deps/icu/source/i18n/quantityformatter.cpp', + '../../deps/icu/source/i18n/quantityformatter.h', + '../../deps/icu/source/i18n/regexcmp.cpp', + '../../deps/icu/source/i18n/regexcmp.h', + '../../deps/icu/source/i18n/regexcst.h', + '../../deps/icu/source/i18n/regeximp.cpp', + '../../deps/icu/source/i18n/regeximp.h', + '../../deps/icu/source/i18n/regexst.cpp', + '../../deps/icu/source/i18n/regexst.h', + '../../deps/icu/source/i18n/regextxt.cpp', + '../../deps/icu/source/i18n/regextxt.h', + '../../deps/icu/source/i18n/region.cpp', + '../../deps/icu/source/i18n/region_impl.h', + '../../deps/icu/source/i18n/reldatefmt.cpp', + '../../deps/icu/source/i18n/reldatefmt.h', + '../../deps/icu/source/i18n/measfmt.h', + '../../deps/icu/source/i18n/measfmt.cpp', + '../../deps/icu/source/i18n/scientificformathelper.cpp', + '../../deps/icu/source/i18n/tmunit.cpp', + '../../deps/icu/source/i18n/tmutamt.cpp', + '../../deps/icu/source/i18n/tmutfmt.cpp', + '../../deps/icu/source/i18n/uregex.cpp', + '../../deps/icu/source/i18n/uregexc.cpp', + '../../deps/icu/source/i18n/uregion.cpp', + '../../deps/icu/source/i18n/uspoof.cpp', + '../../deps/icu/source/i18n/uspoof_build.cpp', + '../../deps/icu/source/i18n/uspoof_conf.cpp', + '../../deps/icu/source/i18n/uspoof_conf.h', + '../../deps/icu/source/i18n/uspoof_impl.cpp', + '../../deps/icu/source/i18n/uspoof_impl.h', + '../../deps/icu/source/i18n/uspoof_wsconf.cpp', + '../../deps/icu/source/i18n/uspoof_wsconf.h', + ]}]], 'include_dirs': [ '../../deps/icu/source/i18n', ], 'defines': [ 'U_I18N_IMPLEMENTATION=1', ], - 'dependencies': [ 'icuucx', 'icu_implementation', 'icu_uconfig' ], + 'dependencies': [ 'icuucx', 'icu_implementation', 'icu_uconfig', 'icu_uconfig_target' ], 'direct_dependent_settings': { 'include_dirs': [ '../../deps/icu/source/i18n', ], }, - 'export_dependent_settings': [ 'icuucx' ], + 'export_dependent_settings': [ 'icuucx', 'icu_uconfig_target' ], }, # This exports actual ICU data { @@ -289,22 +348,42 @@ 'export_dependent_settings': [ 'icuucx', 'icudata' ], }, # This is the 'real' icuuc. - # tools can depend on 'icuuc + stubdata' { 'target_name': 'icuucx', 'type': '<(library)', - 'dependencies': [ 'icu_implementation', 'icu_uconfig' ], + 'dependencies': [ 'icu_implementation', 'icu_uconfig', 'icu_uconfig_target' ], 'toolsets': [ 'target' ], 'sources': [ - '<@(icu_src_common)' + '<@(icu_src_common)', ], + 'conditions': [ + [ 'icu_ver_major == 54', { 'sources!': [ + ## Strip out the following for ICU 54 only. + ## add more conditions in the future? + ## if your compiler can dead-strip, this will + ## make ZERO difference to binary size. + ## Made ICU-specific for future-proofing. + + # bidi- not needed (yet!) + '../../deps/icu/source/common/ubidi.c', + '../../deps/icu/source/common/ubidiimp.h', + '../../deps/icu/source/common/ubidiln.c', + '../../deps/icu/source/common/ubidiwrt.c', + #'../../deps/icu/source/common/ubidi_props.c', + #'../../deps/icu/source/common/ubidi_props.h', + #'../../deps/icu/source/common/ubidi_props_data.h', + # and the callers + '../../deps/icu/source/common/ushape.cpp', + '../../deps/icu/source/common/usprep.cpp', + '../../deps/icu/source/common/uts46.cpp', + ]}]], 'include_dirs': [ '../../deps/icu/source/common', ], 'defines': [ 'U_COMMON_IMPLEMENTATION=1', ], - 'export_dependent_settings': [ 'icu_uconfig' ], + 'export_dependent_settings': [ 'icu_uconfig', 'icu_uconfig_target' ], 'direct_dependent_settings': { 'include_dirs': [ '../../deps/icu/source/common', diff --git a/tools/icu/icu_small.json b/tools/icu/icu_small.json index ddf7d1204e8..038d399d3e8 100644 --- a/tools/icu/icu_small.json +++ b/tools/icu/icu_small.json @@ -28,7 +28,8 @@ "translit": "none", "brkfiles": "none", "brkdict": "none", - "confusables": "none" + "confusables": "none", + "unit": "none" }, "remove": [ "cnvalias.icu", diff --git a/tools/icu/prepare-icu-source.sh b/tools/icu/prepare-icu-source.sh new file mode 100644 index 00000000000..7b38eed49a8 --- /dev/null +++ b/tools/icu/prepare-icu-source.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Call this to regenerate /deps/icu-small +# + +if [ -d `dirname "$0"` ]; +then + cd `dirname "$0"` +fi + +# now we are in /tools/icu +cd ../.. +# now we are in , hopefully +if [ ! -d 'tools/icu/' ]; +then + echo "$0: error: cannot find tools/icu in" `pwd` + exit 1 +fi + +echo "Preparing ICU source in" `pwd` +if [ -d 'deps/icu/' ]; +then + echo "Deleting" 'deps/icu' + rm -rf deps/icu || exit 1 +fi + +# clean +echo 'Delete ./out' +rm -rf ./out +# delete old ICU tgz +echo 'Delete deps/icu-small' +rm -rf ./deps/icu-small + +echo "Configuring node.. hopefully with ninja" +./configure --with-intl=small-icu --ninja || ./configure --with-intl=small-icu || exit 1 +echo "Building node.." +make || ( ./configure --with-intl=small-icu && make ) || exit 1 + +echo "Leaving our mark" +echo "# Generated. This is the SMALL ICU dir. Don't commit changes against it." > deps/icu/is-small-icu.txt +echo "Great. trimming stuff.." +rm -rfv deps/icu/[Aap]* +rm -rfv deps/icu/source/allinone deps/icu/source/config deps/icu/source/test +rm -rfv deps/icu/source/layout* deps/icu/source/samples deps/icu/source/tools/ctestfw +rm -rfv deps/icu/source/tools/gen[bdns]* deps/icu/source/tools/icuinfo +rm -rfv deps/icu/source/tools/m* deps/icu/source/tools/tzcode +rm -rfv deps/icu/source/data/{curr,lang,misc,region,sprep,unidata,unit,zone} +find deps/icu \( -name 'Makefile*' -o -name '*\.vcx*' \) -print0 | xargs -0 rm -fv +# now, the harsh part. Prove you are compiled! +sh tools/icu/trim-uncompiled-source.sh + +# move data +cp ./out/Release/gen/icutmp/icudt*l.dat deps/icu/source/data/in/ + +mv -v deps/icu deps/icu-small + +du -sh deps/icu-small diff --git a/tools/icu/trim-uncompiled-source.sh b/tools/icu/trim-uncompiled-source.sh new file mode 100644 index 00000000000..be6214b52fa --- /dev/null +++ b/tools/icu/trim-uncompiled-source.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Don't call this directly. Called by prepare-icu-source.sh +# + +#./out/Release/obj/deps/icu/source/i18n/icui18n.collationsets.o +#./out/Release/obj/deps/icu/source/i18n/icutools.collationsets.o +for file in `find deps/icu/source -name '*.c' -o -name '*.cpp'`; +do + #echo "# ${file}" + base=`basename ${file} .c` + base=`basename ${base} .cpp` + dir=`dirname ${file}` + #echo ${dir}/${base} + if ls "out/Release/obj/${dir}/"*".${base}.o" 2>/dev/null >/dev/null; + then + true + else + rm -fv ${file} + fi +done diff --git a/vcbuild.bat b/vcbuild.bat index 616b5bb1145..02f97fb7c7e 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -65,6 +65,7 @@ if /i "%1"=="upload" set upload=1&goto arg-ok if /i "%1"=="jslint" set jslint=1&goto arg-ok if /i "%1"=="small-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="full-icu" set i18n_arg=%1&goto arg-ok +if /i "%1"=="intl-none" set i18n_arg=%1&goto arg-ok echo Warning: ignoring invalid command line option `%1`. @@ -85,6 +86,7 @@ if defined noperfctr set noperfctr_arg=--without-perfctr& set noperfctr_msi_arg= if "%i18n_arg%"=="full-icu" set i18n_arg=--with-intl=full-icu if "%i18n_arg%"=="small-icu" set i18n_arg=--with-intl=small-icu +if "%i18n_arg%"=="intl-none" set i18n_arg=--with-intl=none :project-gen @rem Skip project generation if requested. @@ -232,7 +234,7 @@ python tools/closure_linter/closure_linter/gjslint.py --unix_mode --strict --noj goto exit :help -echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [nobuild] [nosign] [x86/x64] +echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [small-icu/full-icu/intl-none] [nobuild] [nosign] [x86/x64] echo Examples: echo vcbuild.bat : builds release build echo vcbuild.bat debug : builds debug build