From 07155a7c0148023ba5546d3be01d6582b185b753 Mon Sep 17 00:00:00 2001 From: Caridy Date: Mon, 29 Feb 2016 00:39:13 -0500 Subject: [PATCH] refactor of the entire build process to work with babel and rollup. grunt is now only used for 262, the rest are just npm commands. --- CONTRIBUTING.md | 20 +- Gruntfile.js | 64 +----- package.json | 28 ++- scripts/build-data.js | 167 ++++++++++++++++ scripts/build-dist.js | 9 +- {tasks => scripts}/utils/extract-calendars.js | 46 ++--- {tasks => scripts}/utils/extract-numbers.js | 24 +-- {tasks => scripts}/utils/locales.js | 56 +++--- {tasks => scripts}/utils/reduce.js | 182 +++++++++--------- tasks/compile-data.js | 105 ---------- tasks/extract-cldr-data.js | 67 ------- 11 files changed, 339 insertions(+), 429 deletions(-) create mode 100644 scripts/build-data.js rename {tasks => scripts}/utils/extract-calendars.js (69%) rename {tasks => scripts}/utils/extract-numbers.js (77%) rename {tasks => scripts}/utils/locales.js (63%) rename {tasks => scripts}/utils/reduce.js (52%) delete mode 100644 tasks/compile-data.js delete mode 100644 tasks/extract-cldr-data.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15f55c36..00e02d93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,35 +14,39 @@ To run the unit tests: To build files in `dist/` and `lib/`: - grunt + npm run build Updating CLDR Data ------------------ -_Note: this step is completely optional._ +To specifically build files in `locale-data`: -Copy fresh CLDR data in `data/`: + npm run build:data - grunt update-cldr-data +Note: this is completely optional since the regular `npm run build` will take care of it. -To build files in `locale-data/` based on CLDR `data/`: +Updating Test 262 +----------------- + +To specifically build files in `tests/test262`: - grunt cldr + grunt update-test262 +Note: be careful when attempting to update the tests. Source Code ----------- All the source code is in `src/` folder, written as ES6 modules, and transpiled -using `es6-module-transpiler` into the `lib/` and `dist/` folders. +using `rollup` and `babel` into the `lib/` and `dist/` folders. The `dist/` is in git because of bower, make sure you commit those files as well. Release checklist ----------------- -* build all files using `grunt` +* build all files using `npm run build` * run all tests using `npm test` * verify that [README.md] is updated * bump the version in [package.json] diff --git a/Gruntfile.js b/Gruntfile.js index b0bab00f..e09cb0c0 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,12 +1,12 @@ +// This file is only needed to update test 262 used to test this polyfill +// by using the command: `$ grunt update-test262` module.exports = function (grunt) { grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), clean: { - cldr : ['locale-data/'], test262: ['tmp/test262**', 'data/test262**', 'tests/test262/'], - lib : ['lib/', 'dist/'], }, curl: { @@ -17,10 +17,6 @@ module.exports = function (grunt) { }, unzip: { - cldr: { - src : 'tmp/cldr.zip', - dest: 'tmp/cldr/', - }, test262: { src : 'tmp/test262.zip', dest: 'tmp/', @@ -38,50 +34,6 @@ module.exports = function (grunt) { 'harness/*.js', ], }, - src: { - expand : true, - flatten: true, - src : ['tmp/src/*.js'], - dest : 'lib/', - }, - }, - - concat: { - complete: { - src : ['dist/Intl.min.js', 'locale-data/complete.js'], - dest: 'dist/Intl.complete.js', - }, - }, - - jshint: { - options: { - eqeqeq: true, - }, - src: ['src/*.js'], - node: ['index.js', '*.json'], - build: ['tasks/**/*.js'], - }, - - bundle_jsnext: { - dest: 'dist/Intl.js', - options: { - namespace: 'IntlPolyfill', - }, - }, - - cjs_jsnext: { - dest: 'tmp/', - }, - - uglify: { - options: { - preserveComments: 'some', - }, - build: { - files: { - 'dist/Intl.min.js': ['dist/Intl.js'], - }, - }, }, }); @@ -89,21 +41,9 @@ module.exports = function (grunt) { grunt.loadTasks('./tasks'); grunt.loadNpmTasks('grunt-contrib-clean'); grunt.loadNpmTasks('grunt-contrib-copy'); - grunt.loadNpmTasks('grunt-contrib-concat'); - grunt.loadNpmTasks('grunt-contrib-jshint'); - grunt.loadNpmTasks('grunt-contrib-uglify'); - grunt.loadNpmTasks('grunt-bundle-jsnext-lib'); grunt.loadNpmTasks('grunt-curl'); grunt.loadNpmTasks('grunt-zip'); - grunt.registerTask('build', [ - 'bundle_jsnext', 'uglify', 'cjs_jsnext', 'copy:src', 'concat:complete', - ]); - - grunt.registerTask('cldr', ['clean:cldr', 'extract-cldr-data', 'compile-data']); - - grunt.registerTask('default', ['jshint', 'clean:lib', 'build']); - grunt.registerTask('update-test262', [ 'clean:test262', 'curl:test262', diff --git a/package.json b/package.json index 038644dd..27be8464 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "version": "1.1.0", "description": "Polyfill the ECMA-402 Intl API (except collation)", "main": "index.js", - "jsnext:main": "src/main.js", "directories": { "test": "tests" }, @@ -13,8 +12,8 @@ }, "devDependencies": { "async": "^0.9.0", - "babel-cli": "^6.2.0", + "babel-eslint": "^5.0.0", "babel-plugin-transform-es2015-modules-commonjs": "^6.4.0", "babel-plugin-transform-es3-member-expression-literals": "^6.3.13", "babel-plugin-transform-es3-property-literals": "^6.3.13", @@ -23,16 +22,6 @@ "babel-preset-es2015": "^6.1.18", "babel-preset-es2015-rollup": "^1.1.1", "babel-register": "^6.2.0", - "rimraf": "^2.4.2", - "rollup": "^0.25.0", - "rollup-plugin-babel": "^2.3.9", - "rollup-plugin-commonjs": "^2.2.0", - "rollup-plugin-memory": "^1.0.0", - "rollup-plugin-npm": "^1.3.0", - "rollup-plugin-replace": "^1.1.0", - "rollup-plugin-uglify": "^0.1.0", - - "babel-eslint": "^5.0.0", "cldr-cal-buddhist-full": "28.0.0", "cldr-cal-chinese-full": "28.0.0", "cldr-cal-coptic-full": "28.0.0", @@ -53,17 +42,22 @@ "finalhandler": "^0.4.0", "glob": "^5.0.3", "grunt": "^0.4.5", - "grunt-bundle-jsnext-lib": "^0.5.0", "grunt-cli": "~0.1.13", "grunt-contrib-clean": "^0.6.0", - "grunt-contrib-concat": "^0.5.0", "grunt-contrib-copy": "^0.5.0", - "grunt-contrib-jshint": "^0.10.0", - "grunt-contrib-uglify": "^0.5.1", "grunt-curl": "^2.1.0", "grunt-zip": "^0.16.2", "jshint": "^2.5.5", + "mkdirp": "^0.5.1", "object.assign": "^1.1.1", + "rimraf": "^2.4.2", + "rollup": "^0.25.0", + "rollup-plugin-babel": "^2.3.9", + "rollup-plugin-commonjs": "^2.2.0", + "rollup-plugin-memory": "^1.0.0", + "rollup-plugin-npm": "^1.3.0", + "rollup-plugin-replace": "^1.1.0", + "rollup-plugin-uglify": "^0.1.0", "sauce-tunnel": "^2.2.3", "serve-static": "^1.10.0", "wd": "^0.3.6" @@ -75,7 +69,7 @@ "build:dist:dev": "NODE_ENV=development babel-node scripts/build-dist", "build:dist:prod": "NODE_ENV=production babel-node scripts/build-dist", "build:dist": "npm run build:dist:dev && npm run build:dist:prod", - "build": "npm run build:lib && npm run build:dist", + "build": "npm run build:data && npm run build:lib && npm run build:dist", "lint": "eslint .", "test": "cd tests && node noderunner.js && node saucelabs.js", "pretest": "npm run lint", diff --git a/scripts/build-data.js b/scripts/build-data.js new file mode 100644 index 00000000..ae037372 --- /dev/null +++ b/scripts/build-data.js @@ -0,0 +1,167 @@ +/* global Promise */ +import * as fs from 'fs'; +import * as p from 'path'; +import {sync as mkdirpSync} from 'mkdirp'; + +function writeFile(filename, contents) { + return new Promise((resolve, reject) => { + fs.writeFile(filename, contents, (err) => { + if (err) { + reject(err); + } else { + resolve(p.resolve(filename)); + } + }); + }); +} + +function mergeData(...sources) { + return sources.reduce((data, source) => { + Object.keys(source || {}).forEach((locale) => { + data[locale] = Object.assign(data[locale] || {}, source[locale]); + }); + + return data; + }, {}); +} + +function reviver (k, v) { + let idx; + + if (k === 'locale') + return v; + + else if (typeof v === 'string') { + idx = prims.indexOf(v); + valCount++; + + if (idx === -1) + idx += prims.push(v); + + return '###prims['+ idx +']###'; + } + + else if (typeof v === 'object' && v !== null) { + const str = JSON.stringify(v); + objCount++; + + if (objStrs.hasOwnProperty(str)) + return objStrs[str]; + + // We need to make sure this object is not added to the same + // array as an object it references (and we need to check + // this recursively) + let depth; + let objDepths = [0]; + + for (let key in v) { + if (typeof v[key] === 'string' && (depth = v[key].match(/^###objs\[(\d+)/))) + objDepths.push(+depth[1] + 1); + } + + depth = Math.max.apply(Math, objDepths); + + if (!Array.isArray(objs[depth])) + objs[depth] = []; + + idx = objs[depth].push(v) - 1; + objStrs[str] = '###objs['+ depth +']['+ idx +']###'; + + return objStrs[str]; + } + + return v; +} + +// ----------------------------------------------------------------------------- + +mkdirpSync('locale-data/'); +mkdirpSync('locale-data/json/'); +mkdirpSync('locale-data/jsonp/'); + +// extracting data into CLDR + +// Regex for converting locale JSON to object grammar, obviously simple and +// incomplete but should be good enough for the CLDR JSON +const jsonpExp = /"(?!default)([\w$][\w\d$]+)":/g; + +import reduceLocaleData from './utils/reduce'; + +import extractCalendars from './utils/extract-calendars'; +import extractNumbersFields from './utils/extract-numbers'; +import {getAllLocales} from './utils/locales'; + +// Default to all CLDR locales. +const locales = getAllLocales(); + +// Each type of data has the structure: `{"": {"": }}`, +// which is well suited for merging into a single object per locale. This +// performs that deep merge and returns the aggregated result. +let locData = mergeData( + extractCalendars(locales), + extractNumbersFields(locales) +); + +let locStringData = {}; + +Object.keys(locData).forEach((locale) => { + // Ignore en-US-POSIX and root + if (locale.toLowerCase() === 'en-us-posix') { + return; + } + + const obj = reduceLocaleData(locale, locData[locale]); + locStringData[locale] = JSON.stringify(obj, null, 4); + const jsonpContent = `IntlPolyfill.__addLocaleData(${JSON.stringify(obj).replace(jsonpExp, '$1:')});`; + writeFile('locale-data/json/' + locale + '.json', locStringData[locale]); + writeFile('locale-data/jsonp/' + locale + '.js', jsonpContent); +}); + +console.log('Total number of locales is ' + Object.keys(locData).length); + +// compiling `locale-date/complete.js` + +function replacer($0, type, loc) { + return (type === 'prims' ? 'a' : 'b') + loc; +} + +let + objStrs = {}, + objs = [], + prims = [], + + valCount = 0, + objCount = 0, + + fileData = ''; + +fileData += '(function(addLocaleData){\n'; + +let locReducedData = {}; +Object.keys(locStringData).forEach((k) => { + const c = locStringData[k]; + locReducedData[k] = JSON.parse(c, reviver); +}); + +fileData += `var a=${JSON.stringify(prims)},b=[];`; +objs.forEach((val, idx) => { + const ref = JSON.stringify(val).replace(/"###(objs|prims)(\[[^#]+)###"/g, replacer); + + fileData += `b[${idx}]=${ref};`; +}); + +for (let k in locReducedData) { + fileData += `addLocaleData(${locReducedData[k].replace(/###(objs|prims)(\[[^#]+)###/, replacer)}); +`; +} + +fileData += `})(IntlPolyfill.__addLocaleData);`; + +// writting the complete optimized bundle +writeFile('locale-data/complete.js', fileData); + +console.log('Total number of reused strings is ' + prims.length + ' (reduced from ' + valCount + ')'); +console.log('Total number of reused objects is ' + Object.keys(objStrs).length + ' (reduced from ' + objCount + ')'); + +process.on('unhandledRejection', (reason) => {throw reason;}); +console.log('Writing locale data files...'); diff --git a/scripts/build-dist.js b/scripts/build-dist.js index 36cccee3..a2f883da 100644 --- a/scripts/build-dist.js +++ b/scripts/build-dist.js @@ -41,6 +41,13 @@ if (isProduction) { } let bundle = Promise.resolve(rollup({entry, plugins})); -bundle.then(({write}) => write(bundleConfig)); +bundle.then(({write}) => write(bundleConfig)).then(() => { + + // special case for locale-data/complete.js + const lib = fs.readFileSync(dest); + const complete = fs.readFileSync(p.resolve('locale-data/complete.js')); + fs.writeFileSync(p.resolve('dist/Intl.complete.js'), `${lib}\n${complete}`); + +}); process.on('unhandledRejection', (reason) => {throw reason;}); diff --git a/tasks/utils/extract-calendars.js b/scripts/utils/extract-calendars.js similarity index 69% rename from tasks/utils/extract-calendars.js rename to scripts/utils/extract-calendars.js index 7ad7f31e..09fe312e 100644 --- a/tasks/utils/extract-calendars.js +++ b/scripts/utils/extract-calendars.js @@ -1,23 +1,19 @@ -/* jshint node:true */ +import * as glob from 'glob'; +import * as path from 'path'; -'use strict'; +import { + getParentLocale, + hasCalendars, + normalizeLocale, +} from './locales'; -var path = require('path'); -var glob = require("glob"); -var assign = require('object.assign'); - -var getParentLocale = require('./locales').getParentLocale; -var hasCalendars = require('./locales').hasCalendars; -var normalizeLocale = require('./locales').normalizeLocale; - -module.exports = function extractCalendars(locales) { - var cache = {}; - var hashes = {}; +export default function extractCalendars(locales) { + let cache = {}; // Loads and caches the date calendars for a given `locale` because loading // and transforming the data is expensive. function getCalendars(locale) { - var calendars = cache[locale]; + let calendars = cache[locale]; if (calendars) { return calendars; } @@ -45,13 +41,13 @@ module.exports = function extractCalendars(locales) { return findLocaleWithCalendar(getParentLocale(locale)); } - return locales.reduce(function (calendars, locale) { + return locales.reduce((calendars, locale) => { locale = normalizeLocale(locale); // Walk the `locale`'s hierarchy to look for suitable ancestor with the // date calendars. If no ancestor is found, the given // `locale` will be returned. - var resolvedLocale = findLocaleWithCalendar(locale); + let resolvedLocale = findLocaleWithCalendar(locale); // Add an entry for the `locale`, which might be an ancestor. If the // locale doesn't have relative fields, then we fallback to the "root" @@ -62,11 +58,11 @@ module.exports = function extractCalendars(locales) { return calendars; }, {}); -}; +} function loadCalendars(locale) { // all NPM packages providing calendars specific data - var pkgs = [ + let pkgs = [ "cldr-dates-full", "cldr-cal-buddhist-full", "cldr-cal-chinese-full", @@ -78,17 +74,17 @@ function loadCalendars(locale) { "cldr-cal-islamic-full", "cldr-cal-japanese-full", "cldr-cal-persian-full", - "cldr-cal-roc-full" + "cldr-cal-roc-full", ]; // walking all packages, selecting calendar files, then // reading the content of each calendar, and concatenating the set - return pkgs.reduce(function (calendars, pkgName) { - var dir = path.resolve(path.dirname(require.resolve(pkgName + '/package.json')), 'main', locale); - var filenames = glob.sync("ca-*.json", { - cwd: dir + return pkgs.reduce((calendars, pkgName) => { + let dir = path.resolve(path.dirname(require.resolve(pkgName + '/package.json')), 'main', locale); + let filenames = glob.sync("ca-*.json", { + cwd: dir, }); - return filenames.reduce(function (calendars, filename) { - return assign(calendars, require(path.join(dir, filename)).main[locale].dates.calendars); + return filenames.reduce((calendars, filename) => { + return Object.assign(calendars, require(path.join(dir, filename)).main[locale].dates.calendars); }, calendars); }, {}); } diff --git a/tasks/utils/extract-numbers.js b/scripts/utils/extract-numbers.js similarity index 77% rename from tasks/utils/extract-numbers.js rename to scripts/utils/extract-numbers.js index 573111bf..8ab99bf9 100644 --- a/tasks/utils/extract-numbers.js +++ b/scripts/utils/extract-numbers.js @@ -1,22 +1,14 @@ -/* jshint node:true */ - -'use strict'; - -var path = require('path'); -var assign = require('object.assign'); - -var getParentLocale = require('./locales').getParentLocale; -var hasNumbersFields = require('./locales').hasNumbersFields; -var normalizeLocale = require('./locales').normalizeLocale; +let getParentLocale = require('./locales').getParentLocale; +let hasNumbersFields = require('./locales').hasNumbersFields; +let normalizeLocale = require('./locales').normalizeLocale; module.exports = function extractNumbersFields(locales) { - var cache = {}; - var hashes = {}; + let cache = {}; // Loads and caches the numbers fields for a given `locale` because loading // and transforming the data is expensive. function getNumbers(locale) { - var numbers = cache[locale]; + let numbers = cache[locale]; if (numbers) { return numbers; } @@ -43,13 +35,13 @@ module.exports = function extractNumbersFields(locales) { return findLocaleWithNumbersFields(getParentLocale(locale)); } - return locales.reduce(function (numbers, locale) { + return locales.reduce((numbers, locale) => { locale = normalizeLocale(locale); // Walk the `locale`'s hierarchy to look for suitable ancestor with the // date calendars. If no ancestor is found, the given // `locale` will be returned. - var resolvedLocale = findLocaleWithNumbersFields(locale); + let resolvedLocale = findLocaleWithNumbersFields(locale); // Add an entry for the `locale`, which might be an ancestor. If the // locale doesn't have relative fields, then we fallback to the "root" @@ -63,7 +55,7 @@ module.exports = function extractNumbersFields(locales) { }; function loadNumbers(locale) { - return assign( + return Object.assign( require('cldr-numbers-full/main/' + locale + '/numbers.json').main[locale].numbers, require('cldr-numbers-full/main/' + locale + '/currencies.json').main[locale].numbers ); diff --git a/tasks/utils/locales.js b/scripts/utils/locales.js similarity index 63% rename from tasks/utils/locales.js rename to scripts/utils/locales.js index 7c615e31..7cefeaf5 100644 --- a/tasks/utils/locales.js +++ b/scripts/utils/locales.js @@ -1,70 +1,60 @@ -/* jshint node:true */ +import * as glob from 'glob'; +import * as path from 'path'; -'use strict'; - -var glob = require('glob'); -var path = require('path'); - -exports.getAllLocales = getAllLocales; -exports.getParentLocale = getParentLocale; -exports.hasCalendars = hasCalendars; -exports.hasNumbersFields = hasNumbersFields; -exports.normalizeLocale = normalizeLocale; - -var CLDR_DATES_DIR = path.dirname(require.resolve('cldr-dates-full/package.json')); -var CLDR_NUMBERS_DIR = path.dirname(require.resolve('cldr-numbers-full/package.json')); +let CLDR_DATES_DIR = path.dirname(require.resolve('cldr-dates-full/package.json')); +let CLDR_NUMBERS_DIR = path.dirname(require.resolve('cldr-numbers-full/package.json')); // These are the exceptions to the default algorithm for determining a locale's // parent locale. -var PARENT_LOCALES_HASH = require('cldr-core/supplemental/parentLocales.json') +let PARENT_LOCALES_HASH = require('cldr-core/supplemental/parentLocales.json') .supplemental.parentLocales.parentLocale; -var CALENDARS_LOCALES_HASH = glob.sync('*/ca-*.json', { +let CALENDARS_LOCALES_HASH = glob.sync('*/ca-*.json', { cwd: path.resolve(CLDR_DATES_DIR, 'main'), -}).reduce(function (hash, filename) { +}).reduce((hash, filename) => { hash[path.dirname(filename)] = true; return hash; }, {}); -var NUMBERS_LOCALES_HASH = glob.sync('*/numbers.json', { +let NUMBERS_LOCALES_HASH = glob.sync('*/numbers.json', { cwd: path.resolve(CLDR_NUMBERS_DIR, 'main'), -}).reduce(function (hash, filename) { +}).reduce((hash, filename) => { hash[path.dirname(filename)] = true; return hash; }, {}); -var CURRENCIES_LOCALES_HASH = glob.sync('*/currencies.json', { +let CURRENCIES_LOCALES_HASH = glob.sync('*/currencies.json', { cwd: path.resolve(CLDR_NUMBERS_DIR, 'main'), -}).reduce(function (hash, filename) { +}).reduce((hash, filename) => { hash[path.dirname(filename)] = true; return hash; }, {}); -var DEFAULT_CONTENT_ARRAY = require('cldr-core/defaultContent.json') - .defaultContent.map(function (value) { +let DEFAULT_CONTENT_ARRAY = require('cldr-core/defaultContent.json') + .defaultContent.map((value) => { return value.replace(/_/g, '-'); }); // Some locales that have a `pluralRuleFunction` don't have a `dateFields.json` // file, and visa versa, so this creates a unique collection of all locales in // the CLDR for which we need data from. -var ALL_LOCALES_HASH = +let ALL_LOCALES_HASH = Object.keys(PARENT_LOCALES_HASH) .concat(Object.keys(CALENDARS_LOCALES_HASH)) .concat(Object.keys(NUMBERS_LOCALES_HASH)) .concat(Object.keys(CURRENCIES_LOCALES_HASH)) .concat(DEFAULT_CONTENT_ARRAY) .sort() - .reduce(function (hash, locale) { + .reduce((hash, locale) => { hash[locale.toLowerCase()] = locale; return hash; }, {}); -function getAllLocales() { +export function getAllLocales() { return Object.keys(ALL_LOCALES_HASH); } -function getParentLocale(locale) { +export function getParentLocale(locale) { locale = normalizeLocale(locale); // If we don't know about the locale, or if it's the "root" locale, then its @@ -75,14 +65,14 @@ function getParentLocale(locale) { // First check the exceptions for locales which don't follow the standard // hierarchical pattern. - var parentLocale = PARENT_LOCALES_HASH[locale]; + let parentLocale = PARENT_LOCALES_HASH[locale]; if (parentLocale) { return parentLocale; } // Be default, the language tags are hierarchal, therefore we can identify // the parent locale by simply popping off the last segment. - var localeParts = locale.split('-'); + let localeParts = locale.split('-'); if (localeParts.length > 1) { localeParts.pop(); return localeParts.join('-'); @@ -92,17 +82,17 @@ function getParentLocale(locale) { return 'root'; } -function hasCalendars(locale) { +export function hasCalendars(locale) { return CALENDARS_LOCALES_HASH.hasOwnProperty(normalizeLocale(locale)); } -function hasNumbersFields(locale) { +export function hasNumbersFields(locale) { return NUMBERS_LOCALES_HASH.hasOwnProperty(normalizeLocale(locale)) && CURRENCIES_LOCALES_HASH.hasOwnProperty(normalizeLocale(locale)); } -function normalizeLocale(locale) { - var normalizedLocale = ALL_LOCALES_HASH[locale.toLowerCase()]; +export function normalizeLocale(locale) { + let normalizedLocale = ALL_LOCALES_HASH[locale.toLowerCase()]; if (normalizedLocale) { return normalizedLocale; } diff --git a/tasks/utils/reduce.js b/scripts/utils/reduce.js similarity index 52% rename from tasks/utils/reduce.js rename to scripts/utils/reduce.js index 4168d2dd..719ef8cc 100644 --- a/tasks/utils/reduce.js +++ b/scripts/utils/reduce.js @@ -1,8 +1,6 @@ -/* jshint node:true */ - function replaceSpecialChars (ptn) { // Matches CLDR number patterns, e.g. #,##0.00, #,##,##0.00, #,##0.##, etc. - var numPtn = /#(?:[\.,]#+)*0(?:[,\.][0#]+)*/; + let numPtn = /#(?:[\.,]#+)*0(?:[,\.][0#]+)*/; return ptn .replace(numPtn, '{number}') @@ -17,10 +15,10 @@ function replaceSpecialChars (ptn) { * Returns an object with positivePattern and negativePattern properties */ function createNumberFormats (ptn) { - var patterns = ptn.split(';'), + let patterns = ptn.split(';'), ret = { - positivePattern: replaceSpecialChars(patterns[0]) + positivePattern: replaceSpecialChars(patterns[0]), }; // Negative patterns aren't always specified, in those cases use '-' + positivePattern @@ -34,88 +32,82 @@ function createNumberFormats (ptn) { /** * Processes an object from CLDR format to an easier-to-parse format */ -module.exports = function (locale, data) { - var - // Sort property name arrays to keep order and minimalise unnecessary file diffs - gopn = function (a) { return Object.getOwnPropertyNames(a).sort(); }, +export default function (locale, data) { + // Sort property name arrays to keep order and minimalise unnecessary file diffs + let gopn = function (a) { return Object.getOwnPropertyNames(a).sort(); }; - test = RegExp.prototype.test, + let test = RegExp.prototype.test; - // Get own property values, useful for converting object map to array when we - // don't care about the keys. Relies on predictable property ordering in V8. - gopv = function (o) { - return o ? Object.getOwnPropertyNames(o).map(function (e) { return o[e]; }) : undefined; - }, - latnNu = 'latn', - - // Copy numbering systems - defaultNu = data.numbers.defaultNumberingSystem, - otherNu = gopn(data.numbers.otherNumberingSystems).map(function(key) { - return data.numbers.otherNumberingSystems[key]; - }).filter(function (key) { - return key !== defaultNu && key !== latnNu; + // Get own property values, useful for converting object map to array when we + // don't care about the keys. Relies on predictable property ordering in V8. + let gopv = function (o) { + return o ? Object.getOwnPropertyNames(o).map((e) => o[e]) : undefined; + }; + let latnNu = 'latn'; + + // Copy numbering systems + let defaultNu = data.numbers.defaultNumberingSystem; + + // Map calendar names to BCP 47 unicode extension 'ca' keys + let caMap = { + 'gregorian': 'gregory', + 'ethiopic-amete-alem': 'ethioaa', + 'islamic-civil': 'islamicc', + }; + + // Default calendar is always gregorian, apparently + let defCa = data.calendars.gregorian; + + // Any of the time format strings can give us some additional information + let defaultTimeFormat = defCa.timeFormats[gopn(defCa.timeFormats)[0]]; + let ampmTimeFormat = defCa.dateTimeFormats.availableFormats.hms; + + // Result object to be returned + let ret = { + // Identifying language tag for this data + locale: locale, + + date: { + // Get supported calendars (as extension keys) + ca: gopn(data.calendars) + .map((cal) => caMap[cal] || cal) + + // Move 'gregory' (the default) to the front, the rest is alphabetical + .sort((a, b) => { + return -(a === 'gregory') + (b === 'gregory') || a.localeCompare(b); + }).filter((cal) => { + return (cal.indexOf('-') < 0); }), - // Map calendar names to BCP 47 unicode extension 'ca' keys - caMap = { - 'gregorian': 'gregory', - 'ethiopic-amete-alem': 'ethioaa', - 'islamic-civil': 'islamicc' - }, + // Boolean value indicating whether hours from 1 to 12 (true) or from 0 to + // 11 (false) should be used. 'h' is 1 to 12, 'k' is 0 to 11. + hourNo0: /h/i.test(ampmTimeFormat), - // Default calendar is always gregorian, apparently - defCa = data.calendars.gregorian, + // Locales defaulting to 24hr time have 'H' or 'k' in their + // default time patterns + hour12: !/H|k/.test(defaultTimeFormat), - // Any of the time format strings can give us some additional information - defaultTimeFormat = defCa.timeFormats[gopn(defCa.timeFormats)[0]], - ampmTimeFormat = defCa.dateTimeFormats.availableFormats.hms, + formats: [], + calendars: {}, + }, + number: { + // Numbering systems, with the default first + nu: (defaultNu === latnNu) ? [ latnNu ] : [ defaultNu, latnNu ], - // Result object to be returned - ret = { - // Identifying language tag for this data - locale: locale, - - date: { - // Get supported calendars (as extension keys) - ca: gopn(data.calendars) - .map(function (cal) { return caMap[cal] || cal; }) - - // Move 'gregory' (the default) to the front, the rest is alphabetical - .sort(function (a, b) { - return -(a === 'gregory') + (b === 'gregory') || a.localeCompare(b); - }).filter(function (cal) { - return (cal.indexOf('-') < 0); - }), - - // Boolean value indicating whether hours from 1 to 12 (true) or from 0 to - // 11 (false) should be used. 'h' is 1 to 12, 'k' is 0 to 11. - hourNo0: /h/i.test(ampmTimeFormat), - - // Locales defaulting to 24hr time have 'H' or 'k' in their - // default time patterns - hour12: !/H|k/.test(defaultTimeFormat), - - formats: [], - calendars: {} - }, - number: { - // Numbering systems, with the default first - nu: (defaultNu === latnNu) ? [ latnNu ] : [ defaultNu, latnNu ], - - // Formatting patterns - patterns: {}, - - // Symbols - symbols: {}, - - currencies: {} - } - }; + // Formatting patterns + patterns: {}, + + // Symbols + symbols: {}, + currencies: {}, + }, + }; + + let ptn; // Copy the numeric symbols for each numbering system - gopn(data.numbers).filter(test.bind(/^symbols-/)).forEach(function (key) { - var ptn, - sym = data.numbers[key]; + gopn(data.numbers).filter(test.bind(/^symbols-/)).forEach((key) => { + const sym = data.numbers[key]; // Currently, Intl 402 only uses these symbols for numbers ret.number.symbols[key.split('-').pop()] = { @@ -125,7 +117,7 @@ module.exports = function (locale, data) { plusSign: sym.plusSign, minusSign: sym.minusSign, percentSign: sym.percentSign, - infinity: sym.infinity + infinity: sym.infinity, }; }); @@ -140,7 +132,7 @@ module.exports = function (locale, data) { ret.number.patterns.percent = createNumberFormats(ptn.standard); // Check the grouping sizes for locales that group irregularly - var pGroupSize = ptn.standard.match(/#+0/)[0].length, + let pGroupSize = ptn.standard.match(/#+0/)[0].length, groups = ptn.standard.split(','); // The pattern in en-US-POSIX doesn't specify group sizes, and the default @@ -153,66 +145,66 @@ module.exports = function (locale, data) { ret.number.patterns.secondaryGroupSize = groups[1].length; // Copy the currency symbols - gopn(data.numbers.currencies).forEach(function (k) { + gopn(data.numbers.currencies).forEach((k) => { if (k !== data.numbers.currencies[k].symbol) ret.number.currencies[k] = data.numbers.currencies[k].symbol; }); // Copy the formatting information - gopn(data.calendars).forEach(function (cal) { - var frmt; - var ca = caMap[cal] || cal; + gopn(data.calendars).forEach((cal) => { + let frmt; + let ca = caMap[cal] || cal; if (ret.date.ca.indexOf(ca) < 0) { // ignoring unknown calendars return; } - var obj = ret.date.calendars[ca] = {}; + let obj = ret.date.calendars[ca] = {}; if ((frmt = data.calendars[cal].months) && (frmt = frmt.format)) { obj.months = { narrow: gopv(frmt.narrow), short: gopv(frmt.abbreviated), - long: gopv(frmt.wide) + long: gopv(frmt.wide), }; } if ((frmt = data.calendars[cal].days) && (frmt = frmt.format)) { obj.days = { narrow: gopv(frmt.narrow), short: gopv(frmt.abbreviated), - long: gopv(frmt.wide) + long: gopv(frmt.wide), }; } if ((frmt = data.calendars[cal].eras)) { obj.eras = { narrow: gopv(frmt.eraNarrow), short: gopv(frmt.eraAbbr), - long: gopv(frmt.eraNames) + long: gopv(frmt.eraNames), }; } if ((frmt = data.calendars[cal].dayPeriods) && (frmt = frmt.format)) { obj.dayPeriods = { am: (frmt.wide || frmt.abbreviated).am, - pm: (frmt.wide || frmt.abbreviated).pm + pm: (frmt.wide || frmt.abbreviated).pm, }; } // Basic Date formats // http://cldr.unicode.org/translation/date-time-patterns#TOC-Basic-Date-Formats - var basicDateFormats = { + let basicDateFormats = { yMMMMEEEEd: defCa.dateFormats.full, yMMMMd: defCa.dateFormats.long, yMMMd: defCa.dateFormats.medium, - yMd: defCa.dateFormats.short + yMd: defCa.dateFormats.short, }; // Basic Time Formats // http://cldr.unicode.org/translation/date-time-patterns#TOC-Basic-Time-Formats - var basicTimeFormats = { + let basicTimeFormats = { hmmsszzzz: defCa.timeFormats.full, hmsz: defCa.timeFormats.long, hms: defCa.timeFormats.medium, - hm: defCa.timeFormats.short + hm: defCa.timeFormats.short, }; ret.date.formats = { @@ -222,9 +214,9 @@ module.exports = function (locale, data) { long: defCa.dateTimeFormats.long, availableFormats: defCa.dateTimeFormats.availableFormats, dateFormats: basicDateFormats, - timeFormats: basicTimeFormats + timeFormats: basicTimeFormats, }; }); return ret; -}; +} diff --git a/tasks/compile-data.js b/tasks/compile-data.js deleted file mode 100644 index 563d6b35..00000000 --- a/tasks/compile-data.js +++ /dev/null @@ -1,105 +0,0 @@ -/* jshint node:true */ - -/** - * Compiles all JSON data into the polyfill and saves it as Intl.complete.js - */ -module.exports = function(grunt) { - - function replacer($0, type, loc) { - return (type === 'prims' ? 'a' : 'b') + loc; - } - - grunt.registerTask('compile-data', 'Compile the data into the polyfill', function() { - var - locData = {}, - objStrs = {}, - objs = [], - prims = [], - - valCount = 0, - objCount = 0, - - fileData = '', - fs = require('fs'), - locales = fs.readdirSync('locale-data/json/'); - - fileData += '(function(addLocaleData){\n'; - - locales.forEach(function (file) { - var c = fs.readFileSync('locale-data/json/' + file), - k = file.slice(0, file.indexOf('.')); - locData[k] = JSON.parse(c, reviver); - }); - function reviver (k, v) { - var idx; - - if (k === 'locale') - return v; - - else if (typeof v === 'string') { - idx = prims.indexOf(v); - valCount++; - - if (idx === -1) - idx += prims.push(v); - - return '###prims['+ idx +']###'; - } - - else if (typeof v === 'object' && v !== null) { - var str = JSON.stringify(v); - objCount++; - - if (objStrs.hasOwnProperty(str)) - return objStrs[str]; - - else { - // We need to make sure this object is not added to the same - // array as an object it references (and we need to check - // this recursively) - var - depth, - objDepths = [ 0 ]; - - for (var key in v) { - if (typeof v[key] === 'string' && (depth = v[key].match(/^###objs\[(\d+)/))) - objDepths.push(+depth[1] + 1); - } - - depth = Math.max.apply(Math, objDepths); - - if (!Array.isArray(objs[depth])) - objs[depth] = []; - - idx = objs[depth].push(v) - 1; - objStrs[str] = '###objs['+ depth +']['+ idx +']###'; - - return objStrs[str]; - } - } - - else - return v; - } - - fileData += 'var a='+ JSON.stringify(prims) +',b=[];'; - objs.forEach(function (val, idx) { - var ref = JSON.stringify(val).replace(/"###(objs|prims)(\[[^#]+)###"/g, replacer); - - fileData += 'b['+ idx +']='+ ref +';'; - }); - - for (var k in locData) { - fileData += 'addLocaleData('+ locData[k].replace(/###(objs|prims)(\[[^#]+)###/, replacer) +');\n'; - } - - fileData += '})(IntlPolyfill.__addLocaleData);'; - - // writting the complete optimized bundle - grunt.file.write('locale-data/complete.js', fileData); - - grunt.log.writeln('Total number of reused strings is ' + prims.length + ' (reduced from ' + valCount + ')'); - grunt.log.writeln('Total number of reused objects is ' + Object.keys(objStrs).length + ' (reduced from ' + objCount + ')'); - }); - -}; diff --git a/tasks/extract-cldr-data.js b/tasks/extract-cldr-data.js deleted file mode 100644 index 932e92c6..00000000 --- a/tasks/extract-cldr-data.js +++ /dev/null @@ -1,67 +0,0 @@ -/* jshint node:true */ - -/** - * Compiles all JSON data into the polyfill and saves it as Intl.complete.js - */ - -var assign = require('object.assign'); -var path = require('path'); - -// Regex for converting locale JSON to object grammar, obviously simple and -// incomplete but should be good enough for the CLDR JSON -var jsonpExp = /"(?!default)([\w$][\w\d$]+)":/g; - -var reduceLocaleData = require('./utils/reduce'); - -function mergeData(/*...sources*/) { - var sources = [].slice.call(arguments); - return sources.reduce(function (data, source) { - Object.keys(source || {}).forEach(function (locale) { - data[locale] = assign(data[locale] || {}, source[locale]); - }); - - return data; - }, {}); -} - -module.exports = function(grunt) { - - grunt.registerTask('extract-cldr-data', 'Extract Numbers and Calendars Data from CLDR', function() { - - try { - require('cldr-core/supplemental/parentLocales.json'); - } catch (e) { - throw new Error('Error locating cldr data, make sure you execute `grunt update-cldr-data` before.'); - } - - var extractCalendars = require('./utils/extract-calendars'); - var extractNumbersFields = require('./utils/extract-numbers'); - // Default to all CLDR locales. - var locales = require('./utils/locales').getAllLocales(); - - // Each type of data has the structure: `{"": {"": }}`, - // which is well suited for merging into a single object per locale. This - // performs that deep merge and returns the aggregated result. - var locData = mergeData( - extractCalendars(locales), - extractNumbersFields(locales) - ); - - Object.keys(locData).forEach(function (locale) { - - // Ignore en-US-POSIX and root - if (locale.toLowerCase() === 'en-us-posix') { - return; - } - - var obj = reduceLocaleData(locale, locData[locale]); - var jsonContent = JSON.stringify(obj, null, 4); - var jsonpContent = 'IntlPolyfill.__addLocaleData(' + JSON.stringify(obj).replace(jsonpExp, '$1:') + ');'; - grunt.file.write('locale-data/json/' + locale + '.json', jsonContent); - grunt.file.write('locale-data/jsonp/' + locale + '.js', jsonpContent); - }); - - grunt.log.writeln('Total number of locales is ' + Object.keys(locData).length); - }); - -};