From e865c0d471d86a6aa5b6fa989f284c9cca840a4e Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 17 Jan 2015 14:07:01 +1100 Subject: [PATCH 1/3] added iojs support, added test --- Makefile | 23 +++++++++++++++++++++++ lib/build.js | 8 +++++--- lib/install.js | 44 ++++++++++++++++++++++++-------------------- package.json | 4 ++++ 4 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..9158e8cafb --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +test: build-buffertools build-leveldown + +build-buffertools: + cd node_modules/buffertools/ && \ + ../../bin/node-gyp.js rebuild + +build-leveldown: + cd node_modules/leveldown/ && \ + ../../bin/node-gyp.js rebuild + +test-docker-node: + docker run \ + --rm=true -i -v `pwd`:/opt/node-gyp/ \ + nodesource/node:wheezy \ + bash -c 'rm -rf /root/.node-gyp/ && cd /opt/node-gyp && make test' + +test-docker-iojs: + docker run \ + --rm=true -i -v `pwd`:/opt/node-gyp/ \ + iojs/iojs:1.0 \ + bash -c 'rm -rf /root/.node-gyp/ && cd /opt/node-gyp && make test' + +test-docker: test-docker-node test-docker-iojs \ No newline at end of file diff --git a/lib/build.js b/lib/build.js index df24aaf454..4dcd6a5e27 100644 --- a/lib/build.js +++ b/lib/build.js @@ -14,6 +14,8 @@ var fs = require('graceful-fs') , mkdirp = require('mkdirp') , exec = require('child_process').exec , win = process.platform == 'win32' + , semver = require('semver') + , runtime = semver.parse(process.version).major < 1 ? 'node' : 'iojs' exports.usage = 'Invokes `' + (win ? 'msbuild' : 'make') + '` and builds the module' @@ -181,15 +183,15 @@ function build (gyp, argv, callback) { if (!win || !copyDevLib) return doBuild() var buildDir = path.resolve(nodeDir, buildType) - , archNodeLibPath = path.resolve(nodeDir, arch, 'node.lib') - , buildNodeLibPath = path.resolve(buildDir, 'node.lib') + , archNodeLibPath = path.resolve(nodeDir, arch, runtime + '.lib') + , buildNodeLibPath = path.resolve(buildDir, runtime + '.lib') mkdirp(buildDir, function (err, isNew) { if (err) return callback(err) log.verbose('"' + buildType + '" dir needed to be created?', isNew) var rs = fs.createReadStream(archNodeLibPath) , ws = fs.createWriteStream(buildNodeLibPath) - log.verbose('copying "node.lib" for ' + arch, buildNodeLibPath) + log.verbose('copying "' + runtime + '.lib" for ' + arch, buildNodeLibPath) rs.pipe(ws) rs.on('error', callback) ws.on('error', callback) diff --git a/lib/install.js b/lib/install.js index 6f72e6a93d..b4091549fa 100644 --- a/lib/install.js +++ b/lib/install.js @@ -21,6 +21,8 @@ var fs = require('graceful-fs') , minimatch = require('minimatch') , mkdir = require('mkdirp') , win = process.platform == 'win32' + , runtime = semver.parse(process.version).major < 1 ? 'node' : 'iojs' + , defaultDisturl = runtime == 'node' ? 'http://nodejs.org/dist' : 'https://iojs.org/dist' function install (gyp, argv, callback) { @@ -39,7 +41,7 @@ function install (gyp, argv, callback) { } } - var distUrl = gyp.opts['dist-url'] || gyp.opts.disturl || 'http://nodejs.org/dist' + var distUrl = gyp.opts['dist-url'] || gyp.opts.disturl || defaultDisturl // Determine which node dev files version we are installing @@ -185,7 +187,7 @@ function install (gyp, argv, callback) { // now download the node tarball var tarPath = gyp.opts['tarball'] - var tarballUrl = tarPath ? tarPath : distUrl + '/v' + version + '/node-v' + version + '.tar.gz' + var tarballUrl = tarPath ? tarPath : distUrl + '/v' + version + '/' + runtime + '-v' + version + '.tar.gz' , badDownload = false , extractCount = 0 , gunzip = zlib.createGunzip() @@ -343,36 +345,38 @@ function install (gyp, argv, callback) { } function downloadNodeLib (done) { - log.verbose('on Windows; need to download `node.lib`...') + log.verbose('on Windows; need to download `' + runtime + '.lib`...') var dir32 = path.resolve(devDir, 'ia32') , dir64 = path.resolve(devDir, 'x64') - , nodeLibPath32 = path.resolve(dir32, 'node.lib') - , nodeLibPath64 = path.resolve(dir64, 'node.lib') - , nodeLibUrl32 = distUrl + '/v' + version + '/node.lib' - , nodeLibUrl64 = distUrl + '/v' + version + '/x64/node.lib' - - log.verbose('32-bit node.lib dir', dir32) - log.verbose('64-bit node.lib dir', dir64) - log.verbose('`node.lib` 32-bit url', nodeLibUrl32) - log.verbose('`node.lib` 64-bit url', nodeLibUrl64) + , nodeLibPath32 = path.resolve(dir32, runtime + '.lib') + , nodeLibPath64 = path.resolve(dir64, runtime + '.lib') + , nodeLibUrlFile32 = (runtime == 'node' ? 'node.lib' : 'win-x86/iojs.lib') + , nodeLibUrlFile64 = (runtime == 'node' ? 'x64/node.lib' : 'win-x64/iojs.lib') + , nodeLibUrl32 = distUrl + '/v' + version + '/' + nodeLibUrlFile32 + , nodeLibUrl64 = distUrl + '/v' + version + '/' + nodeLibUrlFile64 + + log.verbose('32-bit ' + runtime + '.lib dir', dir32) + log.verbose('64-bit ' + runtime + '.lib dir', dir64) + log.verbose('`' + runtime + '.lib` 32-bit url', nodeLibUrl32) + log.verbose('`' + runtime + '.lib` 64-bit url', nodeLibUrl64) var async = 2 mkdir(dir32, function (err) { if (err) return done(err) - log.verbose('streaming 32-bit node.lib to:', nodeLibPath32) + log.verbose('streaming 32-bit ' + runtime + '.lib to:', nodeLibPath32) var req = download(nodeLibUrl32) if (!req) return req.on('error', done) req.on('response', function (res) { if (res.statusCode !== 200) { - done(new Error(res.statusCode + ' status code downloading 32-bit node.lib')) + done(new Error(res.statusCode + ' status code downloading 32-bit ' + runtime + '.lib')) return } getContentSha(res, function (_, checksum) { - contentShasums['node.lib'] = checksum - log.verbose('content checksum', 'node.lib', checksum) + contentShasums[nodeLibUrlFile32] = checksum + log.verbose('content checksum', nodeLibUrlFile32, checksum) }) var ws = fs.createWriteStream(nodeLibPath32) @@ -385,20 +389,20 @@ function install (gyp, argv, callback) { }) mkdir(dir64, function (err) { if (err) return done(err) - log.verbose('streaming 64-bit node.lib to:', nodeLibPath64) + log.verbose('streaming 64-bit ' + runtime + '.lib to:', nodeLibPath64) var req = download(nodeLibUrl64) if (!req) return req.on('error', done) req.on('response', function (res) { if (res.statusCode !== 200) { - done(new Error(res.statusCode + ' status code downloading 64-bit node.lib')) + done(new Error(res.statusCode + ' status code downloading 64-bit ' + runtime + '.lib')) return } getContentSha(res, function (_, checksum) { - contentShasums['x64/node.lib'] = checksum - log.verbose('content checksum', 'x64/node.lib', checksum) + contentShasums[nodeLibUrlFile64] = checksum + log.verbose('content checksum', nodeLibUrlFile64, checksum) }) var ws = fs.createWriteStream(nodeLibPath64) diff --git a/package.json b/package.json index 75d8f9f93b..40ffbbdd0b 100644 --- a/package.json +++ b/package.json @@ -37,5 +37,9 @@ }, "engines": { "node": ">= 0.8.0" + }, + "devDependencies": { + "buffertools": "~2.1.2", + "leveldown": "~1.0.1" } } From 3b49ef8ee45573dae715c519790107c60297769c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sun, 25 Jan 2015 22:16:09 +1100 Subject: [PATCH 2/3] add process.release support inspect process.release object for details about the platform and where to get source files --- lib/configure.js | 30 ++------------- lib/install.js | 91 ++++++++++++++++++--------------------------- lib/util/release.js | 60 ++++++++++++++++++++++++++++++ package.json | 3 ++ 4 files changed, 102 insertions(+), 82 deletions(-) create mode 100644 lib/util/release.js diff --git a/lib/configure.js b/lib/configure.js index 68f26d6e19..6aab4f0292 100644 --- a/lib/configure.js +++ b/lib/configure.js @@ -123,36 +123,12 @@ function configure (gyp, argv, callback) { createBuildDir() } else { - // if no --nodedir specified, ensure node dependencies are installed - var version - var versionStr - - if (gyp.opts.target) { - // if --target was given, then determine a target version to compile for - versionStr = gyp.opts.target - log.verbose('get node dir', 'compiling against --target node version: %s', versionStr) - } else { - // if no --target was specified then use the current host node version - versionStr = process.version - log.verbose('get node dir', 'no --target version specified, falling back to host node version: %s', versionStr) - } - - // make sure we have a valid version - try { - version = semver.parse(versionStr) - } catch (e) { - return callback(e) - } - if (!version) { - return callback(new Error('Invalid version number: ' + versionStr)) - } - // ensure that the target node version's dev files are installed gyp.opts.ensure = true - gyp.commands.install([ versionStr ], function (err, version) { + gyp.commands.install([], function (err, release) { if (err) return callback(err) - log.verbose('get node dir', 'target node version installed:', version) - nodeDir = path.resolve(gyp.devDir, version) + log.verbose('get node dir', 'target node version installed:', release.version) + nodeDir = release.devDir createBuildDir() }) } diff --git a/lib/install.js b/lib/install.js index b4091549fa..550c171459 100644 --- a/lib/install.js +++ b/lib/install.js @@ -15,17 +15,25 @@ var fs = require('graceful-fs') , crypto = require('crypto') , zlib = require('zlib') , log = require('npmlog') - , semver = require('semver') , fstream = require('fstream') , request = require('request') , minimatch = require('minimatch') , mkdir = require('mkdirp') + , getRelease = require('./util/release') , win = process.platform == 'win32' - , runtime = semver.parse(process.version).major < 1 ? 'node' : 'iojs' - , defaultDisturl = runtime == 'node' ? 'http://nodejs.org/dist' : 'https://iojs.org/dist' function install (gyp, argv, callback) { + // Determine which node dev files version we are installing + var release + try { + release = getRelease(gyp, argv[0] || gyp.opts.target) + } catch (e) { + return callback(e) + } + + log.verbose('install', 'input version string %j', release.versionStr) + // ensure no double-callbacks happen function cb (err) { if (cb.done) return @@ -33,34 +41,17 @@ function install (gyp, argv, callback) { if (err) { log.warn('install', 'got an error, rolling back install') // roll-back the install if anything went wrong - gyp.commands.remove([ version ], function (err2) { + gyp.commands.remove([ release.version.version ], function (err2) { callback(err) }) } else { - callback(null, version) + callback(null, release) } } - var distUrl = gyp.opts['dist-url'] || gyp.opts.disturl || defaultDisturl - - - // Determine which node dev files version we are installing - var versionStr = argv[0] || gyp.opts.target || process.version - log.verbose('install', 'input version string %j', versionStr) - - // parse the version to normalize and ensure it's valid - var version = semver.parse(versionStr) - if (!version) { - return callback(new Error('Invalid version number: ' + versionStr)) - } - - if (semver.lt(versionStr, '0.8.0')) { - return callback(new Error('Minimum target version is `0.8.0` or greater. Got: ' + versionStr)) - } - // 0.x.y-pre versions are not published yet and cannot be installed. Bail. - if (version.prerelease[0] === 'pre') { - log.verbose('detected "pre" node version', versionStr) + if (release.version.prerelease[0] === 'pre') { + log.verbose('detected "pre" node version', release.versionStr) if (gyp.opts.nodedir) { log.verbose('--nodedir flag was passed; skipping install', gyp.opts.nodedir) callback() @@ -71,23 +62,16 @@ function install (gyp, argv, callback) { } // flatten version into String - version = version.version - log.verbose('install', 'installing version: %s', version) - - // distributions starting with 0.10.0 contain sha256 checksums - var checksumAlgo = semver.gte(version, '0.10.0') ? 'sha256' : 'sha1' - - // the directory where the dev files will be installed - var devDir = path.resolve(gyp.devDir, version) + log.verbose('install', 'installing version: %s', release.version.version) // If '--ensure' was passed, then don't *always* install the version; // check if it is already installed, and only install when needed if (gyp.opts.ensure) { log.verbose('install', '--ensure was passed, so won\'t reinstall if already installed') - fs.stat(devDir, function (err, stat) { + fs.stat(release.devDir, function (err, stat) { if (err) { if (err.code == 'ENOENT') { - log.verbose('install', 'version not already installed, continuing with install', version) + log.verbose('install', 'version not already installed, continuing with install', release.versionStr) go() } else if (err.code == 'EACCES') { eaccesFallback() @@ -97,7 +81,7 @@ function install (gyp, argv, callback) { return } log.verbose('install', 'version is already installed, need to check "installVersion"') - var installVersionFile = path.resolve(devDir, 'installVersion') + var installVersionFile = path.resolve(release.devDir, 'installVersion') fs.readFile(installVersionFile, 'ascii', function (err, ver) { if (err && err.code != 'ENOENT') { return cb(err) @@ -158,7 +142,7 @@ function install (gyp, argv, callback) { } function getContentSha(res, callback) { - var shasum = crypto.createHash(checksumAlgo) + var shasum = crypto.createHash(release.checksumAlgo) res.on('data', function (chunk) { shasum.update(chunk) }).on('end', function () { @@ -168,10 +152,10 @@ function install (gyp, argv, callback) { function go () { - log.verbose('ensuring nodedir is created', devDir) + log.verbose('ensuring nodedir is created', release.devDir) // first create the dir for the node dev files - mkdir(devDir, function (err, created) { + mkdir(release.devDir, function (err, created) { if (err) { if (err.code == 'EACCES') { eaccesFallback() @@ -187,11 +171,10 @@ function install (gyp, argv, callback) { // now download the node tarball var tarPath = gyp.opts['tarball'] - var tarballUrl = tarPath ? tarPath : distUrl + '/v' + version + '/' + runtime + '-v' + version + '.tar.gz' - , badDownload = false + , badDownload = false , extractCount = 0 , gunzip = zlib.createGunzip() - , extracter = tar.Extract({ path: devDir, strip: 1, filter: isValid }) + , extracter = tar.Extract({ path: release.devDir, strip: 1, filter: isValid }) var contentShasums = {} var expectShasums = {} @@ -199,7 +182,7 @@ function install (gyp, argv, callback) { // checks if a file to be extracted from the tarball is valid. // only .h header files and the gyp files get extracted function isValid () { - var name = this.path.substring(devDir.length + 1) + var name = this.path.substring(release.devDir.length + 1) var isValid = valid(name) if (name === '' && this.type === 'Directory') { // the first directory entry is ok @@ -222,12 +205,12 @@ function install (gyp, argv, callback) { // download the tarball, gunzip and extract! if (tarPath) { - var input = fs.createReadStream(tarballUrl) + var input = fs.createReadStream(release.sourceUrl) input.pipe(gunzip).pipe(extracter) return } - var req = download(tarballUrl) + var req = download(release.sourceUrl) if (!req) return // something went wrong downloading the tarball? @@ -250,7 +233,7 @@ function install (gyp, argv, callback) { } // content checksum getContentSha(res, function (_, checksum) { - var filename = path.basename(tarballUrl).trim() + var filename = path.basename(release.sourceUrl).trim() contentShasums[filename] = checksum log.verbose('content checksum', filename, checksum) }) @@ -276,7 +259,7 @@ function install (gyp, argv, callback) { // write the "installVersion" file async++ - var installVersionPath = path.resolve(devDir, 'installVersion') + var installVersionPath = path.resolve(release.devDir, 'installVersion') fs.writeFile(installVersionPath, gyp.package.installVersion + '\n', deref) // download SHASUMS.txt @@ -308,13 +291,11 @@ function install (gyp, argv, callback) { } function downloadShasums(done) { - var shasumsFile = (checksumAlgo === 'sha256') ? 'SHASUMS256.txt' : 'SHASUMS.txt' - log.verbose('check download content checksum, need to download `' + shasumsFile + '`...') - var shasumsPath = path.resolve(devDir, shasumsFile) - , shasumsUrl = distUrl + '/v' + version + '/' + shasumsFile + log.verbose('check download content checksum, need to download `' + release.shasumsFile + '`...') + var shasumsPath = path.resolve(release.devDir, release.shasumsFile) - log.verbose('checksum url', shasumsUrl) - var req = download(shasumsUrl) + log.verbose('checksum url', release.shasumsUrl) + var req = download(release.shasumsUrl) if (!req) return req.on('error', done) req.on('response', function (res) { @@ -346,8 +327,8 @@ function install (gyp, argv, callback) { function downloadNodeLib (done) { log.verbose('on Windows; need to download `' + runtime + '.lib`...') - var dir32 = path.resolve(devDir, 'ia32') - , dir64 = path.resolve(devDir, 'x64') + var dir32 = path.resolve(release.devDir, 'ia32') + , dir64 = path.resolve(release.devDir, 'x64') , nodeLibPath32 = path.resolve(dir32, runtime + '.lib') , nodeLibPath64 = path.resolve(dir64, runtime + '.lib') , nodeLibUrlFile32 = (runtime == 'node' ? 'node.lib' : 'win-x86/iojs.lib') @@ -441,7 +422,7 @@ function install (gyp, argv, callback) { function eaccesFallback () { var tmpdir = osenv.tmpdir() gyp.devDir = path.resolve(tmpdir, '.node-gyp') - log.warn('EACCES', 'user "%s" does not have permission to access the dev dir "%s"', osenv.user(), devDir) + log.warn('EACCES', 'user "%s" does not have permission to access the dev dir "%s"', osenv.user(), release.devDir) log.warn('EACCES', 'attempting to reinstall using temporary dev dir "%s"', gyp.devDir) if (process.cwd() == tmpdir) { log.verbose('tmpdir == cwd', 'automatically will remove dev files after to save disk space') diff --git a/lib/util/release.js b/lib/util/release.js new file mode 100644 index 0000000000..57a683e763 --- /dev/null +++ b/lib/util/release.js @@ -0,0 +1,60 @@ +var path = require('path') + , url = require('url') + , semver = require('semver') + , win = process.platform == 'win32' + , defaultDistUrl = 'http://nodejs.org/dist/' + +// note: this does not support explicit 'version' setting where the platform +// is not 'node' (i.e. --target=1.0.3 won't pull down io.js) +function getRelease (gyp, version) { + var versionObj = semver.parse(version || process.version) + , versionStr + , release + + if (!versionObj) { + throw new Error('Invalid version number: ' + version) + } + + if (versionObj.compare('0.8.0') < 0) { + throw new Error('Minimum target version is `0.8.0` or greater. Got: ' + version) + } + + versionStr = versionObj && ('v' + versionObj.version) + release = { + version: versionObj, + versionStr: versionStr + } + + if ((!version || version == process.version) && process.release) { + release.name = process.release.name + release.sourceUrl = process.release.sourceUrl + // headersUrl + } else { + release.name = 'node' + release.sourceUrl = defaultDistUrl + versionStr + '/node-' + release.versionStr + '.tar.gz' + } + + // distributions starting with 0.10.0 contain sha256 checksums + release.checksumAlgo = semver.gte(versionObj, '0.10.0') ? 'sha256' : 'sha1' + release.shasumsFile = (release.checksumAlgo === 'sha256') ? 'SHASUMS256.txt' : 'SHASUMS.txt' + + var u = url.parse(release.sourceUrl) + u.pathname = u.pathname.replace(/[^\/]+$/, release.shasumsFile) + release.shasumsUrl = url.format(u) + + if (win) { + if (process.release && process.release.libUrl) + release.libUrl = process.release.libUrl + else + release.libUrl = defaultDistUrl + versionStr + '/' + (process.arch == 'x64' ? 'x64/' : '') + 'node.lib' + } + + // the directory where the dev files will be installed, prefixed with + // the platform name if !node + release.devDir = path.resolve(gyp.devDir, + (release.name != 'node' ? release.name + '-' : '') + release.version.version) + + return release +} + +module.exports = getRelease \ No newline at end of file diff --git a/package.json b/package.json index 40ffbbdd0b..be7d21bd8d 100644 --- a/package.json +++ b/package.json @@ -41,5 +41,8 @@ "devDependencies": { "buffertools": "~2.1.2", "leveldown": "~1.0.1" + }, + "scripts": { + "test": "make test" } } From 55936498fca8f4ae27d55d5ac12c3d714a93742c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 7 Feb 2015 14:01:14 +1100 Subject: [PATCH 3/3] windows support for io.js --- lib/install.js | 12 +++++++----- lib/util/release.js | 23 ++++++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/install.js b/lib/install.js index 550c171459..a764ec5189 100644 --- a/lib/install.js +++ b/lib/install.js @@ -32,6 +32,8 @@ function install (gyp, argv, callback) { return callback(e) } +console.log(release) + log.verbose('install', 'input version string %j', release.versionStr) // ensure no double-callbacks happen @@ -170,8 +172,8 @@ function install (gyp, argv, callback) { } // now download the node tarball - var tarPath = gyp.opts['tarball'] - , badDownload = false + var tarPath = gyp.opts.tarball + , badDownload = false , extractCount = 0 , gunzip = zlib.createGunzip() , extracter = tar.Extract({ path: release.devDir, strip: 1, filter: isValid }) @@ -183,19 +185,19 @@ function install (gyp, argv, callback) { // only .h header files and the gyp files get extracted function isValid () { var name = this.path.substring(release.devDir.length + 1) - var isValid = valid(name) + var validName = valid(name) if (name === '' && this.type === 'Directory') { // the first directory entry is ok return true } - if (isValid) { + if (validName) { log.verbose('extracted file from tarball', name) extractCount++ } else { // invalid log.silly('ignoring from tarball', name) } - return isValid + return validName } gunzip.on('error', cb) diff --git a/lib/util/release.js b/lib/util/release.js index 57a683e763..d6cb50e3b7 100644 --- a/lib/util/release.js +++ b/lib/util/release.js @@ -39,14 +39,23 @@ function getRelease (gyp, version) { release.shasumsFile = (release.checksumAlgo === 'sha256') ? 'SHASUMS256.txt' : 'SHASUMS.txt' var u = url.parse(release.sourceUrl) - u.pathname = u.pathname.replace(/[^\/]+$/, release.shasumsFile) + u.path = u.pathname = u.path.replace(/[^\/]+$/, release.shasumsFile) release.shasumsUrl = url.format(u) - - if (win) { - if (process.release && process.release.libUrl) - release.libUrl = process.release.libUrl - else - release.libUrl = defaultDistUrl + versionStr + '/' + (process.arch == 'x64' ? 'x64/' : '') + 'node.lib' + + if (true || win) { + if (process.release && process.release.lib32Url && process.release.lib64Url) { + release.lib32Url = process.release.lib32Url + release.lib64Url = process.release.lib64Url + u.path = u.pathname = u.path.replace(new RegExp(release.shasumsFile + '$'), '') + console.log(u, url.format(u)) + release.lib32File = release.lib32Url.replace(url.format(u), '') + release.lib64File = release.lib64Url.replace(url.format(u), '') + } else { + release.lib32File = 'node.lib' + release.lib64File = 'x64/node.lib' + release.lib32Url = defaultDistUrl + versionStr + '/' + release.lib32File + release.lib64Url = defaultDistUrl + versionStr + '/' + release.lib64File + } } // the directory where the dev files will be installed, prefixed with