From e90246fee7d225be554b7c0117ce285d27b2a617 Mon Sep 17 00:00:00 2001 From: Mitchell Valine Date: Tue, 28 Jan 2020 17:34:33 -0800 Subject: [PATCH] Use tar Module and Cleanup Use the tar node module instead of shelling out to tar. This allows us to stream the download contents directly to unpacking and removes the need to manually parse the beginning of the commit id hash to traverse into the build directory. --- buildspec.yaml | 1 - packages/@jsii/integ-test/.eslintrc.yaml | 2 + packages/@jsii/integ-test/package.json | 17 +++-- .../@jsii/integ-test/test/build-cdk.test.ts | 72 +++++++++++-------- packages/@jsii/integ-test/utils/index.ts | 28 ++++---- yarn.lock | 9 ++- 6 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 packages/@jsii/integ-test/.eslintrc.yaml diff --git a/buildspec.yaml b/buildspec.yaml index dbeddf51dd..6eb3988768 100644 --- a/buildspec.yaml +++ b/buildspec.yaml @@ -12,7 +12,6 @@ phases: build: commands: - yarn build && yarn test - - yarn test:integ post_build: commands: - '[ ${CODEBUILD_BUILD_SUCCEEDING} = 1 ] && yarn package' diff --git a/packages/@jsii/integ-test/.eslintrc.yaml b/packages/@jsii/integ-test/.eslintrc.yaml new file mode 100644 index 0000000000..e04f6e39ef --- /dev/null +++ b/packages/@jsii/integ-test/.eslintrc.yaml @@ -0,0 +1,2 @@ +--- +extends: ../../../eslint-config.yaml diff --git a/packages/@jsii/integ-test/package.json b/packages/@jsii/integ-test/package.json index 3e0d37c309..513efb9135 100644 --- a/packages/@jsii/integ-test/package.json +++ b/packages/@jsii/integ-test/package.json @@ -1,10 +1,11 @@ { "name": "@jsii/integ-test", - "version": "0.21.2", + "version": "0.22.0", "description": "A suite of integration tests for JSII and related modules.", "private": true, "scripts": { - "build": "tsc --build", + "build": "tsc --build && npm run lint", + "lint": "eslint . --ext .ts --ignore-path=.gitignore", "test:integ": "jest" }, "keywords": [], @@ -13,13 +14,21 @@ "url": "https://aws.amazon.com" }, "license": "Apache-2.0", + "dependencies": { + "tar": "^6.0.1" + }, "devDependencies": { "@octokit/rest": "^16.36.0", + "@types/dotenv": "^8.2.0", + "@types/fs-extra": "^8.0.1", + "@types/tar": "^4.0.3", "dotenv": "^8.2.0", + "eslint": "^6.8.0", "fs-extra": "^8.1.0", "jest": "^25.1.0", - "jsii": "^0.21.2", - "jsii-pacmak": "^0.21.2", + "jsii": "^0.22.0", + "jsii-pacmak": "^0.22.0", + "jsii-rosetta": "^0.22.0", "typescript": "~3.7.5" }, "jest": { diff --git a/packages/@jsii/integ-test/test/build-cdk.test.ts b/packages/@jsii/integ-test/test/build-cdk.test.ts index 141f8ac00f..ec525b6417 100644 --- a/packages/@jsii/integ-test/test/build-cdk.test.ts +++ b/packages/@jsii/integ-test/test/build-cdk.test.ts @@ -1,12 +1,13 @@ -import { mkdtemp, remove } from 'fs-extra'; -import * as path from 'path'; import * as Octokit from '@octokit/rest'; -import { downloadReleaseAsset, minutes, ProcessManager, writeFileStream } from '../utils'; import * as dotenv from 'dotenv'; +import { mkdtemp, remove } from 'fs-extra'; +import * as path from 'path'; +import { downloadReleaseAsset, minutes, ProcessManager, extractFileStream } from '../utils'; dotenv.config(); const JSII_DIR = path.dirname(require.resolve('jsii/package.json')); const JSII_PACMAK_DIR = path.dirname(require.resolve('jsii-pacmak/package.json')); +const JSII_ROSETTA_DIR = path.dirname(require.resolve('jsii-rosetta/package.json')); const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN @@ -28,45 +29,60 @@ describe('Build CDK', () => { test('can build latest cdk release', async () => { // download latest release info - console.time('cdkbuild'); const release = await octokit.repos.getLatestRelease({ owner: 'aws', repo: 'aws-cdk' }); - // save code to tmp dir - const fileName = 'cdk.tar.gz'; - const tarFile = path.join(buildDir, fileName); + // download and extract code const code = await downloadReleaseAsset(`https://api.github.com/repos/aws/aws-cdk/tarball/${release.data.tag_name}`); - - await writeFileStream(code, tarFile); - - // unzip tar archive - await processes.spawn('tar', ['-xzf', fileName], { - cwd: buildDir - }); - - // root dir of extracted src - // `${buildDir}/${owner}-${repo}-${first 7 chars of commit hash} const srcDir = path.join(buildDir, `aws-aws-cdk-${release.data.target_commitish.substring(0, 7)}`); + await extractFileStream(code, buildDir); // install cdk dependencies - await processes.spawn('yarn', ['install'], { + await processes.spawn('yarn', ['install', '--frozen-lockfile'], { cwd: srcDir }); - // link local jsii/jsii-pacmak builds - await processes.spawn('rm', ['-rf', './node_modules/jsii'], { cwd: srcDir }); - await processes.spawn('rm', ['-rf', './node_modules/jsii-pacmak'], { cwd: srcDir }); - await processes.spawn('ln', ['-s', JSII_DIR, './node_modules'], { cwd: srcDir }); - await processes.spawn('ln', ['-s', JSII_PACMAK_DIR, './node_modules'], { cwd: srcDir }); + // build cdk build tools + await processes.spawn('npx', [ + 'lerna', + '--stream', + '--scope', 'cdk-build-tools', + '--scope', 'pkglint', + '--scope', 'awslint', + 'run', 'build', + ], { cwd: srcDir }); - // build cdk - await processes.spawn('npx', ['lerna', 'run', 'build', '--stream'], { cwd: srcDir }); + // build jsii modules + await processes.spawn('npx', [ + 'lerna', + '--stream', + '--scope', '@aws-cdk/*', + '--scope', 'aws-cdk', + 'run', 'build', + '--', '--jsii', path.join(JSII_DIR, 'bin', 'jsii'), + ], { cwd: srcDir }); - // package modules - await processes.spawn('yarn', ['run', 'pack'], { cwd: srcDir }); + // build the rest + await processes.spawn('npx', [ + 'lerna', + '--stream', + '--ignore', '@aws-cdk/*', + '--ignore', 'aws-cdk', + '--ignore', 'cdk-build-tools', + '--ignore', 'pkglint', + '--ignore', 'awslint', + 'run', 'build', + ], { cwd: srcDir }); - console.timeEnd('cdkbuild'); + // package modules + await processes.spawn('./pack.sh', [], { + cwd: srcDir, + env: { + PACMAK: path.join(JSII_PACMAK_DIR, 'bin', 'jsii-pacmak'), + ROSETTA: path.join(JSII_ROSETTA_DIR, 'bin', 'jsii-rosetta') + } + }); }, minutes(60)); }); diff --git a/packages/@jsii/integ-test/utils/index.ts b/packages/@jsii/integ-test/utils/index.ts index 69a6e4062f..c8b1525ed2 100644 --- a/packages/@jsii/integ-test/utils/index.ts +++ b/packages/@jsii/integ-test/utils/index.ts @@ -1,8 +1,8 @@ -import { Readable } from 'stream'; -import { createWriteStream } from 'fs'; import * as cp from 'child_process'; -import * as https from 'https'; import { IncomingMessage } from 'http'; +import * as https from 'https'; +import { Readable } from 'stream'; +import { extract } from 'tar'; /** * @param num a quantity of minutes (could be fractional) @@ -19,17 +19,17 @@ export function minutes(num: number): number { export class ProcessManager { private readonly processes: { [pid: string]: { - proc: cp.ChildProcess, - promise: Promise - } + proc: cp.ChildProcess; + promise: Promise; + }; } = {}; /** * kill all still running processes * - * @param [signal] - signal sent to terminate process + * @param signal sent to terminate process */ - async killAll(signal?: string) { + public async killAll(signal?: string) { const values = Object.values(this.processes); await Promise.all(values.map(({ proc, promise }) => async() => { proc.kill(signal); @@ -53,7 +53,7 @@ export class ProcessManager { * @param arguments passed to command * @param options passed to child process spawn */ - spawn(cmd: string, args: string[], opts: any = {}): Promise { + public async spawn(cmd: string, args: string[], opts: any = {}): Promise { const proc = cp.spawn(cmd, args, { stdio: 'inherit', ...opts }); const promise = new Promise((ok, ko) => { @@ -85,11 +85,11 @@ export class ProcessManager { * write downloaded asset to file * * @param source stream - * @param destination of saved file + * @param destination directory for extracted files */ -export function writeFileStream(source: Readable, destination: string) { +export async function extractFileStream(source: Readable, destination: string) { return new Promise((ok, ko) => { - const destStream = createWriteStream(destination); + const destStream = extract({ cwd: destination }); destStream.once('close', ok); destStream.once('error', ko); source.once('error', ko); @@ -107,7 +107,7 @@ export function writeFileStream(source: Readable, destination: string) { * * @returns readable stream of asset data */ -export function downloadReleaseAsset(url: string): Promise { +export async function downloadReleaseAsset(url: string): Promise { return new Promise((ok, ko) => { const config = { headers: { @@ -131,4 +131,4 @@ export function downloadReleaseAsset(url: string): Promise { return ok(response); }); }); -}; +} diff --git a/yarn.lock b/yarn.lock index 92b46a6c23..de158735d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1374,6 +1374,13 @@ resolved "https://registry.yarnpkg.com/@types/deep-equal/-/deep-equal-1.0.1.tgz#71cfabb247c22bcc16d536111f50c0ed12476b03" integrity sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg== +"@types/dotenv@^8.2.0": + version "8.2.0" + resolved "https://registry.yarnpkg.com/@types/dotenv/-/dotenv-8.2.0.tgz#5cd64710c3c98e82d9d15844375a33bf1b45d053" + integrity sha512-ylSC9GhfRH7m1EUXBXofhgx4lUWmFeQDINW5oLuS+gxWdfUeW4zJdeVTYVkexEW+e2VUvlZR2kGnGGipAWR7kw== + dependencies: + dotenv "*" + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d" @@ -3239,7 +3246,7 @@ dot-prop@^4.2.0: dependencies: is-obj "^1.0.0" -dotenv@^8.2.0: +dotenv@*, dotenv@^8.2.0: version "8.2.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==