Skip to content

Commit

Permalink
feat: stage assets under .cdk.assets (#2182)
Browse files Browse the repository at this point in the history
To ensure that assets are available for the toolchain to deploy after the CDK app
exists, the CLI will, by default, request that the app will stage the assets under
the `.cdk.assets` directory (relative to working directory).

The CDK will then *copy* all assets from their source locations to this staging
directory and will refer to the staging location as the asset path. Assets will
be stored using their content fingerprint (md5 hash) so they will never be copied
twice unless they change.

Docker build context directories will also be staged.

Staging is disabled by default and in cdk-integ.

Added .cdk.staging to all .gitignore files in cdk init templates.

Fixes #1716
Fixes #2096
  • Loading branch information
Elad Ben-Israel authored Apr 8, 2019
1 parent 5d52624 commit 2f74eb4
Show file tree
Hide file tree
Showing 46 changed files with 881 additions and 25 deletions.
17 changes: 12 additions & 5 deletions packages/@aws-cdk/assets-docker/lib/image-asset.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import assets = require('@aws-cdk/assets');
import ecr = require('@aws-cdk/aws-ecr');
import cdk = require('@aws-cdk/cdk');
import cxapi = require('@aws-cdk/cx-api');
Expand Down Expand Up @@ -49,14 +50,20 @@ export class DockerImageAsset extends cdk.Construct {
super(scope, id);

// resolve full path
this.directory = path.resolve(props.directory);
if (!fs.existsSync(this.directory)) {
throw new Error(`Cannot find image directory at ${this.directory}`);
const dir = path.resolve(props.directory);
if (!fs.existsSync(dir)) {
throw new Error(`Cannot find image directory at ${dir}`);
}
if (!fs.existsSync(path.join(this.directory, 'Dockerfile'))) {
throw new Error(`No 'Dockerfile' found in ${this.directory}`);
if (!fs.existsSync(path.join(dir, 'Dockerfile'))) {
throw new Error(`No 'Dockerfile' found in ${dir}`);
}

const staging = new assets.Staging(this, 'Staging', {
sourcePath: dir
});

this.directory = staging.stagedPath;

const imageNameParameter = new cdk.CfnParameter(this, 'ImageName', {
type: 'String',
description: `ECR repository name and tag asset "${this.node.path}"`,
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/assets-docker/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions packages/@aws-cdk/assets-docker/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,15 @@
"@aws-cdk/aws-iam": "^0.28.0",
"@aws-cdk/aws-lambda": "^0.28.0",
"@aws-cdk/aws-s3": "^0.28.0",
"@aws-cdk/assets": "^0.28.0",
"@aws-cdk/cdk": "^0.28.0",
"@aws-cdk/cx-api": "^0.28.0"
},
"homepage": "https://github.com/awslabs/aws-cdk",
"peerDependencies": {
"@aws-cdk/aws-ecr": "^0.28.0",
"@aws-cdk/aws-iam": "^0.28.0",
"@aws-cdk/assets": "^0.28.0",
"@aws-cdk/aws-s3": "^0.28.0",
"@aws-cdk/cdk": "^0.28.0"
},
Expand Down
28 changes: 28 additions & 0 deletions packages/@aws-cdk/assets-docker/test/test.image-asset.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import { expect, haveResource, SynthUtils } from '@aws-cdk/assert';
import iam = require('@aws-cdk/aws-iam');
import cdk = require('@aws-cdk/cdk');
import cxapi = require('@aws-cdk/cx-api');
import fs = require('fs');
import { Test } from 'nodeunit';
import os = require('os');
import path = require('path');
import { DockerImageAsset } from '../lib';

Expand Down Expand Up @@ -143,5 +146,30 @@ export = {
});
}, /No 'Dockerfile' found in/);
test.done();
},

'docker directory is staged if asset staging is enabled'(test: Test) {
const workdir = mkdtempSync();
process.chdir(workdir);

const app = new cdk.App({
context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.stage-me' }
});

const stack = new cdk.Stack(app, 'stack');

new DockerImageAsset(stack, 'MyAsset', {
directory: path.join(__dirname, 'demo-image')
});

app.run();

test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/Dockerfile'));
test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/index.py'));
test.done();
}
};

function mkdtempSync() {
return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets'));
}
18 changes: 13 additions & 5 deletions packages/@aws-cdk/assets/lib/asset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk');
import cxapi = require('@aws-cdk/cx-api');
import fs = require('fs');
import path = require('path');
import { Staging } from './staging';

/**
* Defines the way an asset is packaged before it is uploaded to S3.
Expand Down Expand Up @@ -61,7 +62,10 @@ export class Asset extends cdk.Construct {
public readonly s3Url: string;

/**
* Resolved full-path location of this asset.
* The path to the asset (stringinfied token).
*
* If asset staging is disabled, this will just be the original path.
* If asset staging is enabled it will be the staged path.
*/
public readonly assetPath: string;

Expand All @@ -84,16 +88,20 @@ export class Asset extends cdk.Construct {
constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) {
super(scope, id);

// resolve full path
this.assetPath = path.resolve(props.path);
// stage the asset source (conditionally).
const staging = new Staging(this, 'Stage', {
sourcePath: path.resolve(props.path)
});

this.assetPath = staging.stagedPath;

// sets isZipArchive based on the type of packaging and file extension
const allowedExtensions: string[] = ['.jar', '.zip'];
this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory
? true
: allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext));
: allowedExtensions.some(ext => staging.sourcePath.toLowerCase().endsWith(ext));

validateAssetOnDisk(this.assetPath, props.packaging);
validateAssetOnDisk(staging.sourcePath, props.packaging);

// add parameters for s3 bucket and s3 key. those will be set by
// the toolkit or by CI/CD when the stack is deployed and will include
Expand Down
89 changes: 89 additions & 0 deletions packages/@aws-cdk/assets/lib/fs/copy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import fs = require('fs');
import minimatch = require('minimatch');
import path = require('path');
import { FollowMode } from './follow-mode';

export interface CopyOptions {
/**
* @default External only follows symlinks that are external to the source directory
*/
follow?: FollowMode;

/**
* glob patterns to exclude from the copy.
*/
exclude?: string[];
}

export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
const follow = options.follow !== undefined ? options.follow : FollowMode.External;
const exclude = options.exclude || [];

rootDir = rootDir || srcDir;

if (!fs.statSync(srcDir).isDirectory()) {
throw new Error(`${srcDir} is not a directory`);
}

const files = fs.readdirSync(srcDir);
for (const file of files) {
const sourceFilePath = path.join(srcDir, file);

if (shouldExclude(path.relative(rootDir, sourceFilePath))) {
continue;
}

const destFilePath = path.join(destDir, file);

let stat: fs.Stats | undefined = follow === FollowMode.Always
? fs.statSync(sourceFilePath)
: fs.lstatSync(sourceFilePath);

if (stat && stat.isSymbolicLink()) {
const target = fs.readlinkSync(sourceFilePath);

// determine if this is an external link (i.e. the target's absolute path
// is outside of the root directory).
const targetPath = path.normalize(path.resolve(srcDir, target));
const rootPath = path.normalize(rootDir);
const external = !targetPath.startsWith(rootPath);

if (follow === FollowMode.External && external) {
stat = fs.statSync(sourceFilePath);
} else {
fs.symlinkSync(target, destFilePath);
stat = undefined;
}
}

if (stat && stat.isDirectory()) {
fs.mkdirSync(destFilePath);
copyDirectory(sourceFilePath, destFilePath, options, rootDir);
stat = undefined;
}

if (stat && stat.isFile()) {
fs.copyFileSync(sourceFilePath, destFilePath);
stat = undefined;
}
}

function shouldExclude(filePath: string): boolean {
let excludeOutput = false;

for (const pattern of exclude) {
const negate = pattern.startsWith('!');
const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true });

if (!negate && match) {
excludeOutput = true;
}

if (negate && match) {
excludeOutput = false;
}
}

return excludeOutput;
}
}
86 changes: 86 additions & 0 deletions packages/@aws-cdk/assets/lib/fs/fingerprint.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import crypto = require('crypto');
import fs = require('fs');
import path = require('path');
import { FollowMode } from './follow-mode';

const BUFFER_SIZE = 8 * 1024;

export interface FingerprintOptions {
/**
* Extra information to encode into the fingerprint (e.g. build instructions
* and other inputs)
*/
extra?: string;

/**
* List of exclude patterns (see `CopyOptions`)
* @default include all files
*/
exclude?: string[];

/**
* What to do when we encounter symlinks.
* @default External only follows symlinks that are external to the source
* directory
*/
follow?: FollowMode;
}

/**
* Produces fingerprint based on the contents of a single file or an entire directory tree.
*
* The fingerprint will also include:
* 1. An extra string if defined in `options.extra`.
* 2. The set of exclude patterns, if defined in `options.exclude`
* 3. The symlink follow mode value.
*
* @param fileOrDirectory The directory or file to fingerprint
* @param options Fingerprinting options
*/
export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) {
const follow = options.follow !== undefined ? options.follow : FollowMode.External;
const hash = crypto.createHash('md5');
addToHash(fileOrDirectory);

hash.update(`==follow==${follow}==\n\n`);

if (options.extra) {
hash.update(`==extra==${options.extra}==\n\n`);
}

for (const ex of options.exclude || []) {
hash.update(`==exclude==${ex}==\n\n`);
}

return hash.digest('hex');

function addToHash(pathToAdd: string) {
hash.update('==\n');
const relativePath = path.relative(fileOrDirectory, pathToAdd);
hash.update(relativePath + '\n');
hash.update('~~~~~~~~~~~~~~~~~~\n');
const stat = fs.statSync(pathToAdd);

if (stat.isSymbolicLink()) {
const target = fs.readlinkSync(pathToAdd);
hash.update(target);
} else if (stat.isDirectory()) {
for (const file of fs.readdirSync(pathToAdd)) {
addToHash(path.join(pathToAdd, file));
}
} else {
const file = fs.openSync(pathToAdd, 'r');
const buffer = Buffer.alloc(BUFFER_SIZE);

try {
let bytesRead;
do {
bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null);
hash.update(buffer.slice(0, bytesRead));
} while (bytesRead === BUFFER_SIZE);
} finally {
fs.closeSync(file);
}
}
}
}
29 changes: 29 additions & 0 deletions packages/@aws-cdk/assets/lib/fs/follow-mode.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
export enum FollowMode {
/**
* Never follow symlinks.
*/
Never = 'never',

/**
* Materialize all symlinks, whether they are internal or external to the source directory.
*/
Always = 'always',

/**
* Only follows symlinks that are external to the source directory.
*/
External = 'external',

// ----------------- TODO::::::::::::::::::::::::::::::::::::::::::::
/**
* Forbids source from having any symlinks pointing outside of the source
* tree.
*
* This is the safest mode of operation as it ensures that copy operations
* won't materialize files from the user's file system. Internal symlinks are
* not followed.
*
* If the copy operation runs into an external symlink, it will fail.
*/
BlockExternal = 'internal-only',
}
3 changes: 3 additions & 0 deletions packages/@aws-cdk/assets/lib/fs/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export * from './fingerprint';
export * from './follow-mode';
export * from './copy';
1 change: 1 addition & 0 deletions packages/@aws-cdk/assets/lib/index.ts
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export * from './asset';
export * from './staging';
Loading

0 comments on commit 2f74eb4

Please sign in to comment.