From d990d68809c7bbee84edad8de3966270e3714aa9 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Thu, 31 Oct 2019 14:34:03 +0100 Subject: [PATCH 01/19] feat(rosetta): extract and compile samples into "tablets" Version 2 of the "sampiler" is called "Rosetta". It now provides more control over the compilation of samples found in the source code. It integrates into an existing build by running `jsii-rosetta extract`, which will extract sample code from a jsii assembly, compile it, convert it to all supported languages, and storing the result in a a "tablet file" (effectively, a sample dictionary). Tablet files can then be used by `jsii-pacmak` to look up translations for the code samples it encounters as it is generating language-specific sources. In case the build does not contain a Rosetta step, Pacmak will try to convert samples that are not found in the tablet on the fly. However, the samples will not benefit from compilation, type checking and fixture support. --- .gitignore | 1 + packages/jsii-calc-base-of-base/package.json | 3 +- packages/jsii-calc-base/package.json | 5 +- packages/jsii-calc-lib/package.json | 5 +- packages/jsii-calc/README.md | 17 +- packages/jsii-calc/package.json | 5 +- packages/jsii-calc/rosetta/default.ts-fixture | 3 + .../rosetta/with-calculator.ts-fixture | 4 + packages/jsii-pacmak/bin/jsii-pacmak.ts | 23 +- packages/jsii-pacmak/lib/builder.ts | 9 +- packages/jsii-pacmak/lib/target.ts | 6 + packages/jsii-pacmak/lib/targets/java.ts | 3 +- packages/jsii-pacmak/lib/targets/python.ts | 287 ++++++++++-------- packages/jsii-pacmak/package.json | 3 +- .../test/expected.jsii-calc/python/README.md | 2 +- .../python/src/jsii_calc/__init__.py | 2 +- .../python/src/jsii_calc/_jsii/__init__.py | 2 +- packages/jsii-pacmak/tsconfig.json | 2 +- .../.gitignore | 0 .../.npmignore | 0 packages/jsii-rosetta/README.md | 177 +++++++++++ packages/jsii-rosetta/bin/jsii-rosetta | 2 + packages/jsii-rosetta/bin/jsii-rosetta.ts | 148 +++++++++ .../examples/controlflow.ts | 0 .../examples/incomplete.ts | 0 packages/jsii-rosetta/lib/commands/convert.ts | 36 +++ packages/jsii-rosetta/lib/commands/extract.ts | 37 +++ packages/jsii-rosetta/lib/commands/read.ts | 72 +++++ packages/jsii-rosetta/lib/fixtures.ts | 40 +++ packages/jsii-rosetta/lib/index.ts | 6 + packages/jsii-rosetta/lib/jsii/assemblies.ts | 113 +++++++ .../lib/jsii/jsii-utils.ts | 4 +- packages/jsii-rosetta/lib/jsii/packages.ts | 26 ++ .../lib/languages/default.ts | 86 +++--- packages/jsii-rosetta/lib/languages/index.ts | 9 + .../lib/languages/python.ts | 12 +- .../lib/languages/visualize.ts | 86 +++--- packages/jsii-rosetta/lib/logging.ts | 33 ++ .../lib/markdown/extract-snippets.ts | 21 ++ .../lib/markdown/markdown-renderer.ts | 0 .../lib/markdown/markdown.ts | 0 .../lib/markdown/replace-code-renderer.ts | 2 +- .../markdown/replace-typescript-transform.ts | 37 +++ .../lib/markdown/structure-renderer.ts | 0 .../lib/o-tree.ts | 60 +++- .../lib/renderer.ts} | 102 ++++--- packages/jsii-rosetta/lib/rosetta.ts | 152 ++++++++++ packages/jsii-rosetta/lib/snippet.ts | 106 +++++++ packages/jsii-rosetta/lib/tablets/key.ts | 11 + packages/jsii-rosetta/lib/tablets/schema.ts | 63 ++++ packages/jsii-rosetta/lib/tablets/tablets.ts | 175 +++++++++++ packages/jsii-rosetta/lib/translate.ts | 131 ++++++++ .../lib/typescript/ast-utils.ts | 68 ++++- .../lib/typescript/imports.ts | 6 +- .../lib/typescript/ts-compiler.ts | 4 +- packages/jsii-rosetta/lib/util.ts | 38 +++ .../package.json | 17 +- .../jsii-rosetta/test/jsii/assemblies.test.ts | 120 ++++++++ .../jsii-rosetta/test/jsii/astutils.test.ts | 19 ++ .../jsii/fixtures/rosetta/explicit.ts-fixture | 2 + .../test/markdown/roundtrip.test.ts | 0 .../test/otree.test.ts | 0 .../test/python/calls.test.ts | 0 .../test/python/classes.test.ts | 0 .../test/python/comments.test.ts | 0 .../test/python/expressions.test.ts | 0 .../test/python/hiding.test.ts | 21 ++ .../test/python/imports.test.ts | 0 .../test/python/misc.test.ts | 0 .../test/python/python.ts | 7 +- .../test/python/statements.test.ts | 6 +- packages/jsii-rosetta/test/rosetta.test.ts | 151 +++++++++ .../tsconfig.json | 3 + packages/jsii-sampiler/CHANGELOG.md | 24 -- packages/jsii-sampiler/README.md | 147 --------- packages/jsii-sampiler/bin/jsii-sampiler | 2 - packages/jsii-sampiler/bin/jsii-sampiler.ts | 86 ------ packages/jsii-sampiler/lib/index.ts | 3 - packages/jsii-sampiler/lib/translate.ts | 118 ------- packages/jsii-sampiler/lib/util.ts | 26 -- yarn.lock | 12 + 81 files changed, 2274 insertions(+), 735 deletions(-) create mode 100644 packages/jsii-calc/rosetta/default.ts-fixture create mode 100644 packages/jsii-calc/rosetta/with-calculator.ts-fixture rename packages/{jsii-sampiler => jsii-rosetta}/.gitignore (100%) rename packages/{jsii-sampiler => jsii-rosetta}/.npmignore (100%) create mode 100644 packages/jsii-rosetta/README.md create mode 100755 packages/jsii-rosetta/bin/jsii-rosetta create mode 100644 packages/jsii-rosetta/bin/jsii-rosetta.ts rename packages/{jsii-sampiler => jsii-rosetta}/examples/controlflow.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/examples/incomplete.ts (100%) create mode 100644 packages/jsii-rosetta/lib/commands/convert.ts create mode 100644 packages/jsii-rosetta/lib/commands/extract.ts create mode 100644 packages/jsii-rosetta/lib/commands/read.ts create mode 100644 packages/jsii-rosetta/lib/fixtures.ts create mode 100644 packages/jsii-rosetta/lib/index.ts create mode 100644 packages/jsii-rosetta/lib/jsii/assemblies.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/jsii/jsii-utils.ts (92%) create mode 100644 packages/jsii-rosetta/lib/jsii/packages.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/default.ts (72%) create mode 100644 packages/jsii-rosetta/lib/languages/index.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/python.ts (97%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/visualize.ts (71%) create mode 100644 packages/jsii-rosetta/lib/logging.ts create mode 100644 packages/jsii-rosetta/lib/markdown/extract-snippets.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/markdown-renderer.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/markdown.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/replace-code-renderer.ts (94%) create mode 100644 packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/structure-renderer.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/o-tree.ts (75%) rename packages/{jsii-sampiler/lib/converter.ts => jsii-rosetta/lib/renderer.ts} (82%) create mode 100644 packages/jsii-rosetta/lib/rosetta.ts create mode 100644 packages/jsii-rosetta/lib/snippet.ts create mode 100644 packages/jsii-rosetta/lib/tablets/key.ts create mode 100644 packages/jsii-rosetta/lib/tablets/schema.ts create mode 100644 packages/jsii-rosetta/lib/tablets/tablets.ts create mode 100644 packages/jsii-rosetta/lib/translate.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/ast-utils.ts (85%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/imports.ts (93%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/ts-compiler.ts (95%) create mode 100644 packages/jsii-rosetta/lib/util.ts rename packages/{jsii-sampiler => jsii-rosetta}/package.json (75%) create mode 100644 packages/jsii-rosetta/test/jsii/assemblies.test.ts create mode 100644 packages/jsii-rosetta/test/jsii/astutils.test.ts create mode 100644 packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture rename packages/{jsii-sampiler => jsii-rosetta}/test/markdown/roundtrip.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/otree.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/calls.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/classes.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/comments.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/expressions.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/hiding.test.ts (76%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/imports.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/misc.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/python.ts (87%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/statements.test.ts (88%) create mode 100644 packages/jsii-rosetta/test/rosetta.test.ts rename packages/{jsii-sampiler => jsii-rosetta}/tsconfig.json (96%) delete mode 100644 packages/jsii-sampiler/CHANGELOG.md delete mode 100644 packages/jsii-sampiler/README.md delete mode 100755 packages/jsii-sampiler/bin/jsii-sampiler delete mode 100644 packages/jsii-sampiler/bin/jsii-sampiler.ts delete mode 100644 packages/jsii-sampiler/lib/index.ts delete mode 100644 packages/jsii-sampiler/lib/translate.ts delete mode 100644 packages/jsii-sampiler/lib/util.ts diff --git a/.gitignore b/.gitignore index fb9f4cf428..16334667cc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ tsconfig.tsbuildinfo dist/ .vscode *.tsbuildinfo +.jsii-samples.tabl diff --git a/packages/jsii-calc-base-of-base/package.json b/packages/jsii-calc-base-of-base/package.json index fc67983ba0..b0eb67eccd 100644 --- a/packages/jsii-calc-base-of-base/package.json +++ b/packages/jsii-calc-base-of-base/package.json @@ -24,12 +24,13 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta extract", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, "devDependencies": { "jsii": "^0.20.0", + "jsii-rosetta": "^0.20.0", "jsii-build-tools": "^0.20.0" }, "jsii": { diff --git a/packages/jsii-calc-base/package.json b/packages/jsii-calc-base/package.json index 540d933362..be63851d71 100644 --- a/packages/jsii-calc-base/package.json +++ b/packages/jsii-calc-base/package.json @@ -24,7 +24,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta extract", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, @@ -36,6 +36,7 @@ }, "devDependencies": { "jsii": "^0.20.0", + "jsii-rosetta": "^0.20.0", "jsii-build-tools": "^0.20.0" }, "jsii": { @@ -59,4 +60,4 @@ }, "versionFormat": "short" } -} \ No newline at end of file +} diff --git a/packages/jsii-calc-lib/package.json b/packages/jsii-calc-lib/package.json index bd54bd1d68..f5da0a0523 100644 --- a/packages/jsii-calc-lib/package.json +++ b/packages/jsii-calc-lib/package.json @@ -26,7 +26,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta extract", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, @@ -38,6 +38,7 @@ }, "devDependencies": { "jsii": "^0.20.0", + "jsii-rosetta": "^0.20.0", "jsii-build-tools": "^0.20.0" }, "jsii": { @@ -63,4 +64,4 @@ }, "versionFormat": "short" } -} \ No newline at end of file +} diff --git a/packages/jsii-calc/README.md b/packages/jsii-calc/README.md index 9044031f0a..619c3478d4 100644 --- a/packages/jsii-calc/README.md +++ b/packages/jsii-calc/README.md @@ -2,13 +2,20 @@ This library is used to demonstrate and test the features of JSII -## Sphinx +## How to use running sum API: -This file will be incorporated into the sphinx documentation. +First, create a calculator: -If this file starts with an "H1" line (in our case `# jsii Calculator`), this -heading will be used as the Sphinx topic name. Otherwise, the name of the module -(`jsii-calc`) will be used instead. +```ts +const calculator = new calc.Calculator(); +``` + +Then call some operations: + + +```ts fixture=with-calculator +calculator.add(10); +``` ## Code Samples diff --git a/packages/jsii-calc/package.json b/packages/jsii-calc/package.json index 32f6b38bbc..17bcb50897 100644 --- a/packages/jsii-calc/package.json +++ b/packages/jsii-calc/package.json @@ -25,7 +25,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta extract --compile", "watch": "jsii -w", "test": "node test/test.calc.js && diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" @@ -43,6 +43,7 @@ }, "devDependencies": { "jsii": "^0.20.0", + "jsii-rosetta": "^0.20.0", "jsii-build-tools": "^0.20.0" }, "jsii": { @@ -100,4 +101,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/jsii-calc/rosetta/default.ts-fixture b/packages/jsii-calc/rosetta/default.ts-fixture new file mode 100644 index 0000000000..0dd13f8603 --- /dev/null +++ b/packages/jsii-calc/rosetta/default.ts-fixture @@ -0,0 +1,3 @@ +import calc = require('.'); + +/// here diff --git a/packages/jsii-calc/rosetta/with-calculator.ts-fixture b/packages/jsii-calc/rosetta/with-calculator.ts-fixture new file mode 100644 index 0000000000..1eddf79bd3 --- /dev/null +++ b/packages/jsii-calc/rosetta/with-calculator.ts-fixture @@ -0,0 +1,4 @@ +import calc = require('.'); +const calculator = new calc.Calculator(); + +/// here diff --git a/packages/jsii-pacmak/bin/jsii-pacmak.ts b/packages/jsii-pacmak/bin/jsii-pacmak.ts index 969d7dd4f6..e28e2b5be1 100644 --- a/packages/jsii-pacmak/bin/jsii-pacmak.ts +++ b/packages/jsii-pacmak/bin/jsii-pacmak.ts @@ -2,6 +2,7 @@ import path = require('path'); import process = require('process'); import yargs = require('yargs'); +import { Rosetta } from 'jsii-rosetta'; import logging = require('../lib/logging'); import { Timers } from '../lib/timer'; import { VERSION_DESC } from '../lib/version'; @@ -78,6 +79,15 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; desc: 'Auto-update .npmignore to exclude the output directory and include the .jsii file', default: true }) + .option('samples-tablet', { + type: 'string', + desc: 'Location of a jsii-rosetta tablet with sample translations (created using \'jsii-rosetta extract\')' + }) + .option('live-translation', { + type: 'boolean', + desc: 'Translate code samples on-the-fly if they can\'t be found in the samples tablet', + default: true + }) .version(VERSION_DESC) .strict() .argv; @@ -89,6 +99,11 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; const timers = new Timers(); + const rosetta = new Rosetta({ liveConversion: argv['live-translation'] }); + if (argv['samples-tablet']) { + await rosetta.loadTabletFromFile(argv['samples-tablet']); + } + const modulesToPackage = await findJsiiModules(argv._, argv.recurse); logging.info(`Found ${modulesToPackage.length} modules to package`); if (modulesToPackage.length === 0) { @@ -114,9 +129,12 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; }); await timers.recordAsync('load jsii', () => { - logging.info('Loading jsii assemblies'); + logging.info('Loading jsii assemblies and translations'); return Promise.all(modulesToPackage - .map(m => m.load())); + .map(async m => { + await m.load(); + await rosetta.addAssembly(m.assembly.spec, m.moduleDirectory); + })); }); try { @@ -163,6 +181,7 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; await builder.buildModules(modules, { clean: argv.clean, codeOnly: argv['code-only'], + rosetta, force: argv.force, fingerprint: argv.fingerprint, arguments: argv, diff --git a/packages/jsii-pacmak/lib/builder.ts b/packages/jsii-pacmak/lib/builder.ts index 19d3c140f8..a6aed36a16 100644 --- a/packages/jsii-pacmak/lib/builder.ts +++ b/packages/jsii-pacmak/lib/builder.ts @@ -3,6 +3,7 @@ import logging = require('./logging'); import { JsiiModule } from './packaging'; import { TargetConstructor, Target } from './target'; import { Scratch } from './util'; +import { Rosetta } from 'jsii-rosetta'; export interface BuildOptions { /** @@ -36,6 +37,11 @@ export interface BuildOptions { * Whether to add an additional subdirectory for the target language */ languageSubdirectory?: boolean; + + /** + * The Rosetta instance to load examples from + */ + rosetta: Rosetta; } /** @@ -110,7 +116,8 @@ export class OneByOneBuilder implements TargetBuilder { assembly: module.assembly, fingerprint: options.fingerprint, force: options.force, - arguments: options.arguments + arguments: options.arguments, + rosetta: options.rosetta, }); } diff --git a/packages/jsii-pacmak/lib/target.ts b/packages/jsii-pacmak/lib/target.ts index 43100b2010..a3818154b0 100644 --- a/packages/jsii-pacmak/lib/target.ts +++ b/packages/jsii-pacmak/lib/target.ts @@ -6,6 +6,7 @@ import path = require('path'); import { IGenerator } from './generator'; import logging = require('./logging'); import { resolveDependencyDirectory } from './util'; +import { Rosetta } from 'jsii-rosetta'; export abstract class Target { @@ -15,12 +16,14 @@ export abstract class Target { protected readonly arguments: { [name: string]: any }; protected readonly targetName: string; protected readonly assembly: reflect.Assembly; + protected readonly rosetta: Rosetta; protected abstract readonly generator: IGenerator; public constructor(options: TargetOptions) { this.packageDir = options.packageDir; this.assembly = options.assembly; + this.rosetta = options.rosetta; this.fingerprint = options.fingerprint != null ? options.fingerprint : true; this.force = options.force != null ? options.force : false; this.arguments = options.arguments; @@ -173,6 +176,9 @@ export interface TargetOptions { /** The JSII-reflect assembly for this JSII assembly */ assembly: reflect.Assembly; + /** The Rosetta instance */ + rosetta: Rosetta; + /** * Whether to fingerprint the produced artifacts. * @default true diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index 62df1231a4..8b2561f195 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -127,7 +127,8 @@ export class JavaBuilder implements TargetBuilder { assembly: module.assembly, fingerprint: options.fingerprint, force: options.force, - arguments: options.arguments + arguments: options.arguments, + rosetta: options.rosetta, }); } diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index b8152832f9..40f3b83b4a 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -3,17 +3,26 @@ import path = require('path'); import { CodeMaker, toSnakeCase } from 'codemaker'; import * as escapeStringRegexp from 'escape-string-regexp'; import * as reflect from 'jsii-reflect'; -import * as sampiler from 'jsii-sampiler'; import * as spec from 'jsii-spec'; import { Stability } from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { warn } from '../logging'; import { md2rst } from '../markdown'; -import { Target } from '../target'; +import { Target, TargetOptions } from '../target'; import { shell } from '../util'; +import { Translation, Rosetta, typeScriptSnippetFromSource } from 'jsii-rosetta'; + + +const INCOMPLETE_DISCLAIMER = '# Example automatically generated. See https://github.com/aws/jsii/issues/826'; export default class Python extends Target { - protected readonly generator = new PythonGenerator(); + protected readonly generator: PythonGenerator; + + public constructor(options: TargetOptions) { + super(options); + + this.generator = new PythonGenerator(options.rosetta); + } public async build(sourceDir: string, outDir: string): Promise { // Format our code to make it easier to read, we do this here instead of trying @@ -47,6 +56,7 @@ export default class Python extends Target { } } } + } // ################## @@ -237,7 +247,7 @@ abstract class BasePythonClassType implements PythonType, ISortableType { const bases = classParams.length > 0 ? `(${classParams.join(', ')})` : ''; code.openBlock(`class ${this.pythonName}${bases}`); - emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); this.emitPreamble(code, resolver); @@ -394,7 +404,7 @@ abstract class BaseMethod implements PythonBase { } code.openBlock(`def ${this.pythonName}(${pythonParams.join(', ')}) -> ${returnType}`); - emitDocString(code, this.docs, { arguments: documentableArgs, documentableItem: `method-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { arguments: documentableArgs, documentableItem: `method-${this.pythonName}` }); this.emitBody(code, resolver, renderAbstract, forceEmitBody); code.closeBlock(); } @@ -418,7 +428,7 @@ abstract class BaseMethod implements PythonBase { // We need to build up a list of properties, which are mandatory, these are the // ones we will specifiy to start with in our dictionary literal. - const liftedProps = this.getLiftedProperties(resolver).map(p => new StructField(p)); + const liftedProps = this.getLiftedProperties(resolver).map(p => new StructField(this.generator, p)); const assignments = liftedProps .map(p => p.pythonName) .map(v => `${v}=${v}`); @@ -498,7 +508,9 @@ abstract class BaseProperty implements PythonBase { private readonly immutable: boolean; - public constructor(public readonly pythonName: string, + public constructor( + private readonly generator: PythonGenerator, + public readonly pythonName: string, private readonly jsName: string, private readonly type: spec.OptionalValue, private readonly docs: spec.Docs | undefined, @@ -522,7 +534,7 @@ abstract class BaseProperty implements PythonBase { code.line('@abc.abstractmethod'); } code.openBlock(`def ${this.pythonName}(${this.implicitParameter}) -> ${pythonType}`); - emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); if ((this.shouldEmitBody || forceEmitBody) && (!renderAbstract || !this.abstract)) { code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); } else { @@ -558,7 +570,7 @@ class Interface extends BasePythonClassType { resolver = this.fqn ? resolver.bind(this.fqn) : resolver; const proxyBases: string[] = this.bases.map(b => `jsii.proxy_for(${resolver.resolve({ type: b })})`); code.openBlock(`class ${this.getProxyClassName()}(${proxyBases.join(', ')})`); - emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); code.line(`__jsii_type__ = "${this.fqn}"`); if (this.members.length > 0) { @@ -643,7 +655,7 @@ class Struct extends BasePythonClassType { * Find all fields (inherited as well) */ private get allMembers(): StructField[] { - return this.thisInterface.allProperties.map(x => new StructField(x.spec)); + return this.thisInterface.allProperties.map(x => new StructField(this.generator, x.spec)); } private get thisInterface() { @@ -681,7 +693,7 @@ class Struct extends BasePythonClassType { name: m.pythonName, docs: m.docs, })); - emitDocString(code, this.docs, { arguments: args, documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { arguments: args, documentableItem: `class-${this.pythonName}` }); } private emitGetter(member: StructField, code: CodeMaker, resolver: TypeResolver) { @@ -721,7 +733,7 @@ class StructField implements PythonBase { public readonly docs?: spec.Docs; private readonly type: spec.OptionalValue; - public constructor(public readonly prop: spec.Property) { + public constructor(private readonly generator: PythonGenerator, public readonly prop: spec.Property) { this.pythonName = toPythonPropertyName(prop.name); this.jsiiName = prop.name; this.type = prop; @@ -752,7 +764,7 @@ class StructField implements PythonBase { } public emitDocString(code: CodeMaker) { - emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); } public emit(code: CodeMaker, resolver: TypeResolver) { @@ -930,14 +942,18 @@ class Enum extends BasePythonClassType { } class EnumMember implements PythonBase { - public constructor(public readonly pythonName: string, private readonly value: string, private readonly docs: spec.Docs | undefined) { + public constructor( + private readonly generator: PythonGenerator, + public readonly pythonName: string, + private readonly value: string, + private readonly docs: spec.Docs | undefined) { this.pythonName = pythonName; this.value = value; } public emit(code: CodeMaker, _resolver: TypeResolver) { code.line(`${this.pythonName} = "${this.value}"`); - emitDocString(code, this.docs, { documentableItem: `enum-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `enum-${this.pythonName}` }); } } @@ -1074,7 +1090,7 @@ class Package { private readonly modules: Map; private readonly data: Map; - public constructor(name: string, version: string, metadata: spec.Assembly) { + public constructor(private readonly generator: PythonGenerator, name: string, version: string, metadata: spec.Assembly) { this.name = name; this.version = version; this.metadata = metadata; @@ -1098,7 +1114,7 @@ class Package { public write(code: CodeMaker, resolver: TypeResolver) { if (this.metadata.readme) { // Conversion is expensive, so cache the result in a variable (we need it twice) - this.convertedReadme = convertSnippetsInMarkdown(this.metadata.readme.markdown, 'README.md').trim(); + this.convertedReadme = this.generator.convertMarkdown(this.metadata.readme.markdown).trim(); } const modules = [...this.modules.values()].sort((a, b) => a.pythonName.localeCompare(b.pythonName)); @@ -1436,7 +1452,7 @@ class PythonGenerator extends Generator { private package!: Package; private readonly types: Map; - public constructor(options: GeneratorOptions = {}) { + public constructor(private readonly rosetta: Rosetta, options: GeneratorOptions = {}) { super(options); this.code.openBlockFormatter = s => `${s}:`; @@ -1445,6 +1461,118 @@ class PythonGenerator extends Generator { this.types = new Map(); } + public emitDocString(code: CodeMaker, docs: spec.Docs | undefined, options: { + arguments?: DocumentableArgument[]; + documentableItem?: string; + } = {}) { + if ((!docs || Object.keys(docs).length === 0) && !options.arguments) { return; } + if (!docs) { docs = {}; } + + const lines = new Array(); + + if (docs.summary) { + lines.push(md2rst(docs.summary)); + brk(); + } else { + lines.push(''); + } + + function brk() { + if (lines.length > 0 && lines[lines.length - 1].trim() !== '') { lines.push(''); } + } + + function block(heading: string, content: string, doBrk = true) { + if (doBrk) { brk(); } + lines.push(heading); + const contentLines = md2rst(content).split('\n'); + if (contentLines.length <= 1) { + lines.push(`:${heading}: ${contentLines.join('')}`); + } else { + lines.push(`:${heading}:`); + brk(); + for (const line of contentLines) { + lines.push(`${line}`); + } + } + if (doBrk) { brk(); } + } + + if (docs.remarks) { + brk(); + lines.push(...md2rst(this.convertMarkdown(docs.remarks || '')).split('\n')); + brk(); + } + + if (options.arguments && options.arguments.length > 0) { + brk(); + for (const param of options.arguments) { + // Add a line for every argument. Even if there is no description, we need + // the docstring so that the Sphinx extension can add the type annotations. + lines.push(`:param ${toPythonParameterName(param.name)}: ${onelineDescription(param.docs)}`); + } + brk(); + } + + if (docs.default) { block('default', docs.default); } + if (docs.returns) { block('return', docs.returns); } + if (docs.deprecated) { block('deprecated', docs.deprecated); } + if (docs.see) { block('see', docs.see, false); } + if (docs.stability && shouldMentionStability(docs.stability)) { block('stability', docs.stability, false); } + if (docs.subclassable) { block('subclassable', 'Yes'); } + + for (const [k, v] of Object.entries(docs.custom || {})) { + block(`${k}:`, v, false); + } + + if (docs.example) { + brk(); + lines.push('Example::'); + const exampleText = this.convertExample(docs.example); + + for (const line of exampleText.split('\n')) { + lines.push(` ${line}`); + } + brk(); + } + + while (lines.length > 0 && lines[lines.length - 1] === '') { lines.pop(); } + + if (lines.length === 0) { return; } + + if (lines.length === 1) { + code.line(`"""${lines[0]}"""`); + return; + } + + code.line(`"""${lines[0]}`); + lines.splice(0, 1); + + for (const line of lines) { + code.line(line); + } + + code.line('"""'); + } + + public convertExample(example: string): string { + const snippet = typeScriptSnippetFromSource(example, 'example'); + const translated = this.rosetta.translateSnippet(snippet, 'python'); + if (!translated) { return example; } + return this.prefixDisclaimer(translated); + } + + public convertMarkdown(markdown: string): string { + return this.rosetta.translateSnippetsInMarkdown(markdown, 'python', trans => ({ + language: trans.language, + source: this.prefixDisclaimer(trans) + })); + } + + private prefixDisclaimer(translated: Translation) { + if (translated.didCompile) { return translated.source; } + return `${INCOMPLETE_DISCLAIMER}\n${translated.source}`; + } + public getPythonType(fqn: string): PythonType { const type = this.types.get(fqn); @@ -1461,6 +1589,7 @@ class PythonGenerator extends Generator { protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { this.package = new Package( + this, assm.targets!.python!.distName, assm.version, assm, @@ -1576,6 +1705,7 @@ class PythonGenerator extends Generator { protected onStaticProperty(cls: spec.ClassType, prop: spec.Property) { this.getPythonType(cls.fqn).addMember( new StaticProperty( + this, toPythonPropertyName(prop.name, prop.const), prop.name, prop, @@ -1618,6 +1748,7 @@ class PythonGenerator extends Generator { protected onProperty(cls: spec.ClassType, prop: spec.Property) { this.getPythonType(cls.fqn).addMember( new Property( + this, toPythonPropertyName(prop.name, prop.const, prop.protected), prop.name, prop, @@ -1677,9 +1808,10 @@ class PythonGenerator extends Generator { let ifaceProperty: InterfaceProperty | StructField; if (ifc.datatype) { - ifaceProperty = new StructField(prop); + ifaceProperty = new StructField(this, prop); } else { ifaceProperty = new InterfaceProperty( + this, toPythonPropertyName(prop.name, prop.const, prop.protected), prop.name, prop, @@ -1698,6 +1830,7 @@ class PythonGenerator extends Generator { protected onEnumMember(enm: spec.EnumType, member: spec.EnumMember) { this.getPythonType(enm.fqn).addMember( new EnumMember( + this, toPythonIdentifier(member.name), member.name, member.docs, @@ -1788,99 +1921,6 @@ interface DocumentableArgument { docs?: spec.Docs; } -function emitDocString(code: CodeMaker, docs: spec.Docs | undefined, options: { - arguments?: DocumentableArgument[]; - documentableItem?: string; -} = {}) { - if ((!docs || Object.keys(docs).length === 0) && !options.arguments) { return; } - if (!docs) { docs = {}; } - - const lines = new Array(); - - if (docs.summary) { - lines.push(md2rst(docs.summary)); - brk(); - } else { - lines.push(''); - } - - function brk() { - if (lines.length > 0 && lines[lines.length - 1].trim() !== '') { lines.push(''); } - } - - function block(heading: string, content: string, doBrk = true) { - if (doBrk) { brk(); } - lines.push(heading); - const contentLines = md2rst(content).split('\n'); - if (contentLines.length <= 1) { - lines.push(`:${heading}: ${contentLines.join('')}`); - } else { - lines.push(`:${heading}:`); - brk(); - for (const line of contentLines) { - lines.push(`${line}`); - } - } - if (doBrk) { brk(); } - } - - if (docs.remarks) { - brk(); - lines.push(...md2rst(convertSnippetsInMarkdown(docs.remarks || '', options.documentableItem || 'docstring')).split('\n')); - brk(); - } - - if (options.arguments && options.arguments.length > 0) { - brk(); - for (const param of options.arguments) { - // Add a line for every argument. Even if there is no description, we need - // the docstring so that the Sphinx extension can add the type annotations. - lines.push(`:param ${toPythonParameterName(param.name)}: ${onelineDescription(param.docs)}`); - } - brk(); - } - - if (docs.default) { block('default', docs.default); } - if (docs.returns) { block('return', docs.returns); } - if (docs.deprecated) { block('deprecated', docs.deprecated); } - if (docs.see) { block('see', docs.see, false); } - if (docs.stability && shouldMentionStability(docs.stability)) { block('stability', docs.stability, false); } - if (docs.subclassable) { block('subclassable', 'Yes'); } - - for (const [k, v] of Object.entries(docs.custom || {})) { - block(`${k}:`, v, false); - } - - if (docs.example) { - brk(); - lines.push('Example::'); - const exampleText = convertExample(docs.example, options.documentableItem || 'example'); - - for (const line of exampleText.split('\n')) { - lines.push(` ${line}`); - } - brk(); - } - - while (lines.length > 0 && lines[lines.length - 1] === '') { lines.pop(); } - - if (lines.length === 0) { return; } - - if (lines.length === 1) { - code.line(`"""${lines[0]}"""`); - return; - } - - code.line(`"""${lines[0]}`); - lines.splice(0, 1); - - for (const line of lines) { - code.line(line); - } - - code.line('"""'); -} - /** * Render a one-line description of the given docs, used for method arguments and inlined properties */ @@ -1903,25 +1943,4 @@ function isStruct(typeSystem: reflect.TypeSystem, ref: spec.TypeReference): bool if (!spec.isNamedTypeReference(ref)) { return false; } const type = typeSystem.tryFindFqn(ref.fqn); return type !== undefined && type.isInterfaceType() && type.isDataType(); -} - -const pythonTranslator = new sampiler.PythonVisitor({ - disclaimer: 'Example may have issues. See https://github.com/aws/jsii/issues/826' -}); - -function convertExample(example: string, filename: string): string { - const source = new sampiler.LiteralSource(example, filename); - const result = sampiler.translateTypeScript(source, pythonTranslator); - sampiler.printDiagnostics(result.diagnostics, process.stderr); - return sampiler.renderTree(result.tree); -} - -function convertSnippetsInMarkdown(markdown: string, filename: string): string { - const source = new sampiler.LiteralSource(markdown, filename); - const result = sampiler.translateMarkdown(source, pythonTranslator, { - languageIdentifier: 'python' - }); - // FIXME: This should translate into an exit code somehow - sampiler.printDiagnostics(result.diagnostics, process.stderr); - return sampiler.renderTree(result.tree); -} +} \ No newline at end of file diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index f38a6a7c83..83ef84470d 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -40,8 +40,9 @@ "escape-string-regexp": "^2.0.0", "fs-extra": "^8.1.0", "jsii-reflect": "^0.20.0", - "jsii-sampiler": "^0.20.0", + "jsii-rosetta": "^0.20.0", "jsii-spec": "^0.20.0", + "jsii-rosetta": "^0.20.0", "spdx-license-list": "^6.1.0", "xmlbuilder": "^13.0.2", "yargs": "^14.2.0" diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md index e90486664b..f9f4c183fe 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md +++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md @@ -13,7 +13,7 @@ heading will be used as the Sphinx topic name. Otherwise, the name of the module ## Code Samples ```python -# Example may have issues. See https://github.com/aws/jsii/issues/826 +# Example automatically generated. See https://github.com/aws/jsii/issues/826 # This is totes a magic comment in here, just you wait! foo = "bar" ``` diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py index a829d9365e..8cbfa48ed9 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py +++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py @@ -763,7 +763,7 @@ class ClassWithDocs(metaclass=jsii.JSIIMeta, jsii_type="jsii-calc.ClassWithDocs" :customAttribute:: hasAValue Example:: - # Example may have issues. See https://github.com/aws/jsii/issues/826 + # Example automatically generated. See https://github.com/aws/jsii/issues/826 def an_example(): pass """ diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py index bc1e9e1fc2..0a534bf63b 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py +++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py @@ -14,7 +14,7 @@ ## Code Samples ```python -# Example may have issues. See https://github.com/aws/jsii/issues/826 +# Example automatically generated. See https://github.com/aws/jsii/issues/826 # This is totes a magic comment in here, just you wait! foo = "bar" ``` diff --git a/packages/jsii-pacmak/tsconfig.json b/packages/jsii-pacmak/tsconfig.json index 9c9053dbe9..75e3b8ad29 100644 --- a/packages/jsii-pacmak/tsconfig.json +++ b/packages/jsii-pacmak/tsconfig.json @@ -22,7 +22,7 @@ "references": [ { "path": "../jsii-spec" }, { "path": "../codemaker" }, - { "path": "../jsii-sampiler" }, + { "path": "../jsii-rosetta" }, { "path": "../jsii-reflect" } ] } diff --git a/packages/jsii-sampiler/.gitignore b/packages/jsii-rosetta/.gitignore similarity index 100% rename from packages/jsii-sampiler/.gitignore rename to packages/jsii-rosetta/.gitignore diff --git a/packages/jsii-sampiler/.npmignore b/packages/jsii-rosetta/.npmignore similarity index 100% rename from packages/jsii-sampiler/.npmignore rename to packages/jsii-rosetta/.npmignore diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md new file mode 100644 index 0000000000..7048b72cfe --- /dev/null +++ b/packages/jsii-rosetta/README.md @@ -0,0 +1,177 @@ +# jsii-rosetta: a transpiler for code samples + +Utility to transcribe example code snippets from TypeScript to other +jsii languages. + +Has knowledge about jsii language translation conventions to do the +translations. Only supports a limited set of TypeScript language features. + +## Compilability + +The translator can translate both code that completely compiles and typechecks, +as well as code that doesn't. + +In case of non-compiling samples the translations will be based off of +grammatical parsing only. This has the downside that we do not have the type +information available to the exact right thing in all instances. + +If the samples don't compile or don't have full type information: + +- No way to declare typed variables for Java and C#. +- Can only "see" the fields of structs as far as they are declared in the same + snippet. Inherited fields or structs declared not in the same snippet are + invisible. +- When we explode a struct parameter into keyword parameters and we pass it on + to another callable, we can't know which keyword arguments the called function + actually takes so we just pass all of them (might be too many). +- When structs contain nested structs, Python and other languages need to know + the types of these fields to generate the right calls. +- Object literals are used both to represent structs as well as to represent + dictionaries, and without type information it's impossible to determine + which is which. + +## Hiding code from samples + +In order to make examples compile, boilerplate code may need to be added +that detracts from the example at hand (such as variable declarations +and imports). + +This package supports hiding parts of the original source after +translation. + +To mark special locations in the source tree, we can use one of three mechanisms: + +* Use a `void` expression statement to mark statement locations in the AST. +* Use the `comma` operator combined with a `void` expression to mark expression + locations in the AST. +* Use special directive comments (`/// !hide`, `/// !show`) to mark locations + that span AST nodes. This is less reliable (because the source location of + translated syntax sometimes will have to be estimated) but the only option if + you want to mark non-contiguous nodes (such as hide part of a class + declaration but show statements inside the constructor). + +The `void` expression keyword and or the `comma` operator feature are +little-used JavaScript features that are reliably parsed by TypeScript and do +not affect the semantics of the application in which they appear (so the program +executes the same with or without them). + +A handy mnemonic for this feature is that you can use it to "send your +code into the void". + +### Hiding statements + +Statement hiding looks like this: + +```ts +before(); // will be shown + +void 0; // start hiding (the argument to 'void' doesn't matter) +middle(); // will not be shown +void 'show'; // stop hiding + +after(); // will be shown again +``` + +### Hiding expressions + +For hiding expressions, we use `comma` expressions to attach a `void` +statement to an expression value without changing the meaning of the +code. + +Example: + +```ts +foo(1, 2, (void 1, 3)); +``` + +Will render as + +``` +foo(1, 2) +``` + +Also supports a visible ellipsis: + +```ts +const x = (void '...', 3); +``` + +Renders to: + +``` +x = ... +``` + +### Hiding across AST nodes + +Use special comment directives: + +```ts +before(); +/// !hide +notShown(); +/// !show +after(); +``` + +(May also start with `/// !show` and `/// !hide`). + +## Fixtures + +To avoid having to repeat common setup every time, code samples can use +"fixtures": a source template where the example is inserted. A fixture +must contain the text `/// here` and may look like this: + +```ts +const module = require('@some/dependency'); + +class MyClass { + constructor() { + const obj = new MyObject(); + + /// here + } +} +``` + +The example will be inserted at the location marked as `/// here` and +will have access to `module`, `obj` and `this`. + +The default file loaded as a fixture is called `rosetta/default.ts-fixture` +in the package directory (if it exists). + +Examples can request an alternative fixture by specifying a `fixture` parameter +as part of the code block fence: + + ` ` `ts fixture=some-fixture + ... + +Or opt out of using the default fixture by specifying `nofixture`. + +## Build integration + +Because we want to control the compilation environment when compiling examples, +extracting and compiling all samples can be run as an external build step in a +monorepo. This allows you to set up an environment with all desired packages and +compile all samples in one go. + +The `jsii-rosetta extract` command will take one or more jsii assemblies, +extract the snippets from them, will try to compile them with respect to a +given home directory, and finally store all translations in something called +a "tablet" (which is a lookup map from the original snippet to all of its +translations). + +A translation tablet will automatically be used by `jsii-pacmak` if present, so +it can subsitute the translated examples into the converted source code when +writing out the converted code. When not given a tablet, `jsii-pacmak` can still +live-convert samples, but you will not have the fine control over the +compilation environment that you would have if you were to use the `extract` +command. + +Works like this: + +``` +$ jsii-rosetta extract --compile $(find . -name .jsii) --directory some/dir +$ jsii-pacmak --samples-tablet .jsii-samples.tbl + +``` diff --git a/packages/jsii-rosetta/bin/jsii-rosetta b/packages/jsii-rosetta/bin/jsii-rosetta new file mode 100755 index 0000000000..1ddc3571aa --- /dev/null +++ b/packages/jsii-rosetta/bin/jsii-rosetta @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('./jsii-rosetta.js'); diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts new file mode 100644 index 0000000000..d198c332b4 --- /dev/null +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -0,0 +1,148 @@ +import fs = require('fs-extra'); +import yargs = require('yargs'); +import { TranslateResult, DEFAULT_TABLET_NAME, translateTypeScript } from '../lib'; +import { PythonVisitor } from '../lib/languages/python'; +import { VisualizeAstVisitor } from '../lib/languages/visualize'; +import { extractSnippets } from '../lib/commands/extract'; +import logging = require('../lib/logging'); +import path = require('path'); +import { readTablet as readTablet } from '../lib/commands/read'; +import { translateMarkdown } from '../lib/commands/convert'; +import { File, printDiagnostics, isErrorDiagnostic } from '../lib/util'; + +async function main() { + const argv = yargs + .usage('$0 [args]') + .option('verbose', { + alias: 'v', + type: 'boolean', + desc: 'Increase logging verbosity', + count: true, + default: 0 + }) + .command('snippet FILE', 'Translate a single snippet', command => command + .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' }) + .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' }) + , wrapHandler(async args => { + const result = translateTypeScript( + await makeFileSource(args.file || '-', 'stdin.ts'), + makeVisitor(args)); + renderResult(result); + })) + .command('markdown FILE', 'Translate a MarkDown file', command => command + .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' }) + .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' }) + , wrapHandler(async args => { + const result = translateMarkdown( + await makeFileSource(args.file || '-', 'stdin.md'), + makeVisitor(args)); + renderResult(result); + })) + .command('extract [ASSEMBLY..]', 'Extract code snippets from one or more assemblies into a language tablets', command => command + .positional('ASSEMBLY', { type: 'string', string: true, default: new Array(), describe: 'Assembly or directory to extract from' }) + .option('output', { alias: 'o', type: 'string', describe: 'Output file where to store the sample tablets', default: DEFAULT_TABLET_NAME }) + .option('compile', { alias: 'c', type: 'boolean', describe: 'Try compiling', default: false }) + .option('directory', { alias: 'd', type: 'string', describe: 'Working directory (for require() etc)' }) + .option('fail', { alias: 'f', type: 'boolean', describe: 'Fail if there are compilation errors', default: false }) + , wrapHandler(async args => { + + // Easiest way to get a fixed working directory (for sources) in is to + // chdir, since underneath the in-memory layer we're using a regular TS + // compilerhost. Have to make all file references absolute before we chdir + // though. + const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map(x => path.resolve(x)); + const absOutput = path.resolve(args.output); + if (args.directory) { + process.chdir(args.directory); + } + + const result = await extractSnippets(absAssemblies, absOutput, args.compile); + + printDiagnostics(result.diagnostics, process.stderr); + + if (result.diagnostics.some(isErrorDiagnostic) && args.fail) { + process.exit(1); + } + })) + .command('read [KEY] [LANGUAGE]', 'Read snippets from a language tablet', command => command + .positional('TABLET', { type: 'string', required: true, describe: 'Language tablet to read' }) + .positional('KEY', { type: 'string', describe: 'Snippet key to read' }) + .positional('LANGUAGE', { type: 'string', describe: 'Language ID to read' }) + .demandOption('TABLET') + , wrapHandler(async args => { + await readTablet(args.TABLET, args.KEY, args.LANGUAGE); + })) + .demandCommand() + .help() + .strict() // Error on wrong command + .version(require('../package.json').version) + .showHelpOnFail(false) + .argv; + + // Evaluating .argv triggers the parsing but the command gets implicitly executed, + // so we don't need the output. + Array.isArray(argv); +} + +/** + * Wrap a command's handler with standard pre- and post-work + */ +function wrapHandler(handler: (x: A) => Promise) { + return (argv: A) => { + logging.level = argv.verbose !== undefined ? argv.verbose : 0; + return handler(argv); + }; +} + +function makeVisitor(args: { python?: boolean }) { + if (args.python) { return new PythonVisitor(); } + // Default to visualizing AST, including nodes we don't recognize yet + return new VisualizeAstVisitor(); +} + +async function makeFileSource(fileName: string, stdinName: string): Promise { + if (fileName === '-') { + return { + contents: await readStdin(), + fileName: stdinName + }; + } + return { + contents: await fs.readFile(fileName, { encoding: 'utf-8' }), + fileName: fileName + }; +} + +async function readStdin(): Promise { + process.stdin.setEncoding('utf8'); + + const parts: string[] = []; + + return new Promise((resolve, reject) => { + process.stdin.on('readable', () => { + const chunk = process.stdin.read(); + if (chunk !== null) { parts.push(`${chunk}`); } + }); + + process.stdin.on('error', reject); + process.stdin.on('end', () => resolve(parts.join(''))); + }); +} + +function renderResult(result: TranslateResult) { + process.stdout.write(result.translation + '\n'); + + if (result.diagnostics.length > 0) { + printDiagnostics(result.diagnostics, process.stderr); + + if (result.diagnostics.some(isErrorDiagnostic)) { + process.exit(1); + } + } +} + +main().catch(e => { + // tslint:disable-next-line:no-console + console.error(e); + process.exit(1); +}); \ No newline at end of file diff --git a/packages/jsii-sampiler/examples/controlflow.ts b/packages/jsii-rosetta/examples/controlflow.ts similarity index 100% rename from packages/jsii-sampiler/examples/controlflow.ts rename to packages/jsii-rosetta/examples/controlflow.ts diff --git a/packages/jsii-sampiler/examples/incomplete.ts b/packages/jsii-rosetta/examples/incomplete.ts similarity index 100% rename from packages/jsii-sampiler/examples/incomplete.ts rename to packages/jsii-rosetta/examples/incomplete.ts diff --git a/packages/jsii-rosetta/lib/commands/convert.ts b/packages/jsii-rosetta/lib/commands/convert.ts new file mode 100644 index 0000000000..97b0773c87 --- /dev/null +++ b/packages/jsii-rosetta/lib/commands/convert.ts @@ -0,0 +1,36 @@ +import { AstHandler, AstRendererOptions } from '../renderer'; +import { TranslateResult, Translator } from '../translate'; +import { MarkdownRenderer } from '../markdown/markdown-renderer'; +import { transformMarkdown } from '../markdown/markdown'; +import { File } from '../util'; +import { ReplaceTypeScriptTransform } from '../markdown/replace-typescript-transform'; + +export interface TranslateMarkdownOptions extends AstRendererOptions { + /** + * What language to put in the returned markdown blocks + */ + languageIdentifier?: string; +} + + +export function translateMarkdown(markdown: File, visitor: AstHandler, options: TranslateMarkdownOptions = {}): TranslateResult { + const translator = new Translator(false); + + const translatedMarkdown = transformMarkdown( + markdown.contents, + new MarkdownRenderer(), + new ReplaceTypeScriptTransform(markdown.fileName, tsSnippet => { + const translated = translator.translatorFor(tsSnippet).renderUsing(visitor); + return { + language: options.languageIdentifier || '', + source: translated, + }; + }) + ); + + return { + translation: translatedMarkdown, + diagnostics: translator.diagnostics, + }; +} + diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts new file mode 100644 index 0000000000..503d3b6ffc --- /dev/null +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -0,0 +1,37 @@ +import { loadAssemblies, allTypeScriptSnippets } from '../jsii/assemblies'; +import logging = require('../logging'); +import ts = require('typescript'); +import { LanguageTablet } from '../tablets/tablets'; +import { Translator } from '../translate'; +import { snippetKey } from '../tablets/key'; + +export interface ExtractResult { + diagnostics: ts.Diagnostic[]; +} + +/** + * Extract all samples from the given assemblies into a tablet + */ +export async function extractSnippets(assemblyLocations: string[], outputFile: string, includeCompilerDiagnostics: boolean): Promise { + logging.info(`Loading ${assemblyLocations.length} assemblies`); + const assemblies = await loadAssemblies(assemblyLocations); + + const translator = new Translator(includeCompilerDiagnostics); + + const tablet = new LanguageTablet(); + + logging.info(`Translating`); + const startTime = Date.now(); + + for (const block of allTypeScriptSnippets(assemblies)) { + logging.debug(`Translating ${snippetKey(block)}`); + tablet.addSnippet(translator.translate(block)); + } + + const delta = (Date.now() - startTime) / 1000; + logging.info(`Converted ${tablet.count} snippets in ${delta} seconds (${(delta / tablet.count).toPrecision(3)}s/snippet)`); + logging.info(`Saving language tablet to ${outputFile}`); + await tablet.save(outputFile); + + return { diagnostics: translator.diagnostics }; +} diff --git a/packages/jsii-rosetta/lib/commands/read.ts b/packages/jsii-rosetta/lib/commands/read.ts new file mode 100644 index 0000000000..a44ab6af05 --- /dev/null +++ b/packages/jsii-rosetta/lib/commands/read.ts @@ -0,0 +1,72 @@ +import { LanguageTablet, TranslatedSnippet, Translation } from "../tablets/tablets"; +import { TargetLanguage } from "../languages"; + +export async function readTablet(tabletFile: string, key?: string, lang?: string) { + const tab = new LanguageTablet(); + await tab.load(tabletFile); + + if (key !== undefined) { + const snippet = tab.getSnippet(key); + if (snippet === undefined) { + throw new Error(`No such snippet: ${key}`); + } + displaySnippet(snippet); + } else { + listSnippets(); + } + + function listSnippets() { + for (const key of tab.snippetKeys) { + process.stdout.write(snippetHeader(key) + '\n'); + displaySnippet(tab.getSnippet(key)!); + process.stdout.write('\n'); + } + } + + function displaySnippet(snippet: TranslatedSnippet) { + if (snippet.didCompile !== undefined) { + process.stdout.write(`Compiled: ${snippet.didCompile}\n`); + } + + if (lang !== undefined) { + const translation = snippet.get(lang as TargetLanguage); + if (translation === undefined) { + throw new Error(`No translation for ${lang} in snippet ${snippet.key}`); + } + displayTranslation(translation); + } else { + listTranslations(snippet); + } + } + + function listTranslations(snippet: TranslatedSnippet) { + const original = snippet.originalSource; + if (original !== undefined) { + displayTranslation(original); + } + + for (const lang of snippet.languages) { + process.stdout.write(languageHeader(lang) + '\n'); + displayTranslation(snippet.get(lang)!); + } + } + + function displayTranslation(translation: Translation) { + process.stdout.write(translation.source + '\n'); + } +} + +function snippetHeader(key: string) { + return center(` ${key} `, 100, '='); +} + +function languageHeader(key: string) { + return center(` ${key} `, 30, '-'); +} + +function center(str: string, n: number, fill: string) { + const before = Math.floor((n - str.length) / 2); + const after = Math.ceil((n - str.length) / 2); + + return fill.repeat(Math.max(before, 0)) + str + fill.repeat(Math.max(after, 0)); +} \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/fixtures.ts b/packages/jsii-rosetta/lib/fixtures.ts new file mode 100644 index 0000000000..41dd935546 --- /dev/null +++ b/packages/jsii-rosetta/lib/fixtures.ts @@ -0,0 +1,40 @@ +import fs = require('fs-extra'); +import path = require('path'); +import { TypeScriptSnippet, SnippetParameters } from './snippet'; + +/** + * Complete snippets with fixtures, if required + */ +export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet { + let source = snippet.visibleSource; + const parameters = snippet.parameters || {}; + + const directory = parameters[SnippetParameters.$PROJECT_DIRECTORY]; + if (!directory) { return snippet; } + + if (parameters[SnippetParameters.FIXTURE]) { + // Explicitly request a fixture + source = loadAndSubFixture(directory, parameters.fixture, source, true); + } else if (parameters[SnippetParameters.NO_FIXTURE] === undefined) { + source = loadAndSubFixture(directory, 'default', source, false); + } + + return { visibleSource: snippet.visibleSource, completeSource: source, where: snippet.where, parameters }; +} + +function loadAndSubFixture(directory: string, fixtureName: string, source: string, mustExist: boolean) { + const fixtureFileName = path.join(directory, `rosetta/${fixtureName}.ts-fixture`); + const exists = fs.existsSync(fixtureFileName); + if (!exists && mustExist) { + throw new Error(`Sample uses fixture ${fixtureName}, but not found: ${fixtureFileName}`); + } + if (!exists) { return source; } + const fixtureContents = fs.readFileSync(fixtureFileName, { encoding: 'utf-8' }); + + const subRegex = /\/\/\/ here/i; + if (!subRegex.test(fixtureContents)) { + throw new Error(`Fixture does not contain '/// here': ${fixtureFileName}`); + } + + return fixtureContents.replace(subRegex, `/// !show\n${source}\n/// !hide`); +} diff --git a/packages/jsii-rosetta/lib/index.ts b/packages/jsii-rosetta/lib/index.ts new file mode 100644 index 0000000000..773380721a --- /dev/null +++ b/packages/jsii-rosetta/lib/index.ts @@ -0,0 +1,6 @@ +export * from './translate'; +export { renderTree } from './o-tree'; +export { PythonVisitor } from './languages/python'; +export * from './tablets/tablets' +export * from './rosetta'; +export * from './snippet'; \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts new file mode 100644 index 0000000000..dad5fcb53e --- /dev/null +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -0,0 +1,113 @@ +import spec = require('jsii-spec'); +import fs = require('fs-extra'); +import path = require('path'); +import { TypeScriptSnippet, typeScriptSnippetFromSource, updateParameters, SnippetParameters } from '../snippet'; +import { extractTypescriptSnippetsFromMarkdown } from '../markdown/extract-snippets'; +import { fixturize } from '../fixtures'; + +export interface LoadedAssembly { + assembly: spec.Assembly; + directory: string; +} + +/** + * Load assemblies by filename or directory + */ +export async function loadAssemblies(assemblyLocations: string[]) { + const ret: LoadedAssembly[] = []; + for (const loc of assemblyLocations) { + const stat = await fs.stat(loc); + if (stat.isDirectory()) { + ret.push({ + assembly: await loadAssemblyFromFile(path.join(loc, '.jsii')), + directory: loc + }); + } else { + ret.push({ + assembly: await loadAssemblyFromFile(loc), + directory: path.dirname(loc), + }); + } + } + return ret; +} + +async function loadAssemblyFromFile(filename: string) { + const contents = await fs.readJSON(filename, { encoding: 'utf-8' }); + return spec.validateAssembly(contents); +} + +export type AssemblySnippetSource = { type: 'markdown'; markdown: string; where: string } | { type: 'literal'; source: string; where: string }; + +/** + * Return all markdown and example snippets from the given assembly + */ +export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSource[] { + const ret: AssemblySnippetSource[] = []; + + if (assembly.readme) { + ret.push({ type: 'markdown', markdown: assembly.readme.markdown, where: `${assembly.name}-README` }); + } + + if (assembly.types) { + Object.values(assembly.types).forEach(type => { + emitDocs(type.docs, `${assembly.name}.${type.name}`); + + if (spec.isEnumType(type)) { + type.members.forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}.${m.name}`)); + } + if (spec.isClassOrInterfaceType(type)) { + (type.methods || []).forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`)); + (type.properties || []).forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`)); + } + }); + } + + return ret; + + function emitDocs(docs: spec.Docs | undefined, where: string) { + if (!docs) { return; } + + if (docs.remarks) { ret.push({ 'type': 'markdown', markdown: docs.remarks, where }); } + if (docs.example && exampleLooksLikeSource(docs.example)) { + ret.push({ 'type': 'literal', source: docs.example, where: `${where}-example` }); + } + } +} + +export function* allTypeScriptSnippets(assemblies: Array<{ assembly: spec.Assembly, directory: string }>): IterableIterator { + for (const assembly of assemblies) { + for (const source of allSnippetSources(assembly.assembly)) { + switch (source.type) { + case 'literal': + const snippet = updateParameters(typeScriptSnippetFromSource(source.source, source.where), { + [SnippetParameters.$PROJECT_DIRECTORY]: assembly.directory + }); + yield fixturize(snippet); + break; + case 'markdown': + for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.where)) { + const withDirectory = updateParameters(snippet, { + [SnippetParameters.$PROJECT_DIRECTORY]: assembly.directory + }); + yield fixturize(withDirectory); + } + } + } + } +} + +/** + * See if the given source text looks like a code sample + * + * Many @examples for properties are examples of values (ARNs, formatted strings) + * not code samples, which should not be translated + * + * If the value contains whitespace (newline, space) then we'll assume it's a code + * sample. + */ +function exampleLooksLikeSource(text: string) { + return !!text.trim().match(WHITESPACE); +} + +const WHITESPACE = new RegExp('\\s'); \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/jsii/jsii-utils.ts b/packages/jsii-rosetta/lib/jsii/jsii-utils.ts similarity index 92% rename from packages/jsii-sampiler/lib/jsii/jsii-utils.ts rename to packages/jsii-rosetta/lib/jsii/jsii-utils.ts index 278e90cace..b8cdf8c4d1 100644 --- a/packages/jsii-sampiler/lib/jsii/jsii-utils.ts +++ b/packages/jsii-rosetta/lib/jsii/jsii-utils.ts @@ -1,5 +1,5 @@ import ts = require('typescript'); -import { AstConverter } from '../converter'; +import { AstRenderer } from '../renderer'; export function isStructInterface(name: string) { return !name.startsWith('I'); @@ -22,7 +22,7 @@ export interface StructProperty { questionMark: boolean; } -export function propertiesOfStruct(type: ts.Type, context: AstConverter): StructProperty[] { +export function propertiesOfStruct(type: ts.Type, context: AstRenderer): StructProperty[] { return type.isClassOrInterface() ? type.getProperties().map(s => { let propType; let questionMark = false; diff --git a/packages/jsii-rosetta/lib/jsii/packages.ts b/packages/jsii-rosetta/lib/jsii/packages.ts new file mode 100644 index 0000000000..c3f1c6c233 --- /dev/null +++ b/packages/jsii-rosetta/lib/jsii/packages.ts @@ -0,0 +1,26 @@ + + +/** + * Resolve a package name in an example to a JSII assembly + * + * We assume we've changed directory to the directory where we need to resolve from. + */ +export function resolvePackage(packageName: string) { + try { + const resolved = require.resolve(`${packageName}/package.json`, { paths: [process.cwd()] }); + return require(resolved); + } catch(e) { + return undefined; + } +} + +export function jsiiTargetParam(packageName: string, field: string) { + const pkgJson = resolvePackage(packageName); + + const path = ['jsii', 'targets', ...field.split('.')]; + let r = pkgJson; + while (path.length > 0 && typeof r === 'object' && r !== null) { + r = r[path.splice(0, 1)[0]]; + } + return r; +} \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/languages/default.ts b/packages/jsii-rosetta/lib/languages/default.ts similarity index 72% rename from packages/jsii-sampiler/lib/languages/default.ts rename to packages/jsii-rosetta/lib/languages/default.ts index 0c405ded07..0d778fdb63 100644 --- a/packages/jsii-sampiler/lib/languages/default.ts +++ b/packages/jsii-rosetta/lib/languages/default.ts @@ -1,5 +1,5 @@ import ts = require('typescript'); -import { AstConverter, AstHandler, nimpl } from "../converter"; +import { AstRenderer, AstHandler, nimpl } from "../renderer"; import { OTree } from '../o-tree'; import { ImportStatement } from '../typescript/imports'; @@ -11,58 +11,58 @@ export abstract class DefaultVisitor implements AstHandler { public abstract mergeContext(old: C, update: C): C; - public commentRange(node: ts.CommentRange, context: AstConverter): OTree { + public commentRange(node: ts.CommentRange, context: AstRenderer): OTree { return new OTree([ context.textAt(node.pos, node.end), node.hasTrailingNewLine ? '\n' : '' ]); } - public sourceFile(node: ts.SourceFile, context: AstConverter): OTree { + public sourceFile(node: ts.SourceFile, context: AstRenderer): OTree { return new OTree(context.convertAll(node.statements)); } - public jsDoc(_node: ts.JSDoc, _context: AstConverter): OTree { + public jsDoc(_node: ts.JSDoc, _context: AstRenderer): OTree { // Already handled by other doc handlers return new OTree([]); } - public importStatement(node: ImportStatement, context: AstConverter): OTree { + public importStatement(node: ImportStatement, context: AstRenderer): OTree { return this.notImplemented(node.node, context); } - public functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree { + public functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree { return this.notImplemented(node, children); } - public stringLiteral(node: ts.StringLiteral, _children: AstConverter): OTree { + public stringLiteral(node: ts.StringLiteral, _children: AstRenderer): OTree { return new OTree([JSON.stringify(node.text)]); } - public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, _context: AstConverter): OTree { + public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, _context: AstRenderer): OTree { return new OTree([JSON.stringify(node.text)]); } - public identifier(node: ts.Identifier, _children: AstConverter): OTree { + public identifier(node: ts.Identifier, _children: AstRenderer): OTree { return new OTree([node.text]); } - public block(node: ts.Block, children: AstConverter): OTree { + public block(node: ts.Block, children: AstRenderer): OTree { return new OTree(['{'], ['\n', ...children.convertAll(node.statements)], { indent: 4, suffix: '}', }); } - public parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree { + public parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree { return this.notImplemented(node, children); } - public returnStatement(node: ts.ReturnStatement, children: AstConverter): OTree { + public returnStatement(node: ts.ReturnStatement, children: AstRenderer): OTree { return new OTree(['return ', children.convert(node.expression)]); } - public binaryExpression(node: ts.BinaryExpression, context: AstConverter): OTree { + public binaryExpression(node: ts.BinaryExpression, context: AstRenderer): OTree { return new OTree([ context.convert(node.left), ' ', @@ -72,7 +72,7 @@ export abstract class DefaultVisitor implements AstHandler { ]); } - public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree { + public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree { return new OTree([ UNARY_OPS[node.operator], @@ -80,15 +80,15 @@ export abstract class DefaultVisitor implements AstHandler { ]); } - public ifStatement(node: ts.IfStatement, context: AstConverter): OTree { + public ifStatement(node: ts.IfStatement, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree { + public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree { return new OTree([context.convert(node.expression), '.', context.convert(node.name)]); } - public callExpression(node: ts.CallExpression, context: AstConverter): OTree { + public callExpression(node: ts.CallExpression, context: AstRenderer): OTree { return new OTree([ context.convert(node.expression), '(', @@ -96,112 +96,112 @@ export abstract class DefaultVisitor implements AstHandler { ')']); } - public expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree { + public expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree { return new OTree([context.convert(node.expression)], [], { canBreakLine: true }); } - public token(node: ts.Token, context: AstConverter): OTree { + public token(node: ts.Token, context: AstRenderer): OTree { return new OTree([context.textOf(node)]); } - public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree { + public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public newExpression(node: ts.NewExpression, context: AstConverter): OTree { + public newExpression(node: ts.NewExpression, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree { + public propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public variableStatement(node: ts.VariableStatement, context: AstConverter): OTree { + public variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree { return new OTree([context.convert(node.declarationList)], [], { canBreakLine: true }); } - public variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree { + public variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree { return new OTree([], context.convertAll(node.declarations)); } - public variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree { + public variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree { + public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree { return new OTree(['['], context.convertAll(node.elements), { separator: ', ', suffix: ']', }); } - public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree { + public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree { + public forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree { + public classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree { + public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree { + public propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree { + public methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree { + public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public propertySignature(node: ts.PropertySignature, context: AstConverter): OTree { + public propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public asExpression(node: ts.AsExpression, context: AstConverter): OTree { + public asExpression(node: ts.AsExpression, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public spreadElement(node: ts.SpreadElement, context: AstConverter): OTree { + public spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree { + public spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public ellipsis(_node: ts.SpreadElement | ts.SpreadAssignment, _context: AstConverter): OTree { + public ellipsis(_node: ts.SpreadElement | ts.SpreadAssignment, _context: AstRenderer): OTree { return new OTree(['...']); } - public templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree { + public templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree { return this.notImplemented(node, context); } - public nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree { + public nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree { // We default we drop the non-null assertion return context.convert(node.expression); } - public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree { + public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree { return new OTree(['(', context.convert(node.expression), ')']); } - public maskingVoidExpression(_node: ts.VoidExpression, _context: AstConverter): OTree { + public maskingVoidExpression(_node: ts.VoidExpression, _context: AstRenderer): OTree { // Don't render anything by default when nodes are masked return new OTree([]); } - private notImplemented(node: ts.Node, context: AstConverter) { + private notImplemented(node: ts.Node, context: AstRenderer) { context.reportUnsupported(node); return nimpl(node, context); } diff --git a/packages/jsii-rosetta/lib/languages/index.ts b/packages/jsii-rosetta/lib/languages/index.ts new file mode 100644 index 0000000000..c6a5860811 --- /dev/null +++ b/packages/jsii-rosetta/lib/languages/index.ts @@ -0,0 +1,9 @@ +import { PythonVisitor } from "./python"; +import { AstHandler } from "../renderer"; + +export type TargetLanguage = 'python'; +export type VisitorFactory = () => AstHandler; + +export const TARGET_LANGUAGES: {[key in TargetLanguage]: VisitorFactory} = { + 'python': () => new PythonVisitor() +}; \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/languages/python.ts b/packages/jsii-rosetta/lib/languages/python.ts similarity index 97% rename from packages/jsii-sampiler/lib/languages/python.ts rename to packages/jsii-rosetta/lib/languages/python.ts index 4c871d3291..d234823060 100644 --- a/packages/jsii-sampiler/lib/languages/python.ts +++ b/packages/jsii-rosetta/lib/languages/python.ts @@ -1,11 +1,12 @@ import ts = require('typescript'); -import { AstConverter, nimpl } from "../converter"; +import { AstRenderer, nimpl } from "../renderer"; import { isStructType, parameterAcceptsUndefined, propertiesOfStruct, StructProperty, structPropertyAcceptsUndefined } from '../jsii/jsii-utils'; import { NO_SYNTAX, OTree, renderTree } from "../o-tree"; import { matchAst, nodeOfType, stripCommentMarkers, voidExpressionString } from '../typescript/ast-utils'; import { ImportStatement } from '../typescript/imports'; import { startsWithUppercase } from "../util"; import { DefaultVisitor } from './default'; +import { jsiiTargetParam } from '../jsii/packages'; interface StructVar { variableName: string; @@ -60,7 +61,7 @@ interface PythonLanguageContext { readonly currentMethodName?: string; } -type PythonVisitorContext = AstConverter; +type PythonVisitorContext = AstRenderer; export interface PythonVisitorOptions { disclaimer?: string; @@ -458,7 +459,12 @@ export class PythonVisitor extends DefaultVisitor { } protected convertModuleReference(ref: string) { - return ref.replace(/^@/, '').replace(/\//g, '.').replace(/-/g, '_'); + // Get the Python target name from the referenced package (if available) + const resolvedPackage = jsiiTargetParam(ref, 'python.module'); + + // Return that or some default-derived module name representation + + return resolvedPackage || ref.replace(/^@/, '').replace(/\//g, '.').replace(/-/g, '_'); } /** diff --git a/packages/jsii-sampiler/lib/languages/visualize.ts b/packages/jsii-rosetta/lib/languages/visualize.ts similarity index 71% rename from packages/jsii-sampiler/lib/languages/visualize.ts rename to packages/jsii-rosetta/lib/languages/visualize.ts index 837b0f3c74..ed95eca390 100644 --- a/packages/jsii-sampiler/lib/languages/visualize.ts +++ b/packages/jsii-rosetta/lib/languages/visualize.ts @@ -1,5 +1,5 @@ import ts = require('typescript'); -import { AstConverter, AstHandler, nimpl } from "../converter"; +import { AstRenderer, AstHandler, nimpl } from "../renderer"; import { OTree } from '../o-tree'; import { ImportStatement } from '../typescript/imports'; @@ -13,172 +13,172 @@ export class VisualizeAstVisitor implements AstHandler { return undefined; } - public commentRange(node: ts.CommentRange, context: AstConverter): OTree { + public commentRange(node: ts.CommentRange, context: AstRenderer): OTree { return new OTree(['(Comment', context.textAt(node.pos, node.end)], [], { suffix: ')' }); } - public jsDoc(_node: ts.JSDoc, _context: AstConverter): OTree { + public jsDoc(_node: ts.JSDoc, _context: AstRenderer): OTree { // Already handled by other doc handlers return new OTree([]); } - public sourceFile(node: ts.SourceFile, context: AstConverter): OTree { + public sourceFile(node: ts.SourceFile, context: AstRenderer): OTree { return new OTree(context.convertAll(node.statements)); } - public importStatement(node: ImportStatement, context: AstConverter): OTree { + public importStatement(node: ImportStatement, context: AstRenderer): OTree { return this.defaultNode('importStatement', node.node, context); } - public functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree { + public functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree { return this.defaultNode('functionDeclaration', node, children); } - public stringLiteral(node: ts.StringLiteral, children: AstConverter): OTree { + public stringLiteral(node: ts.StringLiteral, children: AstRenderer): OTree { return this.defaultNode('stringLiteral', node, children); } - public identifier(node: ts.Identifier, children: AstConverter): OTree { + public identifier(node: ts.Identifier, children: AstRenderer): OTree { return this.defaultNode('identifier', node, children); } - public block(node: ts.Block, children: AstConverter): OTree { + public block(node: ts.Block, children: AstRenderer): OTree { return this.defaultNode('block', node, children); } - public parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree { + public parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree { return this.defaultNode('parameterDeclaration', node, children); } - public returnStatement(node: ts.ReturnStatement, children: AstConverter): OTree { + public returnStatement(node: ts.ReturnStatement, children: AstRenderer): OTree { return this.defaultNode('returnStatement', node, children); } - public binaryExpression(node: ts.BinaryExpression, children: AstConverter): OTree { + public binaryExpression(node: ts.BinaryExpression, children: AstRenderer): OTree { return this.defaultNode('binaryExpression', node, children); } - public ifStatement(node: ts.IfStatement, context: AstConverter): OTree { + public ifStatement(node: ts.IfStatement, context: AstRenderer): OTree { return this.defaultNode('ifStatement', node, context); } - public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree { + public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree { return this.defaultNode('propertyAccessExpression', node, context); } - public callExpression(node: ts.CallExpression, context: AstConverter): OTree { + public callExpression(node: ts.CallExpression, context: AstRenderer): OTree { return this.defaultNode('callExpression', node, context); } - public expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree { + public expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree { return this.defaultNode('expressionStatement', node, context); } - public token(node: ts.Token, context: AstConverter): OTree { + public token(node: ts.Token, context: AstRenderer): OTree { return this.defaultNode('token', node, context); } - public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree { + public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree { return this.defaultNode('objectLiteralExpression', node, context); } - public newExpression(node: ts.NewExpression, context: AstConverter): OTree { + public newExpression(node: ts.NewExpression, context: AstRenderer): OTree { return this.defaultNode('newExpression', node, context); } - public propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree { + public propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree { return this.defaultNode('propertyAssignment', node, context); } - public variableStatement(node: ts.VariableStatement, context: AstConverter): OTree { + public variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree { return this.defaultNode('variableStatement', node, context); } - public variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree { + public variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree { return this.defaultNode('variableDeclarationList', node, context); } - public variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree { + public variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree { return this.defaultNode('variableDeclaration', node, context); } - public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree { + public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree { return this.defaultNode('arrayLiteralExpression', node, context); } - public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree { + public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree { return this.defaultNode('shorthandPropertyAssignment', node, context); } - public forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree { + public forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree { return this.defaultNode('forOfStatement', node, context); } - public classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree { + public classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree { return this.defaultNode('classDeclaration', node, context); } - public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree { + public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree { return this.defaultNode('constructorDeclaration', node, context); } - public propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree { + public propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree { return this.defaultNode('propertyDeclaration', node, context); } - public methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree { + public methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree { return this.defaultNode('methodDeclaration', node, context); } - public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree { + public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree { return this.defaultNode('interfaceDeclaration', node, context); } - public propertySignature(node: ts.PropertySignature, context: AstConverter): OTree { + public propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree { return this.defaultNode('propertySignature', node, context); } - public asExpression(node: ts.AsExpression, context: AstConverter): OTree { + public asExpression(node: ts.AsExpression, context: AstRenderer): OTree { return this.defaultNode('asExpression', node, context); } - public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree { + public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree { return this.defaultNode('prefixUnaryExpression', node, context); } - public spreadElement(node: ts.SpreadElement, context: AstConverter): OTree { + public spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree { return this.defaultNode('spreadElement', node, context); } - public spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree { + public spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree { return this.defaultNode('spreadAssignment', node, context); } - public ellipsis(node: ts.SpreadAssignment | ts.SpreadElement, context: AstConverter): OTree { + public ellipsis(node: ts.SpreadAssignment | ts.SpreadElement, context: AstRenderer): OTree { return this.defaultNode('ellipsis', node, context); } - public templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree { + public templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree { return this.defaultNode('templateExpression', node, context); } - public nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree { + public nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree { return this.defaultNode('nonNullExpression', node, context); } - public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree { + public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree { return this.defaultNode('parenthesizedExpression', node, context); } - public maskingVoidExpression(node: ts.VoidExpression, context: AstConverter): OTree { + public maskingVoidExpression(node: ts.VoidExpression, context: AstRenderer): OTree { return this.defaultNode('maskingVoidExpression', node, context); } - public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstConverter): OTree { + public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstRenderer): OTree { return this.defaultNode('noSubstitutionTemplateLiteral', node, context); } - private defaultNode(handlerName: string, node: ts.Node, context: AstConverter): OTree { + private defaultNode(handlerName: string, node: ts.Node, context: AstRenderer): OTree { return nimpl(node, context, { additionalInfo: this.includeHandlerNames ? handlerName : '' }); diff --git a/packages/jsii-rosetta/lib/logging.ts b/packages/jsii-rosetta/lib/logging.ts new file mode 100644 index 0000000000..5266448036 --- /dev/null +++ b/packages/jsii-rosetta/lib/logging.ts @@ -0,0 +1,33 @@ +export enum Level { + WARN = -1, + QUIET = 0, + INFO = 1, + VERBOSE = 2, +} + +export const LEVEL_INFO: number = Level.INFO; +export const LEVEL_VERBOSE: number = Level.VERBOSE; + +/** The minimal logging level for messages to be emitted. */ +/* eslint-disable prefer-const */ +export let level = Level.QUIET; +/* eslint-enable prefer-const */ + +export function warn(fmt: string, ...args: any[]) { + log(Level.WARN, fmt, ...args); +} + +export function info(fmt: string, ...args: any[]) { + log(Level.INFO, fmt, ...args); +} + +export function debug(fmt: string, ...args: any[]) { + log(Level.VERBOSE, fmt, ...args); +} + +function log(messageLevel: Level, fmt: string, ...args: any[]) { + if (level >= messageLevel) { + const levelName = Level[messageLevel]; + console.error(`[jsii-rosetta] [${levelName}]`, fmt, ...args); + } +} diff --git a/packages/jsii-rosetta/lib/markdown/extract-snippets.ts b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts new file mode 100644 index 0000000000..e10033eb45 --- /dev/null +++ b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts @@ -0,0 +1,21 @@ +import cm = require('commonmark'); +import { visitCommonMarkTree } from '../markdown/markdown'; +import { CodeBlock } from '../markdown/replace-code-renderer'; +import { TypeScriptSnippet } from '../snippet'; +import { ReplaceTypeScriptTransform } from './replace-typescript-transform'; + +export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefined; + +export function extractTypescriptSnippetsFromMarkdown(markdown: string, wherePrefix: string): TypeScriptSnippet[] { + const parser = new cm.Parser(); + const doc = parser.parse(markdown); + + const ret: TypeScriptSnippet[] = []; + + visitCommonMarkTree(doc, new ReplaceTypeScriptTransform(wherePrefix, (ts) => { + ret.push(ts); + return undefined; + })); + + return ret; +} \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/markdown/markdown-renderer.ts b/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts similarity index 100% rename from packages/jsii-sampiler/lib/markdown/markdown-renderer.ts rename to packages/jsii-rosetta/lib/markdown/markdown-renderer.ts diff --git a/packages/jsii-sampiler/lib/markdown/markdown.ts b/packages/jsii-rosetta/lib/markdown/markdown.ts similarity index 100% rename from packages/jsii-sampiler/lib/markdown/markdown.ts rename to packages/jsii-rosetta/lib/markdown/markdown.ts diff --git a/packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts similarity index 94% rename from packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts rename to packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts index 4229e4bd7b..fb93793fd8 100644 --- a/packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts +++ b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts @@ -18,7 +18,7 @@ export class ReplaceCodeTransform implements CommonMarkVisitor { public code_block(node: cm.Node) { const ret = this.replacer({ language: node.info || '', source: node.literal || '' }); node.info = ret.language; - node.literal = ret.source; + node.literal = ret.source + (!ret.source || ret.source.endsWith('\n') ? '' : '\n'); } public block_quote(): void { /* nothing */ } diff --git a/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts new file mode 100644 index 0000000000..d89ba5a89a --- /dev/null +++ b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts @@ -0,0 +1,37 @@ +import { ReplaceCodeTransform, CodeBlock } from "./replace-code-renderer"; +import { TypeScriptSnippet, typeScriptSnippetFromSource, parseKeyValueList } from "../snippet"; + +export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefined; + +/** + * A specialization of ReplaceCodeTransform that maintains state about TypeScript snippets + */ +export class ReplaceTypeScriptTransform extends ReplaceCodeTransform { + private readonly wherePrefix: string; + + constructor(wherePrefix: string, replacer: TypeScriptReplacer) { + let count = 0; + super(block => { + const languageParts = block.language ? block.language.split(' ') : []; + if (languageParts[0] !== 'typescript' && languageParts[0] !== 'ts') { return block; } + + count += 1; + const tsSnippet = typeScriptSnippetFromSource( + block.source, + this.addSnippetNumber(count), + parseKeyValueList(languageParts.slice(1)) + ); + + return replacer(tsSnippet) || block; + }); + + this.wherePrefix = wherePrefix; + } + + private addSnippetNumber(snippetNumber: number) { + // First snippet (most cases) will not be numbered + if (snippetNumber === 1) { return this.wherePrefix; } + + return `${this.wherePrefix}-snippet${snippetNumber}`; + } +} diff --git a/packages/jsii-sampiler/lib/markdown/structure-renderer.ts b/packages/jsii-rosetta/lib/markdown/structure-renderer.ts similarity index 100% rename from packages/jsii-sampiler/lib/markdown/structure-renderer.ts rename to packages/jsii-rosetta/lib/markdown/structure-renderer.ts diff --git a/packages/jsii-sampiler/lib/o-tree.ts b/packages/jsii-rosetta/lib/o-tree.ts similarity index 75% rename from packages/jsii-sampiler/lib/o-tree.ts rename to packages/jsii-rosetta/lib/o-tree.ts index a81ae58be5..2bf662c277 100644 --- a/packages/jsii-sampiler/lib/o-tree.ts +++ b/packages/jsii-rosetta/lib/o-tree.ts @@ -1,4 +1,3 @@ - export interface OTreeOptions { /** * Adjust indentation with the given number @@ -50,7 +49,7 @@ export interface OTreeOptions { * Tree-like structure that holds sequences of trees and strings, which * can be rendered to an output stream. */ -export class OTree { +export class OTree implements OTree { public static simplify(xs: Array): Array { return xs.filter(notUndefined).filter(notEmpty); } @@ -59,6 +58,7 @@ export class OTree { private readonly prefix: Array; private readonly children: Array; + private span?: Span; constructor( prefix: Array, @@ -70,25 +70,36 @@ export class OTree { this.attachComment = !!options.canBreakLine; } + /** + * Set the span in the source file this tree node relates to + */ + public setSpan(start: number, end: number) { + this.span = { start, end }; + } + public write(sink: OTreeSink) { if (!sink.tagOnce(this.options.renderOnce)) { return; } + const meVisible = sink.renderingForSpan(this.span); + for (const x of this.prefix) { sink.write(x); } - const popIndent = sink.requestIndentChange(this.options.indent || 0); + const popIndent = sink.requestIndentChange(meVisible ? this.options.indent || 0 : 0); let mark = sink.mark(); for (const child of this.children || []) { - if (this.options.separator && mark.wroteNonWhitespaceSinceMark) { sink.write(this.options.separator); } + if (this.options.separator && mark.wroteNonWhitespaceSinceMark) { + sink.write(this.options.separator); + } mark = sink.mark(); sink.write(child); } - popIndent(); if (this.options.suffix) { + sink.renderingForSpan(this.span); sink.write(this.options.suffix); } } @@ -111,11 +122,19 @@ export interface SinkMark { readonly wroteNonWhitespaceSinceMark: boolean; } +export interface OTreeSinkOptions { + visibleSpans?: Span[]; +} + export class OTreeSink { private readonly indentLevels: number[] = [0]; private readonly fragments = new Array(); private singletonsRendered = new Set(); private pendingIndentChange = 0; + private rendering = true; + + constructor(private readonly options: OTreeSinkOptions = {}) { + } public tagOnce(key: string | undefined): boolean { if (key === undefined) { return true; } @@ -124,6 +143,11 @@ export class OTreeSink { return true; } + /** + * Get a mark for the current sink output location + * + * Marks can be used to query about things that have been written to output. + */ public mark(): SinkMark { const self = this; const markIndex = this.fragments.length; @@ -139,6 +163,8 @@ export class OTreeSink { if (text instanceof OTree) { text.write(this); } else { + if (!this.rendering) { return; } + if (containsNewline(text)) { this.applyPendingIndentChange(); } @@ -146,6 +172,13 @@ export class OTreeSink { } } + public renderingForSpan(span?: Span): boolean { + if (span && this.options.visibleSpans) { + this.rendering = this.options.visibleSpans.some(v => inside(span, v)); + } + return this.rendering; + } + public requestIndentChange(x: number): () => void { if (x === 0) { return () => undefined; } @@ -162,8 +195,8 @@ export class OTreeSink { } public toString() { - // Strip trailing whitespace from every line - return this.fragments.join('').replace(/[ \t]+$/gm, ''); + // Strip trailing whitespace from every line, and empty lines from the start and end + return this.fragments.join('').replace(/[ \t]+$/gm, '').replace(/^\n+/, '').replace(/\n+$/, ''); } private append(x: string) { @@ -190,8 +223,8 @@ function notEmpty(x: OTree | string) { return x instanceof OTree ? !x.isEmpty : x !== ''; } -export function renderTree(tree: OTree): string { - const sink = new OTreeSink(); +export function renderTree(tree: OTree, options?: OTreeSinkOptions): string { + const sink = new OTreeSink(options); tree.write(sink); return sink.toString(); } @@ -199,3 +232,12 @@ export function renderTree(tree: OTree): string { function containsNewline(x: string) { return x.indexOf('\n') !== -1; } + +export interface Span { + start: number; + end: number +} + +function inside(a: Span, b: Span) { + return b.start <= a.start && a.end <= b.end; +} \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/converter.ts b/packages/jsii-rosetta/lib/renderer.ts similarity index 82% rename from packages/jsii-sampiler/lib/converter.ts rename to packages/jsii-rosetta/lib/renderer.ts index 42c540b6b8..5393d554dc 100644 --- a/packages/jsii-sampiler/lib/converter.ts +++ b/packages/jsii-rosetta/lib/renderer.ts @@ -5,14 +5,14 @@ import { commentRangeFromTextRange, extractMaskingVoidExpression, extractShowing import { analyzeImportDeclaration, analyzeImportEquals, ImportStatement } from './typescript/imports'; /** - * AST conversion operation + * Render a TypeScript AST to some other representation (encoded in OTrees) * * Dispatch the actual conversion to a specific handler which will get the * appropriate method called for particular AST nodes. The handler may use * context to modify its own operations when traversing the tree hierarchy, * the type of which should be expressed via the C parameter. */ -export class AstConverter { +export class AstRenderer { public readonly diagnostics = new Array(); public readonly currentContext: C; @@ -20,7 +20,7 @@ export class AstConverter { private readonly sourceFile: ts.SourceFile, private readonly typeChecker: ts.TypeChecker, private readonly handler: AstHandler, - private readonly options: ConvertOptions = {}) { + private readonly options: AstRendererOptions = {}) { this.currentContext = handler.defaultContext; } @@ -28,7 +28,7 @@ export class AstConverter { /** * Merge the new context with the current context and create a new Converter from it */ - public updateContext(contextUpdate: C): AstConverter { + public updateContext(contextUpdate: C): AstRenderer { const newContext = this.handler.mergeContext(this.currentContext, contextUpdate); // Use prototypal inheritance to create a version of 'this' in which only @@ -46,9 +46,12 @@ export class AstConverter { // Basic transform of node const transformed = this.dispatch(node); + transformed.setSpan(node.getStart(this.sourceFile), node.getEnd()); if (!transformed.attachComment) { return transformed; } - return this.attachLeadingTrivia(node, transformed); + const withTrivia = this.attachLeadingTrivia(node, transformed); + withTrivia.setSpan(node.getStart(this.sourceFile), node.getEnd()); + return withTrivia; } /** @@ -236,6 +239,9 @@ export class AstConverter { case 'blockcomment': precede.push(this.handler.commentRange(commentRangeFromTextRange(range), this)); break; + + case 'directive': + break; } } @@ -263,53 +269,53 @@ export interface AstHandler { readonly defaultContext: C; mergeContext(old: C, update: C): C; - sourceFile(node: ts.SourceFile, context: AstConverter): OTree; - commentRange(node: ts.CommentRange, context: AstConverter): OTree; - importStatement(node: ImportStatement, context: AstConverter): OTree; - stringLiteral(node: ts.StringLiteral, children: AstConverter): OTree; - functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree; - identifier(node: ts.Identifier, children: AstConverter): OTree; - block(node: ts.Block, children: AstConverter): OTree; - parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree; - returnStatement(node: ts.ReturnStatement, context: AstConverter): OTree; - binaryExpression(node: ts.BinaryExpression, context: AstConverter): OTree; - ifStatement(node: ts.IfStatement, context: AstConverter): OTree; - propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree; - callExpression(node: ts.CallExpression, context: AstConverter): OTree; - expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree; - token(node: ts.Token, context: AstConverter): OTree; - objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree; - newExpression(node: ts.NewExpression, context: AstConverter): OTree; - propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree; - variableStatement(node: ts.VariableStatement, context: AstConverter): OTree; - variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree; - variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree; - jsDoc(node: ts.JSDoc, context: AstConverter): OTree; - arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree; - shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree; - forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree; - classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree; - constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree; - propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree; - methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree; - interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree; - propertySignature(node: ts.PropertySignature, context: AstConverter): OTree; - asExpression(node: ts.AsExpression, context: AstConverter): OTree; - prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree; - spreadElement(node: ts.SpreadElement, context: AstConverter): OTree; - spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree; - templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree; - nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree; - parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree; - maskingVoidExpression(node: ts.VoidExpression, context: AstConverter): OTree; - noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstConverter): OTree; + sourceFile(node: ts.SourceFile, context: AstRenderer): OTree; + commentRange(node: ts.CommentRange, context: AstRenderer): OTree; + importStatement(node: ImportStatement, context: AstRenderer): OTree; + stringLiteral(node: ts.StringLiteral, children: AstRenderer): OTree; + functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree; + identifier(node: ts.Identifier, children: AstRenderer): OTree; + block(node: ts.Block, children: AstRenderer): OTree; + parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree; + returnStatement(node: ts.ReturnStatement, context: AstRenderer): OTree; + binaryExpression(node: ts.BinaryExpression, context: AstRenderer): OTree; + ifStatement(node: ts.IfStatement, context: AstRenderer): OTree; + propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree; + callExpression(node: ts.CallExpression, context: AstRenderer): OTree; + expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree; + token(node: ts.Token, context: AstRenderer): OTree; + objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree; + newExpression(node: ts.NewExpression, context: AstRenderer): OTree; + propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree; + variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree; + variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree; + variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree; + jsDoc(node: ts.JSDoc, context: AstRenderer): OTree; + arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree; + shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree; + forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree; + classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree; + constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree; + propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree; + methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree; + interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree; + propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree; + asExpression(node: ts.AsExpression, context: AstRenderer): OTree; + prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree; + spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree; + spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree; + templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree; + nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree; + parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree; + maskingVoidExpression(node: ts.VoidExpression, context: AstRenderer): OTree; + noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstRenderer): OTree; // Not a node, called when we recognize a spread element/assignment that is only // '...' and nothing else. - ellipsis(node: ts.SpreadElement | ts.SpreadAssignment, context: AstConverter): OTree; + ellipsis(node: ts.SpreadElement | ts.SpreadAssignment, context: AstRenderer): OTree; } -export function nimpl(node: ts.Node, context: AstConverter, options: { additionalInfo?: string} = {}) { +export function nimpl(node: ts.Node, context: AstRenderer, options: { additionalInfo?: string} = {}) { const children = nodeChildren(node).map(c => context.convert(c)); let syntaxKind = ts.SyntaxKind[node.kind]; @@ -330,7 +336,7 @@ export function nimpl(node: ts.Node, context: AstConverter, options: { add }); } -export interface ConvertOptions { +export interface AstRendererOptions { /** * If enabled, don't translate the text of unknown nodes * diff --git a/packages/jsii-rosetta/lib/rosetta.ts b/packages/jsii-rosetta/lib/rosetta.ts new file mode 100644 index 0000000000..fd19e57f4e --- /dev/null +++ b/packages/jsii-rosetta/lib/rosetta.ts @@ -0,0 +1,152 @@ +import fs = require('fs-extra'); +import path = require('path'); +import spec = require('jsii-spec'); +import { DEFAULT_TABLET_NAME, LanguageTablet, Translation } from "./tablets/tablets"; +import { allTypeScriptSnippets } from './jsii/assemblies'; +import { TargetLanguage } from './languages'; +import { Translator } from './translate'; +import { isError } from 'util'; +import { transformMarkdown } from './markdown/markdown'; +import { MarkdownRenderer } from './markdown/markdown-renderer'; +import { CodeBlock } from './markdown/replace-code-renderer'; +import { TypeScriptSnippet } from './snippet'; +import { printDiagnostics } from './util'; +import { ReplaceTypeScriptTransform } from './markdown/replace-typescript-transform'; + +export interface RosettaOptions { + /** + * Whether or not to live-convert samples + * + * @default false + */ + liveConversion?: boolean; + + /** + * Target languages to use for live conversion + * + * @default All languages + */ + targetLanguages?: TargetLanguage[]; +} + +/** + * Entry point class for consumers for Rosetta functionality + * + * Rosetta can work in one of two modes: + * + * 1. Live translation of snippets. + * 2. Read translations from a pre-translated tablet (prepared using `jsii-rosetta extract` command). + * + * The second method affords more control over the precise circumstances of + * sample compilation and is recommended, but the first method will do + * when the second one is not necessary. + */ +export class Rosetta { + private readonly loadedTablets: LanguageTablet[] = []; + private readonly liveTablet = new LanguageTablet(); + private readonly extractedSnippets: Record = {}; + private readonly translator = new Translator(false); + + constructor(private readonly options: RosettaOptions = {}) { + } + + /** + * Diagnostics encountered while doing live translation + */ + public get diagnostics() { + return this.translator.diagnostics; + } + + /** + * Load a tablet as a source for translateable snippets + */ + public async loadTabletFromFile(tabletFile: string) { + const tablet = new LanguageTablet(); + await tablet.load(tabletFile); + this.addTablet(tablet); + } + + /** + * Directly add a tablet + * + * Should only be needed for testing, use `loadTabletFromFile` and `addAssembly` instead. + */ + public addTablet(tablet: LanguageTablet) { + this.loadedTablets.push(tablet); + } + + /** + * Add an assembly + * + * If a default tablet file is found in the assembly's directory, it will be + * loaded. + * + * Otherwise, if live conversion is enabled, the snippets in the assembly + * become available for live translation later. + * + * (We do it like this to centralize the logic around the "where" calculation, + * otherwise each language generator has to reimplement a way to describe API + * elements while spidering the jsii assembly). + */ + public async addAssembly(assembly: spec.Assembly, assemblyDir: string) { + if (await fs.pathExists(path.join(assemblyDir, DEFAULT_TABLET_NAME))) { + await this.loadTabletFromFile(path.join(assemblyDir, DEFAULT_TABLET_NAME)); + return; + } + + if (this.options.liveConversion) { + for (const tsnip of allTypeScriptSnippets([{ assembly, directory: assemblyDir }])) { + this.extractedSnippets[tsnip.visibleSource] = tsnip; + } + } + } + + public translateSnippet(source: TypeScriptSnippet, targetLang: TargetLanguage): Translation | undefined { + // Look for it in loaded tablets + for (const tab of this.allTablets) { + const ret = tab.lookup(source, targetLang); + if (ret !== undefined) { return ret; } + } + + if (!this.options.liveConversion) { return undefined; } + if (this.options.targetLanguages && !this.options.targetLanguages.includes(targetLang)) { + throw new Error(`Rosetta configured for live conversion to ${this.options.targetLanguages}, but requested ${targetLang}`); + } + + // See if we're going to live-convert it with full source information + const extracted = this.extractedSnippets[source.visibleSource]; + if (extracted !== undefined) { + const snippet = this.translator.translate(extracted, this.options.targetLanguages); + this.liveTablet.addSnippet(snippet); + return snippet.get(targetLang); + } + + // Try to live-convert it on the spot (we won't have "where" information or fixtures) + const snippet = this.translator.translate(source, this.options.targetLanguages); + return snippet.get(targetLang); + } + + public translateSnippetsInMarkdown(markdown: string, targetLang: TargetLanguage, translationToCodeBlock: (x: Translation) => CodeBlock = id): string { + return transformMarkdown(markdown, new MarkdownRenderer(), new ReplaceTypeScriptTransform('markdown', tsSnip => { + const translated = this.translateSnippet(tsSnip, targetLang); + if (!translated) { return undefined; } + + return translationToCodeBlock(translated); + })); + } + + public printDiagnostics(stream: NodeJS.WritableStream) { + printDiagnostics(this.diagnostics, stream); + } + + public get hasErrors() { + return this.diagnostics.some(isError); + }; + + private get allTablets(): LanguageTablet[] { + return [...this.loadedTablets, this.liveTablet]; + } +} + + +function id(x: Translation) { return x; } \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts new file mode 100644 index 0000000000..dd73e88993 --- /dev/null +++ b/packages/jsii-rosetta/lib/snippet.ts @@ -0,0 +1,106 @@ +/** + * A piece of TypeScript code found in an assembly, ready to be translated + */ +export interface TypeScriptSnippet { + /** + * The snippet code that ends up in the JSII assembly + */ + readonly visibleSource: string; + + /** + * A human-readable description of where this snippet was found in the assembly + */ + readonly where: string; + + /** + * When enhanced with a fixture, the snippet's complete source code + */ + readonly completeSource?: string; + + /** + * Parameters for the conversion + */ + readonly parameters?: Record; +} + +/** + * Construct a TypeScript snippet from literal source + * + * Will parse parameters from a directive in the given source. + */ +export function typeScriptSnippetFromSource(typeScriptSource: string, where: string, parameters: Record = {}) { + const [source, sourceParameters] = parametersFromSourceDirectives(typeScriptSource); + return { + visibleSource: source.trimRight(), + where, + parameters: Object.assign({}, parameters, sourceParameters) + }; +} + +export function updateParameters(snippet: TypeScriptSnippet, params: Record): TypeScriptSnippet { + return { + ...snippet, + parameters: Object.assign({}, snippet.parameters || {}, params) + }; +} + +/** + * Get the complete (compilable) source of a snippet + */ +export function completeSource(snippet: TypeScriptSnippet) { + return snippet.completeSource || snippet.visibleSource; +} + +/** + * Extract snippet parameters from the first line of the source if it's a compiler directive + */ +function parametersFromSourceDirectives(source: string): [string, Record] { + const [firstLine, rest] = source.split('\n', 2); + // Also extract parameters from an initial line starting with '/// ' (getting rid of that line). + const m = /\/\/\/(.*)$/.exec(firstLine); + if (m) { + const paramClauses = m[1].trim().split(' ').map(s => s.trim()).filter(s => s !== ''); + return [rest, parseKeyValueList(paramClauses)]; + } + + return [source, {}]; +} + +/** + * Parse a set of 'param param=value' directives into an object + */ +export function parseKeyValueList(parameters: string[]): Record { + const ret: Record = {}; + for (const param of parameters) { + const parts = param.split('=', 2); + if (parts.length === 2) { + ret[parts[0]] = parts[1]; + } else { + ret[parts[0]] = ''; + } + } + + return ret; +} + +/** + * Recognized snippet parameters + */ +export enum SnippetParameters { + /** + * Use fixture with the given name (author parameter) + */ + FIXTURE = 'fixture', + + /** + * Don't use a fixture (author parameter) + */ + NO_FIXTURE = 'nofixture', + + /** + * What directory to resolve fixtures in for this snippet (system parameter) + * + * Attached during processing, should not be used by authors. + */ + $PROJECT_DIRECTORY = '$directory', +}; \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/tablets/key.ts b/packages/jsii-rosetta/lib/tablets/key.ts new file mode 100644 index 0000000000..1d2329859b --- /dev/null +++ b/packages/jsii-rosetta/lib/tablets/key.ts @@ -0,0 +1,11 @@ +import crypto = require('crypto'); +import { TypeScriptSnippet } from '../snippet'; + +/** + * Determine the key for a code block + */ +export function snippetKey(snippet: TypeScriptSnippet) { + const h = crypto.createHash('sha256'); + h.update(snippet.visibleSource); + return h.digest('hex'); +} \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/tablets/schema.ts b/packages/jsii-rosetta/lib/tablets/schema.ts new file mode 100644 index 0000000000..104bc1510e --- /dev/null +++ b/packages/jsii-rosetta/lib/tablets/schema.ts @@ -0,0 +1,63 @@ +/** + * Tablet file schema + */ +export interface TabletSchema { + /** + * Schema version + */ + version: string; + + /** + * What version of the tool this schema was generated with + * + * Hashing algorithms may depend on the tool version, and the tool + * will reject tablets generated by different versions. + * + * Since tablets are designed to be used as scratch space during a build, not + * designed to be stored long-term, this limitation does not impact + * usability. + */ + toolVersion: string; + + /** + * All the snippets in the tablet + */ + snippets: {[key: string]: SnippetSchema}; +} + +export const ORIGINAL_SNIPPET_KEY = '$'; + +/** + * Schema for a snippet + */ +export interface SnippetSchema { + /** + * Translations for each individual language + * + * Since TypeScript is a valid output translation, the original will be + * listed under the key '$'. + */ + translations: {[key: string]: TranslationSchema}; + + /** + * A human-readable description of the location this code snippet was found + */ + where: string; + + /** + * Whether this was compiled without errors + * + * Undefined means no compilation was not attempted. + */ + didCompile?: boolean; +} + +/** + * A single snippet's translation + */ +export interface TranslationSchema { + /** + * The source of a single translation + */ + source: string; +} \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts new file mode 100644 index 0000000000..07254acf12 --- /dev/null +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -0,0 +1,175 @@ +import fs = require('fs-extra'); +import { TabletSchema, SnippetSchema, TranslationSchema, ORIGINAL_SNIPPET_KEY } from './schema'; +import { snippetKey } from './key'; +import { TargetLanguage } from '../languages'; +import { TypeScriptSnippet } from '../snippet'; + +const TOOL_VERSION = require('../../package.json').version; + +export const DEFAULT_TABLET_NAME = '.jsii-samples.tabl'; + +/** + * A tablet containing various snippets in multiple languages + */ +export class LanguageTablet { + private readonly snippets: Record = {}; + + public addSnippet(snippet: TranslatedSnippet) { + const existingSnippet = this.snippets[snippet.key]; + this.snippets[snippet.key] = existingSnippet ? existingSnippet.merge(snippet) : snippet; + } + + public get snippetKeys() { + return Object.keys(this.snippets); + } + + public getSnippet(key: string): TranslatedSnippet | undefined { + return this.snippets[key]; + } + + public lookup(typeScriptSource: TypeScriptSnippet, language: TargetLanguage): Translation | undefined { + const snippet = this.snippets[snippetKey(typeScriptSource)]; + return snippet && snippet.get(language); + } + + public async load(filename: string) { + const obj = await fs.readJson(filename, { encoding: 'utf-8' }); + + if (!obj.toolVersion || !obj.snippets) { + throw new Error(`File '${filename}' does not seem to be a Tablet file`); + } + if (obj.toolVersion !== TOOL_VERSION) { + throw new Error(`Tablet file '${filename}' has been created with version '${obj.toolVersion}', cannot read with current version '${TOOL_VERSION}'`); + } + + Object.assign(this.snippets, mapValues(obj.snippets, (schema: SnippetSchema) => TranslatedSnippet.fromSchema(schema))); + } + + public get count() { + return Object.keys(this.snippets).length; + } + + public async save(filename: string) { + await fs.writeJson(filename, this.toSchema(), { encoding: 'utf-8', spaces: 2 }); + } + + private toSchema(): TabletSchema { + return { + version: '1', + toolVersion: TOOL_VERSION, + snippets: mapValues(this.snippets, s => s.toSchema()) + }; + } +} + +export class TranslatedSnippet { + public static fromSchema(schema: SnippetSchema) { + const ret = new TranslatedSnippet(); + Object.assign(ret.translations, schema.translations); + ret._didCompile = schema.didCompile; + ret._where = schema.where; + return ret; + } + + public static fromSnippet(original: TypeScriptSnippet, didCompile?: boolean) { + const ret = new TranslatedSnippet(); + Object.assign(ret.translations, { [ORIGINAL_SNIPPET_KEY]: { source: original.visibleSource }}); + ret._didCompile = didCompile; + ret._where = original.where; + return ret; + } + + private readonly translations: Record = {}; + private _key?: string; + private _didCompile?: boolean; + private _where: string = ''; + + private constructor() { + } + + public get didCompile() { + return this._didCompile; + } + + public get where() { + return this._where; + } + + public get key() { + if (this._key === undefined) { + this._key = snippetKey(this.asTypescriptSnippet()); + } + return this._key; + } + + public asTypescriptSnippet(): TypeScriptSnippet { + return { + visibleSource: this.translations[ORIGINAL_SNIPPET_KEY].source, + where: this.where, + }; + } + + public get originalSource(): Translation { + return { + source: this.translations[ORIGINAL_SNIPPET_KEY].source, + language: 'typescript', + didCompile: this.didCompile + }; + } + + public addTranslatedSource(language: TargetLanguage, translation: string): Translation { + this.translations[language] = { source: translation }; + + return { + source: translation, + language, + didCompile: this.didCompile + }; + } + + public get languages(): TargetLanguage[] { + return Object.keys(this.translations).filter(x => x !== ORIGINAL_SNIPPET_KEY) as TargetLanguage[]; + } + + public get(language: TargetLanguage): Translation | undefined { + const t = this.translations[language]; + return t && { source: t.source, language, didCompile: this.didCompile }; + } + + public merge(other: TranslatedSnippet) { + const ret = new TranslatedSnippet(); + Object.assign(ret.translations, this.translations, other.translations); + ret._didCompile = this.didCompile; + ret._where = this.where; + return ret; + } + + public toTypeScriptSnippet() { + return { + source: this.originalSource, + where: this.where + }; + } + + public toSchema(): SnippetSchema { + return { + translations: this.translations, + didCompile: this.didCompile, + where: this.where + } + } +} + +export interface Translation { + source: string; + language: string; + didCompile?: boolean; +} + +function mapValues(xs: Record, fn: (x: A) => B): Record { + const ret: Record = {}; + for (const [key, value] of Object.entries(xs)) { + ret[key] = fn(value); + } + return ret; +} \ No newline at end of file diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts new file mode 100644 index 0000000000..576f60cd67 --- /dev/null +++ b/packages/jsii-rosetta/lib/translate.ts @@ -0,0 +1,131 @@ +import ts = require('typescript'); +import { AstRenderer, AstHandler, AstRendererOptions } from './renderer'; +import { renderTree, Span } from './o-tree'; +import { TypeScriptCompiler, CompilationResult } from './typescript/ts-compiler'; +import { TranslatedSnippet } from './tablets/tablets'; +import { TARGET_LANGUAGES, TargetLanguage } from './languages'; +import { calculateVisibleSpans } from './typescript/ast-utils'; +import { File } from './util'; +import { TypeScriptSnippet, completeSource } from './snippet'; + +export function translateTypeScript(source: File, visitor: AstHandler, options: SnippetTranslatorOptions = {}): TranslateResult { + const translator = new SnippetTranslator({ visibleSource: source.contents, where: source.fileName }, options); + const translated = translator.renderUsing(visitor); + + return { + translation: translated, + diagnostics: translator.diagnostics, + }; +} + + +/** + * Translate one or more TypeScript snippets into other languages + * + * Can be configured to fully typecheck the samples, or perform only syntactical + * translation. + */ +export class Translator { + private readonly compiler = new TypeScriptCompiler(); + private readonly translators: Record = {}; + + constructor(private readonly includeCompilerDiagnostics: boolean) { + } + + public translate(snip: TypeScriptSnippet, languages = Object.keys(TARGET_LANGUAGES) as TargetLanguage[]) { + const translator = this.translatorFor(snip); + const snippet = TranslatedSnippet.fromSnippet(snip, this.includeCompilerDiagnostics ? translator.compileDiagnostics.length === 0 : undefined); + + for (const lang of languages) { + const languageConverterFactory = TARGET_LANGUAGES[lang]; + const translated = translator.renderUsing(languageConverterFactory()); + snippet.addTranslatedSource(lang, translated); + } + + return snippet; + } + + public get diagnostics(): ts.Diagnostic[] { + const ret = []; + for (const t of Object.values(this.translators)) { + ret.push(...t.diagnostics); + } + return ret; + } + + /** + * Return the snippet translator for the given snippet + */ + public translatorFor(snippet: TypeScriptSnippet) { + const key = snippet.visibleSource + '-' + snippet.where; + if (!(key in this.translators)) { + const translator = new SnippetTranslator(snippet, { + compiler: this.compiler, + includeCompilerDiagnostics: this.includeCompilerDiagnostics, + }); + + this.diagnostics.push(...translator.compileDiagnostics); + + this.translators[key] = translator; + } + + return this.translators[key]; + } +} + +export interface SnippetTranslatorOptions extends AstRendererOptions { + /** + * Re-use the given compiler if given + */ + readonly compiler?: TypeScriptCompiler; + + /** + * Include compiler errors in return diagnostics + * + * If false, only translation diagnostics will be returned. + * + * @default false + */ + readonly includeCompilerDiagnostics?: boolean; +} + +export interface TranslateResult { + translation: string; + diagnostics: ts.Diagnostic[]; +} + +/** + * Translate a single TypeScript snippet + */ +export class SnippetTranslator { + public readonly translateDiagnostics: ts.Diagnostic[] = []; + public readonly compileDiagnostics: ts.Diagnostic[] = []; + private readonly visibleSpans: Span[]; + private compilation!: CompilationResult; + + constructor(snippet: TypeScriptSnippet, private readonly options: SnippetTranslatorOptions = {}) { + const compiler = options.compiler || new TypeScriptCompiler(); + const source = completeSource(snippet); + this.compilation = compiler.compileInMemory(snippet.where, source); + + // Respect '/// !hide' and '/// !show' directives + this.visibleSpans = calculateVisibleSpans(source); + + // This makes it about 5x slower, so only do it on demand + if (options.includeCompilerDiagnostics) { + const program = this.compilation.program; + this.compileDiagnostics.push(...program.getGlobalDiagnostics(), ...program.getSyntacticDiagnostics(), ...program.getDeclarationDiagnostics(), ...program.getSemanticDiagnostics()); + } + } + + public renderUsing(visitor: AstHandler) { + const converter = new AstRenderer(this.compilation.rootFile, this.compilation.program.getTypeChecker(), visitor, this.options); + const converted = converter.convert(this.compilation.rootFile); + this.translateDiagnostics.push(...converter.diagnostics); + return renderTree(converted, { visibleSpans: this.visibleSpans }); + } + + public get diagnostics() { + return [...this.compileDiagnostics, ...this.translateDiagnostics]; + } +} diff --git a/packages/jsii-sampiler/lib/typescript/ast-utils.ts b/packages/jsii-rosetta/lib/typescript/ast-utils.ts similarity index 85% rename from packages/jsii-sampiler/lib/typescript/ast-utils.ts rename to packages/jsii-rosetta/lib/typescript/ast-utils.ts index 95cefee736..8b95a81373 100644 --- a/packages/jsii-sampiler/lib/typescript/ast-utils.ts +++ b/packages/jsii-rosetta/lib/typescript/ast-utils.ts @@ -1,4 +1,46 @@ import ts = require('typescript'); +import { Span } from '../o-tree'; + +export interface MarkedSpan { + start: number; + end: number; + visible: boolean; +} + +export function calculateVisibleSpans(source: string): Span[] { + return calculateMarkedSpans(source).filter(s => s.visible); +} + +export function calculateMarkedSpans(source: string): MarkedSpan[] { + const regEx = /\/\/\/ (.*)(\r?\n)?$/gm; + + const ret = new Array(); + let match; + let spanStart; + let visible = true; + while ((match = regEx.exec(source)) != null) { + const directiveStart = match.index; + const directive = match[1].trim(); + if (['!hide', '!show'].includes(directive)) { + const isShow = directive === '!show'; + if (spanStart === undefined) { + // Add a span at the start which is the reverse of the actual first directive + ret.push({ start: 0, end: directiveStart, visible: !isShow }); + } else { + // Else add a span for the current directive + ret.push({ start: spanStart, end: directiveStart, visible }); + } + visible = isShow; + spanStart = match.index + match[0].length; + } + } + + // Add the remainder under the last visibility + ret.push({ start: spanStart || 0, end: source.length, visible }); + + // Filter empty spans and return + return ret.filter(s => s.start !== s.end); +} export function stripCommentMarkers(comment: string, multiline: boolean) { if (multiline) { @@ -187,7 +229,7 @@ export function commentRangeFromTextRange(rng: TextRange): ts.CommentRange { interface TextRange { pos: number; end: number; - type: 'linecomment' | 'blockcomment' | 'other'; + type: 'linecomment' | 'blockcomment' | 'other' | 'directive'; hasTrailingNewLine: boolean; } @@ -247,12 +289,24 @@ export function scanText(text: string, start: number, end?: number): TextRange[] function scanSinglelineComment() { const nl = Math.min(findNext('\r', pos + 2), findNext('\n', pos + 2)); - ret.push({ - type: 'linecomment', - hasTrailingNewLine: true, - pos, - end: nl - }); + + if (text[pos + 2] === '/') { + // Special /// comment + ret.push({ + type: 'directive', + hasTrailingNewLine: true, + pos: pos + 1, + end: nl + }); + } else { + // Regular // comment + ret.push({ + type: 'linecomment', + hasTrailingNewLine: true, + pos, + end: nl + }); + } pos = nl + 1; start = pos; } diff --git a/packages/jsii-sampiler/lib/typescript/imports.ts b/packages/jsii-rosetta/lib/typescript/imports.ts similarity index 93% rename from packages/jsii-sampiler/lib/typescript/imports.ts rename to packages/jsii-rosetta/lib/typescript/imports.ts index 55970f355c..f3a5aa7c17 100644 --- a/packages/jsii-sampiler/lib/typescript/imports.ts +++ b/packages/jsii-rosetta/lib/typescript/imports.ts @@ -1,5 +1,5 @@ import ts = require('typescript'); -import { AstConverter } from '../converter'; +import { AstRenderer } from '../renderer'; import { allOfType, matchAst, nodeOfType, stringFromLiteral } from "./ast-utils"; /** @@ -19,7 +19,7 @@ export interface ImportBinding { alias?: string; } -export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: AstConverter): ImportStatement { +export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: AstRenderer): ImportStatement { let moduleName = '???'; matchAst(node.moduleReference, nodeOfType('ref', ts.SyntaxKind.ExternalModuleReference), @@ -34,7 +34,7 @@ export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: A }; } -export function analyzeImportDeclaration(node: ts.ImportDeclaration, context: AstConverter): ImportStatement { +export function analyzeImportDeclaration(node: ts.ImportDeclaration, context: AstRenderer): ImportStatement { const packageName = stringFromLiteral(node.moduleSpecifier); const starBindings = matchAst(node, diff --git a/packages/jsii-sampiler/lib/typescript/ts-compiler.ts b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts similarity index 95% rename from packages/jsii-sampiler/lib/typescript/ts-compiler.ts rename to packages/jsii-rosetta/lib/typescript/ts-compiler.ts index 74397c1250..8154e1c94b 100644 --- a/packages/jsii-sampiler/lib/typescript/ts-compiler.ts +++ b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts @@ -71,8 +71,8 @@ export const STANDARD_COMPILER_OPTIONS: ts.CompilerOptions = { noImplicitAny: true, noImplicitReturns: true, noImplicitThis: true, - noUnusedLocals: true, - noUnusedParameters: true, + noUnusedLocals: false, // Important, becomes super annoying without this + noUnusedParameters: false, // Important, becomes super annoying without this resolveJsonModule: true, strict: true, strictNullChecks: true, diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts new file mode 100644 index 0000000000..79effeb15a --- /dev/null +++ b/packages/jsii-rosetta/lib/util.ts @@ -0,0 +1,38 @@ +import { VisualizeAstVisitor } from './languages/visualize'; +import ts = require('typescript'); +import { translateTypeScript } from './translate'; + +export function startsWithUppercase(x: string) { + return x.match(/^[A-Z]/); +} + +export interface File { + readonly contents: string; + readonly fileName: string; +} + +export function visualizeTypeScriptAst(source: File) { + const vis = translateTypeScript(source, new VisualizeAstVisitor(true), { + bestEffort: false + }); + return vis.translation + '\n'; +} + +export function printDiagnostics(diags: ts.Diagnostic[], stream: NodeJS.WritableStream) { + diags.forEach(d => printDiagnostic(d, stream)); +} + +export function printDiagnostic(diag: ts.Diagnostic, stream: NodeJS.WritableStream) { + const host = { + getCurrentDirectory() { return '.'; }, + getCanonicalFileName(fileName: string) { return fileName; }, + getNewLine() { return '\n'; } + }; + + const message = ts.formatDiagnosticsWithColorAndContext([diag], host); + stream.write(message); +} + +export function isErrorDiagnostic(diag: ts.Diagnostic) { + return diag.category === ts.DiagnosticCategory.Error; +} \ No newline at end of file diff --git a/packages/jsii-sampiler/package.json b/packages/jsii-rosetta/package.json similarity index 75% rename from packages/jsii-sampiler/package.json rename to packages/jsii-rosetta/package.json index a1d634f52a..b38f565e4c 100644 --- a/packages/jsii-sampiler/package.json +++ b/packages/jsii-rosetta/package.json @@ -1,10 +1,10 @@ { - "name": "jsii-sampiler", + "name": "jsii-rosetta", "version": "0.20.0", "description": "Translate TypeScript code snippets to other languages", "main": "lib/index.js", "bin": { - "jsii-sampiler": "bin/jsii-sampiler" + "jsii-rosetta": "bin/jsii-rosetta" }, "scripts": { "build": "tsc --build", @@ -19,13 +19,17 @@ "@types/node": "^10.17.2", "@types/yargs": "^13.0.3", "jest": "^24.9.0", - "jsii-build-tools": "^0.20.0", - "memory-streams": "^0.1.3" + "jsii-build-tools": "^0.20", + "jsii": "^0.20", + "memory-streams": "^0.1.3", + "mock-fs": "^4.10.2", + "@types/mock-fs": "^4.10.0" }, "dependencies": { "commonmark": "^0.29.0", "fs-extra": "^8.1.0", "typescript": "~3.6.4", + "jsii-spec": "^0.20.0", "yargs": "^14.2.0" }, "jest": { @@ -36,10 +40,11 @@ "lib/**/*.js" ], "collectCoverage": true, + "coverageReporters": ["json", "lcov", "text", "clover", "html"], "coverageThreshold": { "global": { "branches": 70, - "statements": 75 + "statements": 70 } } }, @@ -53,7 +58,7 @@ "repository": { "type": "git", "url": "https://github.com/aws/jsii.git", - "directory": "packages/jsii-sampiler" + "directory": "packages/jsii-rosetta" }, "engines": { "node": ">= 10.3.0" diff --git a/packages/jsii-rosetta/test/jsii/assemblies.test.ts b/packages/jsii-rosetta/test/jsii/assemblies.test.ts new file mode 100644 index 0000000000..68a1829abb --- /dev/null +++ b/packages/jsii-rosetta/test/jsii/assemblies.test.ts @@ -0,0 +1,120 @@ +import spec = require('jsii-spec'); +import { allTypeScriptSnippets } from '../../lib/jsii/assemblies'; +import path = require('path'); + +test('Extract snippet from README', () => { + const snippets = Array.from(allTypeScriptSnippets([{ + assembly: fakeAssembly({ + readme: { + markdown: [ + 'Before the example.', + '```ts', + 'someExample();', + '```', + 'After the example.' + ].join('\n') + } + }), + directory: path.join(__dirname, 'fixtures'), + }])); + + expect(snippets[0].visibleSource).toEqual('someExample();'); +}); + +test('Extract snippet from type docstring', () => { + const snippets = Array.from(allTypeScriptSnippets([{ + assembly: fakeAssembly({ + types: { + 'asm.MyType': { + kind: spec.TypeKind.Class, + assembly: 'asm', + fqn: 'asm.MyType', + name: 'MyType', + docs: { + summary: 'My Type', + remarks: [ + 'Before the example.', + '```ts', + 'someExample();', + '```', + 'After the example.' + ].join('\n'), + } + }, + } + }), + directory: path.join(__dirname, 'fixtures'), + }])); + + expect(snippets[0].visibleSource).toEqual('someExample();'); +}); + +test('Snippet can include fixture', () => { + const snippets = Array.from(allTypeScriptSnippets([{ + assembly: fakeAssembly({ + readme: { + markdown: [ + 'Before the example.', + '```ts fixture=explicit', + 'someExample();', + '```', + 'After the example.' + ].join('\n') + } + }), + directory: path.join(__dirname, 'fixtures'), + }])); + + expect(snippets[0].visibleSource).toEqual('someExample();'); + expect(snippets[0].completeSource).toEqual([ + '// This is a fixture', + '/// !show', + 'someExample();', + '/// !hide', + ].join('\n')); +}); + +test('Use fixture from example', () => { + const snippets = Array.from(allTypeScriptSnippets([{ + assembly: fakeAssembly({ + types: { + 'asm.MyType': { + kind: spec.TypeKind.Class, + assembly: 'asm', + fqn: 'asm.MyType', + name: 'MyType', + docs: { + example: [ + '/// fixture=explicit', + 'someExample();', + ].join('\n'), + } + }, + } + }), + directory: path.join(__dirname, 'fixtures'), + }])); + + expect(snippets[0].visibleSource).toEqual('someExample();'); + expect(snippets[0].completeSource).toEqual([ + '// This is a fixture', + '/// !show', + 'someExample();', + '/// !hide', + ].join('\n')); +}); + +export function fakeAssembly(parts: Partial): spec.Assembly { + return Object.assign({ + schema: spec.SchemaVersion.LATEST, + name: '', + description: '', + homepage: '', + repository: { directory: '', type: '', url: '' }, + author: { email: '', name: '', organization: false, roles: [], url: '' }, + fingerprint: '', + version: '', + jsiiVersion: '', + license: '', + }, parts); +} \ No newline at end of file diff --git a/packages/jsii-rosetta/test/jsii/astutils.test.ts b/packages/jsii-rosetta/test/jsii/astutils.test.ts new file mode 100644 index 0000000000..43cc2d0635 --- /dev/null +++ b/packages/jsii-rosetta/test/jsii/astutils.test.ts @@ -0,0 +1,19 @@ +import { calculateVisibleSpans } from "../../lib/typescript/ast-utils"; + +test('full text visible by default', () => { + expect(calculateVisibleSpans('asdf')).toEqual([ + { start: 0, end: 4, visible: true } + ]); +}); + +test('initial span visible if directive is hiding', () => { + expect(calculateVisibleSpans('asdf\n/// !hide\nxyz')).toEqual([ + { start: 0, end: 5, visible: true } + ]); +}); + +test('initial span invisible if directive is showing', () => { + expect(calculateVisibleSpans('asdf\n/// !show\nxyz')).toEqual([ + { start: 14, end: 18, visible: true } + ]); +}); diff --git a/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture b/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture new file mode 100644 index 0000000000..d87a43e2f4 --- /dev/null +++ b/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture @@ -0,0 +1,2 @@ +// This is a fixture +/// here \ No newline at end of file diff --git a/packages/jsii-sampiler/test/markdown/roundtrip.test.ts b/packages/jsii-rosetta/test/markdown/roundtrip.test.ts similarity index 100% rename from packages/jsii-sampiler/test/markdown/roundtrip.test.ts rename to packages/jsii-rosetta/test/markdown/roundtrip.test.ts diff --git a/packages/jsii-sampiler/test/otree.test.ts b/packages/jsii-rosetta/test/otree.test.ts similarity index 100% rename from packages/jsii-sampiler/test/otree.test.ts rename to packages/jsii-rosetta/test/otree.test.ts diff --git a/packages/jsii-sampiler/test/python/calls.test.ts b/packages/jsii-rosetta/test/python/calls.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/calls.test.ts rename to packages/jsii-rosetta/test/python/calls.test.ts diff --git a/packages/jsii-sampiler/test/python/classes.test.ts b/packages/jsii-rosetta/test/python/classes.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/classes.test.ts rename to packages/jsii-rosetta/test/python/classes.test.ts diff --git a/packages/jsii-sampiler/test/python/comments.test.ts b/packages/jsii-rosetta/test/python/comments.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/comments.test.ts rename to packages/jsii-rosetta/test/python/comments.test.ts diff --git a/packages/jsii-sampiler/test/python/expressions.test.ts b/packages/jsii-rosetta/test/python/expressions.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/expressions.test.ts rename to packages/jsii-rosetta/test/python/expressions.test.ts diff --git a/packages/jsii-sampiler/test/python/hiding.test.ts b/packages/jsii-rosetta/test/python/hiding.test.ts similarity index 76% rename from packages/jsii-sampiler/test/python/hiding.test.ts rename to packages/jsii-rosetta/test/python/hiding.test.ts index 76dd0e1a7b..cfd32ac0ca 100644 --- a/packages/jsii-sampiler/test/python/hiding.test.ts +++ b/packages/jsii-rosetta/test/python/hiding.test.ts @@ -57,4 +57,25 @@ test('hide statements with explicit ellipsis', () => { # ... after() `); +}); + +test('hide halfway into class using comments', () => { + expectPython(` + prepare(); + + /// !hide + class Something { + constructor() { + + /// !show + console.log(this, 'it seems to work'); + /// !hide + } + } + `, ` + prepare() + + print(self, "it seems to work") + ` + ); }); \ No newline at end of file diff --git a/packages/jsii-sampiler/test/python/imports.test.ts b/packages/jsii-rosetta/test/python/imports.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/imports.test.ts rename to packages/jsii-rosetta/test/python/imports.test.ts diff --git a/packages/jsii-sampiler/test/python/misc.test.ts b/packages/jsii-rosetta/test/python/misc.test.ts similarity index 100% rename from packages/jsii-sampiler/test/python/misc.test.ts rename to packages/jsii-rosetta/test/python/misc.test.ts diff --git a/packages/jsii-sampiler/test/python/python.ts b/packages/jsii-rosetta/test/python/python.ts similarity index 87% rename from packages/jsii-sampiler/test/python/python.ts rename to packages/jsii-rosetta/test/python/python.ts index 4a92989a10..6ec30d230e 100644 --- a/packages/jsii-sampiler/test/python/python.ts +++ b/packages/jsii-rosetta/test/python/python.ts @@ -1,11 +1,11 @@ -import { LiteralSource, renderTree, translateTypeScript } from "../../lib"; +import { translateTypeScript } from "../../lib"; import { PythonVisitor } from "../../lib/languages/python"; import { visualizeTypeScriptAst } from "../../lib/util"; const DEBUG = false; export function ts2python(source: string): string { - const src = new LiteralSource(source, 'test.ts'); + const src = { contents: source, fileName: 'test.ts' }; if (DEBUG) { // tslint:disable-next-line:no-console @@ -15,8 +15,7 @@ export function ts2python(source: string): string { // Very debug. Much print. // console.log(JSON.stringify(result.tree, undefined, 2)); - - return renderTree(result.tree) + '\n'; + return result.translation + '\n'; } export function expectPython(source: string, expected: string) { diff --git a/packages/jsii-sampiler/test/python/statements.test.ts b/packages/jsii-rosetta/test/python/statements.test.ts similarity index 88% rename from packages/jsii-sampiler/test/python/statements.test.ts rename to packages/jsii-rosetta/test/python/statements.test.ts index 99d1b610bd..015ad3e12f 100644 --- a/packages/jsii-sampiler/test/python/statements.test.ts +++ b/packages/jsii-rosetta/test/python/statements.test.ts @@ -1,5 +1,5 @@ import { expectPython } from "./python"; -import { LiteralSource, PythonVisitor, translateTypeScript, renderTree } from "../../lib"; +import { PythonVisitor, translateTypeScript } from "../../lib"; test('if', () => { expectPython(` @@ -101,13 +101,13 @@ test('whitespace between statements in a block', () => { }); test('prepend disclaimer', () => { - const src = new LiteralSource('console.log("hello");', 'test.ts'); + const src = { contents: 'console.log("hello");', fileName: 'test.ts' }; const result = translateTypeScript(src, new PythonVisitor({ disclaimer: 'Do not write this code' })); - expect(renderTree(result.tree)).toEqual( + expect(result.translation).toEqual( `# Do not write this code print("hello")`); }); \ No newline at end of file diff --git a/packages/jsii-rosetta/test/rosetta.test.ts b/packages/jsii-rosetta/test/rosetta.test.ts new file mode 100644 index 0000000000..7aacb2765b --- /dev/null +++ b/packages/jsii-rosetta/test/rosetta.test.ts @@ -0,0 +1,151 @@ +import { Rosetta, LanguageTablet, TranslatedSnippet, TypeScriptSnippet, DEFAULT_TABLET_NAME } from '../lib'; +import mockfs = require('mock-fs'); +import { TargetLanguage } from '../lib/languages'; +import { fakeAssembly } from './jsii/assemblies.test'; + +const SAMPLE_CODE: TypeScriptSnippet = { + visibleSource: 'callThisFunction();', + where: 'sample', +}; + +test('Rosetta object can do live translation', () => { + // GIVEN + const rosetta = new Rosetta({ + liveConversion: true, + targetLanguages: ["python"] + }); + + // WHEN + const translated = rosetta.translateSnippet(SAMPLE_CODE, "python"); + + // THEN + expect(translated).toMatchObject({ + source: "call_this_function()", + language: "python" + }); +}); + +test('Can use preloaded tablet', () => { + // GIVEN + const rosetta = new Rosetta(); + + const tablet = new LanguageTablet(); + tablet.addSnippet(makeSnippet(SAMPLE_CODE, { + python: 'Not Really Translated' + })); + rosetta.addTablet(tablet); + + // WHEN + const translated = rosetta.translateSnippet(SAMPLE_CODE, "python"); + + // THEN + expect(translated).toMatchObject({ + source: "Not Really Translated", + language: "python" + }); +}); + +test('Rosetta object can do live translation', () => { + // GIVEN + const rosetta = new Rosetta({ + liveConversion: true, + targetLanguages: ["python"] + }); + + // WHEN + const translated = rosetta.translateSnippet(SAMPLE_CODE, "python"); + + // THEN + expect(translated).toMatchObject({ + source: "call_this_function()", + language: "python" + }); +}); + + +test('Rosetta object can do translation and annotation of snippets in MarkDown', () => { + // GIVEN + const rosetta = new Rosetta({ + liveConversion: true, + targetLanguages: ["python"] + }); + + // WHEN + const translated = rosetta.translateSnippetsInMarkdown([ + '# MarkDown Translation', + '', + 'Now follows a snippet:', + '```ts', + SAMPLE_CODE.visibleSource, + '```', + 'That was it, thank you for your attention.' + ].join('\n'), 'python', trans => { + return { ...trans, source: '# We translated something!\n' + trans.source }; + }); + + // THEN + expect(translated).toEqual([ + '# MarkDown Translation', + '', + 'Now follows a snippet:', + '', + '```python', + '# We translated something!', + 'call_this_function()', + '```', + '', + 'That was it, thank you for your attention.' + ].join('\n')); +}); + +describe('with mocked filesystem', () => { + beforeEach(() => { mockfs(); }); + afterEach(() => { mockfs.restore(); }); + + const tablet = new LanguageTablet(); + tablet.addSnippet(makeSnippet(SAMPLE_CODE, { + python: 'My Stored Translation' + })); + + test('Can save language tablet and load it in Rosetta class', async () => { + // GIVEN + await tablet.save('/test.tablet'); + + // WHEN + const rosetta = new Rosetta(); + await rosetta.loadTabletFromFile('/test.tablet'); + const translated = rosetta.translateSnippet(SAMPLE_CODE, "python"); + + // THEN + expect(translated).toMatchObject({ + source: "My Stored Translation", + language: "python" + }); + }); + + test('Rosetta class automatically loads default-named tablets in same directory as assembly', async () => { + // GIVEN + await tablet.save('/' + DEFAULT_TABLET_NAME); + + // WHEN + const rosetta = new Rosetta(); + await rosetta.addAssembly(fakeAssembly({}), '/'); + const translated = rosetta.translateSnippet(SAMPLE_CODE, "python"); + + // THEN + expect(translated).toMatchObject({ + source: "My Stored Translation", + language: "python" + }); + }); + +}); + + +function makeSnippet(original: TypeScriptSnippet, translations: Record) { + const snippet = TranslatedSnippet.fromSnippet(original); + for (const [key, value] of Object.entries(translations)) { + snippet.addTranslatedSource(key as TargetLanguage, value); + } + return snippet; +} \ No newline at end of file diff --git a/packages/jsii-sampiler/tsconfig.json b/packages/jsii-rosetta/tsconfig.json similarity index 96% rename from packages/jsii-sampiler/tsconfig.json rename to packages/jsii-rosetta/tsconfig.json index beb48c6fe6..77418e6ad0 100644 --- a/packages/jsii-sampiler/tsconfig.json +++ b/packages/jsii-rosetta/tsconfig.json @@ -21,5 +21,8 @@ ], "exclude": [ "examples" + ], + "references": [ + { "path": "../jsii-spec" } ] } diff --git a/packages/jsii-sampiler/CHANGELOG.md b/packages/jsii-sampiler/CHANGELOG.md deleted file mode 100644 index 5ed8eb695e..0000000000 --- a/packages/jsii-sampiler/CHANGELOG.md +++ /dev/null @@ -1,24 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. -See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. - -## [0.20.0](https://github.com/aws/jsii/compare/v0.19.0...v0.20.0) (2019-10-30) - -**Note:** Version bump only for package jsii-sampiler - - - - - -# [0.19.0](https://github.com/aws/jsii/compare/v0.18.0...v0.19.0) (2019-10-14) - - -### Bug Fixes - -* **sampiler:** Add missing .npmignore ([#875](https://github.com/aws/jsii/issues/875)) ([b16fc6b](https://github.com/aws/jsii/commit/b16fc6bdaf1825d53629c2a44b769f924ffb91d0)) - - -### Features - -* **sampiler:** translate code samples to Python ([#827](https://github.com/aws/jsii/issues/827)) ([c9a7002](https://github.com/aws/jsii/commit/c9a7002431c0db6224d595eb5555b916036d4575)) diff --git a/packages/jsii-sampiler/README.md b/packages/jsii-sampiler/README.md deleted file mode 100644 index c0ddf4580a..0000000000 --- a/packages/jsii-sampiler/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# jsii-sampiler: a transpiler for code samples - -Utility to transcribe example code snippets from TypeScript to other -jsii languages. - -Has knowledge about jsii language translation conventions to do the -translations. Only supports a limited set of TypeScript language features. - -## Compilability - -The sampiler can translate both code that completely compiles and typechecks, -as well as code that doesn't. - -In case of non-compiling samples the translations will be based off of -grammatical parsing only. This has the downside that we do not have the type -information available to the exact right thing in all instances. - -If the samples don't compile or don't have full type information: - -- No way to declare typed variables for Java and C#. -- Can only "see" the fields of structs as far as they are declared in the same - snippet. Inherited fields or structs declared not in the same snippet are - invisible. -- When we explode a struct parameter into keyword parameters and we pass it on - to another callable, we can't know which keyword arguments the called function - actually takes so we just pass all of them (might be too many). -- When structs contain nested structs, Python and other languages need to know - the types of these fields to generate the right calls. -- Object literals are used both to represent structs as well as to represent - dictionaries, and without type information it's impossible to determine - which is which. - -## Void masking - -In order to make examples compile, boilerplate code may need to be added -that detracts from the example at hand (such as variable declarations -and imports). - -This package supports hiding parts of the original source after -translation. - -To mark special locations in the source tree, we use the `void` -expression keyword and or the `comma` operator feature to attach this -expression to another expression. Both are little-used JavaScript -features that are reliably parsed by TypeScript and do not affect the -semantics of the application in which they appear (so the program -executes the same with or without them). - -A handy mnemonic for this feature is that you can use it to "send your -code into the void". - -### Hiding statements - -Statement hiding looks like this: - -```ts -before(); // will be shown - -void 0; // start hiding (the argument to 'void' doesn't matter) -middle(); // will not be shown -void 'show'; // stop hiding - -after(); // will be shown again -``` - -Void masking only works to the end of the enclosing scope, so in some -cases you can omit the `void 'show'` directive to turn hiding back off. - -To explicit show that code was hidden, pass `'block'` to the void -statement: - - -```ts -before(); -void 'block'; // start hiding, will render a '# ...' -middle(); -``` - -### Hiding expressions - -For hiding expressions, we use `comma` expressions to attach a `void` -statement to an expression value without changing the meaning of the -code. - -Example: - -```ts -foo(1, 2, (void 1, 3)); -``` - -Will render as - -``` -foo(1, 2) -``` - -Also supports a visible ellipsis: - -```ts -const x = (void '...', 3); -``` - -Renders to: - -``` -x = ... -``` - -## Build integration - -This tool has the ability to hide irrelevant parts of the generated code -snippet (see the section called "void masking" below). Because the samples -should be compilable to extract all necessary type information, and because -they could depend on any package, the following steps need to happen: - -* All packages need to be built (by `jsii`). Ideally, the reduced example ends - up in the assembly. -* After all packages have been built, sample snippets should be checked - for compilability and supported language constructs (not all language - features can be translated to other languages). This requires the full - snippets (before reducing). -* After the full samples have been type-checked, their reduced version - can be translated and inserted into the various generated packages by - `jsii-pacmak`. - -To avoid an additional dependency of `jsii` on the `jsii-samples` mechanism, -what we'll do instead is mutating the assembly in-place. So simplified, -the workflow looks like this: - -* All packages get compiled by `jsii`. -* We postprocess all assemblies using `jsii-samples`, extracting code to - a side-archive (`.jsii.samples`) and replacing the original version in the - assembly, and generating all other language versions. This becomes a - translation table, with the key being a hash of the reduced snippet. -* `jsii-pacmak` replaces snippets from the translation table. - -In this process, `jsii-samples` is as much self-contained as possible. It -works on an assembly to produce a lookup file, which `jsii-pacmak` reads. -`jsii-pacmak` has a simple fallback, which is use the unsubtituted example in -case the right example is not available. - -Alternatively, since `jsii` doesn't really provide any facilities to mutate -an assembly in-place, we leave the unreduced examples in the source assembly, -and force all downstream renderers (such as the doc renderer and API tooling) -to use `jsii-samples` to reduce the snippets before presenting them. This is -not ideal but probably the way we're going to go because `jsii` doesn't provide -any tooling to mutate an assembly in-place. diff --git a/packages/jsii-sampiler/bin/jsii-sampiler b/packages/jsii-sampiler/bin/jsii-sampiler deleted file mode 100755 index d778d67448..0000000000 --- a/packages/jsii-sampiler/bin/jsii-sampiler +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env node -require('./jsii-sampiler.js'); diff --git a/packages/jsii-sampiler/bin/jsii-sampiler.ts b/packages/jsii-sampiler/bin/jsii-sampiler.ts deleted file mode 100644 index 7e57c7221d..0000000000 --- a/packages/jsii-sampiler/bin/jsii-sampiler.ts +++ /dev/null @@ -1,86 +0,0 @@ -import yargs = require('yargs'); -import { FileSource, isErrorDiagnostic, LiteralSource, printDiagnostics, - renderTree, translateMarkdown, TranslateResult, translateTypeScript } from '../lib'; -import { PythonVisitor } from '../lib/languages/python'; -import { VisualizeAstVisitor } from '../lib/languages/visualize'; - -async function main() { - const argv = yargs - .usage('$0 [args]') - .command('snippet [file]', 'Translate a single snippet', command => command - .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' }) - .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' }) - , async args => { - const result = translateTypeScript( - await makeFileSource(args.file || '-', 'stdin.ts'), - makeVisitor(args)); - renderResult(result); - }) - .command('markdown ', 'Translate a MarkDown file', command => command - .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' }) - .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' }) - , async args => { - const result = translateMarkdown( - await makeFileSource(args.file || '-', 'stdin.md'), - makeVisitor(args)); - renderResult(result); - return 5; - }) - .demandCommand() - .help() - .strict() // Error on wrong command - .version(require('../package.json').version) - .showHelpOnFail(false) - .argv; - - // Evaluating .argv triggers the parsing but the command gets implicitly executed, - // so we don't need the output. - Array.isArray(argv); -} - -function makeVisitor(args: { python?: boolean }) { - if (args.python) { return new PythonVisitor(); } - // Default to visualizing AST, including nodes we don't recognize yet - return new VisualizeAstVisitor(); -} - -async function makeFileSource(fileName: string, stdinName: string) { - if (fileName === '-') { - return new LiteralSource(await readStdin(), stdinName); - } - return new FileSource(fileName); -} - -async function readStdin(): Promise { - process.stdin.setEncoding('utf8'); - - const parts: string[] = []; - - return new Promise((resolve, reject) => { - process.stdin.on('readable', () => { - const chunk = process.stdin.read(); - if (chunk !== null) { parts.push(`${chunk}`); } - }); - - process.stdin.on('error', reject); - process.stdin.on('end', () => resolve(parts.join(''))); - }); -} - -function renderResult(result: TranslateResult) { - process.stdout.write(renderTree(result.tree) + '\n'); - - if (result.diagnostics.length > 0) { - printDiagnostics(result.diagnostics, process.stderr); - - if (result.diagnostics.some(isErrorDiagnostic)) { - process.exit(1); - } - } -} - -main().catch(e => { - // tslint:disable-next-line:no-console - console.error(e); - process.exit(1); -}); \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/index.ts b/packages/jsii-sampiler/lib/index.ts deleted file mode 100644 index 16b185ccbe..0000000000 --- a/packages/jsii-sampiler/lib/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './translate'; -export { renderTree } from './o-tree'; -export { PythonVisitor } from './languages/python'; \ No newline at end of file diff --git a/packages/jsii-sampiler/lib/translate.ts b/packages/jsii-sampiler/lib/translate.ts deleted file mode 100644 index c9bc42a9ce..0000000000 --- a/packages/jsii-sampiler/lib/translate.ts +++ /dev/null @@ -1,118 +0,0 @@ -import fs = require('fs-extra'); -import ts = require('typescript'); -import { AstConverter, AstHandler, ConvertOptions } from './converter'; -import { transformMarkdown } from './markdown/markdown'; -import { MarkdownRenderer } from './markdown/markdown-renderer'; -import { ReplaceCodeTransform } from './markdown/replace-code-renderer'; -import { OTree, renderTree } from './o-tree'; -import { TypeScriptCompiler } from './typescript/ts-compiler'; -import { inTempDir } from './util'; - -export interface Source { - withFile(fn: (fileName: string) => A): A; - withContents(fn: (fileName: string, contents: string) => A): A; -} - -export class FileSource implements Source { - constructor(private readonly fileName: string) { } - - public withFile(fn: (fileName: string) => A): A { - return fn(this.fileName); - } - - public withContents(fn: (fileName: string, contents: string) => A): A { - const contents = fs.readFileSync(this.fileName, 'utf-8'); - return fn(this.fileName, contents); - } -} - -export class LiteralSource implements Source { - constructor(private readonly source: string, private readonly filenameHint = 'index.ts') { } - - public withFile(fn: (fileName: string) => A): A { - return inTempDir(() => { - fs.writeFileSync(this.filenameHint, this.source); - return fn(this.filenameHint); - }); - } - - public withContents(fn: (fileName: string, contents: string) => A): A { - return fn(this.filenameHint, this.source); - } -} - -export interface TranslateMarkdownOptions extends ConvertOptions { - /** - * What language to put in the returned markdown blocks - */ - languageIdentifier?: string; -} - -export function translateMarkdown(markdown: Source, visitor: AstHandler, options: TranslateMarkdownOptions = {}): TranslateResult { - const compiler = new TypeScriptCompiler(); - - let index = 0; - const diagnostics = new Array(); - - const translatedMarkdown = markdown.withContents((filename, contents) => { - return transformMarkdown(contents, new MarkdownRenderer(), new ReplaceCodeTransform(code => { - if (code.language !== 'typescript' && code.language !== 'ts') { return code; } - - index += 1; - const snippetSource = new LiteralSource(code.source, `${filename}-snippet${index}.ts`); - const snippetTranslation = translateSnippet(snippetSource, compiler, visitor, options); - - diagnostics.push(...snippetTranslation.diagnostics); - - return { language: options.languageIdentifier || '', source: renderTree(snippetTranslation.tree) + '\n' }; - })); - }); - - return { tree: new OTree([translatedMarkdown]), diagnostics }; -} - -export type TranslateOptions = ConvertOptions; - -export function translateTypeScript(source: Source, visitor: AstHandler, options: TranslateOptions = {}): TranslateResult { - const compiler = new TypeScriptCompiler(); - - return translateSnippet(source, compiler, visitor, options); -} - -function translateSnippet(source: Source, compiler: TypeScriptCompiler, visitor: AstHandler, options: TranslateOptions = {}): TranslateResult { - return source.withContents((filename, contents) => { - const result = compiler.compileInMemory(filename, contents); - - const converter = new AstConverter(result.rootFile, result.program.getTypeChecker(), visitor, options); - const converted = converter.convert(result.rootFile); - - return { - tree: converted, - diagnostics: converter.diagnostics - }; - }); -} - -export function printDiagnostics(diags: ts.Diagnostic[], stream: NodeJS.WritableStream) { - diags.forEach(d => printDiagnostic(d, stream)); -} - -export function printDiagnostic(diag: ts.Diagnostic, stream: NodeJS.WritableStream) { - const host = { - getCurrentDirectory() { return '.'; }, - getCanonicalFileName(fileName: string) { return fileName; }, - getNewLine() { return '\n'; } - }; - - const message = ts.formatDiagnosticsWithColorAndContext([diag], host); - stream.write(message); -} - -export function isErrorDiagnostic(diag: ts.Diagnostic) { - return diag.category === ts.DiagnosticCategory.Error; -} - -export interface TranslateResult { - tree: OTree; - diagnostics: ts.Diagnostic[]; -} diff --git a/packages/jsii-sampiler/lib/util.ts b/packages/jsii-sampiler/lib/util.ts deleted file mode 100644 index f4c79948a2..0000000000 --- a/packages/jsii-sampiler/lib/util.ts +++ /dev/null @@ -1,26 +0,0 @@ -import fs = require('fs-extra'); -import os = require('os'); -import path = require('path'); -import { renderTree, Source, translateTypeScript } from '.'; -import { VisualizeAstVisitor } from './languages/visualize'; - -export function startsWithUppercase(x: string) { - return x.match(/^[A-Z]/); -} - -export function inTempDir(block: () => T): T { - const origDir = process.cwd(); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'jsii')); - process.chdir(tmpDir); - const ret = block(); - process.chdir(origDir); - fs.removeSync(tmpDir); - return ret; -} - -export function visualizeTypeScriptAst(source: Source) { - const vis = translateTypeScript(source, new VisualizeAstVisitor(true), { - bestEffort: false - }); - return renderTree(vis.tree) + '\n'; -} diff --git a/yarn.lock b/yarn.lock index 814c1073a9..578d8b38b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1225,6 +1225,13 @@ dependencies: "@types/node" "*" +"@types/mock-fs@^4.10.0": + version "4.10.0" + resolved "https://registry.yarnpkg.com/@types/mock-fs/-/mock-fs-4.10.0.tgz#460061b186993d76856f669d5317cda8a007c24b" + integrity sha512-FQ5alSzmHMmliqcL36JqIA4Yyn9jyJKvRSGV3mvPh108VFatX7naJDzSG4fnFQNZFq9dIx0Dzoe6ddflMB2Xkg== + dependencies: + "@types/node" "*" + "@types/node@*": version "12.11.1" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.11.1.tgz#1fd7b821f798b7fa29f667a1be8f3442bb8922a3" @@ -5413,6 +5420,11 @@ mkdirp@*, mkdirp@^0.5.0, mkdirp@^0.5.1: dependencies: minimist "0.0.8" +mock-fs@^4.10.2: + version "4.10.2" + resolved "https://registry.yarnpkg.com/mock-fs/-/mock-fs-4.10.2.tgz#ee11e5a3288b0235fb345efe8b25610f7dd397b8" + integrity sha512-ewPQ83O4U8/Gd8I15WoB6vgTTmq5khxBskUWCRvswUqjCfOOTREmxllztQOm+PXMWUxATry+VBWXQJloAyxtbQ== + modify-values@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" From 7a0bd5c4aed24e0317b69c39ab51d2adb4cbbde4 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Fri, 1 Nov 2019 11:44:25 +0100 Subject: [PATCH 02/19] Update some expectations --- packages/jsii-calc/test/assembly.jsii | 4 ++-- .../Amazon.JSII.Tests.CalculatorPackageId/.jsii | 4 ++-- .../jsii/tests/calculator/package-info.java | 12 +++++++----- .../test/expected.jsii-calc/python/README.md | 17 +++++++++++------ .../python/src/jsii_calc/__init__.py | 1 - .../python/src/jsii_calc/_jsii/__init__.py | 17 +++++++++++------ 6 files changed, 33 insertions(+), 22 deletions(-) diff --git a/packages/jsii-calc/test/assembly.jsii b/packages/jsii-calc/test/assembly.jsii index 7409ef0b28..a7e1d69e83 100644 --- a/packages/jsii-calc/test/assembly.jsii +++ b/packages/jsii-calc/test/assembly.jsii @@ -195,7 +195,7 @@ }, "name": "jsii-calc", "readme": { - "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## Sphinx\n\nThis file will be incorporated into the sphinx documentation.\n\nIf this file starts with an \"H1\" line (in our case `# jsii Calculator`), this\nheading will be used as the Sphinx topic name. Otherwise, the name of the module\n(`jsii-calc`) will be used instead.\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" + "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## How to use running sum API:\n\nFirst, create a calculator:\n\n```ts\nconst calculator = new calc.Calculator();\n```\n\nThen call some operations:\n\n\n```ts fixture=with-calculator\ncalculator.add(10);\n```\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" }, "repository": { "directory": "packages/jsii-calc", @@ -10328,5 +10328,5 @@ } }, "version": "0.20.0", - "fingerprint": "Gk2mbYSx8tmjrVf/JBPBkNEFEEG9i7jx9ptKC/0sCpE=" + "fingerprint": "WJ8fpBsH7sWh1h86kE1fgaNJf6nLjw6uq0p/aTcpoJY=" } diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii index 7409ef0b28..a7e1d69e83 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii +++ b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii @@ -195,7 +195,7 @@ }, "name": "jsii-calc", "readme": { - "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## Sphinx\n\nThis file will be incorporated into the sphinx documentation.\n\nIf this file starts with an \"H1\" line (in our case `# jsii Calculator`), this\nheading will be used as the Sphinx topic name. Otherwise, the name of the module\n(`jsii-calc`) will be used instead.\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" + "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## How to use running sum API:\n\nFirst, create a calculator:\n\n```ts\nconst calculator = new calc.Calculator();\n```\n\nThen call some operations:\n\n\n```ts fixture=with-calculator\ncalculator.add(10);\n```\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" }, "repository": { "directory": "packages/jsii-calc", @@ -10328,5 +10328,5 @@ } }, "version": "0.20.0", - "fingerprint": "Gk2mbYSx8tmjrVf/JBPBkNEFEEG9i7jx9ptKC/0sCpE=" + "fingerprint": "WJ8fpBsH7sWh1h86kE1fgaNJf6nLjw6uq0p/aTcpoJY=" } diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java b/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java index 0fd07a89d1..be50e425e6 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java +++ b/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java @@ -1,11 +1,13 @@ /** *

jsii Calculator

*

This library is used to demonstrate and test the features of JSII

- *

Sphinx

- *

This file will be incorporated into the sphinx documentation.

- *

If this file starts with an "H1" line (in our case # jsii Calculator), this - * heading will be used as the Sphinx topic name. Otherwise, the name of the module - * (jsii-calc) will be used instead.

+ *

How to use running sum API:

+ *

First, create a calculator:

+ * 
const calculator = new calc.Calculator();
+ *
+ *

Then call some operations:

+ * 
calculator.add(10);
+ *
*

Code Samples

* 
/* This is totes a magic comment in here, just you wait! *{@literal /}
  * const foo = 'bar';
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
index f9f4c183fe..1622efcd58 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
@@ -2,18 +2,23 @@
 
 This library is used to demonstrate and test the features of JSII
 
-## Sphinx
+## How to use running sum API:
 
-This file will be incorporated into the sphinx documentation.
+First, create a calculator:
 
-If this file starts with an "H1" line (in our case `# jsii Calculator`), this
-heading will be used as the Sphinx topic name. Otherwise, the name of the module
-(`jsii-calc`) will be used instead.
+```python
+calculator = calc.Calculator()
+```
+
+Then call some operations:
+
+```python
+calculator.add(10)
+```
 
 ## Code Samples
 
 ```python
-# Example automatically generated. See https://github.com/aws/jsii/issues/826
 # This is totes a magic comment in here, just you wait!
 foo = "bar"
 ```
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
index 8cbfa48ed9..78078c1bc4 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
@@ -763,7 +763,6 @@ class ClassWithDocs(metaclass=jsii.JSIIMeta, jsii_type="jsii-calc.ClassWithDocs"
     :customAttribute:: hasAValue
 
     Example::
-        # Example automatically generated. See https://github.com/aws/jsii/issues/826
         def an_example():
             pass
     """
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py
index 0a534bf63b..1e2732c52e 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/_jsii/__init__.py
@@ -3,18 +3,23 @@
 
 This library is used to demonstrate and test the features of JSII
 
-## Sphinx
+## How to use running sum API:
 
-This file will be incorporated into the sphinx documentation.
+First, create a calculator:
 
-If this file starts with an "H1" line (in our case `# jsii Calculator`), this
-heading will be used as the Sphinx topic name. Otherwise, the name of the module
-(`jsii-calc`) will be used instead.
+```python
+calculator = calc.Calculator()
+```
+
+Then call some operations:
+
+```python
+calculator.add(10)
+```
 
 ## Code Samples
 
 ```python
-# Example automatically generated. See https://github.com/aws/jsii/issues/826
 # This is totes a magic comment in here, just you wait!
 foo = "bar"
 ```

From b90923fb5da1d3e2b366cedca754ce9cad290145 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Fri, 1 Nov 2019 12:48:09 +0100
Subject: [PATCH 03/19] WIP

---
 packages/jsii-rosetta/lib/commands/extract.ts | 8 ++++----
 packages/jsii-rosetta/lib/translate.ts        | 3 +++
 packages/jsii-rosetta/package.json            | 2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts
index 503d3b6ffc..0abdb78753 100644
--- a/packages/jsii-rosetta/lib/commands/extract.ts
+++ b/packages/jsii-rosetta/lib/commands/extract.ts
@@ -3,7 +3,6 @@ import logging = require('../logging');
 import ts = require('typescript');
 import { LanguageTablet } from '../tablets/tablets';
 import { Translator } from '../translate';
-import { snippetKey } from '../tablets/key';
 
 export interface ExtractResult {
   diagnostics: ts.Diagnostic[];
@@ -16,15 +15,16 @@ export async function extractSnippets(assemblyLocations: string[], outputFile: s
   logging.info(`Loading ${assemblyLocations.length} assemblies`);
   const assemblies = await loadAssemblies(assemblyLocations);
 
-  const translator = new Translator(includeCompilerDiagnostics);
+  const snippets = allTypeScriptSnippets(assemblies);
 
   const tablet = new LanguageTablet();
 
   logging.info(`Translating`);
   const startTime = Date.now();
 
-  for (const block of allTypeScriptSnippets(assemblies)) {
-    logging.debug(`Translating ${snippetKey(block)}`);
+  const translator = new Translator(includeCompilerDiagnostics);
+
+  for (const block of snippets) {
     tablet.addSnippet(translator.translate(block));
   }
 
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
index 576f60cd67..87ff30a835 100644
--- a/packages/jsii-rosetta/lib/translate.ts
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -1,3 +1,4 @@
+import logging = require('./logging');
 import ts = require('typescript');
 import { AstRenderer, AstHandler, AstRendererOptions } from './renderer';
 import { renderTree, Span } from './o-tree';
@@ -7,6 +8,7 @@ import { TARGET_LANGUAGES, TargetLanguage } from './languages';
 import { calculateVisibleSpans } from './typescript/ast-utils';
 import { File } from './util';
 import { TypeScriptSnippet, completeSource } from './snippet';
+import { snippetKey } from './tablets/key';
 
 export function translateTypeScript(source: File, visitor: AstHandler, options: SnippetTranslatorOptions = {}): TranslateResult {
   const translator = new SnippetTranslator({ visibleSource: source.contents, where: source.fileName }, options);
@@ -33,6 +35,7 @@ export class Translator {
   }
 
   public translate(snip: TypeScriptSnippet, languages = Object.keys(TARGET_LANGUAGES) as TargetLanguage[]) {
+    logging.debug(`Translating ${snippetKey(snip)}`);
     const translator = this.translatorFor(snip);
     const snippet = TranslatedSnippet.fromSnippet(snip, this.includeCompilerDiagnostics ? translator.compileDiagnostics.length === 0 : undefined);
 
diff --git a/packages/jsii-rosetta/package.json b/packages/jsii-rosetta/package.json
index b38f565e4c..504cea26ca 100644
--- a/packages/jsii-rosetta/package.json
+++ b/packages/jsii-rosetta/package.json
@@ -61,6 +61,6 @@
     "directory": "packages/jsii-rosetta"
   },
   "engines": {
-    "node": ">= 10.3.0"
+    "node": ">= 10.5.0"
   }
 }

From 2c6f2a30e49c33b421ca96df11e0084779b2757b Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Fri, 1 Nov 2019 16:42:26 +0100
Subject: [PATCH 04/19] Make `extract` use worker threads if available

---
 packages/jsii-rosetta/bin/jsii-rosetta.ts     |   4 +
 packages/jsii-rosetta/lib/commands/extract.ts | 100 +++++++++++++++++-
 .../lib/commands/extract_worker.ts            |  37 +++++++
 packages/jsii-rosetta/lib/logging.ts          |   6 +-
 packages/jsii-rosetta/lib/tablets/schema.ts   |   4 +-
 packages/jsii-rosetta/lib/tablets/tablets.ts  |   8 +-
 packages/jsii-rosetta/lib/translate.ts        |  35 +++---
 packages/jsii-rosetta/lib/util.ts             |  14 +++
 8 files changed, 173 insertions(+), 35 deletions(-)
 create mode 100644 packages/jsii-rosetta/lib/commands/extract_worker.ts

diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts
index d198c332b4..8c3be8f07d 100644
--- a/packages/jsii-rosetta/bin/jsii-rosetta.ts
+++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts
@@ -60,6 +60,10 @@ async function main() {
 
       printDiagnostics(result.diagnostics, process.stderr);
 
+      if (result.diagnostics.length > 0) {
+        logging.warn(`${result.diagnostics.length} diagnostics encountered`);
+      }
+
       if (result.diagnostics.some(isErrorDiagnostic) && args.fail) {
         process.exit(1);
       }
diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts
index 0abdb78753..31ba4791cb 100644
--- a/packages/jsii-rosetta/lib/commands/extract.ts
+++ b/packages/jsii-rosetta/lib/commands/extract.ts
@@ -1,8 +1,12 @@
 import { loadAssemblies, allTypeScriptSnippets } from '../jsii/assemblies';
 import logging = require('../logging');
+import os = require('os');
+import path = require('path');
 import ts = require('typescript');
-import { LanguageTablet } from '../tablets/tablets';
+import { LanguageTablet, TranslatedSnippet } from '../tablets/tablets';
 import { Translator } from '../translate';
+import { TypeScriptSnippet } from '../snippet';
+import { divideEvenly } from '../util';
 
 export interface ExtractResult {
   diagnostics: ts.Diagnostic[];
@@ -22,10 +26,10 @@ export async function extractSnippets(assemblyLocations: string[], outputFile: s
   logging.info(`Translating`);
   const startTime = Date.now();
 
-  const translator = new Translator(includeCompilerDiagnostics);
+  const result = await translateAll(snippets, includeCompilerDiagnostics);
 
-  for (const block of snippets) {
-    tablet.addSnippet(translator.translate(block));
+  for (const snippet of result.translatedSnippets) {
+    tablet.addSnippet(snippet);
   }
 
   const delta =  (Date.now() - startTime) / 1000;
@@ -33,5 +37,91 @@ export async function extractSnippets(assemblyLocations: string[], outputFile: s
   logging.info(`Saving language tablet to ${outputFile}`);
   await tablet.save(outputFile);
 
-  return { diagnostics: translator.diagnostics };
+  return { diagnostics: result.diagnostics };
+}
+
+interface TranslateAllResult {
+  translatedSnippets: TranslatedSnippet[];
+  diagnostics: ts.Diagnostic[];
+}
+
+/**
+ * Translate all snippets
+ *
+ * Uses a worker-based parallel translation if available, falling back to a single-threaded workflow if not.
+ */
+async function translateAll(snippets: IterableIterator, includeCompilerDiagnostics: boolean): Promise {
+  try {
+    const worker = await import('worker_threads');
+
+    return workerBasedTranslateAll(worker, snippets, includeCompilerDiagnostics);
+  } catch(e) {
+    if (e.code !== 'MODULE_NOT_FOUND') { throw e; }
+    logging.warn('Worker threads not available (use NodeJS >= 10.5 and --experimental-worker). Working sequentially.');
+
+    return singleThreadedTranslateAll(snippets, includeCompilerDiagnostics);
+  }
+}
+
+/**
+ * Translate the given snippets using a single compiler
+ *
+ * Used both here (directly) and via extract_worker to translate a batch of
+ * snippets in parallel.
+ */
+export function singleThreadedTranslateAll(snippets: IterableIterator, includeCompilerDiagnostics: boolean): TranslateAllResult {
+  const translatedSnippets = new Array();
+
+  const translator = new Translator(includeCompilerDiagnostics);
+  for (const block of snippets) {
+    translatedSnippets.push(translator.translate(block));
+  }
+
+  return { translatedSnippets, diagnostics: translator.diagnostics };
 }
+
+/**
+ * Divide the work evenly over all processors by running 'extract_worker' in Worker Threads, then combine results
+ *
+ * Never include 'extract_worker' directly, only do TypeScript type references (so that in
+ * the script we may assume that 'worker_threads' successfully imports).
+ */
+async function workerBasedTranslateAll(worker: typeof import('worker_threads'), snippets: IterableIterator, includeCompilerDiagnostics: boolean): Promise {
+  // Use about half the advertised cores because hyperthreading doesn't seem to help that
+  // much (on my machine, using more than half the cores actually makes it slower).
+  const N = Math.max(1, Math.ceil(os.cpus().length / 2));
+  const snippetArr = Array.from(snippets);
+  const groups = divideEvenly(N, snippetArr);
+  logging.info(`Translating ${snippetArr.length} snippets using ${groups.length} workers`);
+
+  // Run workers
+  const responses = await Promise.all(groups
+    .map(snippets => ({ snippets, includeCompilerDiagnostics }))
+    .map(runWorker));
+
+  // Combine results
+  const x = responses.reduce((acc, current) => {
+    // Modifying 'acc' in place to not incur useless copying
+    acc.translatedSnippetSchemas.push(...current.translatedSnippetSchemas);
+    acc.diagnostics.push(...current.diagnostics);
+    return acc;
+  }, { translatedSnippetSchemas: [], diagnostics: [] })
+  // Hydrate TranslatedSnippets from data back to objects
+  return { diagnostics: x.diagnostics, translatedSnippets: x.translatedSnippetSchemas.map(s => TranslatedSnippet.fromSchema(s)) };
+
+  /**
+   * Turn running the worker into a nice Promise.
+   */
+  function runWorker(request: import('./extract_worker').TranslateRequest): Promise {
+    return new Promise((resolve, reject) => {
+      const wrk = new worker.Worker(path.join(__dirname, 'extract_worker.js'), { workerData: request });
+      wrk.on('message', resolve);
+      wrk.on('error', reject);
+      wrk.on('exit', code => {
+        if (code !== 0) {
+          reject(new Error(`Worker exited with code ${code}`));
+        }
+      });
+    });
+  }
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/commands/extract_worker.ts b/packages/jsii-rosetta/lib/commands/extract_worker.ts
new file mode 100644
index 0000000000..c2194b0d58
--- /dev/null
+++ b/packages/jsii-rosetta/lib/commands/extract_worker.ts
@@ -0,0 +1,37 @@
+/**
+ * Pool worker for extract.ts
+ */
+import { TypeScriptSnippet } from '../snippet';
+import ts = require('typescript');
+import { singleThreadedTranslateAll } from './extract';
+import worker = require('worker_threads');
+import { TranslatedSnippetSchema } from '../tablets/schema';
+
+export interface TranslateRequest {
+  includeCompilerDiagnostics: boolean;
+  snippets: TypeScriptSnippet[];
+}
+
+export interface TranslateResponse {
+  diagnostics: ts.Diagnostic[];
+  // Cannot be 'TranslatedSnippet' because needs to be serializable
+  translatedSnippetSchemas: TranslatedSnippetSchema[];
+}
+
+function translateSnippet(request: TranslateRequest): TranslateResponse {
+  const result = singleThreadedTranslateAll(request.snippets[Symbol.iterator](), request.includeCompilerDiagnostics);
+
+  return { diagnostics: result.diagnostics, translatedSnippetSchemas: result.translatedSnippets.map(s => s.toSchema()) };
+}
+
+if (worker.isMainThread) {
+  // Throw an error to prevent accidental require() of this module. In principle not a big
+  // deal, but we want to be compatible with run modes where 'worker_threads' is not available
+  // and by doing this people on platforms where 'worker_threads' is available don't accidentally
+  // add a require().
+  throw new Error('This script should be run as a worker, not included directly.');
+}
+
+const request = worker.workerData;
+const response = translateSnippet(request);
+worker.parentPort!.postMessage(response);
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/logging.ts b/packages/jsii-rosetta/lib/logging.ts
index 5266448036..7740acab0a 100644
--- a/packages/jsii-rosetta/lib/logging.ts
+++ b/packages/jsii-rosetta/lib/logging.ts
@@ -1,3 +1,5 @@
+import util = require('util');
+
 export enum Level {
   WARN = -1,
   QUIET = 0,
@@ -28,6 +30,6 @@ export function debug(fmt: string, ...args: any[]) {
 function log(messageLevel: Level, fmt: string, ...args: any[]) {
   if (level >= messageLevel) {
     const levelName = Level[messageLevel];
-    console.error(`[jsii-rosetta] [${levelName}]`, fmt, ...args);
+    process.stderr.write(`[jsii-rosetta] [${levelName}] ${util.format(fmt, ...args)}\n`);
   }
-}
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/tablets/schema.ts b/packages/jsii-rosetta/lib/tablets/schema.ts
index 104bc1510e..7287d3ecdb 100644
--- a/packages/jsii-rosetta/lib/tablets/schema.ts
+++ b/packages/jsii-rosetta/lib/tablets/schema.ts
@@ -22,7 +22,7 @@ export interface TabletSchema {
   /**
    * All the snippets in the tablet
    */
-  snippets: {[key: string]: SnippetSchema};
+  snippets: {[key: string]: TranslatedSnippetSchema};
 }
 
 export const ORIGINAL_SNIPPET_KEY = '$';
@@ -30,7 +30,7 @@ export const ORIGINAL_SNIPPET_KEY = '$';
 /**
  * Schema for a snippet
  */
-export interface SnippetSchema {
+export interface TranslatedSnippetSchema {
   /**
    * Translations for each individual language
    *
diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts
index 07254acf12..8318f160d9 100644
--- a/packages/jsii-rosetta/lib/tablets/tablets.ts
+++ b/packages/jsii-rosetta/lib/tablets/tablets.ts
@@ -1,5 +1,5 @@
 import fs = require('fs-extra');
-import { TabletSchema, SnippetSchema, TranslationSchema, ORIGINAL_SNIPPET_KEY } from './schema';
+import { TabletSchema, TranslatedSnippetSchema, TranslationSchema, ORIGINAL_SNIPPET_KEY } from './schema';
 import { snippetKey } from './key';
 import { TargetLanguage } from '../languages';
 import { TypeScriptSnippet } from '../snippet';
@@ -42,7 +42,7 @@ export class LanguageTablet {
       throw new Error(`Tablet file '${filename}' has been created with version '${obj.toolVersion}', cannot read with current version '${TOOL_VERSION}'`);
     }
 
-    Object.assign(this.snippets, mapValues(obj.snippets, (schema: SnippetSchema) => TranslatedSnippet.fromSchema(schema)));
+    Object.assign(this.snippets, mapValues(obj.snippets, (schema: TranslatedSnippetSchema) => TranslatedSnippet.fromSchema(schema)));
   }
 
   public get count() {
@@ -63,7 +63,7 @@ export class LanguageTablet {
 }
 
 export class TranslatedSnippet {
-  public static fromSchema(schema: SnippetSchema) {
+  public static fromSchema(schema: TranslatedSnippetSchema) {
     const ret = new TranslatedSnippet();
     Object.assign(ret.translations, schema.translations);
     ret._didCompile = schema.didCompile;
@@ -151,7 +151,7 @@ export class TranslatedSnippet {
     };
   }
 
-  public toSchema(): SnippetSchema {
+  public toSchema(): TranslatedSnippetSchema {
     return {
       translations: this.translations,
       didCompile: this.didCompile,
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
index 87ff30a835..631225e207 100644
--- a/packages/jsii-rosetta/lib/translate.ts
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -29,7 +29,7 @@ export function translateTypeScript(source: File, visitor: AstHandler, opti
  */
 export class Translator {
   private readonly compiler = new TypeScriptCompiler();
-  private readonly translators: Record = {};
+  public readonly diagnostics: ts.Diagnostic[] = [];
 
   constructor(private readonly includeCompilerDiagnostics: boolean) {
   }
@@ -45,34 +45,23 @@ export class Translator {
       snippet.addTranslatedSource(lang, translated);
     }
 
-    return snippet;
-  }
+    this.diagnostics.push(...translator.diagnostics);
 
-  public get diagnostics(): ts.Diagnostic[] {
-    const ret = [];
-    for (const t of Object.values(this.translators)) {
-      ret.push(...t.diagnostics);
-    }
-    return ret;
+    return snippet;
   }
 
   /**
    * Return the snippet translator for the given snippet
+   *
+   * We used to cache these, but each translator holds on to quite a bit of memory,
+   * so we don't do that anymore.
    */
   public translatorFor(snippet: TypeScriptSnippet) {
-    const key = snippet.visibleSource + '-' + snippet.where;
-    if (!(key in this.translators)) {
-      const translator = new SnippetTranslator(snippet, {
-        compiler: this.compiler,
-        includeCompilerDiagnostics: this.includeCompilerDiagnostics,
-      });
-
-      this.diagnostics.push(...translator.compileDiagnostics);
-
-      this.translators[key] = translator;
-    }
-
-    return this.translators[key];
+    const translator = new SnippetTranslator(snippet, {
+      compiler: this.compiler,
+      includeCompilerDiagnostics: this.includeCompilerDiagnostics,
+    });
+    return translator;
   }
 }
 
@@ -122,6 +111,8 @@ export class SnippetTranslator {
   }
 
   public renderUsing(visitor: AstHandler) {
+    return '';
+
     const converter = new AstRenderer(this.compilation.rootFile, this.compilation.program.getTypeChecker(), visitor, this.options);
     const converted = converter.convert(this.compilation.rootFile);
     this.translateDiagnostics.push(...converter.diagnostics);
diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts
index 79effeb15a..ee9d8c5a0a 100644
--- a/packages/jsii-rosetta/lib/util.ts
+++ b/packages/jsii-rosetta/lib/util.ts
@@ -35,4 +35,18 @@ export function printDiagnostic(diag: ts.Diagnostic, stream: NodeJS.WritableStre
 
 export function isErrorDiagnostic(diag: ts.Diagnostic) {
   return diag.category === ts.DiagnosticCategory.Error;
+}
+
+/**
+ * Chunk an array of elements into approximately equal groups
+ */
+export function divideEvenly(groups: number, xs: A[]): A[][] {
+  const chunkSize = Math.ceil(xs.length / groups);
+  const ret: A[][] = [];
+
+  for (let i = 0; i < groups; i++) {
+    ret.push(xs.slice(i * chunkSize, (i + 1) * chunkSize));
+  }
+
+  return ret;
 }
\ No newline at end of file

From 78cbf8f46c7926f2a710557273f1a3ea02d933bc Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Mon, 4 Nov 2019 15:08:13 +0100
Subject: [PATCH 05/19] Update README

---
 packages/jsii-rosetta/README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md
index 7048b72cfe..455a834150 100644
--- a/packages/jsii-rosetta/README.md
+++ b/packages/jsii-rosetta/README.md
@@ -175,3 +175,17 @@ $ jsii-rosetta extract --compile $(find . -name .jsii) --directory some/dir
 $ jsii-pacmak --samples-tablet .jsii-samples.tbl
 
 ```
+
+### Running in parallel
+
+Since TypeScript compilation takes a lot of time, much time can be gained
+by using the CPUs in your system effectively. `jsii-rosetta extract` will
+run the compilations in parallel if support for NodeJS Worker Threads is
+detected.
+
+Worker threads are enabled by default on NodeJS 12.x, and can be enabled on
+NodeJS 10.x by using a flag:
+
+```
+$ node --experimental-worker /path/to/jsii-rosetta extract ...
+```

From b34d7368b34f22788ee27a8fd742b367994f7bfc Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Thu, 7 Nov 2019 16:40:45 +0100
Subject: [PATCH 06/19] Review comments

---
 packages/jsii-calc-base-of-base/package.json |  2 +-
 packages/jsii-calc-base/package.json         |  2 +-
 packages/jsii-calc-lib/package.json          |  2 +-
 packages/jsii-calc/package.json              |  2 +-
 packages/jsii-pacmak/bin/jsii-pacmak.ts      | 10 +++++-----
 packages/jsii-rosetta/README.md              |  4 +++-
 packages/jsii-rosetta/bin/jsii-rosetta.ts    |  4 ++--
 packages/jsii-rosetta/lib/commands/read.ts   |  4 ++--
 packages/jsii-rosetta/lib/o-tree.ts          | 11 ++++++++++-
 packages/jsii-rosetta/lib/tablets/tablets.ts |  6 +++---
 packages/jsii-rosetta/lib/translate.ts       |  2 --
 11 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/packages/jsii-calc-base-of-base/package.json b/packages/jsii-calc-base-of-base/package.json
index b0eb67eccd..77b49c3e93 100644
--- a/packages/jsii-calc-base-of-base/package.json
+++ b/packages/jsii-calc-base-of-base/package.json
@@ -24,7 +24,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii && jsii-rosetta extract",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
diff --git a/packages/jsii-calc-base/package.json b/packages/jsii-calc-base/package.json
index be63851d71..8a615c1270 100644
--- a/packages/jsii-calc-base/package.json
+++ b/packages/jsii-calc-base/package.json
@@ -24,7 +24,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii && jsii-rosetta extract",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
diff --git a/packages/jsii-calc-lib/package.json b/packages/jsii-calc-lib/package.json
index f5da0a0523..19c1c9bc24 100644
--- a/packages/jsii-calc-lib/package.json
+++ b/packages/jsii-calc-lib/package.json
@@ -26,7 +26,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii && jsii-rosetta extract",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
diff --git a/packages/jsii-calc/package.json b/packages/jsii-calc/package.json
index 17bcb50897..ac40d4390c 100644
--- a/packages/jsii-calc/package.json
+++ b/packages/jsii-calc/package.json
@@ -25,7 +25,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii && jsii-rosetta extract --compile",
+    "build": "jsii && jsii-rosetta --compile",
     "watch": "jsii -w",
     "test": "node test/test.calc.js && diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
diff --git a/packages/jsii-pacmak/bin/jsii-pacmak.ts b/packages/jsii-pacmak/bin/jsii-pacmak.ts
index e28e2b5be1..a1ff19e025 100644
--- a/packages/jsii-pacmak/bin/jsii-pacmak.ts
+++ b/packages/jsii-pacmak/bin/jsii-pacmak.ts
@@ -79,11 +79,11 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
       desc: 'Auto-update .npmignore to exclude the output directory and include the .jsii file',
       default: true
     })
-    .option('samples-tablet', {
+    .option('rosetta-tablet', {
       type: 'string',
       desc: 'Location of a jsii-rosetta tablet with sample translations (created using \'jsii-rosetta extract\')'
     })
-    .option('live-translation', {
+    .option('rosetta-translate-live', {
       type: 'boolean',
       desc: 'Translate code samples on-the-fly if they can\'t be found in the samples tablet',
       default: true
@@ -99,9 +99,9 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
 
   const timers = new Timers();
 
-  const rosetta = new Rosetta({ liveConversion: argv['live-translation'] });
-  if (argv['samples-tablet']) {
-    await rosetta.loadTabletFromFile(argv['samples-tablet']);
+  const rosetta = new Rosetta({ liveConversion: argv['rosetta-translate-live'] });
+  if (argv['rosetta-tablet']) {
+    await rosetta.loadTabletFromFile(argv['rosetta-tablet']);
   }
 
   const modulesToPackage = await findJsiiModules(argv._, argv.recurse);
diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md
index 455a834150..d6e541102a 100644
--- a/packages/jsii-rosetta/README.md
+++ b/packages/jsii-rosetta/README.md
@@ -173,9 +173,11 @@ Works like this:
 ```
 $ jsii-rosetta extract --compile $(find . -name .jsii) --directory some/dir
 $ jsii-pacmak --samples-tablet .jsii-samples.tbl
-
 ```
 
+(The `extract` command is the default and may be omitted, but if you're passing
+assembliess as arguments you should terminate the option list by passing `--`).
+
 ### Running in parallel
 
 Since TypeScript compilation takes a lot of time, much time can be gained
diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts
index 8c3be8f07d..4c73642dce 100644
--- a/packages/jsii-rosetta/bin/jsii-rosetta.ts
+++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts
@@ -38,7 +38,7 @@ async function main() {
         makeVisitor(args));
       renderResult(result);
     }))
-    .command('extract [ASSEMBLY..]', 'Extract code snippets from one or more assemblies into a language tablets', command => command
+    .command(['extract [ASSEMBLY..]', '$0 [ASSEMBLY..]'], 'Extract code snippets from one or more assemblies into a language tablets', command => command
       .positional('ASSEMBLY', { type: 'string', string: true, default: new Array(), describe: 'Assembly or directory to extract from' })
       .option('output', { alias: 'o', type: 'string', describe: 'Output file where to store the sample tablets', default: DEFAULT_TABLET_NAME })
       .option('compile', { alias: 'c', type: 'boolean', describe: 'Try compiling', default: false })
@@ -68,7 +68,7 @@ async function main() {
         process.exit(1);
       }
     }))
-    .command('read  [KEY] [LANGUAGE]', 'Read snippets from a language tablet', command => command
+    .command('read  [KEY] [LANGUAGE]', 'Display snippets in a language tablet file', command => command
       .positional('TABLET', { type: 'string', required: true, describe: 'Language tablet to read' })
       .positional('KEY', { type: 'string', describe: 'Snippet key to read' })
       .positional('LANGUAGE', { type: 'string', describe: 'Language ID to read' })
diff --git a/packages/jsii-rosetta/lib/commands/read.ts b/packages/jsii-rosetta/lib/commands/read.ts
index a44ab6af05..c1592b529e 100644
--- a/packages/jsii-rosetta/lib/commands/read.ts
+++ b/packages/jsii-rosetta/lib/commands/read.ts
@@ -6,7 +6,7 @@ export async function readTablet(tabletFile: string, key?: string, lang?: string
   await tab.load(tabletFile);
 
   if (key !== undefined) {
-    const snippet = tab.getSnippet(key);
+    const snippet = tab.tryGetSnippet(key);
     if (snippet === undefined) {
       throw new Error(`No such snippet: ${key}`);
     }
@@ -18,7 +18,7 @@ export async function readTablet(tabletFile: string, key?: string, lang?: string
   function listSnippets() {
     for (const key of tab.snippetKeys) {
       process.stdout.write(snippetHeader(key) + '\n');
-      displaySnippet(tab.getSnippet(key)!);
+      displaySnippet(tab.tryGetSnippet(key)!);
       process.stdout.write('\n');
     }
   }
diff --git a/packages/jsii-rosetta/lib/o-tree.ts b/packages/jsii-rosetta/lib/o-tree.ts
index 2bf662c277..127bdf44db 100644
--- a/packages/jsii-rosetta/lib/o-tree.ts
+++ b/packages/jsii-rosetta/lib/o-tree.ts
@@ -47,7 +47,7 @@ export interface OTreeOptions {
  * "Output" Tree
  *
  * Tree-like structure that holds sequences of trees and strings, which
- * can be rendered to an output stream.
+ * can be rendered to an output sink.
  */
 export class OTree implements OTree {
   public static simplify(xs: Array): Array {
@@ -126,6 +126,15 @@ export interface OTreeSinkOptions {
   visibleSpans?: Span[];
 }
 
+/**
+ * Output sink for OTree objects
+ *
+ * Maintains state about what has been rendered supports suppressing code
+ * fragments based on their tagged source location.
+ *
+ * Basically: manages the state that was too hard to manage in the
+ * tree :).
+ */
 export class OTreeSink {
   private readonly indentLevels: number[] = [0];
   private readonly fragments = new Array();
diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts
index 8318f160d9..77f766d8a4 100644
--- a/packages/jsii-rosetta/lib/tablets/tablets.ts
+++ b/packages/jsii-rosetta/lib/tablets/tablets.ts
@@ -6,7 +6,7 @@ import { TypeScriptSnippet } from '../snippet';
 
 const TOOL_VERSION = require('../../package.json').version;
 
-export const DEFAULT_TABLET_NAME = '.jsii-samples.tabl';
+export const DEFAULT_TABLET_NAME = '.jsii.tabl.json';
 
 /**
  * A tablet containing various snippets in multiple languages
@@ -23,14 +23,14 @@ export class LanguageTablet {
     return Object.keys(this.snippets);
   }
 
-  public getSnippet(key: string): TranslatedSnippet | undefined {
+  public tryGetSnippet(key: string): TranslatedSnippet | undefined {
     return this.snippets[key];
   }
 
   public lookup(typeScriptSource: TypeScriptSnippet, language: TargetLanguage): Translation | undefined {
     const snippet = this.snippets[snippetKey(typeScriptSource)];
     return snippet && snippet.get(language);
-  }
+}
 
   public async load(filename: string) {
     const obj = await fs.readJson(filename, { encoding: 'utf-8' });
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
index 631225e207..00eaff1c56 100644
--- a/packages/jsii-rosetta/lib/translate.ts
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -111,8 +111,6 @@ export class SnippetTranslator {
   }
 
   public renderUsing(visitor: AstHandler) {
-    return '';
-
     const converter = new AstRenderer(this.compilation.rootFile, this.compilation.program.getTypeChecker(), visitor, this.options);
     const converted = converter.convert(this.compilation.rootFile);
     this.translateDiagnostics.push(...converter.diagnostics);

From 51a6baea8dd910c38c896df61b60cb25ebcb8171 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Thu, 7 Nov 2019 18:04:54 +0100
Subject: [PATCH 07/19] Try to make Python build succeed from a dirty state

---
 packages/jsii-python-runtime/bin/generate-calc | 5 +++++
 packages/jsii-python-runtime/package.json      | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/jsii-python-runtime/bin/generate-calc b/packages/jsii-python-runtime/bin/generate-calc
index 990e02fcbb..f4605e14ef 100755
--- a/packages/jsii-python-runtime/bin/generate-calc
+++ b/packages/jsii-python-runtime/bin/generate-calc
@@ -3,6 +3,11 @@ import os
 import subprocess
 import sys
 
+# Clean out this directory, as it otherwise may
+# accumuluate multiple versions of the same library
+# and pip will complain.
+subprocess.run(['rm', '-rf', '.env/jsii-calc'], check=True)
+
 subprocess.run(
     [
         "jsii-pacmak",
diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json
index 99497f21e0..a01375b261 100644
--- a/packages/jsii-python-runtime/package.json
+++ b/packages/jsii-python-runtime/package.json
@@ -24,7 +24,7 @@
   "scripts": {
     "generate": "python3 bin/generate",
     "deps": "python3 -m venv .env && .env/bin/pip install pip==19.0.1 setuptools==40.7.0 wheel==0.32.3 && .env/bin/pip install -r requirements.txt",
-    "build": "cp ../../README.md . && npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
+    "build": "cp ../../README.md . && rm -f jsii-*.whl npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
     "package": "package-python",
     "test": ".env/bin/python bin/generate-calc && .env/bin/py.test -v --mypy"
   },

From 224037f5afc1d30d4c3a8963c2c35559973e835b Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Mon, 11 Nov 2019 11:17:12 +0100
Subject: [PATCH 08/19] Missing &&

---
 packages/jsii-python-runtime/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json
index 39c7b809f8..418d4100ef 100644
--- a/packages/jsii-python-runtime/package.json
+++ b/packages/jsii-python-runtime/package.json
@@ -24,7 +24,7 @@
   "scripts": {
     "generate": "python3 bin/generate",
     "deps": "python3 -m venv .env && .env/bin/pip install pip==19.0.1 setuptools==40.7.0 wheel==0.32.3 && .env/bin/pip install -r requirements.txt",
-    "build": "cp ../../README.md . && rm -f jsii-*.whl npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
+    "build": "cp ../../README.md . && rm -f jsii-*.whl && npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
     "package": "package-python",
     "test": ".env/bin/python bin/generate-calc && .env/bin/py.test -v --mypy",
     "test:update": "UPDATE_DIFF=1 .env/bin/python bin/generate-calc && .env/bin/py.test -v --mypy"

From 538099153228a6056ca29f0fb6b6e8b35416e8c2 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 14:44:53 +0100
Subject: [PATCH 09/19] Fix pacmak unified build sanity

For both .NET and Java, build in a temporary directory and copy
artifacts out to final destination.

Fix .NET projects to build all in one solution, and use project
references to automatically fix build order.
---
 packages/jsii-pacmak/lib/builder.ts           |   9 -
 packages/jsii-pacmak/lib/targets/dotnet.ts    | 200 +++++++++++-------
 .../lib/targets/dotnet/filegenerator.ts       |   7 +-
 packages/jsii-pacmak/lib/targets/index.ts     |   4 +-
 packages/jsii-pacmak/lib/targets/java.ts      | 158 +++++++++-----
 packages/jsii-pacmak/lib/util.ts              |   7 +
 packages/jsii-rosetta/bin/jsii-rosetta.ts     |   2 +-
 packages/jsii-rosetta/lib/commands/extract.ts |   3 +-
 8 files changed, 234 insertions(+), 156 deletions(-)

diff --git a/packages/jsii-pacmak/lib/builder.ts b/packages/jsii-pacmak/lib/builder.ts
index a6aed36a16..5417d27258 100644
--- a/packages/jsii-pacmak/lib/builder.ts
+++ b/packages/jsii-pacmak/lib/builder.ts
@@ -53,15 +53,6 @@ export interface TargetBuilder {
   buildModules(modules: JsiiModule[], options: BuildOptions): Promise;
 }
 
-/**
- * Return the output directory if all modules have the same directory
- */
-export function allOutputDirectoriesTheSame(modules: JsiiModule[]): boolean {
-  if (modules.length === 0) { return true; }
-  const ret = modules[0].outputDirectory;
-  return modules.every(m => m.outputDirectory === ret);
-}
-
 /**
  * Builds the targets for the given language sequentially
  */
diff --git a/packages/jsii-pacmak/lib/targets/dotnet.ts b/packages/jsii-pacmak/lib/targets/dotnet.ts
index 176e157c1a..9b7c2dc0fd 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet.ts
@@ -1,11 +1,125 @@
 import * as fs from 'fs-extra';
 import * as spec from 'jsii-spec';
 import * as path from 'path';
-import * as xmlbuilder from 'xmlbuilder';
 import * as logging from '../logging';
 import { PackageInfo, Target } from '../target';
-import { shell } from '../util';
+import { shell, Scratch } from '../util';
 import { DotNetGenerator } from './dotnet/dotnetgenerator';
+import { TargetBuilder, BuildOptions } from '../builder';
+import { JsiiModule } from '../packaging';
+
+/**
+ * Build .NET packages all together, by generating an aggregate solution file
+ */
+export class DotnetBuilder implements TargetBuilder {
+  private readonly targetName = 'dotnet';
+
+  public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise {
+    if (modules.length === 0) { return; }
+
+    if (options.codeOnly) {
+      // Simple, just generate code to respective output dirs
+      for (const module of modules) {
+        await this.generateModuleCode(module, options, module.outputDirectory);
+      }
+      return;
+    }
+
+    // Otherwise make a single tempdir to hold all sources, build them together and copy them back out
+    const scratchDirs: Array> = [];
+    try {
+      const tempSourceDir = await this.generateAggregateSourceDir(modules, options);
+      scratchDirs.push(tempSourceDir);
+
+      // Build solution
+      logging.debug(`Building .NET`);
+      await shell('dotnet', ['build', '-c', 'Release'], { cwd: tempSourceDir.directory });
+
+      await this.copyOutArtifacts(tempSourceDir.object, options);
+    } finally {
+      if (options.clean) {
+        Scratch.cleanupAll(scratchDirs);
+      }
+    }
+  }
+
+  private async generateAggregateSourceDir(modules: JsiiModule[], options: BuildOptions): Promise>> {
+    return Scratch.make(async (tmpDir: string) => {
+      logging.debug(`Generating aggregate .NET source dir at ${tmpDir}`);
+
+      const csProjs = [];
+      const ret: TemporaryDotnetPackage[] = [];
+
+      for (const module of modules) {
+        // Code generator will make its own subdirectory
+        await this.generateModuleCode(module, options, tmpDir);
+        const loc = projectLocation(module);
+        csProjs.push(loc.projectFile);
+        ret.push({
+          outputTargetDirectory: module.outputDirectory,
+          artifactsDir: path.join(tmpDir, loc.projectDir, 'bin', 'Release')
+        });
+      }
+
+      // Use 'dotnet' command line tool to build a solution file from these csprojs
+      await shell('dotnet', ['new', 'sln', '-n', 'JsiiBuild'], { cwd: tmpDir });
+      await shell('dotnet', ['sln', 'add', ...csProjs], { cwd: tmpDir });
+
+      return ret;
+    });
+  }
+
+  private async copyOutArtifacts(packages: TemporaryDotnetPackage[], options: BuildOptions) {
+    logging.debug(`Copying out .NET artifacts`);
+    for (const pkg of packages) {
+      const targetDirectory = path.join(pkg.outputTargetDirectory, options.languageSubdirectory ? this.targetName : '');
+
+      await fs.mkdirp(targetDirectory);
+      await fs.copy(pkg.artifactsDir, targetDirectory, { recursive: true });
+
+      // This copies more than we need, remove the directory with the bare assembly again
+      await fs.remove(path.join(targetDirectory, 'netcoreapp3.0'));
+    }
+  }
+
+  private async generateModuleCode(module: JsiiModule, options: BuildOptions, where: string): Promise {
+    const target = this.makeTarget(module, options);
+    logging.debug(`Generating ${this.targetName} code into ${where}`);
+    await target.generateCode(where, module.tarball);
+  }
+
+  private makeTarget(module: JsiiModule, options: BuildOptions): Dotnet {
+    return new Dotnet({
+      targetName: this.targetName,
+      packageDir: module.moduleDirectory,
+      assembly: module.assembly,
+      fingerprint: options.fingerprint,
+      force: options.force,
+      arguments: options.arguments,
+      rosetta: options.rosetta,
+    });
+  }
+}
+
+interface TemporaryDotnetPackage {
+  /**
+   * Where the artifacts will be stored after build (relative to build dir)
+   */
+  artifactsDir: string;
+
+  /**
+   * Where the artifacts ought to go for this particular module
+   */
+  outputTargetDirectory: string;
+}
+
+function projectLocation(module: JsiiModule) {
+  const packageId: string = module.assembly.targets!.dotnet!.packageId;
+  return {
+    projectDir: packageId,
+    projectFile: path.join(packageId, `${packageId}.csproj`)
+  };
+}
 
 export default class Dotnet extends Target {
   public static toPackageInfos(assm: spec.Assembly): { [language: string]: PackageInfo } {
@@ -40,85 +154,7 @@ export default class Dotnet extends Target {
 
   protected readonly generator = new DotNetGenerator();
 
-  public async build(sourceDir: string, outDir: string): Promise {
-    await this.generateNuGetConfigForLocalDeps(sourceDir, outDir);
-    const pkg = await fs.readJson(path.join(this.packageDir, 'package.json'));
-    const packageId: string = pkg.jsii.targets.dotnet.packageId;
-    const project: string = path.join(packageId, `${packageId}.csproj`);
-
-    await shell(
-      'dotnet',
-      ['build', project, '-c', 'Release'],
-      { cwd: sourceDir }
-    );
-
-    await this.copyFiles(
-      path.join(sourceDir, packageId, 'bin', 'Release'),
-      outDir);
-    await fs.remove(path.join(outDir, 'netcoreapp3.0'));
-  }
-
-  private async generateNuGetConfigForLocalDeps(sourceDirectory: string, currentOutputDirectory: string): Promise {
-    // Traverse the dependency graph of this module and find all modules that have
-    // an /dotnet directory. We will add those as local NuGet repositories.
-    // This enables building against local modules.
-    const localRepos = await this.findLocalDepsOutput(this.packageDir);
-
-    // Add the current output directory as a local repo for the case where we build multiple packages
-    // into the same output. NuGet throws an error if a source directory doesn't exist, so we check
-    // before adding it to the list.
-    if (await fs.pathExists(currentOutputDirectory)) {
-      localRepos.push(path.resolve(process.cwd(), currentOutputDirectory));
-    }
-
-    // If dotnet-jsonmodel is checked-out and we can find a local repository, add it to the list.
-    try {
-      /* eslint-disable @typescript-eslint/no-var-requires */
-      const jsiiDotNetJsonModel = require('jsii-dotnet-jsonmodel');
-      /* eslint-enable @typescript-eslint/no-var-requires */
-      const localDotNetJsonModel = jsiiDotNetJsonModel.repository;
-      if (await fs.pathExists(localDotNetJsonModel)) {
-        localRepos.push(localDotNetJsonModel);
-      }
-    } catch {
-      // Couldn't locate jsii-dotnet-jsonmodel, which is owkay!
-    }
-
-    // If dotnet-runtime is checked-out and we can find a local repository, add it to the list.
-    try {
-      /* eslint-disable @typescript-eslint/no-var-requires */
-      const jsiiDotNetRuntime = require('jsii-dotnet-runtime');
-      /* eslint-enable @typescript-eslint/no-var-requires */
-      const localDotNetRuntime = jsiiDotNetRuntime.repository;
-      if (await fs.pathExists(localDotNetRuntime)) {
-        localRepos.push(localDotNetRuntime);
-      }
-    } catch {
-      // Couldn't locate jsii-dotnet-runtime, which is owkay!
-    }
-
-    logging.debug('local NuGet repos:', localRepos);
-
-    // Construct XML content.
-    const configuration = xmlbuilder.create('configuration', { encoding: 'UTF-8' });
-    const packageSources = configuration.ele('packageSources');
-
-    const nugetOrgAdd = packageSources.ele('add');
-    nugetOrgAdd.att('key', 'nuget.org');
-    nugetOrgAdd.att('value', 'https://api.nuget.org/v3/index.json');
-    nugetOrgAdd.att('protocolVersion', '3');
-
-    localRepos.forEach((repo, index) => {
-      const add = packageSources.ele('add');
-      add.att('key', `local-${index}`);
-      add.att('value', path.join(repo));
-    });
-
-    const xml = configuration.end({ pretty: true });
-
-    // Write XML content to NuGet.config.
-    const filePath = path.join(sourceDirectory, 'NuGet.config');
-    logging.debug(`Generated ${filePath}`);
-    await fs.writeFile(filePath, xml);
+  public async build(_sourceDir: string, _outDir: string): Promise {
+    throw new Error('Should not be called; use builder instead');
   }
 }
diff --git a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
index e9c96281b5..c9624b6a81 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
@@ -106,9 +106,10 @@ export class FileGenerator {
     packageReference.att('Version', `[${jsiiVersion},${jsiiVersionNextMajor})`);
 
     dependencies.forEach((value: DotNetDependency) => {
-      const dependencyReference = itemGroup2.ele('PackageReference');
-      dependencyReference.att('Include', value.packageId);
-      dependencyReference.att('Version', value.version);
+      const dependencyReference = itemGroup2.ele('ProjectReference');
+      // dependencyReference.att('Include', value.packageId);
+      // dependencyReference.att('Version', value.version);
+      dependencyReference.att('Include', `../${value.packageId}/${value.packageId}.csproj`);
     });
 
     const xml = rootNode.end({ pretty: true, spaceBeforeSlash: true });
diff --git a/packages/jsii-pacmak/lib/targets/index.ts b/packages/jsii-pacmak/lib/targets/index.ts
index 263a9d405d..7f45a08dbf 100644
--- a/packages/jsii-pacmak/lib/targets/index.ts
+++ b/packages/jsii-pacmak/lib/targets/index.ts
@@ -1,6 +1,6 @@
 import { OneByOneBuilder, TargetBuilder } from '../builder';
 
-import Dotnet from './dotnet';
+import { DotnetBuilder } from './dotnet';
 import { JavaBuilder } from './java';
 import JavaScript from './js';
 import Python from './python';
@@ -9,7 +9,7 @@ import Ruby from './ruby';
 export type TargetName = 'dotnet' | 'java' | 'js' | 'python' | 'ruby';
 
 export const ALL_BUILDERS: {[key in TargetName]: TargetBuilder} = {
-  dotnet: new OneByOneBuilder('dotnet', Dotnet),
+  dotnet: new DotnetBuilder(),
   java: new JavaBuilder(),
   js: new OneByOneBuilder('js', JavaScript),
   python: new OneByOneBuilder('python', Python),
diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts
index 8b2561f195..4bb53f901c 100644
--- a/packages/jsii-pacmak/lib/targets/java.ts
+++ b/packages/jsii-pacmak/lib/targets/java.ts
@@ -9,9 +9,9 @@ import { Generator } from '../generator';
 import logging = require('../logging');
 import { md2html } from '../markdown';
 import { PackageInfo, Target } from '../target';
-import { shell, Scratch } from '../util';
+import { shell, Scratch, slugify } from '../util';
 import { VERSION, VERSION_DESC } from '../version';
-import { TargetBuilder, BuildOptions, allOutputDirectoriesTheSame, OneByOneBuilder } from '../builder';
+import { TargetBuilder, BuildOptions } from '../builder';
 import { JsiiModule } from '../packaging';
 
 /* eslint-disable @typescript-eslint/no-var-requires */
@@ -21,7 +21,13 @@ const spdxLicenseList = require('spdx-license-list');
 const BUILDER_CLASS_NAME = 'Builder';
 
 /**
- * Build Java packages in parallel, by generating an aggregate POM
+ * Build Java packages all together, by generating an aggregate POM
+ *
+ * This will make the Java build a lot more efficient (~300%).
+ *
+ * Do this by copying the code into a temporary directory, generating an aggregate
+ * POM there, and then copying the artifacts back into the respective output
+ * directories.
  */
 export class JavaBuilder implements TargetBuilder {
   private readonly targetName = 'java';
@@ -29,51 +35,68 @@ export class JavaBuilder implements TargetBuilder {
   public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise {
     if (modules.length === 0) { return; }
 
-    // We can only do the optimized build if '-o' was specified, which we will notice
-    // as all module outputdirectories being the same. (Maybe we can build to per-package
-    // dist dirs as well, but this is the smallest delta from what we had and we will
-    // always specify the output dir anyway).
-    if (!allOutputDirectoriesTheSame(modules)) {
-      logging.warn('Single output directory not specified, doing (slower) one-by-one build for Java');
-      await new OneByOneBuilder(this.targetName, Java).buildModules(modules, options);
+    if (options.codeOnly) {
+      // Simple, just generate code to respective output dirs
+      for (const module of modules) {
+        await this.generateModuleCode(module, options, module.outputDirectory);
+      }
       return;
     }
 
-    const singleOutputDir = this.finalOutputDir(modules[0], options);
-
-    const moduleDirectories = [];
-
-    for (const module of modules) {
-      moduleDirectories.push(await this.generateModuleCode(module, options, options.codeOnly));
-    }
+    // Otherwise make a single tempdir to hold all sources, build them together and copy them back out
+    const scratchDirs: Array> = [];
+    try {
+      const tempSourceDir = await this.generateAggregateSourceDir(modules, options);
+      scratchDirs.push(tempSourceDir);
 
-    if (!options.codeOnly && modules.length > 0) {
-      // Need a module to get a target
-      const pomDirectory = modules.length > 1
-        ? await this.generateAggregatePom(moduleDirectories)
-        : moduleDirectories[0].directory;
+      // Need any old module object to make a target to be able to invoke build, though none of its settings
+      // will be used.
       const target = this.makeTarget(modules[0], options);
+      const tempOutputDir = await Scratch.make(async dir => {
+        logging.debug(`Building Java code to ${dir}`);
+        await target.build(tempSourceDir.directory, dir);
+      });
+      scratchDirs.push(tempOutputDir);
+
+      await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object, options);
 
-      await target.build(pomDirectory, singleOutputDir);
+    } finally {
+      if (options.clean) {
+        Scratch.cleanupAll(scratchDirs);
+      }
     }
   }
 
-  private async generateModuleCode(module: JsiiModule, options: BuildOptions, finalDirectory?: boolean): Promise> {
+  private async generateModuleCode(module: JsiiModule, options: BuildOptions, where: string): Promise {
     const target = this.makeTarget(module, options);
+    logging.debug(`Generating Java code into ${where}`);
+    await target.generateCode(where, module.tarball);
+  }
+
+  private async generateAggregateSourceDir(modules: JsiiModule[], options: BuildOptions): Promise>> {
+    return Scratch.make(async (tmpDir: string) => {
+      logging.debug(`Generating aggregate Java source dir at ${tmpDir}`);
+      const ret: TemporaryJavaPackage[] = [];
+      for (const module of modules) {
+        const relativeName = slugify(module.name);
+        const sourceDir = path.join(tmpDir, relativeName);
+        await this.generateModuleCode(module, options, sourceDir);
+
+        module.assembly
+        ret.push({
+          relativeSourceDir: relativeName,
+          relativeArtifactsDir: moduleArtifactsSubdir(module),
+          outputTargetDirectory: module.outputDirectory
+        });
+      }
 
-    const srcDir = finalDirectory
-      ? Scratch.fake(this.finalOutputDir(module, options), undefined)
-      : await Scratch.make(_ => undefined);
-
-    logging.debug(`Generating ${this.targetName} code into ${srcDir.directory}`);
-    await target.generateCode(srcDir.directory, module.tarball);
+      await this.generateAggregatePom(tmpDir, ret.map(m => m.relativeSourceDir));
 
-    return srcDir;
+      return ret;
+    });
   }
 
-  private async generateAggregatePom(sourceDirectories: Array>) {
-    const parentDir = this.findSharedParentDirectory(sourceDirectories.map(s => s.directory));
-
+  private async generateAggregatePom(where: string, moduleNames: string[]) {
     const aggregatePom = xmlbuilder.create({
       project: {
         '@xmlns': 'http://maven.apache.org/POM/4.0.0',
@@ -91,32 +114,32 @@ export class JavaBuilder implements TargetBuilder {
         'version': '1.0.0',
 
         'modules': {
-          module: sourceDirectories.map(s => path.relative(parentDir, s.directory))
+          module: moduleNames,
         }
       }
     }, { encoding: 'UTF-8' }).end({ pretty: true });
 
-    logging.debug(`Generated ${parentDir}/pom.xml`);
-    await fs.writeFile(path.join(parentDir, 'pom.xml'), aggregatePom);
-    return parentDir;
+    logging.debug(`Generated ${where}/pom.xml`);
+    await fs.writeFile(path.join(where, 'pom.xml'), aggregatePom);
   }
 
-  /**
-   * Find the longest shared given a set of directories
-   */
-  private findSharedParentDirectory(dirs: string[]) {
-    if (dirs.length === 0) { return ''; }
-    const dirParts = dirs.map(dir => dir.split(path.sep));
+  private async copyOutArtifacts(artifactsRoot: string, packages: TemporaryJavaPackage[], options: BuildOptions) {
+    logging.debug(`Copying out Java artifacts`);
+    // The artifacts directory looks like this:
+    //  /tmp/XXX/software/amazon/awscdk/something/v1.2.3
+    //                                 /else/v1.2.3
+    //                                 /entirely/v1.2.3
+    //
+    // We get the 'software/amazon/awscdk/something' path from the package, identifying
+    // the files we need to copy, including Maven metadata. But we need to recreate
+    // the whole path in the target directory.
 
-    return dirParts.reduce(longestPrefix).join(path.sep);
+    for (const pkg of packages) {
+      const artifactsSource = path.join(artifactsRoot, pkg.relativeArtifactsDir);
+      const artifactsDest = path.join(pkg.outputTargetDirectory, options.languageSubdirectory ? this.targetName : '', pkg.relativeArtifactsDir);
 
-    function longestPrefix(accumulator: string[], current: string[]) {
-      const len = Math.min(accumulator.length, current.length);
-      let i = 0;
-      while (i < len && accumulator[i] === current[i]) {
-        i++;
-      }
-      return accumulator.slice(0, i);
+      await fs.mkdirp(artifactsDest);
+      await fs.copy(artifactsSource, artifactsDest, { recursive: true });
     }
   }
 
@@ -131,13 +154,32 @@ export class JavaBuilder implements TargetBuilder {
       rosetta: options.rosetta,
     });
   }
+}
 
-  private finalOutputDir(module: JsiiModule, options: BuildOptions): string {
-    if (options.languageSubdirectory) {
-      return path.join(module.outputDirectory, this.targetName);
-    }
-    return module.outputDirectory;
-  }
+interface TemporaryJavaPackage {
+  /**
+   * Where the sources are (relative to the source root)
+   */
+  relativeSourceDir: string;
+
+  /**
+   * Where the artifacts will be stored after build (relative to build dir)
+   */
+  relativeArtifactsDir: string;
+
+  /**
+   * Where the artifacts ought to go for this particular module
+   */
+  outputTargetDirectory: string;
+}
+
+/**
+ * Return the subdirectory of the output directory where the artifacts for this particular package are produced
+ */
+function moduleArtifactsSubdir(module: JsiiModule) {
+  const groupId = module.assembly.targets!.java!.maven.groupId;
+  const artifactId = module.assembly.targets!.java!.maven.artifactId;
+  return `${groupId.replace(/\./g, '/')}/${artifactId}`;
 }
 
 export default class Java extends Target {
diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts
index 0dbdafe0a9..132173f470 100644
--- a/packages/jsii-pacmak/lib/util.ts
+++ b/packages/jsii-pacmak/lib/util.ts
@@ -96,6 +96,13 @@ export async function loadAssembly(modulePath: string): Promise {
   return spec.validateAssembly(await fs.readJson(assmPath));
 }
 
+/**
+ * Strip filesystem unsafe characters from a string
+ */
+export function slugify(x: string) {
+  return x.replace(/[^a-zA-Z0-9_-]/g, '_');
+}
+
 /**
  * Class that makes a temporary directory and holds on to an operation object
  */
diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts
index 4c73642dce..852fd4b4b9 100644
--- a/packages/jsii-rosetta/bin/jsii-rosetta.ts
+++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts
@@ -61,7 +61,7 @@ async function main() {
       printDiagnostics(result.diagnostics, process.stderr);
 
       if (result.diagnostics.length > 0) {
-        logging.warn(`${result.diagnostics.length} diagnostics encountered`);
+        logging.warn(`${result.diagnostics.length} diagnostics encountered in ${result.tablet.count} snippets`);
       }
 
       if (result.diagnostics.some(isErrorDiagnostic) && args.fail) {
diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts
index 31ba4791cb..7bd2b4c052 100644
--- a/packages/jsii-rosetta/lib/commands/extract.ts
+++ b/packages/jsii-rosetta/lib/commands/extract.ts
@@ -10,6 +10,7 @@ import { divideEvenly } from '../util';
 
 export interface ExtractResult {
   diagnostics: ts.Diagnostic[];
+  tablet: LanguageTablet;
 }
 
 /**
@@ -37,7 +38,7 @@ export async function extractSnippets(assemblyLocations: string[], outputFile: s
   logging.info(`Saving language tablet to ${outputFile}`);
   await tablet.save(outputFile);
 
-  return { diagnostics: result.diagnostics };
+  return { diagnostics: result.diagnostics, tablet };
 }
 
 interface TranslateAllResult {

From 6efc55c293684774b378bb690082ad29d6207c05 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 15:31:01 +0100
Subject: [PATCH 10/19] Render HTML comments as-is

---
 packages/jsii-rosetta/lib/markdown/markdown-renderer.ts | 4 ++--
 packages/jsii-rosetta/test/markdown/roundtrip.test.ts   | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts b/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
index c876a38b86..c7873d831d 100644
--- a/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
+++ b/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
@@ -41,8 +41,8 @@ export class MarkdownRenderer implements CommonMarkRenderer {
     return node.literal || '';
   }
 
-  public html_block(_node: cm.Node, context: RendererContext) {
-    return `${context.content()}`;
+  public html_block(node: cm.Node, _context: RendererContext) {
+    return node.literal || '';
   }
 
   public link(node: cm.Node, context: RendererContext) {
diff --git a/packages/jsii-rosetta/test/markdown/roundtrip.test.ts b/packages/jsii-rosetta/test/markdown/roundtrip.test.ts
index 5307b2576f..b23ca8f8e2 100644
--- a/packages/jsii-rosetta/test/markdown/roundtrip.test.ts
+++ b/packages/jsii-rosetta/test/markdown/roundtrip.test.ts
@@ -152,6 +152,15 @@ test('headings', () => {
   `);
 });
 
+test('HTML comments', () => {
+  expectOutput(`
+
+  `, `
+
+  `);
+});
+
+
 function expectOutput(source: string, expected: string) {
   if (DEBUG) {
     const struct = new StructureRenderer();

From 965848ff4ca6e0de1f239a747cdfc53be970436e Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 16:29:24 +0100
Subject: [PATCH 11/19] Add backwards compatible support for literate examples

---
 packages/jsii-rosetta/lib/fixtures.ts         | 19 ++++++++++-
 packages/jsii-rosetta/lib/jsii/assemblies.ts  | 14 ++++++--
 packages/jsii-rosetta/lib/snippet.ts          | 16 ++++++++++
 packages/jsii-rosetta/lib/translate.ts        |  8 +++--
 .../lib/typescript/ts-compiler.ts             |  8 ++---
 .../jsii-rosetta/test/jsii/assemblies.test.ts | 32 +++++++++++++++++++
 packages/jsii/lib/literate.ts                 | 18 ++++++-----
 packages/jsii/test/literate.test.ts           |  6 ++--
 8 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/packages/jsii-rosetta/lib/fixtures.ts b/packages/jsii-rosetta/lib/fixtures.ts
index 41dd935546..ce0566e1f9 100644
--- a/packages/jsii-rosetta/lib/fixtures.ts
+++ b/packages/jsii-rosetta/lib/fixtures.ts
@@ -12,16 +12,33 @@ export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet {
   const directory = parameters[SnippetParameters.$PROJECT_DIRECTORY];
   if (!directory) { return snippet; }
 
-  if (parameters[SnippetParameters.FIXTURE]) {
+  const literateSource = parameters[SnippetParameters.LITERATE_SOURCE];
+  if (literateSource) {
+    // Compatibility with the "old school" example inclusion mechanism.
+    // Completely load this file and attach a parameter with its directory.
+    source = loadLiterateSource(directory, literateSource);
+    parameters[SnippetParameters.$COMPILATION_DIRECTORY] = path.join(directory, path.dirname(literateSource));
+  } else if (parameters[SnippetParameters.FIXTURE]) {
     // Explicitly request a fixture
     source = loadAndSubFixture(directory, parameters.fixture, source, true);
   } else if (parameters[SnippetParameters.NO_FIXTURE] === undefined) {
+    // Don't explicitly request no fixture
     source = loadAndSubFixture(directory, 'default', source, false);
   }
 
   return { visibleSource: snippet.visibleSource, completeSource: source, where: snippet.where, parameters };
 }
 
+function loadLiterateSource(directory: string, literateFileName: string) {
+  const fullPath = path.join(directory, literateFileName);
+  const exists = fs.existsSync(fullPath);
+  if (!exists) {
+    // This couldn't really happen in practice, but do the check anyway
+    throw new Error(`Sample uses literate source ${literateFileName}, but not found: ${fullPath}`);
+  }
+  return fs.readFileSync(fullPath, { encoding: 'utf-8' });
+}
+
 function loadAndSubFixture(directory: string, fixtureName: string, source: string, mustExist: boolean) {
   const fixtureFileName = path.join(directory, `rosetta/${fixtureName}.ts-fixture`);
   const exists = fs.existsSync(fixtureFileName);
diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts
index dad5fcb53e..4100c044ab 100644
--- a/packages/jsii-rosetta/lib/jsii/assemblies.ts
+++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts
@@ -46,7 +46,7 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc
   const ret: AssemblySnippetSource[] = [];
 
   if (assembly.readme) {
-    ret.push({ type: 'markdown', markdown: assembly.readme.markdown, where: `${assembly.name}-README` });
+    ret.push({ type: 'markdown', markdown: assembly.readme.markdown, where: removeSlashes(`${assembly.name}-README`) });
   }
 
   if (assembly.types) {
@@ -68,13 +68,21 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc
   function emitDocs(docs: spec.Docs | undefined, where: string) {
     if (!docs) { return; }
 
-    if (docs.remarks) { ret.push({ 'type': 'markdown', markdown: docs.remarks, where }); }
+    if (docs.remarks) { ret.push({ 'type': 'markdown', markdown: docs.remarks, where: removeSlashes(where) }); }
     if (docs.example && exampleLooksLikeSource(docs.example)) {
-      ret.push({ 'type': 'literal', source: docs.example, where: `${where}-example` });
+      ret.push({ 'type': 'literal', source: docs.example, where: removeSlashes(`${where}-example`) });
     }
   }
 }
 
+/**
+ * Remove slashes from a "where" description, as the TS compiler will interpret it as a directory
+ * and we can't have that for compiling literate files
+ */
+function removeSlashes(x: string) {
+  return x.replace(/\//g, '.');
+}
+
 export function* allTypeScriptSnippets(assemblies: Array<{ assembly: spec.Assembly, directory: string }>): IterableIterator {
   for (const assembly of assemblies) {
     for (const source of allSnippetSources(assembly.assembly)) {
diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts
index dd73e88993..186664e0ee 100644
--- a/packages/jsii-rosetta/lib/snippet.ts
+++ b/packages/jsii-rosetta/lib/snippet.ts
@@ -97,10 +97,26 @@ export enum SnippetParameters {
    */
   NO_FIXTURE = 'nofixture',
 
+  /**
+   * Snippet was extracted from this literate file (backwards compatibility)
+   *
+   * Parameter attached by 'jsii'; load the given file instead of any fixture,
+   * process as usual.
+   */
+  LITERATE_SOURCE = 'lit',
+
   /**
    * What directory to resolve fixtures in for this snippet (system parameter)
    *
    * Attached during processing, should not be used by authors.
    */
   $PROJECT_DIRECTORY = '$directory',
+
+  /**
+   * What directory to pretend the file is in (system parameter)
+   *
+   * Attached when compiling a literate file, as they compile in
+   * the location where they are stored.
+   */
+  $COMPILATION_DIRECTORY = '$compilation',
 };
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
index 00eaff1c56..8de08bb14e 100644
--- a/packages/jsii-rosetta/lib/translate.ts
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -7,7 +7,7 @@ import { TranslatedSnippet } from './tablets/tablets';
 import { TARGET_LANGUAGES, TargetLanguage } from './languages';
 import { calculateVisibleSpans } from './typescript/ast-utils';
 import { File } from './util';
-import { TypeScriptSnippet, completeSource } from './snippet';
+import { TypeScriptSnippet, completeSource, SnippetParameters } from './snippet';
 import { snippetKey } from './tablets/key';
 
 export function translateTypeScript(source: File, visitor: AstHandler, options: SnippetTranslatorOptions = {}): TranslateResult {
@@ -35,7 +35,7 @@ export class Translator {
   }
 
   public translate(snip: TypeScriptSnippet, languages = Object.keys(TARGET_LANGUAGES) as TargetLanguage[]) {
-    logging.debug(`Translating ${snippetKey(snip)}`);
+    logging.debug(`Translating ${snippetKey(snip)} ${Object.entries(snip.parameters || {})}`);
     const translator = this.translatorFor(snip);
     const snippet = TranslatedSnippet.fromSnippet(snip, this.includeCompilerDiagnostics ? translator.compileDiagnostics.length === 0 : undefined);
 
@@ -98,7 +98,9 @@ export class SnippetTranslator {
   constructor(snippet: TypeScriptSnippet, private readonly options: SnippetTranslatorOptions = {}) {
     const compiler = options.compiler || new TypeScriptCompiler();
     const source = completeSource(snippet);
-    this.compilation = compiler.compileInMemory(snippet.where, source);
+
+    const fakeCurrentDirectory = snippet.parameters && snippet.parameters[SnippetParameters.$COMPILATION_DIRECTORY];
+    this.compilation = compiler.compileInMemory(snippet.where, source, fakeCurrentDirectory);
 
     // Respect '/// !hide' and '/// !show' directives
     this.visibleSpans = calculateVisibleSpans(source);
diff --git a/packages/jsii-rosetta/lib/typescript/ts-compiler.ts b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts
index 8154e1c94b..4889f72824 100644
--- a/packages/jsii-rosetta/lib/typescript/ts-compiler.ts
+++ b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts
@@ -7,14 +7,14 @@ export class TypeScriptCompiler {
     this.realHost = ts.createCompilerHost(STANDARD_COMPILER_OPTIONS, true);
   }
 
-  public createInMemoryCompilerHost(sourcePath: string, sourceContents: string): ts.CompilerHost {
+  public createInMemoryCompilerHost(sourcePath: string, sourceContents: string, currentDirectory?: string): ts.CompilerHost {
     const realHost = this.realHost;
     const sourceFile = ts.createSourceFile(sourcePath, sourceContents, ts.ScriptTarget.Latest);
 
     return {
       fileExists: filePath => filePath === sourcePath || realHost.fileExists(filePath),
       directoryExists: realHost.directoryExists && realHost.directoryExists.bind(realHost),
-      getCurrentDirectory: realHost.getCurrentDirectory.bind(realHost),
+      getCurrentDirectory: () => currentDirectory || realHost.getCurrentDirectory(),
       getDirectories: realHost.getDirectories && realHost.getDirectories.bind(realHost),
       getCanonicalFileName: fileName => realHost.getCanonicalFileName(fileName),
       getNewLine: realHost.getNewLine.bind(realHost),
@@ -30,7 +30,7 @@ export class TypeScriptCompiler {
     };
   }
 
-  public compileInMemory(filename: string, contents: string): CompilationResult {
+  public compileInMemory(filename: string, contents: string, currentDirectory?: string): CompilationResult {
     if (!filename.endsWith('.ts')) {
       // Necessary or the TypeScript compiler won't compile the file.
       filename += '.ts';
@@ -39,7 +39,7 @@ export class TypeScriptCompiler {
     const program = ts.createProgram({
       rootNames: [filename],
       options: STANDARD_COMPILER_OPTIONS,
-      host: this.createInMemoryCompilerHost(filename, contents),
+      host: this.createInMemoryCompilerHost(filename, contents, currentDirectory),
     });
 
     const rootFiles = program.getSourceFiles().filter(f => f.fileName === filename);
diff --git a/packages/jsii-rosetta/test/jsii/assemblies.test.ts b/packages/jsii-rosetta/test/jsii/assemblies.test.ts
index 68a1829abb..37731c0d58 100644
--- a/packages/jsii-rosetta/test/jsii/assemblies.test.ts
+++ b/packages/jsii-rosetta/test/jsii/assemblies.test.ts
@@ -1,6 +1,8 @@
+import mockfs = require('mock-fs');
 import spec = require('jsii-spec');
 import { allTypeScriptSnippets } from '../../lib/jsii/assemblies';
 import path = require('path');
+import { SnippetParameters } from '../../lib/snippet';
 
 test('Extract snippet from README', () => {
   const snippets = Array.from(allTypeScriptSnippets([{
@@ -104,6 +106,36 @@ test('Use fixture from example', () => {
   ].join('\n'));
 });
 
+
+test('Backwards compatibility with literate integ tests', () => {
+  mockfs({
+    '/package/test/integ.example.lit.ts': '# Some literate source file'
+  });
+
+  try {
+    const snippets = Array.from(allTypeScriptSnippets([{
+      assembly: fakeAssembly({
+        readme: {
+          markdown: [
+            'Before the example.',
+            '```ts lit=test/integ.example.lit.ts',
+            'someExample();',
+            '```',
+            'After the example.'
+          ].join('\n')
+        }
+      }),
+      directory: '/package'
+    }]));
+
+    expect(snippets[0].visibleSource).toEqual('someExample();');
+    expect(snippets[0].completeSource).toEqual('# Some literate source file');
+    expect(snippets[0].parameters && snippets[0].parameters[SnippetParameters.$COMPILATION_DIRECTORY]).toEqual('/package/test');
+  } finally {
+    mockfs.restore();
+  }
+});
+
 export function fakeAssembly(parts: Partial): spec.Assembly {
   return Object.assign({
     schema: spec.SchemaVersion.LATEST,
diff --git a/packages/jsii/lib/literate.ts b/packages/jsii/lib/literate.ts
index 1eaeef642f..dfda600d4d 100644
--- a/packages/jsii/lib/literate.ts
+++ b/packages/jsii/lib/literate.ts
@@ -62,9 +62,9 @@ import path = require('path');
 /**
  * Convert an annotated TypeScript source file to MarkDown
  */
-export function typescriptSourceToMarkdown(lines: string[]): string[] {
+export function typescriptSourceToMarkdown(lines: string[], codeBlockAnnotations: string[]): string[] {
   const relevantLines = findRelevantLines(lines);
-  const markdownLines = markdownify(relevantLines);
+  const markdownLines = markdownify(relevantLines, codeBlockAnnotations);
   return markdownLines;
 }
 
@@ -87,9 +87,11 @@ export async function includeAndRenderExamples(lines: string[], loader: FileLoad
     if (m) {
       // Found an include
       /* eslint-disable no-await-in-loop */
-      const source = await loader(m[2]);
+      const filename = m[2];
+      const source = await loader(filename);
       /* eslint-enable no-await-in-loop */
-      const imported = typescriptSourceToMarkdown(source);
+      // 'lit' source attribute will make snippet compiler know to extract the same source
+      const imported = typescriptSourceToMarkdown(source, [`lit=${filename}`]);
       ret.push(...imported);
     } else {
       ret.push(line);
@@ -165,7 +167,7 @@ function stripCommonIndent(lines: string[]): string[] {
 /**
  * Turn source lines into Markdown, starting in TypeScript mode
  */
-function markdownify(lines: string[]): string[] {
+function markdownify(lines: string[], codeBlockAnnotations: string[]): string[] {
   const typescriptLines: string[] = [];
   const ret: string[] = [];
 
@@ -185,11 +187,11 @@ function markdownify(lines: string[]): string[] {
   return ret;
 
   /**
-     * Flush typescript lines with a triple-backtick-ts block around it.
-     */
+   * Flush typescript lines with a triple-backtick-ts block around it.
+   */
   function flushTS() {
     if (typescriptLines.length !== 0) {
-      ret.push('```ts', ...typescriptLines, '```');
+      ret.push('```ts' + (codeBlockAnnotations.length > 0 ? ' ' + codeBlockAnnotations.join(' ') : ''), ...typescriptLines, '```');
       typescriptLines.splice(0); // Clear
     }
   }
diff --git a/packages/jsii/test/literate.test.ts b/packages/jsii/test/literate.test.ts
index 7855a942c5..ddd17554d9 100644
--- a/packages/jsii/test/literate.test.ts
+++ b/packages/jsii/test/literate.test.ts
@@ -115,11 +115,11 @@ test('can do example inclusion', async () => {
 
   expect(rendered).toEqual([
     'This is a preamble',
-    '```ts',
+    '```ts lit=test/something.lit.ts',
     'const x = 1;',
     '```',
     'This is how we print x',
-    '```ts',
+    '```ts lit=test/something.lit.ts',
     'console.log(x);',
     '```',
     'This is a postamble'
@@ -127,6 +127,6 @@ test('can do example inclusion', async () => {
 });
 
 function assertRendersTo(source: string[], expected: string[]) {
-  const rendered = typescriptSourceToMarkdown(source);
+  const rendered = typescriptSourceToMarkdown(source, []);
   expect(expected).toEqual(rendered);
 }

From b059c5c6f0cf67bbf28573cc7789afd925c227e6 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 18:06:42 +0100
Subject: [PATCH 12/19] Fix dependencies on packages outside the compile set

---
 packages/jsii-calc/test/assembly.jsii         |   2 +-
 packages/jsii-pacmak/bin/jsii-pacmak.ts       |   8 +-
 packages/jsii-pacmak/lib/builder.ts           |  18 +-
 packages/jsii-pacmak/lib/target.ts            |  64 ++++---
 packages/jsii-pacmak/lib/targets/dotnet.ts    | 134 ++++++++++++---
 .../lib/targets/dotnet/dotnetgenerator.ts     |   3 +-
 .../lib/targets/dotnet/dotnettyperesolver.ts  |  19 ++-
 .../lib/targets/dotnet/filegenerator.ts       |  28 ++--
 packages/jsii-pacmak/lib/targets/index.ts     |  17 +-
 packages/jsii-pacmak/lib/targets/java.ts      | 157 +++++++++---------
 packages/jsii-pacmak/lib/util.ts              |   7 +
 11 files changed, 291 insertions(+), 166 deletions(-)

diff --git a/packages/jsii-calc/test/assembly.jsii b/packages/jsii-calc/test/assembly.jsii
index 9d6371eefb..d441e0d331 100644
--- a/packages/jsii-calc/test/assembly.jsii
+++ b/packages/jsii-calc/test/assembly.jsii
@@ -10913,5 +10913,5 @@
     }
   },
   "version": "0.20.3",
-  "fingerprint": "1F+uskR3++T5mjRcWge9oG3H/jJvXm1C3IhR1AwsBTE="
+  "fingerprint": "BNhgAwuQfsvttsEliEal3aNXXFY0yeCY6npreFwNwpg="
 }
diff --git a/packages/jsii-pacmak/bin/jsii-pacmak.ts b/packages/jsii-pacmak/bin/jsii-pacmak.ts
index 896980874f..97817f2d5d 100644
--- a/packages/jsii-pacmak/bin/jsii-pacmak.ts
+++ b/packages/jsii-pacmak/bin/jsii-pacmak.ts
@@ -173,12 +173,12 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
 
   async function buildTargetsForLanguage(targetLanguage: string, modules: JsiiModule[], perLanguageDirectory: boolean) {
     // ``argv.target`` is guaranteed valid by ``yargs`` through the ``choices`` directive.
-    const builder = ALL_BUILDERS[targetLanguage as TargetName];
-    if (!builder) {
+    const factory = ALL_BUILDERS[targetLanguage as TargetName];
+    if (!factory) {
       throw new Error(`Unsupported target: '${targetLanguage}'`);
     }
 
-    await builder.buildModules(modules, {
+    await factory(modules, {
       clean: argv.clean,
       codeOnly: argv['code-only'],
       rosetta,
@@ -186,7 +186,7 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
       fingerprint: argv.fingerprint,
       arguments: argv,
       languageSubdirectory: perLanguageDirectory,
-    });
+    }).buildModules();
   }
 })().catch(err => {
   process.stderr.write(`${err.stack}\n`);
diff --git a/packages/jsii-pacmak/lib/builder.ts b/packages/jsii-pacmak/lib/builder.ts
index 5417d27258..33d4e15357 100644
--- a/packages/jsii-pacmak/lib/builder.ts
+++ b/packages/jsii-pacmak/lib/builder.ts
@@ -50,23 +50,27 @@ export interface BuildOptions {
  * Building can happen one target at a time, or multiple targets at a time.
  */
 export interface TargetBuilder {
-  buildModules(modules: JsiiModule[], options: BuildOptions): Promise;
+  buildModules(): Promise;
 }
 
 /**
  * Builds the targets for the given language sequentially
  */
 export class OneByOneBuilder implements TargetBuilder {
-  public constructor(private readonly targetName: string, private readonly targetConstructor: TargetConstructor) {
+  public constructor(
+    private readonly targetName: string,
+    private readonly targetConstructor: TargetConstructor,
+    private readonly modules: JsiiModule[],
+    private readonly options: BuildOptions) {
 
   }
 
-  public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise {
-    for (const module of modules) {
-      if (options.codeOnly) {
-        await this.generateModuleCode(module, options);
+  public async buildModules(): Promise {
+    for (const module of this.modules) {
+      if (this.options.codeOnly) {
+        await this.generateModuleCode(module, this.options);
       } else {
-        await this.buildModule(module, options);
+        await this.buildModule(module, this.options);
       }
     }
   }
diff --git a/packages/jsii-pacmak/lib/target.ts b/packages/jsii-pacmak/lib/target.ts
index a3818154b0..6c62e39799 100644
--- a/packages/jsii-pacmak/lib/target.ts
+++ b/packages/jsii-pacmak/lib/target.ts
@@ -72,35 +72,45 @@ export abstract class Target {
      * @param packageDir The directory of the package to resolve from.
      */
   protected async findLocalDepsOutput(rootPackageDir: string) {
-    const results = new Set();
-
-    async function recurse(this: Target, packageDir: string, isRoot: boolean) {
-      const pkg = await fs.readJson(path.join(packageDir, 'package.json'));
-
-      // no jsii or jsii.outdir - either a misconfigured jsii package or a non-jsii dependency. either way, we are done here.
-      if (!pkg.jsii || !pkg.jsii.outdir) {
-        return;
-      }
-
-      // if an output directory exists for this module, then we add it to our
-      // list of results (unless it's the root package, which we are currently building)
-      const outdir = path.join(packageDir, pkg.jsii.outdir, this.targetName);
-      if (results.has(outdir)) { return; } // Already visited, don't recurse again
-
-      if (!isRoot && await fs.pathExists(outdir)) {
-        logging.debug(`Found ${outdir} as a local dependency output`);
-        results.add(outdir);
-      }
-
-      // now descend to dependencies
-      await Promise.all(Object.keys(pkg.dependencies || {}).map(dependencyName => {
-        const dependencyDir = resolveDependencyDirectory(packageDir, dependencyName);
-        return recurse.call(this, dependencyDir, false);
-      }));
+    return findLocalBuildDirs(rootPackageDir, this.targetName);
+  }
+}
+
+/**
+   * Traverses the dep graph and returns a list of pacmak output directories
+   * available locally for this specific target. This allows target builds to
+   * take local dependencies in case a dependency is checked-out.
+   *
+   * @param packageDir The directory of the package to resolve from.
+   */
+export async function findLocalBuildDirs(rootPackageDir: string, targetName: string) {
+  const results = new Set();
+  await recurse(rootPackageDir, true);
+  return Array.from(results);
+
+  async function recurse(packageDir: string, isRoot: boolean) {
+    const pkg = await fs.readJson(path.join(packageDir, 'package.json'));
+
+    // no jsii or jsii.outdir - either a misconfigured jsii package or a non-jsii dependency. either way, we are done here.
+    if (!pkg.jsii || !pkg.jsii.outdir) {
+      return;
+    }
+
+    // if an output directory exists for this module, then we add it to our
+    // list of results (unless it's the root package, which we are currently building)
+    const outdir = path.join(packageDir, pkg.jsii.outdir, targetName);
+    if (results.has(outdir)) { return; } // Already visited, don't recurse again
+
+    if (!isRoot && await fs.pathExists(outdir)) {
+      logging.debug(`Found ${outdir} as a local dependency output`);
+      results.add(outdir);
     }
 
-    await recurse.call(this, rootPackageDir, true);
-    return Array.from(results);
+    // now descend to dependencies
+    await Promise.all(Object.keys(pkg.dependencies || {}).map(dependencyName => {
+      const dependencyDir = resolveDependencyDirectory(packageDir, dependencyName);
+      return recurse(dependencyDir, false);
+    }));
   }
 }
 
diff --git a/packages/jsii-pacmak/lib/targets/dotnet.ts b/packages/jsii-pacmak/lib/targets/dotnet.ts
index 72f4f1e364..74ce621ec0 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet.ts
@@ -2,8 +2,9 @@ import * as fs from 'fs-extra';
 import * as spec from 'jsii-spec';
 import * as path from 'path';
 import * as logging from '../logging';
-import { PackageInfo, Target } from '../target';
-import { shell, Scratch } from '../util';
+import xmlbuilder = require('xmlbuilder');
+import { PackageInfo, Target, TargetOptions, findLocalBuildDirs } from '../target';
+import { shell, Scratch, setExtend } from '../util';
 import { DotNetGenerator } from './dotnet/dotnetgenerator';
 import { TargetBuilder, BuildOptions } from '../builder';
 import { JsiiModule } from '../packaging';
@@ -14,13 +15,16 @@ import { JsiiModule } from '../packaging';
 export class DotnetBuilder implements TargetBuilder {
   private readonly targetName = 'dotnet';
 
-  public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise {
-    if (modules.length === 0) { return; }
+  public constructor(private readonly modules: JsiiModule[], private readonly options: BuildOptions) {
+  }
+
+  public async buildModules(): Promise {
+    if (this.modules.length === 0) { return; }
 
-    if (options.codeOnly) {
+    if (this.options.codeOnly) {
       // Simple, just generate code to respective output dirs
-      for (const module of modules) {
-        await this.generateModuleCode(module, options, module.outputDirectory);
+      for (const module of this.modules) {
+        await this.generateModuleCode(module, module.outputDirectory);
       }
       return;
     }
@@ -28,22 +32,24 @@ export class DotnetBuilder implements TargetBuilder {
     // Otherwise make a single tempdir to hold all sources, build them together and copy them back out
     const scratchDirs: Array> = [];
     try {
-      const tempSourceDir = await this.generateAggregateSourceDir(modules, options);
+      const tempSourceDir = await this.generateAggregateSourceDir(this.modules);
       scratchDirs.push(tempSourceDir);
 
       // Build solution
       logging.debug('Building .NET');
       await shell('dotnet', ['build', '-c', 'Release'], { cwd: tempSourceDir.directory });
 
-      await this.copyOutArtifacts(tempSourceDir.object, options);
-    } finally {
-      if (options.clean) {
+      await this.copyOutArtifacts(tempSourceDir.object);
+      if (this.options.clean) {
         await Scratch.cleanupAll(scratchDirs);
       }
+    } catch(e) {
+      logging.warn(`Exception occurred, not cleaning up ${scratchDirs.map(s => s.directory)}`);
+      throw e;
     }
   }
 
-  private async generateAggregateSourceDir(modules: JsiiModule[], options: BuildOptions): Promise> {
+  private async generateAggregateSourceDir(modules: JsiiModule[]): Promise> {
     return Scratch.make(async (tmpDir: string) => {
       logging.debug(`Generating aggregate .NET source dir at ${tmpDir}`);
 
@@ -52,7 +58,7 @@ export class DotnetBuilder implements TargetBuilder {
 
       for (const module of modules) {
         // Code generator will make its own subdirectory
-        await this.generateModuleCode(module, options, tmpDir);
+        await this.generateModuleCode(module, tmpDir);
         const loc = projectLocation(module);
         csProjs.push(loc.projectFile);
         ret.push({
@@ -65,14 +71,16 @@ export class DotnetBuilder implements TargetBuilder {
       await shell('dotnet', ['new', 'sln', '-n', 'JsiiBuild'], { cwd: tmpDir });
       await shell('dotnet', ['sln', 'add', ...csProjs], { cwd: tmpDir });
 
+      await this.generateNuGetConfigForLocalDeps(tmpDir);
+
       return ret;
     });
   }
 
-  private async copyOutArtifacts(packages: TemporaryDotnetPackage[], options: BuildOptions) {
+  private async copyOutArtifacts(packages: TemporaryDotnetPackage[]) {
     logging.debug('Copying out .NET artifacts');
     for (const pkg of packages) {
-      const targetDirectory = path.join(pkg.outputTargetDirectory, options.languageSubdirectory ? this.targetName : '');
+      const targetDirectory = path.join(pkg.outputTargetDirectory, this.options.languageSubdirectory ? this.targetName : '');
 
       await fs.mkdirp(targetDirectory);
       await fs.copy(pkg.artifactsDir, targetDirectory, { recursive: true });
@@ -82,22 +90,92 @@ export class DotnetBuilder implements TargetBuilder {
     }
   }
 
-  private async generateModuleCode(module: JsiiModule, options: BuildOptions, where: string): Promise {
-    const target = this.makeTarget(module, options);
+  private async generateModuleCode(module: JsiiModule, where: string): Promise {
+    const target = this.makeTarget(module);
     logging.debug(`Generating ${this.targetName} code into ${where}`);
     await target.generateCode(where, module.tarball);
   }
 
-  private makeTarget(module: JsiiModule, options: BuildOptions): Dotnet {
+  /**
+   * Write a NuGet.config that will include build directories for local packages not in the current build
+   *
+   */
+  private async generateNuGetConfigForLocalDeps(where: string): Promise {
+    // Traverse the dependency graph of this module and find all modules that have
+    // an /dotnet directory. We will add those as local NuGet repositories.
+    // This enables building against local modules.
+    const allDepsOutputDirs = new Set();
+    for (const module of this.modules) {
+      setExtend(allDepsOutputDirs, await findLocalBuildDirs(module.moduleDirectory, this.targetName));
+
+      // Also include output directory where we're building to, in case we build multiple packages into
+      // the same output directory.
+      allDepsOutputDirs.add(path.join(module.outputDirectory, this.options.languageSubdirectory ? this.targetName : ''));
+    }
+
+    const localRepos = Array.from(allDepsOutputDirs);
+
+    // If dotnet-jsonmodel is checked-out and we can find a local repository, add it to the list.
+    try {
+      /* eslint-disable @typescript-eslint/no-var-requires */
+      const jsiiDotNetJsonModel = require('jsii-dotnet-jsonmodel');
+      /* eslint-enable @typescript-eslint/no-var-requires */
+      const localDotNetJsonModel = jsiiDotNetJsonModel.repository;
+      if (await fs.pathExists(localDotNetJsonModel)) {
+        localRepos.push(localDotNetJsonModel);
+      }
+    } catch {
+      // Couldn't locate jsii-dotnet-jsonmodel, which is owkay!
+    }
+
+    // If dotnet-runtime is checked-out and we can find a local repository, add it to the list.
+    try {
+      /* eslint-disable @typescript-eslint/no-var-requires */
+      const jsiiDotNetRuntime = require('jsii-dotnet-runtime');
+      /* eslint-enable @typescript-eslint/no-var-requires */
+      const localDotNetRuntime = jsiiDotNetRuntime.repository;
+      if (await fs.pathExists(localDotNetRuntime)) {
+        localRepos.push(localDotNetRuntime);
+      }
+    } catch {
+      // Couldn't locate jsii-dotnet-runtime, which is owkay!
+    }
+
+    logging.debug('local NuGet repos:', localRepos);
+
+    // Construct XML content.
+    const configuration = xmlbuilder.create('configuration', { encoding: 'UTF-8' });
+    const packageSources = configuration.ele('packageSources');
+
+    const nugetOrgAdd = packageSources.ele('add');
+    nugetOrgAdd.att('key', 'nuget.org');
+    nugetOrgAdd.att('value', 'https://api.nuget.org/v3/index.json');
+    nugetOrgAdd.att('protocolVersion', '3');
+
+    localRepos.forEach((repo, index) => {
+      const add = packageSources.ele('add');
+      add.att('key', `local-${index}`);
+      add.att('value', path.join(repo));
+    });
+
+    const xml = configuration.end({ pretty: true });
+
+    // Write XML content to NuGet.config.
+    const filePath = path.join(where, 'NuGet.config');
+    logging.debug(`Generated ${filePath}`);
+    await fs.writeFile(filePath, xml);
+  }
+
+  private makeTarget(module: JsiiModule): Dotnet {
     return new Dotnet({
       targetName: this.targetName,
       packageDir: module.moduleDirectory,
       assembly: module.assembly,
-      fingerprint: options.fingerprint,
-      force: options.force,
-      arguments: options.arguments,
-      rosetta: options.rosetta,
-    });
+      fingerprint: this.options.fingerprint,
+      force: this.options.force,
+      arguments: this.options.arguments,
+      rosetta: this.options.rosetta,
+    }, this.modules.map(m => m.name));
   }
 }
 
@@ -152,11 +230,17 @@ export default class Dotnet extends Target {
     };
   }
 
-  protected readonly generator = new DotNetGenerator();
+  protected readonly generator: DotNetGenerator;
+
+  public constructor(options: TargetOptions, assembliesCurrentlyBeingCompiled: string[]) {
+    super(options);
+
+    this.generator = new DotNetGenerator(assembliesCurrentlyBeingCompiled);
+  }
 
   /* eslint-disable @typescript-eslint/require-await */
   public async build(_sourceDir: string, _outDir: string): Promise {
     throw new Error('Should not be called; use builder instead');
   }
   /* eslint-enable @typescript-eslint/require-await */
-}
+}
\ No newline at end of file
diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts
index 1811ba1df8..d556e50293 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts
@@ -28,7 +28,7 @@ export class DotNetGenerator extends Generator {
 
   private dotnetDocGenerator!: DotNetDocGenerator;
 
-  public constructor() {
+  public constructor(private readonly assembliesCurrentlyBeingCompiled: string[]) {
     super();
 
     // Override the openBlock to get a correct C# looking code block with the curly brace after the line
@@ -50,6 +50,7 @@ export class DotNetGenerator extends Generator {
     this.typeresolver = new DotNetTypeResolver(this.assembly,
       (fqn: string) => this.findModule(fqn),
       (fqn: string) => this.findType(fqn),
+      this.assembliesCurrentlyBeingCompiled
     );
 
     this.dotnetRuntimeGenerator = new DotNetRuntimeGenerator(this.code, this.typeresolver);
diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts
index d8e9c1702b..20e33536b3 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts
@@ -17,7 +17,9 @@ export class DotNetTypeResolver {
 
   public constructor(assembly: spec.Assembly,
     findModule: FindModuleCallback,
-    findType: FindTypeCallback) {
+    findType: FindTypeCallback,
+    private readonly assembliesCurrentlyBeingCompiled: string[]
+  ) {
     this.assembly = assembly;
     this.findModule = findModule;
     this.findType = findType;
@@ -58,10 +60,10 @@ export class DotNetTypeResolver {
         return `${actualNamespace}.${typeName}`;
       }
       return `${dotnetNamespace}.${type.namespace}.${typeName}`;
-    } 
+    }
     // When undefined, the type is located at the root of the assembly
     return `${dotnetNamespace}.${typeName}`;
-    
+
 
   }
 
@@ -82,7 +84,12 @@ export class DotNetTypeResolver {
           // suffix is guaranteed to start with a leading `-`
           version = `${depInfo.version}${suffix}`;
         }
-        this.namespaceDependencies.set(depName, new DotNetDependency(namespace, packageId, depName, version));
+        this.namespaceDependencies.set(depName, new DotNetDependency(
+          namespace,
+          packageId,
+          depName,
+          version,
+          this.assembliesCurrentlyBeingCompiled.includes(depName)));
       }
     }
   }
@@ -115,9 +122,9 @@ export class DotNetTypeResolver {
       return this.toNativeFqn(typeref.fqn);
     } else if (typeref.union) {
       return 'object';
-    } 
+    }
     throw new Error(`Invalid type reference: ${JSON.stringify(typeref)}`);
-    
+
   }
 
   /**
diff --git a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
index c9624b6a81..4e2147a613 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts
@@ -8,16 +8,12 @@ import { nextMajorVersion } from '../../util';
 
 // Represents a dependency in the dependency tree.
 export class DotNetDependency {
-  public namespace: string;
-  public packageId: string;
-  public fqn: string;
-  public version: string;
-
-  public constructor(namespace: string, packageId: string, fqn: string, version: string) {
-    this.namespace = namespace;
-    this.packageId = packageId;
-    this.fqn = fqn;
-    this.version = version;
+  public constructor(
+    public readonly namespace: string,
+    public readonly packageId: string,
+    public readonly fqn: string,
+    public readonly version: string,
+    public readonly partOfCompilation: boolean) {
   }
 }
 
@@ -106,10 +102,14 @@ export class FileGenerator {
     packageReference.att('Version', `[${jsiiVersion},${jsiiVersionNextMajor})`);
 
     dependencies.forEach((value: DotNetDependency) => {
-      const dependencyReference = itemGroup2.ele('ProjectReference');
-      // dependencyReference.att('Include', value.packageId);
-      // dependencyReference.att('Version', value.version);
-      dependencyReference.att('Include', `../${value.packageId}/${value.packageId}.csproj`);
+      if (value.partOfCompilation) {
+        const dependencyReference = itemGroup2.ele('ProjectReference');
+        dependencyReference.att('Include', `../${value.packageId}/${value.packageId}.csproj`);
+      } else {
+        const dependencyReference = itemGroup2.ele('PackageReference');
+        dependencyReference.att('Include', value.packageId);
+        dependencyReference.att('Version', value.version);
+      }
     });
 
     const xml = rootNode.end({ pretty: true, spaceBeforeSlash: true });
diff --git a/packages/jsii-pacmak/lib/targets/index.ts b/packages/jsii-pacmak/lib/targets/index.ts
index 7f45a08dbf..94ca1c82bc 100644
--- a/packages/jsii-pacmak/lib/targets/index.ts
+++ b/packages/jsii-pacmak/lib/targets/index.ts
@@ -1,19 +1,22 @@
-import { OneByOneBuilder, TargetBuilder } from '../builder';
+import { OneByOneBuilder, TargetBuilder, BuildOptions } from '../builder';
 
 import { DotnetBuilder } from './dotnet';
 import { JavaBuilder } from './java';
 import JavaScript from './js';
 import Python from './python';
 import Ruby from './ruby';
+import { JsiiModule } from '../packaging';
 
 export type TargetName = 'dotnet' | 'java' | 'js' | 'python' | 'ruby';
+export type BuilderFactory = (modules: JsiiModule[], options: BuildOptions) => TargetBuilder;
 
-export const ALL_BUILDERS: {[key in TargetName]: TargetBuilder} = {
-  dotnet: new DotnetBuilder(),
-  java: new JavaBuilder(),
-  js: new OneByOneBuilder('js', JavaScript),
-  python: new OneByOneBuilder('python', Python),
-  ruby: new OneByOneBuilder('ruby', Ruby),
+
+export const ALL_BUILDERS: {[key in TargetName]: BuilderFactory} = {
+  dotnet: (ms, o) => new DotnetBuilder(ms, o),
+  java: (ms, o) => new JavaBuilder(ms, o),
+  js: (ms, o) => new OneByOneBuilder('js', JavaScript, ms, o),
+  python: (ms, o) => new OneByOneBuilder('python', Python, ms, o),
+  ruby: (ms, o) => new OneByOneBuilder('ruby', Ruby, ms, o),
 };
 
 
diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts
index 8a9a07b105..e41e0b8788 100644
--- a/packages/jsii-pacmak/lib/targets/java.ts
+++ b/packages/jsii-pacmak/lib/targets/java.ts
@@ -8,8 +8,8 @@ import xmlbuilder = require('xmlbuilder');
 import { Generator } from '../generator';
 import logging = require('../logging');
 import { md2html } from '../markdown';
-import { PackageInfo, Target } from '../target';
-import { shell, Scratch, slugify } from '../util';
+import { PackageInfo, Target, findLocalBuildDirs } from '../target';
+import { shell, Scratch, slugify, setExtend } from '../util';
 import { VERSION, VERSION_DESC } from '../version';
 import { TargetBuilder, BuildOptions } from '../builder';
 import { JsiiModule } from '../packaging';
@@ -32,13 +32,16 @@ const BUILDER_CLASS_NAME = 'Builder';
 export class JavaBuilder implements TargetBuilder {
   private readonly targetName = 'java';
 
-  public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise {
-    if (modules.length === 0) { return; }
+  public constructor(private readonly modules: JsiiModule[], private readonly options: BuildOptions) {
+  }
+
+  public async buildModules(): Promise {
+    if (this.modules.length === 0) { return; }
 
-    if (options.codeOnly) {
+    if (this.options.codeOnly) {
       // Simple, just generate code to respective output dirs
-      for (const module of modules) {
-        await this.generateModuleCode(module, options, module.outputDirectory);
+      for (const module of this.modules) {
+        await this.generateModuleCode(module, this.options, module.outputDirectory);
       }
       return;
     }
@@ -46,24 +49,26 @@ export class JavaBuilder implements TargetBuilder {
     // Otherwise make a single tempdir to hold all sources, build them together and copy them back out
     const scratchDirs: Array> = [];
     try {
-      const tempSourceDir = await this.generateAggregateSourceDir(modules, options);
+      const tempSourceDir = await this.generateAggregateSourceDir(this.modules, this.options);
       scratchDirs.push(tempSourceDir);
 
       // Need any old module object to make a target to be able to invoke build, though none of its settings
       // will be used.
-      const target = this.makeTarget(modules[0], options);
+      const target = this.makeTarget(this.modules[0], this.options);
       const tempOutputDir = await Scratch.make(async dir => {
         logging.debug(`Building Java code to ${dir}`);
         await target.build(tempSourceDir.directory, dir);
       });
       scratchDirs.push(tempOutputDir);
 
-      await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object, options);
+      await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object, this.options);
 
-    } finally {
-      if (options.clean) {
+      if (this.options.clean) {
         await Scratch.cleanupAll(scratchDirs);
       }
+    } catch(e) {
+      logging.warn(`Exception occurred, not cleaning up ${scratchDirs.map(s => s.directory)}`);
+      throw e;
     }
   }
 
@@ -90,6 +95,7 @@ export class JavaBuilder implements TargetBuilder {
       }
 
       await this.generateAggregatePom(tmpDir, ret.map(m => m.relativeSourceDir));
+      await this.generateMavenSettingsForLocalDeps(tmpDir);
 
       return ret;
     });
@@ -142,6 +148,70 @@ export class JavaBuilder implements TargetBuilder {
     }
   }
 
+  /**
+   * Generates maven settings file for this build.
+   * @param where The generated sources directory. This is where user.xml will be placed.
+   * @param currentOutputDirectory The current output directory. Will be added as a local maven repo.
+   */
+  private async generateMavenSettingsForLocalDeps(where: string) {
+    const filePath = path.join(where, 'user.xml');
+
+    // traverse the dep graph of this module and find all modules that have
+    // an /java directory. we will add those as local maven
+    // repositories which will resolve instead of Maven Central for those
+    // module. this enables building against local modules (i.e. in lerna
+    // repositories or linked modules).
+    const allDepsOutputDirs = new Set();
+    for (const module of this.modules) {
+      setExtend(allDepsOutputDirs, await findLocalBuildDirs(module.moduleDirectory, this.targetName));
+
+      // Also include output directory where we're building to, in case we build multiple packages into
+      // the same output directory.
+      allDepsOutputDirs.add(path.join(module.outputDirectory, this.options.languageSubdirectory ? this.targetName : ''));
+    }
+
+    const localRepos = Array.from(allDepsOutputDirs);
+
+    // if java-runtime is checked-out and we can find a local repository,
+    // add it to the list.
+    const localJavaRuntime = await findJavaRuntimeLocalRepository();
+    if (localJavaRuntime) {
+      localRepos.push(localJavaRuntime);
+    }
+
+    logging.debug('local maven repos:', localRepos);
+
+    const profileName = 'local-jsii-modules';
+    const settings = xmlbuilder.create({
+      settings: {
+        '@xmlns': 'http://maven.apache.org/POM/4.0.0',
+        '@xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
+        '@xsi:schemaLocation': 'http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd',
+        '#comment': [
+          `Generated by jsii-pacmak@${VERSION_DESC} on ${new Date().toISOString()}`,
+        ],
+        'profiles': {
+          profile: {
+            id: profileName,
+            repositories: {
+              repository: localRepos.map((repo, index) => ({
+                id: `local${index}`,
+                url: `file://${repo}`
+              }))
+            }
+          }
+        },
+        'activeProfiles': {
+          activeProfile: profileName
+        }
+      }
+    }, { encoding: 'UTF-8' }).end({ pretty: true });
+
+    logging.debug(`Generated ${filePath}`);
+    await fs.writeFile(filePath, settings);
+    return filePath;
+  }
+
   private makeTarget(module: JsiiModule, options: BuildOptions): Target {
     return new Java({
       targetName: this.targetName,
@@ -227,10 +297,9 @@ export default class Java extends Target {
       mvnArguments.push(this.arguments[arg].toString());
     }
 
-    const userXml = await this.generateMavenSettingsForLocalDeps(sourceDir, outDir);
     await shell(
       'mvn',
-      [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=${userXml}`],
+      [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=user.xml`],
       {
         cwd: sourceDir,
         env: {
@@ -241,66 +310,6 @@ export default class Java extends Target {
       }
     );
   }
-
-  /**
-     * Generates maven settings file for this build.
-     * @param sourceDir The generated sources directory. This is where user.xml will be placed.
-     * @param currentOutputDirectory The current output directory. Will be added as a local maven repo.
-     */
-  private async generateMavenSettingsForLocalDeps(sourceDir: string, currentOutputDirectory: string) {
-    const filePath = path.join(sourceDir, 'user.xml');
-
-    // traverse the dep graph of this module and find all modules that have
-    // an /java directory. we will add those as local maven
-    // repositories which will resolve instead of Maven Central for those
-    // module. this enables building against local modules (i.e. in lerna
-    // repositories or linked modules).
-    const localRepos = await this.findLocalDepsOutput(this.packageDir);
-
-    // add the current output directory as a local repo as well for the case
-    // where we build multiple packages into the same output.
-    localRepos.push(currentOutputDirectory);
-
-    // if java-runtime is checked-out and we can find a local repository,
-    // add it to the list.
-    const localJavaRuntime = await findJavaRuntimeLocalRepository();
-    if (localJavaRuntime) {
-      localRepos.push(localJavaRuntime);
-    }
-
-    logging.debug('local maven repos:', localRepos);
-
-    const profileName = 'local-jsii-modules';
-    const settings = xmlbuilder.create({
-      settings: {
-        '@xmlns': 'http://maven.apache.org/POM/4.0.0',
-        '@xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
-        '@xsi:schemaLocation': 'http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd',
-        '#comment': [
-          `Generated by jsii-pacmak@${VERSION_DESC} on ${new Date().toISOString()}`,
-        ],
-        'profiles': {
-          profile: {
-            id: profileName,
-            repositories: {
-              repository: localRepos.map((repo, index) => ({
-                id: `local${index}`,
-                url: `file://${repo}`
-              }))
-            }
-          }
-        },
-        'activeProfiles': {
-          activeProfile: profileName
-        }
-      }
-    }, { encoding: 'UTF-8' }).end({ pretty: true });
-
-    logging.debug(`Generated ${filePath}`);
-    await fs.writeFile(filePath, settings);
-    return filePath;
-  }
-
 }
 
 // ##################
diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts
index 132173f470..0b13002e46 100644
--- a/packages/jsii-pacmak/lib/util.ts
+++ b/packages/jsii-pacmak/lib/util.ts
@@ -154,3 +154,10 @@ export function nextMajorVersion(version: string): string {
   }
   return v.inc('patch').version;
 }
+
+
+export function setExtend(xs: Set, els: Iterable) {
+  for (const el of els) {
+    xs.add(el);
+  }
+}
\ No newline at end of file

From c489266d650c18df001918bb2a59629908275890 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 18:13:15 +0100
Subject: [PATCH 13/19] Use the right output dir for source-only mode

---
 packages/jsii-pacmak/lib/targets/dotnet.ts        | 13 ++++++++++---
 packages/jsii-pacmak/lib/targets/java.ts          | 15 +++++++++++----
 .../Amazon.JSII.Tests.CalculatorPackageId/.jsii   |  2 +-
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/packages/jsii-pacmak/lib/targets/dotnet.ts b/packages/jsii-pacmak/lib/targets/dotnet.ts
index 74ce621ec0..d569bc14bd 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet.ts
@@ -24,7 +24,7 @@ export class DotnetBuilder implements TargetBuilder {
     if (this.options.codeOnly) {
       // Simple, just generate code to respective output dirs
       for (const module of this.modules) {
-        await this.generateModuleCode(module, module.outputDirectory);
+        await this.generateModuleCode(module, this.outputDir(module.outputDirectory));
       }
       return;
     }
@@ -80,7 +80,7 @@ export class DotnetBuilder implements TargetBuilder {
   private async copyOutArtifacts(packages: TemporaryDotnetPackage[]) {
     logging.debug('Copying out .NET artifacts');
     for (const pkg of packages) {
-      const targetDirectory = path.join(pkg.outputTargetDirectory, this.options.languageSubdirectory ? this.targetName : '');
+      const targetDirectory = this.outputDir(pkg.outputTargetDirectory);
 
       await fs.mkdirp(targetDirectory);
       await fs.copy(pkg.artifactsDir, targetDirectory, { recursive: true });
@@ -96,6 +96,13 @@ export class DotnetBuilder implements TargetBuilder {
     await target.generateCode(where, module.tarball);
   }
 
+  /**
+   * Decide whether or not to append 'dotnet' to the given output directory
+   */
+  private outputDir(declaredDir: string) {
+    return this.options.languageSubdirectory ? path.join(declaredDir, this.targetName) : declaredDir;
+  }
+
   /**
    * Write a NuGet.config that will include build directories for local packages not in the current build
    *
@@ -110,7 +117,7 @@ export class DotnetBuilder implements TargetBuilder {
 
       // Also include output directory where we're building to, in case we build multiple packages into
       // the same output directory.
-      allDepsOutputDirs.add(path.join(module.outputDirectory, this.options.languageSubdirectory ? this.targetName : ''));
+      allDepsOutputDirs.add(this.outputDir(module.outputDirectory));
     }
 
     const localRepos = Array.from(allDepsOutputDirs);
diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts
index e41e0b8788..56fd2fe993 100644
--- a/packages/jsii-pacmak/lib/targets/java.ts
+++ b/packages/jsii-pacmak/lib/targets/java.ts
@@ -41,7 +41,7 @@ export class JavaBuilder implements TargetBuilder {
     if (this.options.codeOnly) {
       // Simple, just generate code to respective output dirs
       for (const module of this.modules) {
-        await this.generateModuleCode(module, this.options, module.outputDirectory);
+        await this.generateModuleCode(module, this.options, this.outputDir(module.outputDirectory));
       }
       return;
     }
@@ -61,7 +61,7 @@ export class JavaBuilder implements TargetBuilder {
       });
       scratchDirs.push(tempOutputDir);
 
-      await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object, this.options);
+      await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object);
 
       if (this.options.clean) {
         await Scratch.cleanupAll(scratchDirs);
@@ -128,7 +128,7 @@ export class JavaBuilder implements TargetBuilder {
     await fs.writeFile(path.join(where, 'pom.xml'), aggregatePom);
   }
 
-  private async copyOutArtifacts(artifactsRoot: string, packages: TemporaryJavaPackage[], options: BuildOptions) {
+  private async copyOutArtifacts(artifactsRoot: string, packages: TemporaryJavaPackage[]) {
     logging.debug('Copying out Java artifacts');
     // The artifacts directory looks like this:
     //  /tmp/XXX/software/amazon/awscdk/something/v1.2.3
@@ -141,13 +141,20 @@ export class JavaBuilder implements TargetBuilder {
 
     for (const pkg of packages) {
       const artifactsSource = path.join(artifactsRoot, pkg.relativeArtifactsDir);
-      const artifactsDest = path.join(pkg.outputTargetDirectory, options.languageSubdirectory ? this.targetName : '', pkg.relativeArtifactsDir);
+      const artifactsDest = path.join(this.outputDir(pkg.outputTargetDirectory), pkg.relativeArtifactsDir);
 
       await fs.mkdirp(artifactsDest);
       await fs.copy(artifactsSource, artifactsDest, { recursive: true });
     }
   }
 
+  /**
+   * Decide whether or not to append 'java' to the given output directory
+   */
+  private outputDir(declaredDir: string) {
+    return this.options.languageSubdirectory ? path.join(declaredDir, this.targetName) : declaredDir;
+  }
+
   /**
    * Generates maven settings file for this build.
    * @param where The generated sources directory. This is where user.xml will be placed.
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii
index 9d6371eefb..d441e0d331 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii
@@ -10913,5 +10913,5 @@
     }
   },
   "version": "0.20.3",
-  "fingerprint": "1F+uskR3++T5mjRcWge9oG3H/jJvXm1C3IhR1AwsBTE="
+  "fingerprint": "BNhgAwuQfsvttsEliEal3aNXXFY0yeCY6npreFwNwpg="
 }

From d30c3eff34fb31266d82e95c0468ef4ac1b98752 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Tue, 12 Nov 2019 18:32:22 +0100
Subject: [PATCH 14/19] Add test for different pacmak build modes

---
 packages/jsii-calc-lib/.npmignore        |  4 +++
 packages/jsii-pacmak/lib/targets/java.ts |  2 +-
 packages/jsii-pacmak/test/build-test.sh  | 33 ++++++++++++++++++++++--
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/packages/jsii-calc-lib/.npmignore b/packages/jsii-calc-lib/.npmignore
index d2284ef6ef..5a7c6524f1 100644
--- a/packages/jsii-calc-lib/.npmignore
+++ b/packages/jsii-calc-lib/.npmignore
@@ -6,3 +6,7 @@
 
 # Include .jsii
 !.jsii
+
+
+# Exclude jsii outdir
+dist
diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts
index 56fd2fe993..5d92a6de63 100644
--- a/packages/jsii-pacmak/lib/targets/java.ts
+++ b/packages/jsii-pacmak/lib/targets/java.ts
@@ -306,7 +306,7 @@ export default class Java extends Target {
 
     await shell(
       'mvn',
-      [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=user.xml`],
+      [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, '--settings=user.xml'],
       {
         cwd: sourceDir,
         env: {
diff --git a/packages/jsii-pacmak/test/build-test.sh b/packages/jsii-pacmak/test/build-test.sh
index 2eab08894d..8585c49838 100755
--- a/packages/jsii-pacmak/test/build-test.sh
+++ b/packages/jsii-pacmak/test/build-test.sh
@@ -2,7 +2,36 @@
 set -euo pipefail
 cd $(dirname $0)
 
-outdir=$(mktemp -d)
-../bin/jsii-pacmak -o ${outdir} --recurse ../../jsii-calc -vv
+# Test various build modes for jsii-pacmak (these are all ways in
+# which users can decide to run jsii-pacmak)
+#
+# The following list of packages must be toposorted, like a monorepo
+# manager would order the individual builds.
+packagedirs="\
+    ../../jsii-calc-base-of-base \
+    ../../jsii-calc-base \
+    ../../jsii-calc-lib \
+    ../../jsii-calc \
+    "
+clean_dists() {
+    for dir in $packagedirs; do rm -rf $dir/dist; done
+}
 
+# Single target, recursive build to a certain location
+outdir=$(mktemp -d)
+clean_dists
+echo "Testing SINGLE TARGET, RECURSIVE build."
+../bin/jsii-pacmak -o ${outdir} --recurse ../../jsii-calc
 rm -rf ${outdir}
+
+# Multiple targets, build one-by-one into own directory
+clean_dists
+echo "Testing ONE-BY-ONE build."
+for dir in $packagedirs; do
+    ../bin/jsii-pacmak $dir -vv
+done
+
+# Multiple targets, build all at once into own directory
+clean_dists
+echo "Testing ALL-AT-ONCE build."
+../bin/jsii-pacmak $packagedirs

From 9e97489eac1678eb77b1718756a173f15a8caec6 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Wed, 13 Nov 2019 10:21:25 +0100
Subject: [PATCH 15/19] Only include existing directories

---
 packages/jsii-pacmak/lib/targets/dotnet.ts | 19 ++++++++-----------
 packages/jsii-pacmak/lib/util.ts           | 10 ++++++++++
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/packages/jsii-pacmak/lib/targets/dotnet.ts b/packages/jsii-pacmak/lib/targets/dotnet.ts
index d569bc14bd..1e8b401f5f 100644
--- a/packages/jsii-pacmak/lib/targets/dotnet.ts
+++ b/packages/jsii-pacmak/lib/targets/dotnet.ts
@@ -4,7 +4,7 @@ import * as path from 'path';
 import * as logging from '../logging';
 import xmlbuilder = require('xmlbuilder');
 import { PackageInfo, Target, TargetOptions, findLocalBuildDirs } from '../target';
-import { shell, Scratch, setExtend } from '../util';
+import { shell, Scratch, setExtend, filterAsync } from '../util';
 import { DotNetGenerator } from './dotnet/dotnetgenerator';
 import { TargetBuilder, BuildOptions } from '../builder';
 import { JsiiModule } from '../packaging';
@@ -127,10 +127,7 @@ export class DotnetBuilder implements TargetBuilder {
       /* eslint-disable @typescript-eslint/no-var-requires */
       const jsiiDotNetJsonModel = require('jsii-dotnet-jsonmodel');
       /* eslint-enable @typescript-eslint/no-var-requires */
-      const localDotNetJsonModel = jsiiDotNetJsonModel.repository;
-      if (await fs.pathExists(localDotNetJsonModel)) {
-        localRepos.push(localDotNetJsonModel);
-      }
+      localRepos.push(jsiiDotNetJsonModel.repository);
     } catch {
       // Couldn't locate jsii-dotnet-jsonmodel, which is owkay!
     }
@@ -140,15 +137,15 @@ export class DotnetBuilder implements TargetBuilder {
       /* eslint-disable @typescript-eslint/no-var-requires */
       const jsiiDotNetRuntime = require('jsii-dotnet-runtime');
       /* eslint-enable @typescript-eslint/no-var-requires */
-      const localDotNetRuntime = jsiiDotNetRuntime.repository;
-      if (await fs.pathExists(localDotNetRuntime)) {
-        localRepos.push(localDotNetRuntime);
-      }
+      localRepos.push(jsiiDotNetRuntime.repository);
     } catch {
       // Couldn't locate jsii-dotnet-runtime, which is owkay!
     }
 
-    logging.debug('local NuGet repos:', localRepos);
+    // Filter out nonexistant directories, .NET will be unhappy if paths don't exist
+    const existingLocalRepos = await filterAsync(localRepos, fs.pathExists);
+
+    logging.debug('local NuGet repos:', existingLocalRepos);
 
     // Construct XML content.
     const configuration = xmlbuilder.create('configuration', { encoding: 'UTF-8' });
@@ -159,7 +156,7 @@ export class DotnetBuilder implements TargetBuilder {
     nugetOrgAdd.att('value', 'https://api.nuget.org/v3/index.json');
     nugetOrgAdd.att('protocolVersion', '3');
 
-    localRepos.forEach((repo, index) => {
+    existingLocalRepos.forEach((repo, index) => {
       const add = packageSources.ele('add');
       add.att('key', `local-${index}`);
       add.att('value', path.join(repo));
diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts
index 0b13002e46..49f23bb112 100644
--- a/packages/jsii-pacmak/lib/util.ts
+++ b/packages/jsii-pacmak/lib/util.ts
@@ -160,4 +160,14 @@ export function setExtend(xs: Set, els: Iterable) {
   for (const el of els) {
     xs.add(el);
   }
+}
+
+export async function filterAsync(xs: Array, pred: (x: A) => Promise): Promise> {
+  const ret = new Array();
+  for (const x of xs) {
+    if (await pred(x)) {
+      ret.push(x);
+    }
+  }
+  return ret;
 }
\ No newline at end of file

From 4eb11e37269e1843281742a178916422dbf89278 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Wed, 13 Nov 2019 11:32:57 +0100
Subject: [PATCH 16/19] Satisfy eslint

---
 packages/jsii-pacmak/lib/util.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts
index 49f23bb112..33c78ea52b 100644
--- a/packages/jsii-pacmak/lib/util.ts
+++ b/packages/jsii-pacmak/lib/util.ts
@@ -162,7 +162,7 @@ export function setExtend(xs: Set, els: Iterable) {
   }
 }
 
-export async function filterAsync(xs: Array, pred: (x: A) => Promise): Promise> {
+export async function filterAsync(xs: A[], pred: (x: A) => Promise): Promise {
   const ret = new Array();
   for (const x of xs) {
     if (await pred(x)) {

From 42512e2d6a67f3d18d7c77f5cbb685d863d91b61 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Wed, 13 Nov 2019 13:47:14 +0100
Subject: [PATCH 17/19] Handle @ signs in examples

---
 packages/jsii/lib/docs.ts       | 69 +++++++++++++++++++++++++++------
 packages/jsii/test/docs.test.ts | 18 +++++++++
 2 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/packages/jsii/lib/docs.ts b/packages/jsii/lib/docs.ts
index 7fce0d3650..969b552203 100644
--- a/packages/jsii/lib/docs.ts
+++ b/packages/jsii/lib/docs.ts
@@ -33,9 +33,27 @@
 import spec = require('jsii-spec');
 import ts = require('typescript');
 
+/**
+ * Tags that we recognize
+ */
+enum DocTag {
+  PARAM = 'param',
+  DEFAULT = 'default',
+  DEFAULT_VALUE = 'defaultValue',
+  DEPRECATED = 'deprecated',
+  RETURNS = 'returns',
+  RETURN = 'return',
+  STABLE = 'stable',
+  EXPERIMENTAL = 'experimental',
+  SEE = 'see',
+  SUBCLASSABLE = 'subclassable',
+  EXAMPLE = 'example',
+  STABILITY = 'stability',
+}
+
 export function parseSymbolDocumentation(sym: ts.Symbol, typeChecker: ts.TypeChecker): DocsParsingResult {
   const comment = ts.displayPartsToString(sym.getDocumentationComment(typeChecker)).trim();
-  const tags = sym.getJsDocTags();
+  const tags = reabsorbExampleTags(sym.getJsDocTags());
 
   // Right here we'll just guess that the first declaration site is the most important one.
   return parseDocParts(comment, tags);
@@ -47,7 +65,7 @@ export function parseSymbolDocumentation(sym: ts.Symbol, typeChecker: ts.TypeChe
 export function getReferencedDocParams(sym: ts.Symbol): string[] {
   const ret = new Array();
   for (const tag of sym.getJsDocTags()) {
-    if (tag.name === 'param') {
+    if (tag.name === DocTag.PARAM) {
       const parts = (tag.text || '').split(' ');
       ret.push(parts[0]);
     }
@@ -64,7 +82,7 @@ function parseDocParts(comments: string | undefined, tags: ts.JSDocTagInfo[]): D
   const tagNames = new Map();
   for (const tag of tags) {
     // 'param' gets parsed as a tag and as a comment for a method
-    if (tag.name !== 'param') { tagNames.set(tag.name, tag.text); }
+    if (tag.name !== DocTag.PARAM) { tagNames.set(tag.name, tag.text); }
   }
 
   function eatTag(...names: string[]): string | undefined {
@@ -78,17 +96,17 @@ function parseDocParts(comments: string | undefined, tags: ts.JSDocTagInfo[]): D
     return undefined;
   }
 
-  docs.default = eatTag('default', 'defaultValue');
-  docs.deprecated = eatTag('deprecated');
-  docs.example = eatTag('example');
-  docs.returns = eatTag('returns', 'return');
-  docs.see = eatTag('see');
-  docs.subclassable = eatTag('subclassable') !== undefined ? true : undefined;
+  docs.default = eatTag(DocTag.DEFAULT, DocTag.DEFAULT_VALUE);
+  docs.deprecated = eatTag(DocTag.DEPRECATED);
+  docs.example = eatTag(DocTag.EXAMPLE);
+  docs.returns = eatTag(DocTag.RETURNS, DocTag.RETURN);
+  docs.see = eatTag(DocTag.SEE);
+  docs.subclassable = eatTag(DocTag.SUBCLASSABLE) !== undefined ? true : undefined;
 
-  docs.stability = parseStability(eatTag('stability'), diagnostics);
+  docs.stability = parseStability(eatTag(DocTag.STABILITY), diagnostics);
   //  @experimental is a shorthand for '@stability experimental', same for '@stable'
-  const experimental = eatTag('experimental') !== undefined;
-  const stable = eatTag('stable') !== undefined;
+  const experimental = eatTag(DocTag.EXPERIMENTAL) !== undefined;
+  const stable = eatTag(DocTag.STABLE) !== undefined;
   // Can't combine them
   if (countBools(docs.stability !== undefined, experimental, stable) > 1) {
     diagnostics.push('Use only one of @stability, @experimental or @stable');
@@ -205,3 +223,30 @@ function parseStability(s: string | undefined, diagnostics: string[]): spec.Stab
   diagnostics.push(`Unrecognized @stability: '${s}'`);
   return undefined;
 }
+
+
+/**
+ * Unrecognized tags that follow an '@ example' tag will be absorbed back into the example value
+ *
+ * The TypeScript parser by itself is naive and will start parsing a new tag there.
+ *
+ * We do this until we encounter a supported @ keyword.
+ */
+function reabsorbExampleTags(tags: ts.JSDocTagInfo[]): ts.JSDocTagInfo[] {
+  const recognizedTags: string[] = Object.values(DocTag);
+  const ret = [...tags];
+
+  let i = 0;
+  while (i < ret.length) {
+    if (ret[i].name === 'example') {
+      while (i + 1 < ret.length && !recognizedTags.includes(ret[i + 1].name)) {
+        // Incorrectly classified as @tag, absorb back into example
+        ret[i].text += '@' + ret[i + 1].name + ret[i + 1].text;
+        ret.splice(i + 1, 1);
+      }
+    }
+    i++;
+  }
+
+  return ret;
+}
diff --git a/packages/jsii/test/docs.test.ts b/packages/jsii/test/docs.test.ts
index d14f184d8e..394638b854 100644
--- a/packages/jsii/test/docs.test.ts
+++ b/packages/jsii/test/docs.test.ts
@@ -333,3 +333,21 @@ test('stability is inherited from parent type', async () => {
     expect(method.docs!.stability).toBe(stability);
   }
 });
+
+// ----------------------------------------------------------------------
+test('@example can contain @ sign', async () => {
+const assembly = await compile(`
+  /**
+   * An IAM role to associate with the instance profile assigned to this Auto Scaling Group.
+   *
+   * @example
+   *
+   * import x = require('\@banana');
+   */
+  export class Foo {
+  }
+`);
+
+const classType = assembly.types!['testpkg.Foo'] as spec.ClassType;
+expect(classType.docs!.example).toBe('import x = require(\'@banana\');');
+});
\ No newline at end of file

From 9f597f37fb05f039dd6ee91409370e37739dfbea Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Wed, 13 Nov 2019 14:04:56 +0100
Subject: [PATCH 18/19] Satisfy eslint

---
 packages/jsii/lib/docs.ts       |  2 +-
 packages/jsii/test/docs.test.ts | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/packages/jsii/lib/docs.ts b/packages/jsii/lib/docs.ts
index 969b552203..34b7cbe980 100644
--- a/packages/jsii/lib/docs.ts
+++ b/packages/jsii/lib/docs.ts
@@ -241,7 +241,7 @@ function reabsorbExampleTags(tags: ts.JSDocTagInfo[]): ts.JSDocTagInfo[] {
     if (ret[i].name === 'example') {
       while (i + 1 < ret.length && !recognizedTags.includes(ret[i + 1].name)) {
         // Incorrectly classified as @tag, absorb back into example
-        ret[i].text += '@' + ret[i + 1].name + ret[i + 1].text;
+        ret[i].text += `@${ret[i + 1].name}${ret[i + 1].text}`;
         ret.splice(i + 1, 1);
       }
     }
diff --git a/packages/jsii/test/docs.test.ts b/packages/jsii/test/docs.test.ts
index 394638b854..b1a7f7782d 100644
--- a/packages/jsii/test/docs.test.ts
+++ b/packages/jsii/test/docs.test.ts
@@ -336,18 +336,18 @@ test('stability is inherited from parent type', async () => {
 
 // ----------------------------------------------------------------------
 test('@example can contain @ sign', async () => {
-const assembly = await compile(`
-  /**
-   * An IAM role to associate with the instance profile assigned to this Auto Scaling Group.
-   *
-   * @example
-   *
-   * import x = require('\@banana');
-   */
-  export class Foo {
-  }
-`);
+  const assembly = await compile(`
+    /**
+     * An IAM role to associate with the instance profile assigned to this Auto Scaling Group.
+     *
+     * @example
+     *
+     * import x = require('@banana');
+     */
+    export class Foo {
+    }
+  `);
 
-const classType = assembly.types!['testpkg.Foo'] as spec.ClassType;
-expect(classType.docs!.example).toBe('import x = require(\'@banana\');');
+  const classType = assembly.types!['testpkg.Foo'] as spec.ClassType;
+  expect(classType.docs!.example).toBe('import x = require(\'@banana\');');
 });
\ No newline at end of file

From a63e55e53320352538c868cd4be8d6ebc7b75925 Mon Sep 17 00:00:00 2001
From: Rico Huijbers 
Date: Wed, 13 Nov 2019 14:05:03 +0100
Subject: [PATCH 19/19] Don't report diagnostics originating from hidden source

---
 packages/jsii-rosetta/lib/o-tree.ts    |  8 ++++++--
 packages/jsii-rosetta/lib/renderer.ts  |  4 +++-
 packages/jsii-rosetta/lib/translate.ts | 11 +++++++++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/packages/jsii-rosetta/lib/o-tree.ts b/packages/jsii-rosetta/lib/o-tree.ts
index 127bdf44db..6ca3feb8f1 100644
--- a/packages/jsii-rosetta/lib/o-tree.ts
+++ b/packages/jsii-rosetta/lib/o-tree.ts
@@ -183,7 +183,7 @@ export class OTreeSink {
 
   public renderingForSpan(span?: Span): boolean {
     if (span && this.options.visibleSpans) {
-      this.rendering = this.options.visibleSpans.some(v => inside(span, v));
+      this.rendering = this.options.visibleSpans.some(v => spanInside(span, v));
     }
     return this.rendering;
   }
@@ -247,6 +247,10 @@ export interface Span {
   end: number
 }
 
-function inside(a: Span, b: Span) {
+export function spanInside(a: Span, b: Span) {
   return b.start <= a.start && a.end <= b.end;
+}
+
+export function spanContains(a: Span, position: number) {
+  return a.start <= position && position < a.end;
 }
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/renderer.ts b/packages/jsii-rosetta/lib/renderer.ts
index 5393d554dc..e4f3d9897c 100644
--- a/packages/jsii-rosetta/lib/renderer.ts
+++ b/packages/jsii-rosetta/lib/renderer.ts
@@ -105,7 +105,9 @@ export class AstRenderer {
 
   public report(node: ts.Node, messageText: string, category: ts.DiagnosticCategory = ts.DiagnosticCategory.Error) {
     this.diagnostics.push({
-      category, code: 0,
+      category,
+      code: 0,
+      source: 'rosetta',
       messageText,
       file: this.sourceFile,
       start: node.getStart(this.sourceFile),
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
index 8de08bb14e..5f0f76c156 100644
--- a/packages/jsii-rosetta/lib/translate.ts
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -1,7 +1,7 @@
 import logging = require('./logging');
 import ts = require('typescript');
 import { AstRenderer, AstHandler, AstRendererOptions } from './renderer';
-import { renderTree, Span } from './o-tree';
+import { renderTree, Span, spanContains } from './o-tree';
 import { TypeScriptCompiler, CompilationResult } from './typescript/ts-compiler';
 import { TranslatedSnippet } from './tablets/tablets';
 import { TARGET_LANGUAGES, TargetLanguage } from './languages';
@@ -115,7 +115,7 @@ export class SnippetTranslator {
   public renderUsing(visitor: AstHandler) {
     const converter = new AstRenderer(this.compilation.rootFile, this.compilation.program.getTypeChecker(), visitor, this.options);
     const converted = converter.convert(this.compilation.rootFile);
-    this.translateDiagnostics.push(...converter.diagnostics);
+    this.translateDiagnostics.push(...filterVisibleDiagnostics(converter.diagnostics, this.visibleSpans));
     return renderTree(converted, { visibleSpans: this.visibleSpans });
   }
 
@@ -123,3 +123,10 @@ export class SnippetTranslator {
     return [...this.compileDiagnostics, ...this.translateDiagnostics];
   }
 }
+
+/**
+ * Hide diagnostics that are rosetta-sourced if they are reported against a non-visible span
+ */
+function filterVisibleDiagnostics(diags: ts.Diagnostic[], visibleSpans: Span[]): ts.Diagnostic[] {
+  return diags.filter(d => d.source !== 'rosetta' || d.start === undefined || visibleSpans.some(s => spanContains(s, d.start!)));
+}
\ No newline at end of file