diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts index 5e2ebb5491..70711df95d 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnetdocgenerator.ts @@ -5,8 +5,8 @@ import { TargetLanguage, Translation, enforcesStrictMode, - typeScriptSnippetFromSource, markDownToXmlDoc, + ApiLocation, } from 'jsii-rosetta'; import * as xmlbuilder from 'xmlbuilder'; @@ -44,7 +44,7 @@ export class DotNetDocGenerator { * Returns * Remarks (includes examples, links, deprecated) */ - public emitDocs(obj: spec.Documentable): void { + public emitDocs(obj: spec.Documentable, apiLocation: ApiLocation): void { const docs = obj.docs; // The docs may be undefined at the method level but not the parameters level @@ -76,7 +76,7 @@ export class DotNetDocGenerator { // Remarks does not use emitXmlDoc() because the remarks can contain code blocks // which are fenced with tags, which would be escaped to // <code> if we used the xml builder. - const remarks = this.renderRemarks(docs); + const remarks = this.renderRemarks(docs, apiLocation); if (remarks.length > 0) { this.code.line('/// '); remarks.forEach((r) => this.code.line(`/// ${r}`.trimRight())); @@ -85,18 +85,21 @@ export class DotNetDocGenerator { if (docs.example) { this.code.line('/// '); - this.emitXmlDoc('code', this.convertExample(docs.example)); + this.emitXmlDoc('code', this.convertExample(docs.example, apiLocation)); this.code.line('/// '); } } - public emitMarkdownAsRemarks(markdown: string | undefined) { + public emitMarkdownAsRemarks( + markdown: string | undefined, + apiLocation: ApiLocation, + ) { if (!markdown) { return; } const translated = markDownToXmlDoc( - this.convertSamplesInMarkdown(markdown), + this.convertSamplesInMarkdown(markdown, apiLocation), ); const lines = translated.split('\n'); @@ -110,12 +113,12 @@ export class DotNetDocGenerator { /** * Returns the lines that should go into the section */ - private renderRemarks(docs: spec.Docs): string[] { + private renderRemarks(docs: spec.Docs, apiLocation: ApiLocation): string[] { const ret: string[] = []; if (docs.remarks) { const translated = markDownToXmlDoc( - this.convertSamplesInMarkdown(docs.remarks), + this.convertSamplesInMarkdown(docs.remarks, apiLocation), ); ret.push(...translated.split('\n')); ret.push(''); @@ -161,24 +164,19 @@ export class DotNetDocGenerator { } } - private convertExample(example: string): string { - const snippet = typeScriptSnippetFromSource( + private convertExample(example: string, apiLocation: ApiLocation): string { + const translated = this.rosetta.translateExample( + apiLocation, example, - 'example', - enforcesStrictMode(this.assembly), - ); - const translated = this.rosetta.translateSnippet( - snippet, TargetLanguage.CSHARP, + enforcesStrictMode(this.assembly), ); - if (!translated) { - return example; - } return this.prefixDisclaimer(translated); } - private convertSamplesInMarkdown(markdown: string): string { + private convertSamplesInMarkdown(markdown: string, api: ApiLocation): string { return this.rosetta.translateSnippetsInMarkdown( + api, markdown, TargetLanguage.CSHARP, enforcesStrictMode(this.assembly), diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts index ca39ab6dd3..fe41b807a1 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts @@ -160,7 +160,7 @@ export class DotNetGenerator extends Generator { this.assembly.name, jsiiNs, ); - this.emitNamespaceDocs(dotnetNs, submodule); + this.emitNamespaceDocs(dotnetNs, jsiiNs, submodule); } } @@ -174,7 +174,7 @@ export class DotNetGenerator extends Generator { const namespace = this.namespaceFor(this.assembly, ifc); this.openFileIfNeeded(interfaceName, namespace, this.isNested(ifc)); - this.dotnetDocGenerator.emitDocs(ifc); + this.dotnetDocGenerator.emitDocs(ifc, { api: 'type', fqn: ifc.fqn }); this.dotnetRuntimeGenerator.emitAttributesForInterface(ifc); if (implementations.length > 0) { @@ -204,7 +204,11 @@ export class DotNetGenerator extends Generator { } protected onInterfaceMethod(ifc: spec.InterfaceType, method: spec.Method) { - this.dotnetDocGenerator.emitDocs(method); + this.dotnetDocGenerator.emitDocs(method, { + api: 'member', + fqn: ifc.fqn, + memberName: method.name, + }); this.dotnetRuntimeGenerator.emitAttributesForMethod(ifc, method); const returnType = method.returns ? this.typeresolver.toDotNetType(method.returns.type) @@ -243,7 +247,11 @@ export class DotNetGenerator extends Generator { } this.emitNewLineIfNecessary(); - this.dotnetDocGenerator.emitDocs(prop); + this.dotnetDocGenerator.emitDocs(prop, { + api: 'member', + fqn: ifc.fqn, + memberName: prop.name, + }); this.dotnetRuntimeGenerator.emitAttributesForProperty(prop); const propType = this.typeresolver.toDotNetType(prop.type); @@ -308,7 +316,11 @@ export class DotNetGenerator extends Generator { const implementsExpr = ` : ${baseTypeNames.join(', ')}`; - this.dotnetDocGenerator.emitDocs(cls); + this.dotnetDocGenerator.emitDocs(cls, { + api: 'type', + fqn: cls.fqn, + }); + this.dotnetRuntimeGenerator.emitAttributesForClass(cls); this.code.openBlock( @@ -320,7 +332,10 @@ export class DotNetGenerator extends Generator { let parametersBase = ''; const initializer = cls.initializer; if (initializer) { - this.dotnetDocGenerator.emitDocs(initializer); + this.dotnetDocGenerator.emitDocs(initializer, { + api: 'initializer', + fqn: cls.fqn, + }); this.dotnetRuntimeGenerator.emitDeprecatedAttributeIfNecessary( initializer, ); @@ -452,7 +467,10 @@ export class DotNetGenerator extends Generator { const namespace = this.namespaceFor(this.assembly, enm); this.openFileIfNeeded(enumName, namespace, this.isNested(enm)); this.emitNewLineIfNecessary(); - this.dotnetDocGenerator.emitDocs(enm); + this.dotnetDocGenerator.emitDocs(enm, { + api: 'type', + fqn: enm.fqn, + }); this.dotnetRuntimeGenerator.emitAttributesForEnum(enm, enumName); this.code.openBlock(`public enum ${enm.name}`); } @@ -465,7 +483,11 @@ export class DotNetGenerator extends Generator { } protected onEnumMember(enm: spec.EnumType, member: spec.EnumMember) { - this.dotnetDocGenerator.emitDocs(member); + this.dotnetDocGenerator.emitDocs(member, { + api: 'member', + fqn: enm.fqn, + memberName: member.name, + }); const enumMemberName = this.nameutils.convertEnumMemberName(member.name); this.dotnetRuntimeGenerator.emitAttributesForEnumMember( enumMemberName, @@ -544,7 +566,11 @@ export class DotNetGenerator extends Generator { method, )})`; - this.dotnetDocGenerator.emitDocs(method); + this.dotnetDocGenerator.emitDocs(method, { + api: 'member', + fqn: cls.fqn, + memberName: method.name, + }); this.dotnetRuntimeGenerator.emitAttributesForMethod( cls, method /*, emitForProxyOrDatatype*/, @@ -669,7 +695,10 @@ export class DotNetGenerator extends Generator { this.openFileIfNeeded(name, namespace, isNested); this.code.line(); - this.dotnetDocGenerator.emitDocs(ifc); + this.dotnetDocGenerator.emitDocs(ifc, { + api: 'type', + fqn: ifc.fqn, + }); this.dotnetRuntimeGenerator.emitAttributesForInterfaceProxy(ifc); const interfaceFqn = this.typeresolver.toNativeFqn(ifc.fqn); @@ -758,7 +787,10 @@ export class DotNetGenerator extends Generator { this.code.line(); } - this.dotnetDocGenerator.emitDocs(ifc); + this.dotnetDocGenerator.emitDocs(ifc, { + api: 'type', + fqn: ifc.fqn, + }); const suffix = `: ${this.typeresolver.toNativeFqn(ifc.fqn)}`; this.dotnetRuntimeGenerator.emitAttributesForInterfaceDatatype(ifc); this.code.openBlock(`public class ${name} ${suffix}`); @@ -888,7 +920,11 @@ export class DotNetGenerator extends Generator { const staticKeyWord = prop.static ? 'static ' : ''; const propName = this.nameutils.convertPropertyName(prop.name); - this.dotnetDocGenerator.emitDocs(prop); + this.dotnetDocGenerator.emitDocs(prop, { + api: 'member', + fqn: cls.fqn, + memberName: prop.name, + }); if (prop.optional) { this.code.line('[JsiiOptional]'); } @@ -969,7 +1005,11 @@ export class DotNetGenerator extends Generator { this.flagFirstMemberWritten(true); const propType = this.typeresolver.toDotNetType(prop.type); const isOptional = prop.optional ? '?' : ''; - this.dotnetDocGenerator.emitDocs(prop); + this.dotnetDocGenerator.emitDocs(prop, { + api: 'member', + fqn: cls.fqn, + memberName: prop.name, + }); this.dotnetRuntimeGenerator.emitAttributesForProperty(prop); const access = this.renderAccessLevel(prop); const propName = this.nameutils.convertPropertyName(prop.name); @@ -1086,6 +1126,7 @@ export class DotNetGenerator extends Generator { private emitAssemblyDocs() { this.emitNamespaceDocs( this.assembly.targets!.dotnet!.namespace, + this.assembly.name, this.assembly, ); } @@ -1102,7 +1143,11 @@ export class DotNetGenerator extends Generator { * In any case, we need a place to attach the docs where they can be transported around, * might as well be this method. */ - private emitNamespaceDocs(namespace: string, docSource: spec.Targetable) { + private emitNamespaceDocs( + namespace: string, + jsiiFqn: string, + docSource: spec.Targetable, + ) { if (!docSource.readme) { return; } @@ -1110,7 +1155,10 @@ export class DotNetGenerator extends Generator { const className = 'NamespaceDoc'; this.openFileIfNeeded(className, namespace, false, false); - this.dotnetDocGenerator.emitMarkdownAsRemarks(docSource.readme.markdown); + this.dotnetDocGenerator.emitMarkdownAsRemarks(docSource.readme.markdown, { + api: 'moduleReadme', + moduleFqn: jsiiFqn, + }); this.emitHideAttribute(); // Traditionally this class is made 'internal', but that interacts poorly with DocFX's default filters // which aren't overridable. So we make it public, but use attributes to hide it from users' IntelliSense, diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index 85fe231fde..93a63c3ea6 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -6,10 +6,10 @@ import * as reflect from 'jsii-reflect'; import { Rosetta, TargetLanguage, - typeScriptSnippetFromSource, Translation, enforcesStrictMode, markDownToJavaDoc, + ApiLocation, } from 'jsii-rosetta'; import * as path from 'path'; import * as xmlbuilder from 'xmlbuilder'; @@ -641,7 +641,7 @@ class JavaGenerator extends Generator { protected onBeginClass(cls: spec.ClassType, abstract: boolean) { this.openFileIfNeeded(cls); - this.addJavaDocs(cls); + this.addJavaDocs(cls, { api: 'type', fqn: cls.fqn }); const classBase = this.getClassBase(cls); const extendsExpression = classBase ? ` extends ${classBase}` : ''; @@ -688,7 +688,7 @@ class JavaGenerator extends Generator { this.code.line(); // If needed, patching up the documentation to point users at the builder pattern - this.addJavaDocs(method); + this.addJavaDocs(method, { api: 'initializer', fqn: cls.fqn }); this.emitStabilityAnnotations(method); // Abstract classes should have protected initializers @@ -734,7 +734,7 @@ class JavaGenerator extends Generator { protected onStaticProperty(cls: spec.ClassType, prop: spec.Property) { if (prop.const) { - this.emitConstProperty(prop); + this.emitConstProperty(cls, prop); } else { this.emitProperty(cls, prop); } @@ -777,7 +777,7 @@ class JavaGenerator extends Generator { protected onBeginEnum(enm: spec.EnumType) { this.openFileIfNeeded(enm); - this.addJavaDocs(enm); + this.addJavaDocs(enm, { api: 'type', fqn: enm.fqn }); if (!this.isNested(enm)) { this.emitGeneratedAnnotation(); } @@ -791,8 +791,12 @@ class JavaGenerator extends Generator { this.code.closeBlock(); this.closeFileIfNeeded(enm); } - protected onEnumMember(_: spec.EnumType, member: spec.EnumMember) { - this.addJavaDocs(member); + protected onEnumMember(parentType: spec.EnumType, member: spec.EnumMember) { + this.addJavaDocs(member, { + api: 'member', + fqn: parentType.fqn, + memberName: member.name, + }); this.emitStabilityAnnotations(member); this.code.line(`${member.name},`); } @@ -815,7 +819,7 @@ class JavaGenerator extends Generator { protected onBeginInterface(ifc: spec.InterfaceType) { this.openFileIfNeeded(ifc); - this.addJavaDocs(ifc); + this.addJavaDocs(ifc, { api: 'type', fqn: ifc.fqn }); // all interfaces always extend JsiiInterface so we can identify that it is a jsii interface. const interfaces = ifc.interfaces ?? []; @@ -863,12 +867,16 @@ class JavaGenerator extends Generator { this.closeFileIfNeeded(ifc); } - protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { + protected onInterfaceMethod(ifc: spec.InterfaceType, method: spec.Method) { this.code.line(); const returnType = method.returns ? this.toDecoratedJavaType(method.returns) : 'void'; - this.addJavaDocs(method); + this.addJavaDocs(method, { + api: 'member', + fqn: ifc.fqn, + memberName: method.name, + }); this.emitStabilityAnnotations(method); this.code.line( `${returnType} ${method.name}(${this.renderMethodParameters(method)});`, @@ -883,7 +891,7 @@ class JavaGenerator extends Generator { this.onInterfaceMethod(ifc, overload); } - protected onInterfaceProperty(_ifc: spec.InterfaceType, prop: spec.Property) { + protected onInterfaceProperty(ifc: spec.InterfaceType, prop: spec.Property) { const getterType = this.toDecoratedJavaType(prop); const propName = jsiiToPascalCase( JavaGenerator.safeJavaPropertyName(prop.name), @@ -891,7 +899,11 @@ class JavaGenerator extends Generator { // for unions we only generate overloads for setters, not getters. this.code.line(); - this.addJavaDocs(prop); + this.addJavaDocs(prop, { + api: 'member', + fqn: ifc.fqn, + memberName: prop.name, + }); this.emitStabilityAnnotations(prop); if (prop.optional) { if (prop.overrides) { @@ -908,7 +920,11 @@ class JavaGenerator extends Generator { const setterTypes = this.toDecoratedJavaTypes(prop); for (const type of setterTypes) { this.code.line(); - this.addJavaDocs(prop); + this.addJavaDocs(prop, { + api: 'member', + fqn: ifc.fqn, + memberName: prop.name, + }); if (prop.optional) { if (prop.overrides) { this.code.line('@Override'); @@ -997,7 +1013,10 @@ class JavaGenerator extends Generator { this.code.line('/**'); if (mod.readme) { for (const line of markDownToJavaDoc( - this.convertSamplesInMarkdown(mod.readme.markdown), + this.convertSamplesInMarkdown(mod.readme.markdown, { + api: 'moduleReadme', + moduleFqn: mod.name, + }), ).split('\n')) { this.code.line(` * ${line.replace(/\*\//g, '*{@literal /}')}`); } @@ -1028,7 +1047,10 @@ class JavaGenerator extends Generator { this.code.line('/**'); if (mod.readme) { for (const line of markDownToJavaDoc( - this.convertSamplesInMarkdown(mod.readme.markdown), + this.convertSamplesInMarkdown(mod.readme.markdown, { + api: 'moduleReadme', + moduleFqn, + }), ).split('\n')) { this.code.line(` * ${line.replace(/\*\//g, '*{@literal /}')}`); } @@ -1324,13 +1346,17 @@ class JavaGenerator extends Generator { return this.code.toSnakeCase(prop.name).toLocaleUpperCase(); // java consts are SNAKE_UPPER_CASE } - private emitConstProperty(prop: spec.Property) { + private emitConstProperty(parentType: spec.Type, prop: spec.Property) { const propType = this.toJavaType(prop.type); const propName = this.renderConstName(prop); const access = this.renderAccessLevel(prop); this.code.line(); - this.addJavaDocs(prop); + this.addJavaDocs(prop, { + api: 'member', + fqn: parentType.fqn, + memberName: prop.name, + }); this.emitStabilityAnnotations(prop); this.code.line(`${access} final static ${propType} ${propName};`); } @@ -1368,7 +1394,11 @@ class JavaGenerator extends Generator { // for unions we only generate overloads for setters, not getters. if (includeGetter) { this.code.line(); - this.addJavaDocs(prop); + this.addJavaDocs(prop, { + api: 'member', + fqn: cls.fqn, + memberName: prop.name, + }); if (overrides && !prop.static) { this.code.line('@Override'); } @@ -1401,7 +1431,11 @@ class JavaGenerator extends Generator { if (!prop.immutable) { for (const type of setterTypes) { this.code.line(); - this.addJavaDocs(prop); + this.addJavaDocs(prop, { + api: 'member', + fqn: cls.fqn, + memberName: prop.name, + }); if (overrides && !prop.static) { this.code.line('@Override'); } @@ -1457,7 +1491,11 @@ class JavaGenerator extends Generator { method, )})`; this.code.line(); - this.addJavaDocs(method); + this.addJavaDocs(method, { + api: 'member', + fqn: cls.fqn, + memberName: method.name, + }); this.emitStabilityAnnotations(method); if (overrides && !method.static) { this.code.line('@Override'); @@ -1738,7 +1776,11 @@ class JavaGenerator extends Generator { name: 'create', parameters: params.map((param) => param.param), }; - this.addJavaDocs(dummyMethod); + this.addJavaDocs(dummyMethod, { + api: 'member', + fqn: cls.fqn, + memberName: dummyMethod.name, + }); this.emitStabilityAnnotations(cls.initializer); this.code.openBlock( `public static ${BUILDER_CLASS_NAME} create(${params @@ -1817,7 +1859,11 @@ class JavaGenerator extends Generator { for (const javaType of this.toJavaTypes(prop.type.spec!, { covariant: true, })) { - this.addJavaDocs(setter); + this.addJavaDocs(setter, { + api: 'member', + fqn: cls.fqn, + memberName: setter.name, + }); this.emitStabilityAnnotations(prop.spec); this.code.openBlock( `public ${BUILDER_CLASS_NAME} ${fieldName}(final ${javaType} ${fieldName})`, @@ -1875,13 +1921,13 @@ class JavaGenerator extends Generator { private emitBuilderSetter( prop: JavaProp, builderName: string, - builtType: string, + parentType: spec.InterfaceType, ) { for (const type of prop.javaTypes) { this.code.line(); this.code.line('/**'); this.code.line( - ` * Sets the value of {@link ${builtType}#${getterFor( + ` * Sets the value of {@link ${parentType.name}#${getterFor( prop.fieldName, )}}`, ); @@ -1892,7 +1938,11 @@ class JavaGenerator extends Generator { if (prop.docs?.remarks != null) { const indent = ' '.repeat(7 + prop.fieldName.length); const remarks = markDownToJavaDoc( - this.convertSamplesInMarkdown(prop.docs.remarks), + this.convertSamplesInMarkdown(prop.docs.remarks, { + api: 'member', + fqn: parentType.fqn, + memberName: prop.jsiiName, + }), ).trimRight(); for (const line of remarks.split('\n')) { this.code.line(` * ${indent} ${line}`); @@ -1962,7 +2012,7 @@ class JavaGenerator extends Generator { this.code.line(`private ${prop.fieldJavaType} ${prop.fieldName};`), ); props.forEach((prop) => - this.emitBuilderSetter(prop, BUILDER_CLASS_NAME, classSpec.name), + this.emitBuilderSetter(prop, BUILDER_CLASS_NAME, classSpec), ); // Start build() @@ -2301,7 +2351,11 @@ class JavaGenerator extends Generator { } // eslint-disable-next-line complexity - private addJavaDocs(doc: spec.Documentable, defaultText?: string) { + private addJavaDocs( + doc: spec.Documentable, + apiLoc: ApiLocation, + defaultText?: string, + ) { if ( !defaultText && Object.keys(doc.docs ?? {}).length === 0 && @@ -2325,7 +2379,7 @@ class JavaGenerator extends Generator { if (docs.remarks) { paras.push( markDownToJavaDoc( - this.convertSamplesInMarkdown(docs.remarks), + this.convertSamplesInMarkdown(docs.remarks, apiLoc), ).trimRight(), ); } @@ -2337,7 +2391,7 @@ class JavaGenerator extends Generator { if (docs.example) { paras.push('Example:'); - const convertedExample = this.convertExample(docs.example); + const convertedExample = this.convertExample(docs.example, apiLoc); // We used to use the slightly nicer `
{@code ...}
`, which doesn't // require (and therefore also doesn't allow) escaping special characters. @@ -2923,24 +2977,19 @@ class JavaGenerator extends Generator { ); } - private convertExample(example: string): string { - const snippet = typeScriptSnippetFromSource( + private convertExample(example: string, api: ApiLocation): string { + const translated = this.rosetta.translateExample( + api, example, - 'example', - enforcesStrictMode(this.assembly), - ); - const translated = this.rosetta.translateSnippet( - snippet, TargetLanguage.JAVA, + enforcesStrictMode(this.assembly), ); - if (!translated) { - return example; - } return this.prefixDisclaimer(translated); } - private convertSamplesInMarkdown(markdown: string): string { + private convertSamplesInMarkdown(markdown: string, api: ApiLocation): string { return this.rosetta.translateSnippetsInMarkdown( + api, markdown, TargetLanguage.JAVA, enforcesStrictMode(this.assembly), diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 8c052b67f6..fe695983c3 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -9,7 +9,7 @@ import { Translation, Rosetta, enforcesStrictMode, - typeScriptSnippetFromSource, + ApiLocation, } from 'jsii-rosetta'; import * as path from 'path'; @@ -383,13 +383,22 @@ abstract class BasePythonClassType implements PythonType, ISortableType { this.members.push(member); } + public get apiLocation(): ApiLocation { + if (!this.fqn) { + throw new Error( + `Cannot make apiLocation for ${this.pythonName}, does not have FQN`, + ); + } + return { api: 'type', fqn: this.fqn }; + } + public emit(code: CodeMaker, context: EmitContext) { context = nestedContext(context, this.fqn); const classParams = this.getClassParams(context); openSignature(code, 'class', this.pythonName, classParams); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { documentableItem: `class-${this.pythonName}`, trailingNewLine: true, }); @@ -464,6 +473,10 @@ abstract class BaseMethod implements PythonBase { this.parent = opts.parent; } + public get apiLocation(): ApiLocation { + return { api: 'member', fqn: this.parent.fqn, memberName: this.jsiiMethod }; + } + public requiredImports(context: EmitContext): PythonImports { return mergePythonImports( toTypeName(this.returns).requiredImports(context), @@ -628,7 +641,7 @@ abstract class BaseMethod implements PythonBase { false, returnType, ); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { arguments: documentableArgs, documentableItem: `method-${this.pythonName}`, }); @@ -688,7 +701,7 @@ abstract class BaseMethod implements PythonBase { // We need to build up a list of properties, which are mandatory, these are the // ones we will specifiy to start with in our dictionary literal. const liftedProps = this.getLiftedProperties(context.resolver).map( - (p) => new StructField(this.generator, p), + (p) => new StructField(this.generator, p, this.parent), ); const assignments = liftedProps .map((p) => p.pythonName) @@ -814,6 +827,7 @@ abstract class BaseProperty implements PythonBase { protected readonly shouldEmitBody: boolean = true; private readonly immutable: boolean; + private readonly parent: spec.NamedTypeReference; public constructor( private readonly generator: PythonGenerator, @@ -828,6 +842,11 @@ abstract class BaseProperty implements PythonBase { this.abstract = abstract; this.immutable = immutable; this.isStatic = isStatic; + this.parent = opts.parent; + } + + public get apiLocation(): ApiLocation { + return { api: 'member', fqn: this.parent.fqn, memberName: this.jsName }; } public requiredImports(context: EmitContext): PythonImports { @@ -856,7 +875,7 @@ abstract class BaseProperty implements PythonBase { true, pythonType, ); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { documentableItem: `prop-${this.pythonName}`, }); if ( @@ -925,7 +944,7 @@ class Interface extends BasePythonClassType { })}) # type: ignore[misc]`, ); openSignature(code, 'class', this.proxyClassName, proxyBases); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { documentableItem: `class-${this.pythonName}`, trailingNewLine: true, }); @@ -1039,7 +1058,7 @@ class Struct extends BasePythonClassType { */ private get allMembers(): StructField[] { return this.thisInterface.allProperties.map( - (x) => new StructField(this.generator, x.spec), + (x) => new StructField(this.generator, x.spec, this.thisInterface), ); } @@ -1108,7 +1127,7 @@ class Struct extends BasePythonClassType { name: m.pythonName, docs: m.docs, })); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { arguments: args, documentableItem: `class-${this.pythonName}`, }); @@ -1180,6 +1199,7 @@ class StructField implements PythonBase { public constructor( private readonly generator: PythonGenerator, public readonly prop: spec.Property, + private readonly parent: spec.NamedTypeReference, ) { this.pythonName = toPythonPropertyName(prop.name); this.jsiiName = prop.name; @@ -1187,6 +1207,10 @@ class StructField implements PythonBase { this.docs = prop.docs; } + public get apiLocation(): ApiLocation { + return { api: 'member', fqn: this.parent.fqn, memberName: this.jsiiName }; + } + public get optional(): boolean { return !!this.type.optional; } @@ -1215,7 +1239,7 @@ class StructField implements PythonBase { } public emitDocString(code: CodeMaker) { - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { documentableItem: `prop-${this.pythonName}`, }); } @@ -1438,18 +1462,23 @@ class EnumMember implements PythonBase { public readonly pythonName: string, private readonly value: string, public readonly docs: spec.Docs | undefined, + private readonly parent: spec.NamedTypeReference, ) { this.pythonName = pythonName; this.value = value; } + public get apiLocation(): ApiLocation { + return { api: 'member', fqn: this.parent.fqn, memberName: this.value }; + } + public dependsOnModules() { return new Set(); } public emit(code: CodeMaker, _context: EmitContext) { code.line(`${this.pythonName} = "${this.value}"`); - this.generator.emitDocString(code, this.docs, { + this.generator.emitDocString(code, this.apiLocation, this.docs, { documentableItem: `enum-${this.pythonName}`, }); } @@ -2186,6 +2215,7 @@ class PythonGenerator extends Generator { // eslint-disable-next-line complexity public emitDocString( code: CodeMaker, + apiLocation: ApiLocation, docs: spec.Docs | undefined, options: { arguments?: DocumentableArgument[]; @@ -2237,7 +2267,9 @@ class PythonGenerator extends Generator { if (docs.remarks) { brk(); lines.push( - ...md2rst(this.convertMarkdown(docs.remarks ?? '')).split('\n'), + ...md2rst(this.convertMarkdown(docs.remarks ?? '', apiLocation)).split( + '\n', + ), ); brk(); } @@ -2283,7 +2315,7 @@ class PythonGenerator extends Generator { brk(); lines.push('Example::'); lines.push(''); - const exampleText = this.convertExample(docs.example); + const exampleText = this.convertExample(docs.example, apiLocation); for (const line of exampleText.split('\n')) { lines.push(` ${line}`); @@ -2316,24 +2348,19 @@ class PythonGenerator extends Generator { } } - public convertExample(example: string): string { - const snippet = typeScriptSnippetFromSource( + public convertExample(example: string, apiLoc: ApiLocation): string { + const translated = this.rosetta.translateExample( + apiLoc, example, - 'example', - enforcesStrictMode(this.assembly), - ); - const translated = this.rosetta.translateSnippet( - snippet, TargetLanguage.PYTHON, + enforcesStrictMode(this.assembly), ); - if (!translated) { - return example; - } return this.prefixDisclaimer(translated); } - public convertMarkdown(markdown: string): string { + public convertMarkdown(markdown: string, apiLoc: ApiLocation): string { return this.rosetta.translateSnippetsInMarkdown( + apiLoc, markdown, TargetLanguage.PYTHON, enforcesStrictMode(this.assembly), @@ -2418,12 +2445,17 @@ class PythonGenerator extends Generator { ? this.assembly : this.assembly.submodules?.[ns]; + const readmeLocation: ApiLocation = { api: 'moduleReadme', moduleFqn: ns }; + const module = new PythonModule(toPackageName(ns, this.assembly), ns, { assembly: this.assembly, assemblyFilename: this.getAssemblyFileName(), package: this.package, moduleDocumentation: submoduleLike?.readme - ? this.convertMarkdown(submoduleLike.readme?.markdown).trim() + ? this.convertMarkdown( + submoduleLike.readme?.markdown, + readmeLocation, + ).trim() : undefined, }); @@ -2635,7 +2667,7 @@ class PythonGenerator extends Generator { let ifaceProperty: InterfaceProperty | StructField; if (ifc.datatype) { - ifaceProperty = new StructField(this, prop); + ifaceProperty = new StructField(this, prop, ifc); } else { ifaceProperty = new InterfaceProperty( this, @@ -2663,6 +2695,7 @@ class PythonGenerator extends Generator { toPythonIdentifier(member.name), member.name, member.docs, + enm, ), ); } diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts index a8a54eea2d..e32902ea60 100644 --- a/packages/jsii-rosetta/bin/jsii-rosetta.ts +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -14,7 +14,7 @@ import { TargetLanguage } from '../lib/languages'; import { PythonVisitor } from '../lib/languages/python'; import { VisualizeAstVisitor } from '../lib/languages/visualize'; import * as logging from '../lib/logging'; -import { File, printDiagnostics } from '../lib/util'; +import { File, fmap, printDiagnostics } from '../lib/util'; function main() { const argv = yargs @@ -163,6 +163,14 @@ function main() { describe: 'Whether to validate loaded assemblies or not (this can be slow)', default: false, }) + .option('cache-from', { + alias: 'C', + type: 'string', + // eslint-disable-next-line prettier/prettier + describe: 'Reuse translations from the given tablet file if the snippet and type definitions did not change', + requiresArg: true, + default: undefined, + }) .option('strict', { alias: 'S', type: 'boolean', @@ -184,6 +192,7 @@ function main() { // though. const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map((x) => path.resolve(x)); const absOutput = path.resolve(args.output); + const absCache = fmap(args['cache-from'], path.resolve); if (args.directory) { process.chdir(args.directory); } @@ -193,6 +202,7 @@ function main() { includeCompilerDiagnostics: !!args.compile, validateAssemblies: args['validate-assemblies'], only: args.include, + cacheTabletFile: absCache, }); handleDiagnostics(result.diagnostics, args.fail, result.tablet.count); diff --git a/packages/jsii-rosetta/lib/commands/convert.ts b/packages/jsii-rosetta/lib/commands/convert.ts index 39d423005c..7620b98a65 100644 --- a/packages/jsii-rosetta/lib/commands/convert.ts +++ b/packages/jsii-rosetta/lib/commands/convert.ts @@ -2,7 +2,7 @@ import { transformMarkdown } from '../markdown/markdown'; import { MarkdownRenderer } from '../markdown/markdown-renderer'; import { ReplaceTypeScriptTransform } from '../markdown/replace-typescript-transform'; import { AstHandler, AstRendererOptions } from '../renderer'; -import { TranslateResult, Translator, rosettaDiagFromTypescript } from '../translate'; +import { TranslateResult, Translator } from '../translate'; import { File } from '../util'; export interface TranslateMarkdownOptions extends AstRendererOptions { @@ -24,10 +24,12 @@ export function translateMarkdown( ): TranslateResult { const translator = new Translator(false); + const location = { api: 'file', fileName: markdown.fileName } as const; + const translatedMarkdown = transformMarkdown( markdown.contents, new MarkdownRenderer(), - new ReplaceTypeScriptTransform(markdown.fileName, opts.strict ?? false, (tsSnippet) => { + new ReplaceTypeScriptTransform(location, opts.strict ?? false, (tsSnippet) => { const translated = translator.translatorFor(tsSnippet).renderUsing(visitor); return { language: opts.languageIdentifier ?? '', @@ -38,6 +40,6 @@ export function translateMarkdown( return { translation: translatedMarkdown, - diagnostics: translator.diagnostics.map(rosettaDiagFromTypescript), + diagnostics: translator.diagnostics, }; } diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index b478ab5e28..c896367ea8 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -1,14 +1,15 @@ import * as os from 'os'; import * as path from 'path'; -import * as ts from 'typescript'; import * as workerpool from 'workerpool'; import { loadAssemblies, allTypeScriptSnippets } from '../jsii/assemblies'; +import { TypeFingerprinter } from '../jsii/fingerprinting'; +import { TARGET_LANGUAGES } from '../languages'; import * as logging from '../logging'; -import { TypeScriptSnippet } from '../snippet'; +import { TypeScriptSnippet, completeSource } from '../snippet'; import { snippetKey } from '../tablets/key'; import { LanguageTablet, TranslatedSnippet } from '../tablets/tablets'; -import { RosettaDiagnostic, Translator, rosettaDiagFromTypescript } from '../translate'; +import { RosettaDiagnostic, Translator, makeRosettaDiagnostic } from '../translate'; import type { TranslateBatchRequest, TranslateBatchResponse } from './extract_worker'; export interface ExtractResult { @@ -17,12 +18,26 @@ export interface ExtractResult { } export interface ExtractOptions { - outputFile: string; - includeCompilerDiagnostics: boolean; - validateAssemblies: boolean; - only?: string[]; + readonly outputFile: string; + readonly includeCompilerDiagnostics: boolean; + readonly validateAssemblies: boolean; + readonly only?: string[]; + + /** + * A tablet file to be loaded and used as a source for caching + */ + readonly cacheTabletFile?: string; + + /** + * Call the given translation function on the snippets. + * + * Optional, only for testing. Uses `translateAll` by default. + */ + readonly translationFunction?: TranslationFunc; } +type TranslationFunc = typeof translateAll; + /** * Extract all samples from the given assemblies into a tablet */ @@ -35,31 +50,45 @@ export async function extractSnippets( logging.info(`Loading ${assemblyLocations.length} assemblies`); const assemblies = await loadAssemblies(assemblyLocations, options.validateAssemblies); + const fingerprinter = new TypeFingerprinter(assemblies.map((a) => a.assembly)); - let snippets = allTypeScriptSnippets(assemblies, loose); + let snippets = Array.from(allTypeScriptSnippets(assemblies, loose)); if (only.length > 0) { snippets = filterSnippets(snippets, only); } const tablet = new LanguageTablet(); - logging.info('Translating'); - const startTime = Date.now(); + if (options.cacheTabletFile) { + await reuseTranslationsFromCache(snippets, tablet, options.cacheTabletFile, fingerprinter); + } + + const translateCount = snippets.length; + const diagnostics = []; + if (translateCount > 0) { + logging.info('Translating'); + const startTime = Date.now(); + + const result = await (options.translationFunction ?? translateAll)(snippets, options.includeCompilerDiagnostics); - const result = await translateAll(snippets, options.includeCompilerDiagnostics); + for (const snippet of result.translatedSnippets) { + const fingerprinted = snippet.withFingerprint(fingerprinter.fingerprintAll(snippet.fqnsReferenced())); + tablet.addSnippet(fingerprinted); + } - for (const snippet of result.translatedSnippets) { - tablet.addSnippet(snippet); + const delta = (Date.now() - startTime) / 1000; + logging.info( + `Translated ${translateCount} snippets in ${delta} seconds (${(delta / tablet.count).toPrecision(3)}s/snippet)`, + ); + diagnostics.push(...result.diagnostics); + } else { + logging.info('Nothing left to translate'); } - const delta = (Date.now() - startTime) / 1000; - logging.info( - `Converted ${tablet.count} snippets in ${delta} seconds (${(delta / tablet.count).toPrecision(3)}s/snippet)`, - ); logging.info(`Saving language tablet to ${options.outputFile}`); await tablet.save(options.outputFile); - return { diagnostics: result.diagnostics, tablet }; + return { diagnostics, tablet }; } interface TranslateAllResult { @@ -70,12 +99,8 @@ interface TranslateAllResult { /** * Only yield the snippets whose id exists in a whitelist */ -function* filterSnippets(ts: IterableIterator, includeIds: string[]) { - for (const t of ts) { - if (includeIds.includes(snippetKey(t))) { - yield t; - } - } +function filterSnippets(ts: TypeScriptSnippet[], includeIds: string[]) { + return ts.filter((t) => includeIds.includes(snippetKey(t))); } /** @@ -84,7 +109,7 @@ function* filterSnippets(ts: IterableIterator, includeIds: st * We are now always using workers, as we are targeting Node 12+. */ async function translateAll( - snippets: IterableIterator, + snippets: TypeScriptSnippet[], includeCompilerDiagnostics: boolean, ): Promise { return workerBasedTranslateAll(snippets, includeCompilerDiagnostics); @@ -97,32 +122,27 @@ async function translateAll( * snippets in parallel. */ export function singleThreadedTranslateAll( - snippets: IterableIterator, + snippets: TypeScriptSnippet[], includeCompilerDiagnostics: boolean, ): TranslateAllResult { const translatedSnippets = new Array(); - const failures = new Array(); + const failures = new Array(); const translator = new Translator(includeCompilerDiagnostics); for (const block of snippets) { try { translatedSnippets.push(translator.translate(block)); } catch (e) { - failures.push({ - category: ts.DiagnosticCategory.Error, - code: 999, - messageText: `rosetta: error translating snippet: ${e}\n${e.stack}\n${block.completeSource}`, - file: undefined, - start: undefined, - length: undefined, - }); + failures.push( + makeRosettaDiagnostic(true, `rosetta: error translating snippet: ${e}\n${e.stack}\n${block.completeSource}`), + ); } } return { translatedSnippets, - diagnostics: [...translator.diagnostics, ...failures].map(rosettaDiagFromTypescript), + diagnostics: [...translator.diagnostics, ...failures], }; } @@ -137,7 +157,7 @@ export function singleThreadedTranslateAll( * the script we may assume that 'worker_threads' successfully imports). */ async function workerBasedTranslateAll( - snippets: IterableIterator, + snippets: TypeScriptSnippet[], includeCompilerDiagnostics: boolean, ): Promise { // Use about half the advertised cores because hyperthreading doesn't seem to @@ -191,3 +211,65 @@ function batchSnippets( return ret; } + +/** + * Try and read as many snippet translations from the cache as possible, adding them to the target tablet + * + * Removes the already translated snippets from the input array. + */ +async function reuseTranslationsFromCache( + snippets: TypeScriptSnippet[], + tablet: LanguageTablet, + cacheFile: string, + fingerprinter: TypeFingerprinter, +) { + try { + const cache = await LanguageTablet.fromFile(cacheFile); + + let snippetsFromCacheCtr = 0; + let i = 0; + while (i < snippets.length) { + const fromCache = tryReadFromCache(snippets[i], cache, fingerprinter); + if (fromCache) { + tablet.addSnippet(fromCache); + snippets.splice(i, 1); + snippetsFromCacheCtr += 1; + } else { + i += 1; + } + } + + logging.info(`Reused ${snippetsFromCacheCtr} translations from cache ${cacheFile}`); + } catch (e) { + logging.warn(`Error reading cache ${cacheFile}: ${e.message}`); + } +} + +/** + * Try to find the translation for the given snippet in the given cache + * + * Rules for cacheability are: + * - id is the same (== visible source didn't change) + * - complete source is the same (== fixture didn't change) + * - all types involved have the same fingerprint (== API surface didn't change) + * - the versions of all translations match the versions on the available translators (== translator itself didn't change) + * + * For the versions check: we could have selectively picked some translations + * from the cache while performing others. However, since the big work is in + * parsing the TypeScript, and the rendering itself is peanutes (assumption), it + * doesn't really make a lot of difference. So, for simplification's sake, + * we'll regen all translations if there's at least one that's outdated. + */ +function tryReadFromCache(sourceSnippet: TypeScriptSnippet, cache: LanguageTablet, fingerprinter: TypeFingerprinter) { + const fromCache = cache.tryGetSnippet(snippetKey(sourceSnippet)); + + const cacheable = + fromCache && + completeSource(sourceSnippet) === fromCache.snippet.fullSource && + Object.entries(TARGET_LANGUAGES).every( + ([lang, translator]) => fromCache.snippet.translations?.[lang]?.version === translator.version, + ) && + fingerprinter.fingerprintAll(fromCache.fqnsReferenced()) === fromCache.snippet.fqnsFingerprint; + + return cacheable ? fromCache : undefined; +} diff --git a/packages/jsii-rosetta/lib/commands/extract_worker.ts b/packages/jsii-rosetta/lib/commands/extract_worker.ts index 8316c1548c..9a46ac924e 100644 --- a/packages/jsii-rosetta/lib/commands/extract_worker.ts +++ b/packages/jsii-rosetta/lib/commands/extract_worker.ts @@ -20,10 +20,10 @@ export interface TranslateBatchResponse { } function translateBatch(request: TranslateBatchRequest): TranslateBatchResponse { - const result = singleThreadedTranslateAll(request.snippets[Symbol.iterator](), request.includeCompilerDiagnostics); + const result = singleThreadedTranslateAll(request.snippets, request.includeCompilerDiagnostics); return { - translatedSchemas: result.translatedSnippets.map((s) => s.toSchema()), + translatedSchemas: result.translatedSnippets.map((s) => s.snippet), diagnostics: result.diagnostics, }; } diff --git a/packages/jsii-rosetta/lib/commands/infuse.ts b/packages/jsii-rosetta/lib/commands/infuse.ts index 1cbb582a73..1bf2b86a57 100644 --- a/packages/jsii-rosetta/lib/commands/infuse.ts +++ b/packages/jsii-rosetta/lib/commands/infuse.ts @@ -159,7 +159,7 @@ function mapFqns(tab: LanguageTablet): Record { for (const key of tab.snippetKeys) { const snippet = tab.tryGetSnippet(key)!; - for (const fqn of snippet.fqnsReferenced) { + for (const fqn of snippet.snippet.fqnsReferenced ?? []) { fqnsReferencedMap.add(fqn, snippet); } } diff --git a/packages/jsii-rosetta/lib/commands/read.ts b/packages/jsii-rosetta/lib/commands/read.ts index a998f78ae6..6668681f33 100644 --- a/packages/jsii-rosetta/lib/commands/read.ts +++ b/packages/jsii-rosetta/lib/commands/read.ts @@ -24,8 +24,8 @@ export async function readTablet(tabletFile: string, key?: string, lang?: string } function displaySnippet(snippet: TranslatedSnippet) { - if (snippet.didCompile !== undefined) { - process.stdout.write(`Compiled: ${snippet.didCompile}\n`); + if (snippet.snippet.didCompile !== undefined) { + process.stdout.write(`Compiled: ${snippet.snippet.didCompile}\n`); } if (lang !== undefined) { diff --git a/packages/jsii-rosetta/lib/commands/transliterate.ts b/packages/jsii-rosetta/lib/commands/transliterate.ts index 5b3cfbb013..23432d9c31 100644 --- a/packages/jsii-rosetta/lib/commands/transliterate.ts +++ b/packages/jsii-rosetta/lib/commands/transliterate.ts @@ -6,7 +6,7 @@ import { fixturize } from '../fixtures'; import { TargetLanguage } from '../languages'; import { debug } from '../logging'; import { Rosetta } from '../rosetta'; -import { SnippetParameters, typeScriptSnippetFromSource } from '../snippet'; +import { SnippetParameters, typeScriptSnippetFromSource, ApiLocation } from '../snippet'; import { Translation } from '../tablets/tablets'; export interface TransliterateAssemblyOptions { @@ -69,6 +69,7 @@ export async function transliterateAssembly( const result = await loadAssembly(); if (result.readme?.markdown) { result.readme.markdown = rosetta.translateSnippetsInMarkdown( + { api: 'moduleReadme', moduleFqn: result.name }, result.readme.markdown, language, true /* strict */, @@ -161,29 +162,35 @@ function transliterateType( workingDirectory: string, loose = false, ): void { - transliterateDocs(type.docs); + transliterateDocs({ api: 'type', fqn: type.fqn }, type.docs, workingDirectory); switch (type.kind) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore 7029 case TypeKind.Class: - transliterateDocs(type?.initializer?.docs); + if (type.initializer) { + transliterateDocs({ api: 'initializer', fqn: type.fqn }, type.initializer.docs, workingDirectory); + } // fallthrough case TypeKind.Interface: for (const method of type.methods ?? []) { - transliterateDocs(method.docs); + transliterateDocs({ api: 'member', fqn: type.fqn, memberName: method.name }, method.docs, workingDirectory); for (const parameter of method.parameters ?? []) { - transliterateDocs(parameter.docs); + transliterateDocs( + { api: 'parameter', fqn: type.fqn, methodName: method.name, parameterName: parameter.name }, + parameter.docs, + workingDirectory, + ); } } for (const property of type.properties ?? []) { - transliterateDocs(property.docs); + transliterateDocs({ api: 'member', fqn: type.fqn, memberName: property.name }, property.docs, workingDirectory); } break; case TypeKind.Enum: for (const member of type.members) { - transliterateDocs(member.docs); + transliterateDocs({ api: 'member', fqn: type.fqn, memberName: member.name }, member.docs, workingDirectory); } break; @@ -191,10 +198,25 @@ function transliterateType( throw new Error(`Unsupported type kind: ${(type as any).kind}`); } - function transliterateDocs(docs: Docs | undefined) { + function transliterateDocs(api: ApiLocation, docs: Docs | undefined, workingDirectory: string) { + if (docs?.remarks) { + docs.remarks = rosetta.translateSnippetsInMarkdown( + api, + docs.remarks, + language, + true /* strict */, + (translation) => ({ + language: translation.language, + source: prefixDisclaimer(translation), + }), + workingDirectory, + ); + } + if (docs?.example) { + const location = { api, field: { field: 'example' } } as const; const snippet = fixturize( - typeScriptSnippetFromSource(docs.example, 'example', true /* strict */, { + typeScriptSnippetFromSource(docs.example, location, true /* strict */, { [SnippetParameters.$PROJECT_DIRECTORY]: workingDirectory, }), loose, diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts index 338d4ab2cd..f3cceb2d2a 100644 --- a/packages/jsii-rosetta/lib/jsii/assemblies.ts +++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts @@ -5,7 +5,13 @@ import * as path from 'path'; import { fixturize } from '../fixtures'; import { extractTypescriptSnippetsFromMarkdown } from '../markdown/extract-snippets'; -import { TypeScriptSnippet, typeScriptSnippetFromSource, updateParameters, SnippetParameters } from '../snippet'; +import { + TypeScriptSnippet, + typeScriptSnippetFromSource, + updateParameters, + SnippetParameters, + ApiLocation, +} from '../snippet'; import { enforcesStrictMode } from '../strict'; // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports @@ -43,8 +49,8 @@ async function loadAssemblyFromFile(filename: string, validate: boolean): Promis } export type AssemblySnippetSource = - | { type: 'markdown'; markdown: string; where: string } - | { type: 'literal'; source: string; where: string }; + | { type: 'markdown'; markdown: string; location: ApiLocation } + | { type: 'example'; source: string; location: ApiLocation }; /** * Return all markdown and example snippets from the given assembly @@ -56,7 +62,7 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc ret.push({ type: 'markdown', markdown: assembly.readme.markdown, - where: removeSlashes(`${assembly.name}-README`), + location: { api: 'moduleReadme', moduleFqn: assembly.name }, }); } @@ -65,28 +71,28 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc ret.push({ type: 'markdown', markdown: submodule.readme.markdown, - where: removeSlashes(`${submoduleFqn}-README`), + location: { api: 'moduleReadme', moduleFqn: submoduleFqn }, }); } } if (assembly.types) { Object.values(assembly.types).forEach((type) => { - emitDocs(type.docs, `${assembly.name}.${type.name}`); + emitDocs(type.docs, { api: 'type', fqn: type.fqn }); if (spec.isEnumType(type)) { - type.members.forEach((m) => emitDocs(m.docs, `${assembly.name}.${type.name}.${m.name}`)); + type.members.forEach((m) => emitDocs(m.docs, { api: 'member', fqn: type.fqn, memberName: m.name })); } if (spec.isClassOrInterfaceType(type)) { - (type.methods ?? []).forEach((m) => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`)); - (type.properties ?? []).forEach((m) => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`)); + (type.methods ?? []).forEach((m) => emitDocs(m.docs, { api: 'member', fqn: type.fqn, memberName: m.name })); + (type.properties ?? []).forEach((m) => emitDocs(m.docs, { api: 'member', fqn: type.fqn, memberName: m.name })); } }); } return ret; - function emitDocs(docs: spec.Docs | undefined, where: string) { + function emitDocs(docs: spec.Docs | undefined, location: ApiLocation) { if (!docs) { return; } @@ -95,51 +101,45 @@ export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSourc ret.push({ type: 'markdown', markdown: docs.remarks, - where: removeSlashes(where), + location, }); } if (docs.example && exampleLooksLikeSource(docs.example)) { ret.push({ - type: 'literal', + type: 'example', source: docs.example, - where: removeSlashes(`${where}-example`), + location, }); } } } -/** - * Remove slashes from a "where" description, as the TS compiler will interpret it as a directory - * and we can't have that for compiling literate files - */ -function removeSlashes(x: string) { - return x.replace(/\//g, '.'); -} +export function allTypeScriptSnippets(assemblies: readonly LoadedAssembly[], loose = false): TypeScriptSnippet[] { + const ret = new Array(); -export function* allTypeScriptSnippets( - assemblies: readonly LoadedAssembly[], - loose = false, -): IterableIterator { for (const { assembly, directory } of assemblies) { const strict = enforcesStrictMode(assembly); for (const source of allSnippetSources(assembly)) { switch (source.type) { - case 'literal': - const snippet = updateParameters(typeScriptSnippetFromSource(source.source, source.where, strict), { + case 'example': + const location = { api: source.location, field: { field: 'example' } } as const; + + const snippet = updateParameters(typeScriptSnippetFromSource(source.source, location, strict), { [SnippetParameters.$PROJECT_DIRECTORY]: directory, }); - yield fixturize(snippet, loose); + ret.push(fixturize(snippet, loose)); break; case 'markdown': - for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.where, strict)) { + for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.location, strict)) { const withDirectory = updateParameters(snippet, { [SnippetParameters.$PROJECT_DIRECTORY]: directory, }); - yield fixturize(withDirectory, loose); + ret.push(fixturize(withDirectory, loose)); } } } } + return ret; } /** @@ -172,7 +172,7 @@ export async function replaceAssembly(assembly: spec.Assembly, directory: string * We should make sure not to change one without changing the other as well. */ function _fingerprint(assembly: spec.Assembly): spec.Assembly { - delete assembly.fingerprint; + delete (assembly as any).fingerprint; assembly = sortJson(assembly); const fingerprint = crypto.createHash('sha256').update(JSON.stringify(assembly)).digest('base64'); return { ...assembly, fingerprint }; diff --git a/packages/jsii-rosetta/lib/jsii/fingerprinting.ts b/packages/jsii-rosetta/lib/jsii/fingerprinting.ts new file mode 100644 index 0000000000..36215782c3 --- /dev/null +++ b/packages/jsii-rosetta/lib/jsii/fingerprinting.ts @@ -0,0 +1,158 @@ +import * as spec from '@jsii/spec'; +import * as crypto from 'crypto'; + +/** + * Return a fingerprint for a type. + * + * The fingerprint will change if the API of the given type changes. + * + * The fingerprint is an approximation, it's not exhaustive. It will not trace + * into types from assemblies it can't see, for example. For the purposes of Rosetta, + * we'll assume this is Good Enough™. + */ +export class TypeFingerprinter { + private readonly cache = new Map(); + private readonly assemblies = new Map(); + + public constructor(assemblies: spec.Assembly[]) { + for (const assembly of assemblies) { + this.assemblies.set(assembly.name, assembly); + } + } + + /** + * Return a single fingerprint that encompasses all fqns in the list + */ + public fingerprintAll(fqns: string[]) { + const hash = crypto.createHash('sha256'); + for (const fqn of fqns) { + hash.update(this.fingerprintType(fqn)); + } + return hash.digest('hex'); + } + + /** + * Return the fingerprint for the given FQN in the assembly of this fingerprinter + * + * The fingerprint is always going to contain the FQN, even if the type doesn't exist + * in this assembly. + */ + public fingerprintType(fqn: string) { + return this.doFingerprint(fqn, new Set([fqn])); + } + + private doFingerprint(fqn: string, recursionBreaker: Set) { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + const existing = this.cache.get(fqn); + if (existing) { + return existing; + } + + const hash = crypto.createHash('sha256'); + hash.update(fqn); + + const type = this.findType(fqn); + if (type) { + hash.update(type.kind); + switch (type.kind) { + case spec.TypeKind.Enum: + for (const member of sortedByName(type.members)) { + hash.update(member.name); + } + break; + case spec.TypeKind.Class: + case spec.TypeKind.Interface: + if (type.kind === spec.TypeKind.Class) { + visitType(type.base); + visitCallable(type.initializer); + } + + for (const prop of sortedByName(type.properties ?? [])) { + hash.update(prop.name); + visitBools(prop.immutable, prop.static, prop.optional, prop.protected); + visitTypeReference(prop.type); + } + for (const method of sortedByName(type.methods ?? [])) { + hash.update(method.name); + visitCallable(method); + visitBools(method.returns?.optional); + visitTypeReference(method.returns?.type); + } + for (const implint of type.interfaces ?? []) { + visitType(implint); + } + + break; + } + } + + const ret = hash.digest('hex'); + this.cache.set(fqn, ret); + return ret; + + function visitType(fqn?: string) { + if (!fqn) { + return; + } + + if (recursionBreaker.has(fqn)) { + hash.update('$RECURSION$'); + return; + } + + recursionBreaker.add(fqn); + hash.update(self.doFingerprint(fqn, recursionBreaker)); + recursionBreaker.delete(fqn); + } + + function visitCallable(callable?: spec.Callable) { + if (!callable) { + return; + } + + visitBools(callable.protected); + for (const param of callable.parameters ?? []) { + visitBools(param.optional, param.variadic); + visitTypeReference(param.type); + } + } + + function visitTypeReference(typeRef?: spec.TypeReference) { + if (!typeRef) { + return; + } + + if (spec.isPrimitiveTypeReference(typeRef)) { + hash.update(typeRef.primitive); + } + if (spec.isNamedTypeReference(typeRef)) { + visitType(typeRef.fqn); + } + if (spec.isCollectionTypeReference(typeRef)) { + hash.update(typeRef.collection.kind); + visitTypeReference(typeRef.collection.elementtype); + } + if (spec.isUnionTypeReference(typeRef)) { + for (const type of typeRef.union.types) { + visitTypeReference(type); + } + } + } + + function visitBools(...vs: Array) { + hash.update(vs.map((v) => (v ? '1' : '0')).join('')); + } + } + + private findType(fqn: string) { + const assemblyName = fqn.split('.')[0]; + return this.assemblies.get(assemblyName)?.types?.[fqn]; + } +} + +function sortedByName(xs: A[]): A[] { + xs.sort((a, b) => a.name.localeCompare(b.name)); + return xs; +} diff --git a/packages/jsii-rosetta/lib/languages/csharp.ts b/packages/jsii-rosetta/lib/languages/csharp.ts index c3ad10fbaf..813e3c1843 100644 --- a/packages/jsii-rosetta/lib/languages/csharp.ts +++ b/packages/jsii-rosetta/lib/languages/csharp.ts @@ -70,6 +70,14 @@ interface CSharpLanguageContext { type CSharpRenderer = AstRenderer; export class CSharpVisitor extends DefaultVisitor { + /** + * Translation version + * + * Bump this when you change something in the implementation to invalidate + * existing cached translations. + */ + public static readonly VERSION = '1'; + public readonly language = TargetLanguage.CSHARP; public readonly defaultContext = { diff --git a/packages/jsii-rosetta/lib/languages/index.ts b/packages/jsii-rosetta/lib/languages/index.ts index 8bfa9c34cd..28bcf72141 100644 --- a/packages/jsii-rosetta/lib/languages/index.ts +++ b/packages/jsii-rosetta/lib/languages/index.ts @@ -6,10 +6,22 @@ import { TargetLanguage } from './target-language'; export { TargetLanguage }; -export type VisitorFactory = () => AstHandler; +export interface VisitorFactory { + readonly version: string; + createVisitor(): AstHandler; +} export const TARGET_LANGUAGES: { [key in TargetLanguage]: VisitorFactory } = { - python: () => new PythonVisitor(), - csharp: () => new CSharpVisitor(), - java: () => new JavaVisitor(), + python: { + version: PythonVisitor.VERSION, + createVisitor: () => new PythonVisitor(), + }, + csharp: { + version: CSharpVisitor.VERSION, + createVisitor: () => new CSharpVisitor(), + }, + java: { + version: JavaVisitor.VERSION, + createVisitor: () => new JavaVisitor(), + }, }; diff --git a/packages/jsii-rosetta/lib/languages/java.ts b/packages/jsii-rosetta/lib/languages/java.ts index f0bdc77231..22ff74db03 100644 --- a/packages/jsii-rosetta/lib/languages/java.ts +++ b/packages/jsii-rosetta/lib/languages/java.ts @@ -93,6 +93,14 @@ interface InsideTypeDeclaration { type JavaRenderer = AstRenderer; export class JavaVisitor extends DefaultVisitor { + /** + * Translation version + * + * Bump this when you change something in the implementation to invalidate + * existing cached translations. + */ + public static readonly VERSION = '1'; + public readonly language = TargetLanguage.JAVA; public readonly defaultContext = {}; diff --git a/packages/jsii-rosetta/lib/languages/python.ts b/packages/jsii-rosetta/lib/languages/python.ts index 39e065f6e0..08e90692ef 100644 --- a/packages/jsii-rosetta/lib/languages/python.ts +++ b/packages/jsii-rosetta/lib/languages/python.ts @@ -88,6 +88,14 @@ export interface PythonVisitorOptions { } export class PythonVisitor extends DefaultVisitor { + /** + * Translation version + * + * Bump this when you change something in the implementation to invalidate + * existing cached translations. + */ + public static readonly VERSION = '1'; + public readonly language = TargetLanguage.PYTHON; public readonly defaultContext = {}; diff --git a/packages/jsii-rosetta/lib/markdown/extract-snippets.ts b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts index 5f09699704..279f45cd39 100644 --- a/packages/jsii-rosetta/lib/markdown/extract-snippets.ts +++ b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts @@ -1,7 +1,7 @@ import * as cm from 'commonmark'; import { visitCommonMarkTree } from '../markdown/markdown'; -import { TypeScriptSnippet } from '../snippet'; +import { TypeScriptSnippet, ApiLocation } from '../snippet'; import { ReplaceTypeScriptTransform } from './replace-typescript-transform'; import { CodeBlock } from './types'; @@ -9,7 +9,7 @@ export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefi export function extractTypescriptSnippetsFromMarkdown( markdown: string, - wherePrefix: string, + location: ApiLocation, strict: boolean, ): TypeScriptSnippet[] { const parser = new cm.Parser(); @@ -19,7 +19,7 @@ export function extractTypescriptSnippetsFromMarkdown( visitCommonMarkTree( doc, - new ReplaceTypeScriptTransform(wherePrefix, strict, (ts) => { + new ReplaceTypeScriptTransform(location, strict, (ts) => { ret.push(ts); return undefined; }), diff --git a/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts index 2ea2a97d8c..c7d0acbb9e 100644 --- a/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts +++ b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts @@ -3,7 +3,7 @@ import * as cm from 'commonmark'; import { CommonMarkVisitor } from './markdown'; import { CodeBlock } from './types'; -export type CodeReplacer = (code: CodeBlock) => CodeBlock; +export type CodeReplacer = (code: CodeBlock, line: number) => CodeBlock; /** * Renderer that replaces code blocks in a MarkDown document @@ -12,10 +12,14 @@ export class ReplaceCodeTransform implements CommonMarkVisitor { public constructor(private readonly replacer: CodeReplacer) {} public code_block(node: cm.Node) { - const ret = this.replacer({ - language: node.info ?? '', - source: node.literal ?? '', - }); + const line = node.sourcepos[0][0]; + const ret = this.replacer( + { + language: node.info ?? '', + source: node.literal ?? '', + }, + line, + ); node.info = ret.language; node.literal = ret.source + (!ret.source || ret.source.endsWith('\n') ? '' : '\n'); } diff --git a/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts index 113a835fe2..51a0b614e0 100644 --- a/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts +++ b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts @@ -1,4 +1,4 @@ -import { TypeScriptSnippet, typeScriptSnippetFromSource, parseKeyValueList } from '../snippet'; +import { TypeScriptSnippet, typeScriptSnippetFromSource, parseKeyValueList, ApiLocation } from '../snippet'; import { ReplaceCodeTransform } from './replace-code-renderer'; import { CodeBlock } from './types'; @@ -8,36 +8,21 @@ export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefi * A specialization of ReplaceCodeTransform that maintains state about TypeScript snippets */ export class ReplaceTypeScriptTransform extends ReplaceCodeTransform { - private readonly wherePrefix: string; - - public constructor(wherePrefix: string, strict: boolean, replacer: TypeScriptReplacer) { - let count = 0; - super((block) => { + public constructor(api: ApiLocation, strict: boolean, replacer: TypeScriptReplacer) { + super((block, line) => { const languageParts = block.language ? block.language.split(' ') : []; if (languageParts[0] !== 'typescript' && languageParts[0] !== 'ts') { return block; } - count += 1; const tsSnippet = typeScriptSnippetFromSource( block.source, - this.addSnippetNumber(count), + { api, field: { field: 'markdown', line } }, strict, parseKeyValueList(languageParts.slice(1)), ); return replacer(tsSnippet) ?? block; }); - - this.wherePrefix = wherePrefix; - } - - private addSnippetNumber(snippetNumber: number) { - // First snippet (most cases) will not be numbered - if (snippetNumber === 1) { - return this.wherePrefix; - } - - return `${this.wherePrefix}-snippet${snippetNumber}`; } } diff --git a/packages/jsii-rosetta/lib/rosetta.ts b/packages/jsii-rosetta/lib/rosetta.ts index 8d92cb5f92..3c542364d3 100644 --- a/packages/jsii-rosetta/lib/rosetta.ts +++ b/packages/jsii-rosetta/lib/rosetta.ts @@ -9,9 +9,15 @@ import { transformMarkdown } from './markdown/markdown'; import { MarkdownRenderer } from './markdown/markdown-renderer'; import { ReplaceTypeScriptTransform } from './markdown/replace-typescript-transform'; import { CodeBlock } from './markdown/types'; -import { SnippetParameters, TypeScriptSnippet, updateParameters } from './snippet'; +import { + SnippetParameters, + TypeScriptSnippet, + updateParameters, + ApiLocation, + typeScriptSnippetFromSource, +} from './snippet'; import { DEFAULT_TABLET_NAME, LanguageTablet, Translation } from './tablets/tablets'; -import { Translator, rosettaDiagFromTypescript } from './translate'; +import { Translator } from './translate'; import { printDiagnostics } from './util'; export interface RosettaOptions { @@ -56,8 +62,15 @@ export interface RosettaOptions { * when the second one is not necessary. */ export class Rosetta { + /** + * Newly translated samples + * + * In case live translation has been enabled, all samples that have been translated on-the-fly + * are added to this tablet. + */ + public readonly liveTablet = new LanguageTablet(); + private readonly loadedTablets: LanguageTablet[] = []; - private readonly liveTablet = new LanguageTablet(); private readonly extractedSnippets = new Map(); private readonly translator: Translator; private readonly loose: boolean; @@ -71,7 +84,7 @@ export class Rosetta { * Diagnostics encountered while doing live translation */ public get diagnostics() { - return this.translator.diagnostics.map(rosettaDiagFromTypescript); + return this.translator.diagnostics; } /** @@ -99,11 +112,9 @@ export class Rosetta { * loaded. * * Otherwise, if live conversion is enabled, the snippets in the assembly - * become available for live translation later. - * - * (We do it like this to centralize the logic around the "where" calculation, - * otherwise each language generator has to reimplement a way to describe API - * elements while spidering the jsii assembly). + * become available for live translation later. This is necessary because we probably + * need to fixture snippets for successful compilation, and the information + * pacmak sends our way later on is not going to be enough to do that. */ public async addAssembly(assembly: spec.Assembly, assemblyDir: string) { if (await fs.pathExists(path.join(assemblyDir, DEFAULT_TABLET_NAME))) { @@ -118,6 +129,18 @@ export class Rosetta { } } + /** + * Translate the given snippet for the given target language + * + * This will either: + * + * - Find an existing translation in a tablet and return that, if available. + * - Otherwise, find a fixturized version of this snippet in an assembly that + * was loaded beforehand, and translate it on-the-fly. Finding the fixture + * will be based on the snippet key, which consists of a hash of the + * visible source and the API location. + * - Otherwise, translate the snippet as-is (without fixture information). + */ public translateSnippet(source: TypeScriptSnippet, targetLang: TargetLanguage): Translation | undefined { // Look for it in loaded tablets for (const tab of this.allTablets) { @@ -151,7 +174,37 @@ export class Rosetta { return snippet.get(targetLang); } + /** + * Translate a snippet found in the "@ example" section of a jsii assembly + * + * Behaves exactly like `translateSnippet`, so see that method for documentation. + */ + public translateExample( + apiLocation: ApiLocation, + example: string, + targetLang: TargetLanguage, + strict: boolean, + compileDirectory = process.cwd(), + ): Translation { + const location = { api: apiLocation, field: { field: 'example' } } as const; + + const snippet = typeScriptSnippetFromSource(example, location, strict, { + [SnippetParameters.$COMPILATION_DIRECTORY]: compileDirectory, + }); + + const translated = this.translateSnippet(snippet, targetLang); + + return translated ?? { language: 'typescript', source: example }; + } + + /** + * Translate all TypeScript snippets found in a block of Markdown text + * + * For each snippet, behaves exactly like `translateSnippet`, so see that + * method for documentation. + */ public translateSnippetsInMarkdown( + apiLocation: ApiLocation, markdown: string, targetLang: TargetLanguage, strict: boolean, @@ -161,7 +214,7 @@ export class Rosetta { return transformMarkdown( markdown, new MarkdownRenderer(), - new ReplaceTypeScriptTransform('markdown', strict, (tsSnip) => { + new ReplaceTypeScriptTransform(apiLocation, strict, (tsSnip) => { const translated = this.translateSnippet( updateParameters(tsSnip, { [SnippetParameters.$COMPILATION_DIRECTORY]: compileDirectory, diff --git a/packages/jsii-rosetta/lib/snippet-selectors.ts b/packages/jsii-rosetta/lib/snippet-selectors.ts index 127057b2e5..0d3728d88c 100644 --- a/packages/jsii-rosetta/lib/snippet-selectors.ts +++ b/packages/jsii-rosetta/lib/snippet-selectors.ts @@ -61,7 +61,7 @@ export function mean(snippets: TranslatedSnippet[]): TranslatedSnippet { // Find mean counter. const counters: Array> = []; snippets.map((snippet) => { - counters.push(snippet.syntaxKindCounter); + counters.push(snippet.snippet.syntaxKindCounter ?? {}); }); const meanCounter = findCenter(counters); // Find counter with closest euclidian distance. diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts index d585270fa1..c8d1a6d07c 100644 --- a/packages/jsii-rosetta/lib/snippet.ts +++ b/packages/jsii-rosetta/lib/snippet.ts @@ -8,9 +8,9 @@ export interface TypeScriptSnippet { readonly visibleSource: string; /** - * A human-readable description of where this snippet was found in the assembly + * Description of where the snippet was found */ - readonly where: string; + readonly location: SnippetLocation; /** * When enhanced with a fixture, the snippet's complete source code @@ -30,6 +30,76 @@ export interface TypeScriptSnippet { readonly strict?: boolean; } +/** + * Description of a location where the snippet is found + * + * The location does not necessarily indicate an exact source file, + * but it will generally refer to a location that can contain one or more + * snippets. + */ +export interface SnippetLocation { + /** + * The jsii API with which this snippet is associated + */ + readonly api: ApiLocation; + + /** + * The API field in which the snippet is found, if any + * + * Absence of this field is appropriate for source files (or tests), + * but for Markdown files 'field' should really be set to a Markdown + * location. + */ + readonly field?: FieldLocation; +} + +export type ApiLocation = + | { readonly api: 'file'; readonly fileName: string } + | { readonly api: 'moduleReadme'; readonly moduleFqn: string } + | { readonly api: 'type'; readonly fqn: string } + | { readonly api: 'initializer'; readonly fqn: string } + | { readonly api: 'member'; readonly fqn: string; readonly memberName: string } + | { readonly api: 'parameter'; readonly fqn: string; readonly methodName: string; readonly parameterName: string }; + +export type FieldLocation = { readonly field: 'markdown'; readonly line: number } | { readonly field: 'example' }; + +/** + * Render an API location to a human readable representation + */ +export function formatLocation(location: SnippetLocation): string { + switch (location.field?.field) { + case 'example': + return `${renderApiLocation(location.api)}-example`; + case 'markdown': + return `${renderApiLocation(location.api)}-L${location.field.line}`; + case undefined: + return renderApiLocation(location.api); + } +} + +/** + * Render an API location to an unique string + * + * This function is used in hashing examples for reuse, and so the formatting + * here should not be changed lightly. + */ +export function renderApiLocation(apiLoc: ApiLocation): string { + switch (apiLoc.api) { + case 'file': + return apiLoc.fileName; + case 'moduleReadme': + return `${apiLoc.moduleFqn}-README`; + case 'type': + return apiLoc.fqn; + case 'initializer': + return `${apiLoc.fqn}#initializer`; + case 'member': + return `${apiLoc.fqn}#${apiLoc.memberName}`; + case 'parameter': + return `${apiLoc.fqn}#${apiLoc.methodName}!#${apiLoc.parameterName}`; + } +} + /** * Construct a TypeScript snippet from literal source * @@ -37,14 +107,14 @@ export interface TypeScriptSnippet { */ export function typeScriptSnippetFromSource( typeScriptSource: string, - where: string, + location: SnippetLocation, strict: boolean, parameters: Record = {}, ): TypeScriptSnippet { const [source, sourceParameters] = parametersFromSourceDirectives(typeScriptSource); return { visibleSource: source.trimRight(), - where, + location, parameters: Object.assign({}, parameters, sourceParameters), strict, }; diff --git a/packages/jsii-rosetta/lib/tablets/key.ts b/packages/jsii-rosetta/lib/tablets/key.ts index 09a5a7fe6e..04bbcab003 100644 --- a/packages/jsii-rosetta/lib/tablets/key.ts +++ b/packages/jsii-rosetta/lib/tablets/key.ts @@ -1,12 +1,15 @@ import * as crypto from 'crypto'; -import { TypeScriptSnippet } from '../snippet'; +import { TypeScriptSnippet, renderApiLocation } from '../snippet'; /** * Determine the key for a code block */ export function snippetKey(snippet: TypeScriptSnippet) { const h = crypto.createHash('sha256'); + // Mix in API location to distinguish between similar snippets + h.update(renderApiLocation(snippet.location.api)); + h.update(':'); h.update(snippet.visibleSource); return h.digest('hex'); } diff --git a/packages/jsii-rosetta/lib/tablets/schema.ts b/packages/jsii-rosetta/lib/tablets/schema.ts index 1c06d2b6da..f17d524625 100644 --- a/packages/jsii-rosetta/lib/tablets/schema.ts +++ b/packages/jsii-rosetta/lib/tablets/schema.ts @@ -1,3 +1,5 @@ +import { SnippetLocation } from '../snippet'; + /** * Tablet file schema */ @@ -37,34 +39,47 @@ export interface TranslatedSnippetSchema { * Since TypeScript is a valid output translation, the original will be * listed under the key '$'. */ - translations: { [key: string]: TranslationSchema }; + readonly translations: { [key: string]: TranslationSchema }; /** - * A human-readable description of the location this code snippet was found + * A description of the location this code snippet was found */ - where: string; + readonly location: SnippetLocation; /** * Whether this was compiled without errors * * Undefined means compilation was not attempted. */ - didCompile?: boolean; + readonly didCompile?: boolean; /** * FQNs of classes and functions referenced in this snippet. */ - fqnsReferenced?: string[]; + readonly fqnsReferenced?: string[]; + + /** + * A fingerprint of the types referenced in `fqnsReferenced`. + * + * This can be used to validate/invalidate previous compilations of a snippet. + * + * A snippet needs to be recompiled if: + * + * - Its source text changes: hash will be different + * - Its fixture changes: fullSource will be different + * - The referenced types have changed: fingerprint will be different + */ + readonly fqnsFingerprint?: string; /** * Counts the number of instances each kind of Typescript object shows up in the snippet AST. */ - syntaxKindCounter?: { [key: number]: number }; + readonly syntaxKindCounter?: { [key: string]: number }; /** * The full source (with fixture) that was compiled */ - fullSource?: string; + readonly fullSource?: string; } /** @@ -74,5 +89,10 @@ export interface TranslationSchema { /** * The source of a single translation */ - source: string; + readonly source: string; + + /** + * The version of the translator used to obtain this translation + */ + readonly version: string; } diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index fb4be70b9d..d11accd3f4 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -3,23 +3,32 @@ import * as path from 'path'; import { TargetLanguage } from '../languages'; import { TypeScriptSnippet } from '../snippet'; +import { mapValues } from '../util'; import { snippetKey } from './key'; -import { TabletSchema, TranslatedSnippetSchema, TranslationSchema, ORIGINAL_SNIPPET_KEY } from './schema'; +import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './schema'; // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires const TOOL_VERSION = require('../../package.json').version; export const DEFAULT_TABLET_NAME = '.jsii.tabl.json'; +export const CURRENT_SCHEMA_VERSION = '2'; + /** * A tablet containing various snippets in multiple languages */ export class LanguageTablet { + public static async fromFile(filename: string) { + const ret = new LanguageTablet(); + await ret.load(filename); + return ret; + } + private readonly snippets: Record = {}; public addSnippet(snippet: TranslatedSnippet) { const existingSnippet = this.snippets[snippet.key]; - this.snippets[snippet.key] = existingSnippet ? existingSnippet.merge(snippet) : snippet; + this.snippets[snippet.key] = existingSnippet ? existingSnippet.mergeTranslations(snippet) : snippet; } public get snippetKeys() { @@ -36,14 +45,17 @@ export class LanguageTablet { } public async load(filename: string) { - const obj = await fs.readJson(filename, { encoding: 'utf-8' }); + const obj = (await fs.readJson(filename, { encoding: 'utf-8' })) as TabletSchema; if (!obj.toolVersion || !obj.snippets) { throw new Error(`File '${filename}' does not seem to be a Tablet file`); } - if (obj.toolVersion !== TOOL_VERSION && TOOL_VERSION !== '0.0.0') { + + if (obj.version !== CURRENT_SCHEMA_VERSION) { + // If we're ever changing the schema version in a backwards incompatible way, + // do upconversion here. throw new Error( - `Tablet file '${filename}' has been created with version '${obj.toolVersion}', cannot read with current version '${TOOL_VERSION}'`, + `Tablet file '${filename}' has schema version '${obj.version}', this program expects '${CURRENT_SCHEMA_VERSION}'`, ); } @@ -67,60 +79,43 @@ export class LanguageTablet { private toSchema(): TabletSchema { return { - version: '1', + version: CURRENT_SCHEMA_VERSION, toolVersion: TOOL_VERSION, - snippets: mapValues(this.snippets, (s) => s.toSchema()), + snippets: mapValues(this.snippets, (s) => s.snippet), }; } } +/** + * Mutable operations on an underlying TranslatedSnippetSchema + */ export class TranslatedSnippet { public static fromSchema(schema: TranslatedSnippetSchema) { - const ret = new TranslatedSnippet(); - Object.assign(ret.translations, schema.translations); - Object.assign(ret._fqnsReferenced, schema.fqnsReferenced); - Object.assign(ret._syntaxKindCounter, schema.syntaxKindCounter); - ret._didCompile = schema.didCompile; - ret._where = schema.where; - ret.fullSource = schema.fullSource; - return ret; + if (!schema.translations[ORIGINAL_SNIPPET_KEY]) { + throw new Error(`Input schema must have '${ORIGINAL_SNIPPET_KEY}' key set in translations`); + } + return new TranslatedSnippet(schema); } - public static fromSnippet(original: TypeScriptSnippet, didCompile?: boolean) { - const ret = new TranslatedSnippet(); - Object.assign(ret.translations, { - [ORIGINAL_SNIPPET_KEY]: { source: original.visibleSource }, + public static fromTypeScript(original: TypeScriptSnippet, didCompile?: boolean) { + return new TranslatedSnippet({ + translations: { + [ORIGINAL_SNIPPET_KEY]: { source: original.visibleSource, version: '0' }, + }, + didCompile: didCompile, + location: original.location, + fullSource: original.completeSource, }); - ret._didCompile = didCompile; - ret._where = original.where; - ret.fullSource = original.completeSource; - return ret; } - private readonly translations: Record = {}; - private readonly _fqnsReferenced = new Array(); - private readonly _syntaxKindCounter: Record = {}; - private _key?: string; - private _didCompile?: boolean; - private _where = ''; - private fullSource?: string; - - private constructor() {} - - public get didCompile() { - return this._didCompile; - } + public readonly snippet: TranslatedSnippetSchema; - public get where() { - return this._where; - } - - public get fqnsReferenced() { - return this._fqnsReferenced; - } + private readonly _snippet: Mutable; + private _key?: string; - public get syntaxKindCounter() { - return this._syntaxKindCounter; + private constructor(snippet: TranslatedSnippetSchema) { + this._snippet = { ...snippet }; + this.snippet = this._snippet; } public get key() { @@ -130,82 +125,65 @@ export class TranslatedSnippet { return this._key; } - public asTypescriptSnippet(): TypeScriptSnippet { - return { - visibleSource: this.translations[ORIGINAL_SNIPPET_KEY].source, - where: this.where, - }; - } - public get originalSource(): Translation { return { - source: this.translations[ORIGINAL_SNIPPET_KEY].source, + source: this.snippet.translations[ORIGINAL_SNIPPET_KEY].source, language: 'typescript', - didCompile: this.didCompile, + didCompile: this.snippet.didCompile, }; } - public addTranslatedSource(language: TargetLanguage, translation: string): Translation { - this.translations[language] = { source: translation }; + public addTranslation(language: TargetLanguage, translation: string, version: string): Translation { + this.snippet.translations[language] = { source: translation, version }; return { source: translation, language, - didCompile: this.didCompile, + didCompile: this.snippet.didCompile, }; } - public addFqnsReferenced(fqnsReferenced: string[]) { - this.fqnsReferenced.push(...fqnsReferenced); + public fqnsReferenced() { + return this._snippet.fqnsReferenced ?? []; } public addSyntaxKindCounter(syntaxKindCounter: Record) { + if (!this._snippet.syntaxKindCounter) { + this._snippet.syntaxKindCounter = {}; + } for (const [key, value] of Object.entries(syntaxKindCounter)) { - this.syntaxKindCounter[key] = value + (this.syntaxKindCounter[key] ?? 0); + const x = this._snippet.syntaxKindCounter[key] ?? 0; + this._snippet.syntaxKindCounter[key] = value + x; } } - public setFullSource(fullSource: string) { - this.fullSource = fullSource; - } - public get languages(): TargetLanguage[] { - return Object.keys(this.translations).filter((x) => x !== ORIGINAL_SNIPPET_KEY) as TargetLanguage[]; + return Object.keys(this.snippet.translations).filter((x) => x !== ORIGINAL_SNIPPET_KEY) as TargetLanguage[]; } public get(language: TargetLanguage): Translation | undefined { - const t = this.translations[language]; - return t && { source: t.source, language, didCompile: this.didCompile }; - } - - public merge(other: TranslatedSnippet) { - const ret = new TranslatedSnippet(); - Object.assign(ret.translations, this.translations, other.translations); - ret._didCompile = this.didCompile; - ret._where = this.where; - ret.fqnsReferenced.splice( - 0, - ret.fqnsReferenced.length, - ...new Set([...this.fqnsReferenced, ...other.fqnsReferenced]), - ); - return ret; + const t = this.snippet.translations[language]; + return t && { source: t.source, language, didCompile: this.snippet.didCompile }; } - public toTypeScriptSnippet() { - return { - source: this.originalSource, - where: this.where, - }; + public mergeTranslations(other: TranslatedSnippet) { + return new TranslatedSnippet({ + ...this.snippet, + translations: { ...this.snippet.translations, ...other.snippet.translations }, + }); + } + + public withFingerprint(fp: string) { + return new TranslatedSnippet({ + ...this.snippet, + fqnsFingerprint: fp, + }); } - public toSchema(): TranslatedSnippetSchema { + private asTypescriptSnippet(): TypeScriptSnippet { return { - translations: this.translations, - didCompile: this.didCompile, - where: this.where, - fqnsReferenced: this.fqnsReferenced, - syntaxKindCounter: this.syntaxKindCounter, - fullSource: this.fullSource, + visibleSource: this.snippet.translations[ORIGINAL_SNIPPET_KEY].source, + location: this.snippet.location, }; } } @@ -216,10 +194,4 @@ export interface Translation { didCompile?: boolean; } -function mapValues(xs: Record, fn: (x: A) => B): Record { - const ret: Record = {}; - for (const [key, value] of Object.entries(xs)) { - ret[key] = fn(value); - } - return ret; -} +type Mutable = { -readonly [P in keyof T]: Mutable }; diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts index b3ae85e554..c12ae6046d 100644 --- a/packages/jsii-rosetta/lib/translate.ts +++ b/packages/jsii-rosetta/lib/translate.ts @@ -6,20 +6,24 @@ import { RecordReferencesVisitor } from './languages/record-references'; import * as logging from './logging'; import { renderTree } from './o-tree'; import { AstRenderer, AstHandler, AstRendererOptions } from './renderer'; -import { TypeScriptSnippet, completeSource, SnippetParameters } from './snippet'; +import { TypeScriptSnippet, completeSource, SnippetParameters, formatLocation } from './snippet'; import { snippetKey } from './tablets/key'; +import { ORIGINAL_SNIPPET_KEY } from './tablets/schema'; import { TranslatedSnippet } from './tablets/tablets'; import { SyntaxKindCounter } from './typescript/syntax-kind-counter'; import { TypeScriptCompiler, CompilationResult } from './typescript/ts-compiler'; import { Spans } from './typescript/visible-spans'; -import { annotateStrictDiagnostic, File, hasStrictBranding } from './util'; +import { annotateStrictDiagnostic, File, hasStrictBranding, mkDict } from './util'; export function translateTypeScript( source: File, visitor: AstHandler, options: SnippetTranslatorOptions = {}, ): TranslateResult { - const translator = new SnippetTranslator({ visibleSource: source.contents, where: source.fileName }, options); + const translator = new SnippetTranslator( + { visibleSource: source.contents, location: { api: { api: 'file', fileName: source.fileName } } }, + options, + ); const translated = translator.renderUsing(visitor); return { @@ -37,34 +41,39 @@ export function translateTypeScript( export class Translator { private readonly compiler = new TypeScriptCompiler(); // eslint-disable-next-line @typescript-eslint/explicit-member-accessibility - #diagnostics: readonly ts.Diagnostic[] = []; + #diagnostics: ts.Diagnostic[] = []; public constructor(private readonly includeCompilerDiagnostics: boolean) {} public translate(snip: TypeScriptSnippet, languages: readonly TargetLanguage[] = Object.values(TargetLanguage)) { logging.debug(`Translating ${snippetKey(snip)} ${inspect(snip.parameters ?? {})}`); const translator = this.translatorFor(snip); - const snippet = TranslatedSnippet.fromSnippet( - snip, - this.includeCompilerDiagnostics ? translator.compileDiagnostics.length === 0 : undefined, - ); - - for (const lang of languages) { - const languageConverterFactory = TARGET_LANGUAGES[lang]; - const translated = translator.renderUsing(languageConverterFactory()); - snippet.addTranslatedSource(lang, translated); - } - snippet.addFqnsReferenced(translator.fqnsReferenced()); - snippet.addSyntaxKindCounter(translator.syntaxKindCounter()); + const translations = mkDict( + languages.map((lang) => { + const languageConverterFactory = TARGET_LANGUAGES[lang]; + const translated = translator.renderUsing(languageConverterFactory.createVisitor()); + return [lang, { source: translated, version: languageConverterFactory.version }] as const; + }), + ); - this.#diagnostics = ts.sortAndDeduplicateDiagnostics(this.#diagnostics.concat(translator.diagnostics)); + this.#diagnostics.push(...translator.diagnostics); - return snippet; + return TranslatedSnippet.fromSchema({ + translations: { + ...translations, + [ORIGINAL_SNIPPET_KEY]: { source: snip.visibleSource, version: '0' }, + }, + location: snip.location, + didCompile: translator.didSuccessfullyCompile, + fqnsReferenced: translator.fqnsReferenced(), + fullSource: snip.completeSource, + syntaxKindCounter: translator.syntaxKindCounter(), + }); } - public get diagnostics(): readonly ts.Diagnostic[] { - return Array.from(this.#diagnostics); + public get diagnostics(): readonly RosettaDiagnostic[] { + return ts.sortAndDeduplicateDiagnostics(this.#diagnostics).map(rosettaDiagFromTypescript); } /** @@ -131,6 +140,10 @@ export interface RosettaDiagnostic { readonly formattedMessage: string; } +export function makeRosettaDiagnostic(isError: boolean, formattedMessage: string): RosettaDiagnostic { + return { isError, formattedMessage, isFromStrictAssembly: false }; +} + /** * Translate a single TypeScript snippet */ @@ -147,7 +160,11 @@ export class SnippetTranslator { const fakeCurrentDirectory = snippet.parameters?.[SnippetParameters.$COMPILATION_DIRECTORY] ?? snippet.parameters?.[SnippetParameters.$PROJECT_DIRECTORY]; - this.compilation = compiler.compileInMemory(snippet.where, source, fakeCurrentDirectory); + this.compilation = compiler.compileInMemory( + removeSlashes(formatLocation(snippet.location)), + source, + fakeCurrentDirectory, + ); // Respect '/// !hide' and '/// !show' directives this.visibleSpans = Spans.visibleSpansFromSource(source); @@ -179,13 +196,22 @@ export class SnippetTranslator { try { return call(...args); } catch (err) { - console.error(`Failed to execute ${call.name}: ${err}`); + const isExpectedTypescriptError = err.message.includes('Error: Debug Failure'); + + if (!isExpectedTypescriptError) { + console.error(`Failed to execute ${call.name}: ${err}`); + } + return []; } }; } } + public get didSuccessfullyCompile() { + return this.compileDiagnostics.length === 0; + } + public renderUsing(visitor: AstHandler) { const converter = new AstRenderer( this.compilation.rootFile, @@ -249,3 +275,11 @@ const DIAG_HOST = { return '\n'; }, }; + +/** + * Remove slashes from a "where" description, as the TS compiler will interpret it as a directory + * and we can't have that for compiling literate files + */ +function removeSlashes(x: string) { + return x.replace(/\/|\\/g, '.'); +} diff --git a/packages/jsii-rosetta/lib/typescript/ts-compiler.ts b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts index 4d58a27d68..0ad5b91637 100644 --- a/packages/jsii-rosetta/lib/typescript/ts-compiler.ts +++ b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts @@ -38,7 +38,7 @@ export class TypeScriptCompiler { const rootFile = program.getSourceFile(filename); if (rootFile == null) { - throw new Error("Oopsie -- couldn't find root file back"); + throw new Error(`Oopsie -- couldn't find root file back: ${filename}`); } return { program, rootFile }; diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts index 00e1ff69d6..9642757a9d 100644 --- a/packages/jsii-rosetta/lib/util.ts +++ b/packages/jsii-rosetta/lib/util.ts @@ -84,3 +84,28 @@ export function setExtend(xs: Set, els: Iterable) { xs.add(el); } } + +export function mkDict(xs: Array): Record { + const ret: any = {}; + for (const [key, value] of xs) { + ret[key] = value; + } + return ret; +} + +export function fmap(value: NonNullable, fn: (x: A) => B): B; +export function fmap(value: undefined, fn: (x: A) => B): undefined; +export function fmap(value: A, fn: (x: A) => B): B | undefined { + if (value === undefined) { + return undefined; + } + return fn(value); +} + +export function mapValues(xs: Record, fn: (x: A) => B): Record { + const ret: Record = {}; + for (const [key, value] of Object.entries(xs)) { + ret[key] = fn(value); + } + return ret; +} diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts new file mode 100644 index 0000000000..30bfc9948e --- /dev/null +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -0,0 +1,99 @@ +import * as path from 'path'; + +import { LanguageTablet } from '../../lib'; +import * as extract from '../../lib/commands/extract'; +import { TARGET_LANGUAGES } from '../../lib/languages'; +import { AssemblyFixture, DUMMY_ASSEMBLY_TARGETS } from '../testutil'; + +const DUMMY_README = ` + Here is an example of how to use ClassA: + + \`\`\`ts + import * as ass from 'my_assembly'; + const aClass = new ass.ClassA(); + aClass.someMethod(); + \`\`\` +`; + +const defaultExtractOptions = { + includeCompilerDiagnostics: false, + validateAssemblies: false, +}; + +let assembly: AssemblyFixture; +beforeAll(async () => { + // Create an assembly in a temp directory + assembly = await AssemblyFixture.fromSource( + { + 'index.ts': ` + export class ClassA { + public someMethod() { + } + } + `, + 'README.md': DUMMY_README, + }, + { + name: 'my_assembly', + jsii: DUMMY_ASSEMBLY_TARGETS, + }, + ); +}); + +afterAll(async () => assembly.cleanup()); + +test('extract samples from test assembly', async () => { + const outputFile = path.join(assembly.directory, 'test.tabl.json'); + await extract.extractSnippets([assembly.directory], { + outputFile, + ...defaultExtractOptions, + }); + + const tablet = new LanguageTablet(); + await tablet.load(outputFile); + + expect(tablet.snippetKeys.length).toEqual(1); +}); + +describe('with cache file', () => { + let cacheTabletFile: string; + beforeAll(async () => { + cacheTabletFile = path.join(assembly.directory, 'cache.tabl.json'); + await extract.extractSnippets([assembly.directory], { + outputFile: cacheTabletFile, + ...defaultExtractOptions, + }); + }); + + test('translation does not happen if it can be read from cache', async () => { + const translationFunction = jest.fn().mockResolvedValue({ diagnostics: [], translatedSnippets: [] }); + + await extract.extractSnippets([assembly.directory], { + outputFile: path.join(assembly.directory, 'dummy.tabl.json'), + cacheTabletFile, + translationFunction, + ...defaultExtractOptions, + }); + + expect(translationFunction).not.toHaveBeenCalled(); + }); + + test('translation does happen if translator version is different', async () => { + const translationFunction = jest.fn().mockResolvedValue({ diagnostics: [], translatedSnippets: [] }); + + const oldJavaVersion = TARGET_LANGUAGES.java.version; + (TARGET_LANGUAGES.java as any).version = '999'; + try { + await extract.extractSnippets([assembly.directory], { + outputFile: path.join(assembly.directory, 'dummy.tabl.json'), + cacheTabletFile, + translationFunction, + ...defaultExtractOptions, + }); + + expect(translationFunction).toHaveBeenCalled(); + } finally { + (TARGET_LANGUAGES.java as any).version = oldJavaVersion; + } + }); +}); diff --git a/packages/jsii-rosetta/test/fixtures.test.ts b/packages/jsii-rosetta/test/fixtures.test.ts index f11b6927bf..d7b259bc65 100644 --- a/packages/jsii-rosetta/test/fixtures.test.ts +++ b/packages/jsii-rosetta/test/fixtures.test.ts @@ -1,11 +1,14 @@ import { fixturize } from '../lib/fixtures'; import { SnippetParameters } from '../lib/snippet'; +import { testSnippetLocation } from './testutil'; + +const location = testSnippetLocation('where'); describe('fixturize', () => { test('snippet retains properties', () => { const snippet = { visibleSource: 'visibleSource', - where: 'where', + location, parameters: { [SnippetParameters.$PROJECT_DIRECTORY]: 'directory', [SnippetParameters.NO_FIXTURE]: '', @@ -23,7 +26,7 @@ declare const mock: Tpe; const val = new Cls();`; const snippet = { visibleSource: source, - where: 'where', + location, parameters: { [SnippetParameters.$PROJECT_DIRECTORY]: 'test', }, diff --git a/packages/jsii-rosetta/test/rosetta.test.ts b/packages/jsii-rosetta/test/rosetta.test.ts index 5b9eba7e89..0730766dcd 100644 --- a/packages/jsii-rosetta/test/rosetta.test.ts +++ b/packages/jsii-rosetta/test/rosetta.test.ts @@ -3,10 +3,11 @@ import * as mockfs from 'mock-fs'; import { Rosetta, LanguageTablet, TranslatedSnippet, TypeScriptSnippet, DEFAULT_TABLET_NAME } from '../lib'; import { TargetLanguage } from '../lib/languages'; import { fakeAssembly } from './jsii/fake-assembly'; +import { testSnippetLocation } from './testutil'; const SAMPLE_CODE: TypeScriptSnippet = { visibleSource: 'callThisFunction();', - where: 'sample', + location: testSnippetLocation('sample'), }; test('Rosetta object can do live translation', () => { @@ -76,6 +77,7 @@ test('Rosetta object can do translation and annotation of snippets in MarkDown', // WHEN const translated = rosetta.translateSnippetsInMarkdown( + { api: 'file', fileName: 'markdown' }, [ '# MarkDown Translation', '', @@ -163,9 +165,9 @@ describe('with mocked filesystem', () => { }); function makeSnippet(original: TypeScriptSnippet, translations: Record) { - const snippet = TranslatedSnippet.fromSnippet(original); + const snippet = TranslatedSnippet.fromTypeScript(original); for (const [key, value] of Object.entries(translations)) { - snippet.addTranslatedSource(key as TargetLanguage, value); + snippet.addTranslation(key as TargetLanguage, value, 'x'); } return snippet; } diff --git a/packages/jsii-rosetta/test/snippet-selectors.test.ts b/packages/jsii-rosetta/test/snippet-selectors.test.ts index 10124b8156..58f50b43ad 100644 --- a/packages/jsii-rosetta/test/snippet-selectors.test.ts +++ b/packages/jsii-rosetta/test/snippet-selectors.test.ts @@ -1,6 +1,7 @@ import { typeScriptSnippetFromSource } from '../lib/snippet'; import { longest, mean, meanLength, shortest } from '../lib/snippet-selectors'; import { TranslatedSnippet } from '../lib/tablets/tablets'; +import { testSnippetLocation } from './testutil'; const snippets: TranslatedSnippet[] = []; const sources: string[] = [ @@ -28,7 +29,9 @@ const sources: string[] = [ ]; beforeAll(() => { for (const source of sources) { - const snippet = TranslatedSnippet.fromSnippet(typeScriptSnippetFromSource(source, 'selectors', false)); + const snippet = TranslatedSnippet.fromTypeScript( + typeScriptSnippetFromSource(source, testSnippetLocation('selectors'), false), + ); snippets.push(snippet); } }); diff --git a/packages/jsii-rosetta/test/testutil.ts b/packages/jsii-rosetta/test/testutil.ts index 14290c2e82..c779bd655c 100644 --- a/packages/jsii-rosetta/test/testutil.ts +++ b/packages/jsii-rosetta/test/testutil.ts @@ -3,7 +3,13 @@ import { PackageInfo, compileJsiiForTest } from 'jsii'; import * as os from 'os'; import * as path from 'path'; -import { typeScriptSnippetFromSource, SnippetTranslator, SnippetParameters, rosettaDiagFromTypescript } from '../lib'; +import { + typeScriptSnippetFromSource, + SnippetTranslator, + SnippetParameters, + rosettaDiagFromTypescript, + SnippetLocation, +} from '../lib'; export type MultipleSources = { [key: string]: string; 'index.ts': string }; @@ -46,7 +52,8 @@ export class AssemblyFixture { * Make a snippet translator for the given source w.r.t this compiled assembly */ public successfullyCompile(source: string) { - const snippet = typeScriptSnippetFromSource(source, 'testutil', false, { + const location = testSnippetLocation('testutil'); + const snippet = typeScriptSnippetFromSource(source, location, false, { [SnippetParameters.$COMPILATION_DIRECTORY]: this.directory, }); const ret = new SnippetTranslator(snippet, { @@ -66,6 +73,10 @@ export class AssemblyFixture { } } +export function testSnippetLocation(fileName: string): SnippetLocation { + return { api: { api: 'file', fileName }, field: { field: 'example' } }; +} + export const DUMMY_ASSEMBLY_TARGETS = { dotnet: { namespace: 'Example.Test.Demo', diff --git a/packages/jsii-rosetta/test/translate.test.ts b/packages/jsii-rosetta/test/translate.test.ts index d39a846f70..e3b2733b6e 100644 --- a/packages/jsii-rosetta/test/translate.test.ts +++ b/packages/jsii-rosetta/test/translate.test.ts @@ -1,11 +1,17 @@ import { SnippetTranslator, TypeScriptSnippet, PythonVisitor } from '../lib'; import { VisualizeAstVisitor } from '../lib/languages/visualize'; +import { snippetKey } from '../lib/tablets/key'; + +const location = { + api: { api: 'moduleReadme', moduleFqn: '@aws-cdk/aws-apigateway' }, + field: { field: 'example' }, +} as const; test('does not fail on "Debug Failure"', () => { // GIVEN const snippet: TypeScriptSnippet = { completeSource: 'Missing literate source file test/integ.restapi-import.lit.ts', - where: '@aws-cdk.aws-apigateway-README-snippet4', + location, visibleSource: "import { App, CfnOutput, NestedStack, NestedStackProps, Stack } from '@aws-cdk/core';\nimport { Construct } from 'constructs';\nimport { Deployment, Method, MockIntegration, PassthroughBehavior, RestApi, Stage } from '../lib';\n\n/**\n * This file showcases how to split up a RestApi's Resources and Methods across nested stacks.\n *\n * The root stack 'RootStack' first defines a RestApi.\n * Two nested stacks BooksStack and PetsStack, create corresponding Resources '/books' and '/pets'.\n * They are then…;\n\n readonly methods?: Method[];\n}\n\nclass DeployStack extends NestedStack {\n constructor(scope: Construct, props: DeployStackProps) {\n super(scope, 'integ-restapi-import-DeployStack', props);\n\n const deployment = new Deployment(this, 'Deployment', {\n api: RestApi.fromRestApiId(this, 'RestApi', props.restApiId),\n });\n (props.methods ?? []).forEach((method) => deployment.node.addDependency(method));\n new Stage(this, 'Stage', { deployment });\n }\n}\n\nnew RootStack(new App());", parameters: { lit: 'test/integ.restapi-import.lit.ts' }, @@ -41,7 +47,7 @@ test('does not fail on "Debug Failure"', () => { test('rejects ?? operator', () => { const snippet: TypeScriptSnippet = { completeSource: 'const x = false ?? true;', - where: '@aws-cdk.aws-apigateway-README-snippet4', + location, visibleSource: 'const x = false ?? true;', parameters: { lit: 'test/integ.restapi-import.lit.ts' }, strict: false, @@ -59,7 +65,7 @@ test('rejects ?? operator', () => { test('rejects function declarations in object literals', () => { const snippet: TypeScriptSnippet = { completeSource: 'const x = { method() { return 1; } }', - where: '@aws-cdk.aws-apigateway-README-snippet4', + location, visibleSource: 'const x = { method() { return 1; } }', parameters: { lit: 'test/integ.restapi-import.lit.ts' }, strict: false, @@ -75,3 +81,34 @@ test('rejects function declarations in object literals', () => { 'Use of MethodDeclaration in an object literal is not supported', ); }); + +test('completeSource does not impact the snippet key', () => { + const snippet1: TypeScriptSnippet = { + visibleSource: '// hello', + location: { api: { api: 'member', fqn: 'my.class', memberName: 'member1' } }, + completeSource: '// i do not like to say\n// hello', + }; + + const snippet2 = { + ...snippet1, + completeSource: undefined, + }; + + expect(snippetKey(snippet1)).toEqual(snippetKey(snippet2)); +}); + +test('Snippets from different locations have different keys', () => { + const visibleSource = 'console.log("banana");'; + + const snippet1: TypeScriptSnippet = { + visibleSource, + location: { api: { api: 'member', fqn: 'my.class', memberName: 'member1' } }, + }; + + const snippet2: TypeScriptSnippet = { + visibleSource, + location: { api: { api: 'type', fqn: 'my.class' } }, + }; + + expect(snippetKey(snippet1)).not.toEqual(snippetKey(snippet2)); +}); diff --git a/packages/jsii-rosetta/test/translations.test.ts b/packages/jsii-rosetta/test/translations.test.ts index 87aa067d0a..c37075ae60 100644 --- a/packages/jsii-rosetta/test/translations.test.ts +++ b/packages/jsii-rosetta/test/translations.test.ts @@ -5,6 +5,7 @@ import { JavaVisitor, PythonVisitor, SnippetTranslator } from '../lib'; import { CSharpVisitor } from '../lib/languages/csharp'; import { VisualizeAstVisitor } from '../lib/languages/visualize'; import { AstHandler } from '../lib/renderer'; +import { testSnippetLocation } from './testutil'; // This iterates through all subdirectories of this directory, // and creates a Jest test for each, by translating the TypeScript file it finds there, @@ -64,7 +65,7 @@ for (const typeScriptTest of typeScriptTests) { beforeAll(() => { translator = new SnippetTranslator({ visibleSource: typeScriptSource, - where: typeScriptTest, + location: testSnippetLocation(typeScriptTest), }); });