Add the ability to emit index entries. (#271)

* Add the ability to emit index entries.

I'm prototyping a Kythe TypeScript indexer plugin and things are fitting together pretty well.

This starts by adding a method on the analyzer that will yield all of the information that the analyzer has determined about a file. I started with element and attribute references as they seem like the most valuable. What else could we emit here?

* Add a test that `indexFile` creates a well-formed `HtmlNodeIndexEntry` for a locally defined element.

* Add a test that element references are found across different files.

* Test that entries are only generated for elements in template strings with a template tag named `html`.

* Remove some empty lines.

* Use `HtmlNodeKind` enum property instead of literal `"NODE"`.

* Test that entries are created for attributes referencing properties.

* Test that other types of attribute-position syntax produces references of the right kind.

* Rename some tests.

* Add more test cases for element references.

* Add `name` for attribute entry definition targets.

* Add tests that entries are created for different types of attribute definitions.

* Use `type: Boolean` to define a property referenced by a boolean attribute.

* Make `kind` a required property of `HtmlMemberBase`.

* Use `.ts` for test files using types.

* Add a change log entry.

Co-authored-by: Peter Burns <rictic@google.com>

CHANGELOG.md

@@ -7,6 +7,12 @@ All notable changes to this project will be documented in this file.
 <!-- ### Removed -->
 <!-- ### Fixed -->
 
+## [Unreleased]
+
+### Added
+
+- Added the ability to emit 'index entries'. ([#271](https://github.com/runem/lit-analyzer/pull/271))
+
 ## [2.0.0-pre.2]
 
 ### Fixed

packages/lit-analyzer/src/lib/analyze/document-analyzer/html/definition/definition-for-html-attr.ts

@@ -16,7 +16,8 @@ export function definitionForHtmlAttr(htmlAttr: HtmlNodeAttr, { htmlStore, ts }:
 			fromRange: rangeFromHtmlNodeAttr(htmlAttr),
 			target: {
 				kind: "node",
-				node: getNodeIdentifier(node, ts) || node
+				node: getNodeIdentifier(node, ts) || node,
+				name: target.name
 			}
 		};
 	} else if (isHtmlEvent(target) && target.declaration != null) {
@@ -26,7 +27,8 @@ export function definitionForHtmlAttr(htmlAttr: HtmlNodeAttr, { htmlStore, ts }:
 			fromRange: rangeFromHtmlNodeAttr(htmlAttr),
 			target: {
 				kind: "node",
-				node: getNodeIdentifier(node, ts) || node
+				node: getNodeIdentifier(node, ts) || node,
+				name: target.name
 			}
 		};
 	}

packages/lit-analyzer/src/lib/analyze/document-analyzer/html/lit-html-document-analyzer.ts

@@ -1,8 +1,8 @@
 import { FormatCodeSettings } from "typescript";
 import { LitAnalyzerContext } from "../../lit-analyzer-context.js";
 import { HtmlDocument } from "../../parse/document/text-document/html-document/html-document.js";
-import { isHTMLAttr } from "../../types/html-node/html-node-attr-types.js";
-import { isHTMLNode } from "../../types/html-node/html-node-types.js";
+import { isHTMLAttr, HtmlNodeAttr } from "../../types/html-node/html-node-attr-types.js";
+import { isHTMLNode, HtmlNode } from "../../types/html-node/html-node-types.js";
 import { LitClosingTagInfo } from "../../types/lit-closing-tag-info.js";
 import { LitCodeFix } from "../../types/lit-code-fix.js";
 import { LitCompletion } from "../../types/lit-completion.js";
@@ -148,4 +148,33 @@ export class LitHtmlDocumentAnalyzer {
 	getFormatEdits(document: HtmlDocument, settings: FormatCodeSettings): LitFormatEdit[] {
 		return this.vscodeHtmlService.format(document, settings);
 	}
+
+	*indexFile(document: HtmlDocument, context: LitAnalyzerContext): IterableIterator<LitIndexEntry> {
+		for (const node of document.nodes()) {
+			const definition = definitionForHtmlNode(node, context);
+			if (definition != null) {
+				yield { kind: "NODE-REFERENCE", node, document, definition };
+			}
+			for (const attribute of node.attributes) {
+				const definition = definitionForHtmlAttr(attribute, context);
+				if (definition != null) {
+					yield { kind: "ATTRIBUTE-REFERENCE", attribute, document, definition };
+				}
+			}
+		}
+	}
+}
+
+export type LitIndexEntry = HtmlNodeIndexEntry | HtmlNodeAttrIndexEntry;
+interface HtmlNodeIndexEntry {
+	kind: "NODE-REFERENCE";
+	node: HtmlNode;
+	document: HtmlDocument;
+	definition: LitDefinition;
+}
+interface HtmlNodeAttrIndexEntry {
+	kind: "ATTRIBUTE-REFERENCE";
+	attribute: HtmlNodeAttr;
+	document: HtmlDocument;
+	definition: LitDefinition;
 }

packages/lit-analyzer/src/lib/analyze/lit-analyzer.ts

@@ -1,7 +1,7 @@
 import { SourceFile } from "typescript";
 import { ComponentAnalyzer } from "./component-analyzer/component-analyzer.js";
 import { LitCssDocumentAnalyzer } from "./document-analyzer/css/lit-css-document-analyzer.js";
-import { LitHtmlDocumentAnalyzer } from "./document-analyzer/html/lit-html-document-analyzer.js";
+import { LitHtmlDocumentAnalyzer, LitIndexEntry } from "./document-analyzer/html/lit-html-document-analyzer.js";
 import { renameLocationsForTagName } from "./document-analyzer/html/rename-locations/rename-locations-for-tag-name.js";
 import { LitAnalyzerContext } from "./lit-analyzer-context.js";
 import { CssDocument } from "./parse/document/text-document/css-document/css-document.js";
@@ -72,6 +72,23 @@ export class LitAnalyzer {
 		return;
 	}
 
+	/**
+	 * Yields entries that describe regions of code in the given file, and
+	 * what the analyzer knows about them.
+	 *
+	 * This is useful for generating a static index of analysis output. Two such
+	 * indexing systems are Kythe and the Language Server Index Format.
+	 */
+	*indexFile(file: SourceFile): IterableIterator<LitIndexEntry> {
+		this.context.updateComponents(file);
+		const documents = this.getDocumentsInFile(file);
+		for (const document of documents) {
+			if (document instanceof HtmlDocument) {
+				yield* this.litHtmlDocumentAnalyzer.indexFile(document, this.context);
+			}
+		}
+	}
+
 	getQuickInfoAtPosition(file: SourceFile, position: SourceFilePosition): LitQuickInfo | undefined {
 		this.context.setContextBase({ file });
 

packages/lit-analyzer/src/lib/analyze/parse/convert-component-definitions-to-html-collection.ts

@@ -1,8 +1,8 @@
-import { isSimpleType, SimpleType, toSimpleType } from "ts-simple-type";
+import { isSimpleType, SimpleType, SimpleTypeAny, toSimpleType } from "ts-simple-type";
 import { TypeChecker } from "typescript";
 import { AnalyzerResult, ComponentDeclaration, ComponentDefinition, ComponentFeatures } from "web-component-analyzer";
 import { lazy } from "../util/general-util.js";
-import { HtmlDataCollection, HtmlDataFeatures, HtmlMemberBase, HtmlTag } from "./parse-html-data/html-tag.js";
+import { HtmlDataCollection, HtmlDataFeatures, HtmlTag } from "./parse-html-data/html-tag.js";
 
 export interface AnalyzeResultConversionOptions {
 	addDeclarationPropertiesAsAttributes?: boolean;
@@ -156,14 +156,14 @@ export function convertComponentFeaturesToHtml(
 			continue;
 		}
 
-		const base: HtmlMemberBase = {
+		const base = {
 			declaration: member,
 			description: member.jsDoc?.description,
 			getType: lazy(() => {
 				const type = member.type?.();
 
 				if (type == null) {
-					return { kind: "ANY" };
+					return { kind: "ANY" } as SimpleTypeAny;
 				}
 
 				return isSimpleType(type) ? type : toSimpleType(type, checker);

packages/lit-analyzer/src/lib/analyze/parse/document/text-document/html-document/html-document.ts

@@ -104,20 +104,16 @@ export class HtmlDocument extends TextDocument {
 		return items;
 	}
 
-	private mapFindOne<T>(map: (node: HtmlNode) => T | undefined): T | undefined {
-		function innerTest(node: HtmlNode): T | undefined {
-			const res = map(node);
-			if (res) return res;
-
-			for (const childNode of node.children || []) {
-				const found = innerTest(childNode);
-				if (found != null) return found;
-			}
-			return;
+	*nodes(roots = this.rootNodes): IterableIterator<HtmlNode> {
+		for (const root of roots) {
+			yield root;
+			yield* this.nodes(root.children);
 		}
+	}
 
-		for (const rootNode of this.rootNodes || []) {
-			const found = innerTest(rootNode);
+	private mapFindOne<T>(map: (node: HtmlNode) => T | undefined): T | undefined {
+		for (const node of this.nodes()) {
+			const found = map(node);
 			if (found != null) {
 				return found;
 			}

packages/lit-analyzer/src/lib/analyze/parse/parse-html-data/html-tag.ts

@@ -37,7 +37,7 @@ export interface HtmlTag extends HtmlDataFeatures {
 export type HtmlTagMemberKind = "attribute" | "property";
 
 export interface HtmlMemberBase {
-	kind?: HtmlTagMemberKind;
+	kind: HtmlTagMemberKind;
 	builtIn?: boolean;
 	required?: boolean;
 	description?: string;

packages/lit-analyzer/src/test/helpers/analyze.ts

@@ -8,6 +8,7 @@ import { compileFiles, TestFile } from "./compile-files.js";
 import { getCurrentTsModule } from "./ts-test.js";
 import { Range } from "../../lib/analyze/types/range.js";
 import { LitCodeFix } from "../../lib/analyze/types/lit-code-fix.js";
+import { LitIndexEntry } from "../../lib/analyze/document-analyzer/html/lit-html-document-analyzer.js";
 
 /**
  * Prepares both the Typescript program and the LitAnalyzer
@@ -60,6 +61,7 @@ export function getDiagnostics(
 /**
  * Returns code fixes in 'virtual' files using the LitAnalyzer
  * @param inputFiles
+ * @param range
  * @param config
  */
 export function getCodeFixesAtRange(
@@ -75,3 +77,20 @@ export function getCodeFixesAtRange(
 		sourceFile
 	};
 }
+
+/**
+ * @param inputFiles
+ * @param config
+ */
+export function getIndexEntries(
+	inputFiles: TestFile[] | TestFile,
+	config: Partial<LitAnalyzerConfig> = {}
+): { indexEntries: IterableIterator<LitIndexEntry>; program: Program; sourceFile: SourceFile } {
+	const { analyzer, sourceFile, program } = prepareAnalyzer(inputFiles, config);
+
+	return {
+		indexEntries: analyzer.indexFile(sourceFile),
+		program,
+		sourceFile
+	};
+}