diff --git a/ark/attest/__tests__/assertions.test.ts b/ark/attest/__tests__/assertions.test.ts index 851972f735..17f3c1303d 100644 --- a/ark/attest/__tests__/assertions.test.ts +++ b/ark/attest/__tests__/assertions.test.ts @@ -3,7 +3,7 @@ import * as assert from "node:assert/strict" const o = { ark: "type" } -specify(() => { +specify("type assertions", () => { it("type parameter", () => { attest<{ ark: string }>(o) assert.throws( diff --git a/ark/attest/__tests__/benchExpectedOutput.ts b/ark/attest/__tests__/benchExpectedOutput.ts index c3c0240738..db78710ffc 100644 --- a/ark/attest/__tests__/benchExpectedOutput.ts +++ b/ark/attest/__tests__/benchExpectedOutput.ts @@ -37,7 +37,7 @@ type makeComplexType = bench("bench type", () => { return {} as makeComplexType<"defenestration"> -}).types([177, "instantiations"]) +}).types([176, "instantiations"]) bench("bench type from external module", () => { return {} as externalmakeComplexType<"defenestration"> @@ -51,6 +51,6 @@ bench( fakeCallOptions ) .mean([2, "ms"]) - .types([345, "instantiations"]) + .types([344, "instantiations"]) bench("empty", () => {}).types([0, "instantiations"]) diff --git a/ark/attest/__tests__/demo.test.ts b/ark/attest/__tests__/demo.test.ts index a00a45598e..c4e5aba0aa 100644 --- a/ark/attest/__tests__/demo.test.ts +++ b/ark/attest/__tests__/demo.test.ts @@ -77,17 +77,17 @@ contextualize(() => { // works for keys or index access as well (may need prettier-ignore to // avoid removing quotes) // prettier-ignore - attest({ f: "🐐" } as Legends).completions({ f: ["faker"] }) - }) - - it("integrate runtime logic with type assertions", () => { - const arrayOf = type("", "t[]") - const numericArray = arrayOf("number | bigint") - // flexibly combine runtime logic with type assertions to customize your - // tests beyond what is possible from pure static-analysis based type testing tools - if (getPrimaryTsVersionUnderTest().startsWith("5")) { - // this assertion will only occur when testing TypeScript 5+! - attest<(number | bigint)[]>(numericArray.infer) - } + attest({ "f": "🐐" } as Legends).completions({ f: ["faker"] }) }) + // TODO: reenable once generics are finished + // it("integrate runtime logic with type assertions", () => { + // const arrayOf = type("", "t[]") + // const numericArray = arrayOf("number | bigint") + // // flexibly combine runtime logic with type assertions to customize your + // // tests beyond what is possible from pure static-analysis based type testing tools + // if (getPrimaryTsVersionUnderTest().startsWith("5")) { + // // this assertion will only occur when testing TypeScript 5+! + // attest<(number | bigint)[]>(numericArray.infer) + // } + // }) }) diff --git a/ark/attest/__tests__/instantiations.test.ts b/ark/attest/__tests__/instantiations.test.ts new file mode 100644 index 0000000000..3c4df29928 --- /dev/null +++ b/ark/attest/__tests__/instantiations.test.ts @@ -0,0 +1,10 @@ +import { attest, contextualize } from "@arktype/attest" +import { type } from "arktype" +import { it } from "mocha" + +contextualize(() => { + it("Inline instantiations", () => { + type("string") + attest.instantiations([1968, "instantiations"]) + }) +}) diff --git a/ark/attest/__tests__/snapExpectedOutput.ts b/ark/attest/__tests__/snapExpectedOutput.ts index 404aee2e4f..d661034701 100644 --- a/ark/attest/__tests__/snapExpectedOutput.ts +++ b/ark/attest/__tests__/snapExpectedOutput.ts @@ -2,25 +2,25 @@ import { attest, cleanup, setup } from "@arktype/attest" setup() -attest({ re: "do" }).equals({ re: "do" }).type.toString.snap(`{ re: string; }`) +attest({ re: "do" }).equals({ re: "do" }).type.toString.snap("{ re: string; }") attest(5).snap(5) -attest({ re: "do" }).snap({ re: `do` }) +attest({ re: "do" }).snap({ re: "do" }) // @ts-expect-error (using internal updateSnapshots hook) -attest({ re: "dew" }, { updateSnapshots: true }).snap({ re: `dew` }) +attest({ re: "dew" }, { cfg: { updateSnapshots: true } }).snap({ re: "dew" }) // @ts-expect-error (using internal updateSnapshots hook) -attest(5, { updateSnapshots: true }).snap(5) +attest(5, { cfg: { updateSnapshots: true } }).snap(5) -attest(undefined).snap(`(undefined)`) +attest(undefined).snap("(undefined)") -attest({ a: undefined }).snap({ a: `(undefined)` }) +attest({ a: undefined }).snap({ a: "(undefined)" }) attest("multiline\nmultiline").snap(`multiline multiline`) -attest("with `quotes`").snap(`with \`quotes\``) +attest("with `quotes`").snap("with `quotes`") cleanup() diff --git a/ark/attest/__tests__/snapPopulation.test.ts b/ark/attest/__tests__/snapPopulation.test.ts index bedced28a0..2b8bea9eee 100644 --- a/ark/attest/__tests__/snapPopulation.test.ts +++ b/ark/attest/__tests__/snapPopulation.test.ts @@ -10,7 +10,7 @@ contextualize(() => { fromHere("benchExpectedOutput.ts") ).replaceAll("\r\n", "\n") equal(actual, expectedOutput) - }) + }).timeout(10000) it("snap populates file", () => { const actual = runThenGetContents(fromHere("snapTemplate.ts")) @@ -18,5 +18,5 @@ contextualize(() => { fromHere("snapExpectedOutput.ts") ).replaceAll("\r\n", "\n") equal(actual, expectedOutput) - }) + }).timeout(10000) }) diff --git a/ark/attest/__tests__/snapTemplate.ts b/ark/attest/__tests__/snapTemplate.ts index bcabe87801..c2d584b236 100644 --- a/ark/attest/__tests__/snapTemplate.ts +++ b/ark/attest/__tests__/snapTemplate.ts @@ -9,10 +9,10 @@ attest(5).snap() attest({ re: "do" }).snap() // @ts-expect-error (using internal updateSnapshots hook) -attest({ re: "dew" }, { updateSnapshots: true }).snap() +attest({ re: "dew" }, { cfg: { updateSnapshots: true } }).snap({ re: "do" }) // @ts-expect-error (using internal updateSnapshots hook) -attest(5, { updateSnapshots: true }).snap(6) +attest(5, { cfg: { updateSnapshots: true } }).snap(6) attest(undefined).snap() diff --git a/ark/attest/assert/assertions.ts b/ark/attest/assert/assertions.ts index 2fcbe3ee46..f6c0813cdf 100644 --- a/ark/attest/assert/assertions.ts +++ b/ark/attest/assert/assertions.ts @@ -1,7 +1,7 @@ import { printable, throwInternalError } from "@arktype/util" import { AssertionError } from "node:assert" import * as assert from "node:assert/strict" -import type { TypeAssertionData } from "../cache/writeAssertionCache.js" +import type { TypeRelationshipAssertionData } from "../cache/writeAssertionCache.js" import type { AssertionContext } from "./attest.js" export type ThrowAssertionErrorContext = { @@ -34,7 +34,7 @@ export type MappedTypeAssertionResult = { export class TypeAssertionMapping { constructor( public fn: ( - data: TypeAssertionData, + data: TypeRelationshipAssertionData, ctx: AssertionContext ) => MappedTypeAssertionResult ) {} @@ -44,12 +44,12 @@ export const versionableAssertion = (fn: AssertFn): AssertFn => (expected, actual, ctx) => { if (actual instanceof TypeAssertionMapping) { - if (!ctx.typeAssertionEntries) { + if (!ctx.typeRelationshipAssertionEntries) { throwInternalError( `Unexpected missing typeAssertionEntries when passed a TypeAssertionMapper` ) } - for (const [version, data] of ctx.typeAssertionEntries!) { + for (const [version, data] of ctx.typeRelationshipAssertionEntries) { let errorMessage = "" try { const mapped = actual.fn(data, ctx) diff --git a/ark/attest/assert/attest.ts b/ark/attest/assert/attest.ts index ec26920954..84c39510b9 100644 --- a/ark/attest/assert/attest.ts +++ b/ark/attest/assert/attest.ts @@ -1,9 +1,16 @@ import { caller, getCallStack, type SourcePosition } from "@arktype/fs" import type { inferTypeRoot, validateTypeRoot } from "arktype" +import { getBenchCtx } from "../bench/bench.js" +import type { Measure } from "../bench/measure.js" +import { instantiationDataHandler } from "../bench/type.js" import { - getTypeAssertionsAtPosition, + getTypeRelationshipAssertionsAtPosition, type VersionedTypeAssertion } from "../cache/getCachedAssertions.js" +import type { + TypeBenchmarkingAssertionData, + TypeRelationshipAssertionData +} from "../cache/writeAssertionCache.js" import { getConfig, type AttestConfig } from "../config.js" import { assertEquals, typeEqualityMapping } from "./assertions.js" import { @@ -22,6 +29,8 @@ export type AttestFn = { def: validateTypeRoot ): asserts actual is unknown extends actual ? inferTypeRoot & actual : Extract> + + instantiations: (count?: Measure<"instantiations"> | undefined) => void } export type AssertionContext = { @@ -32,7 +41,8 @@ export type AssertionContext = { position: SourcePosition defaultExpected?: unknown assertionStack: string - typeAssertionEntries?: VersionedTypeAssertion[] + typeRelationshipAssertionEntries?: VersionedTypeAssertion[] + typeBenchmarkingAssertionEntries?: VersionedTypeAssertion[] lastSnapName?: string } @@ -57,8 +67,9 @@ export const attestInternal = ( ...ctxHooks } if (!cfg.skipTypes) { - ctx.typeAssertionEntries = getTypeAssertionsAtPosition(position) - if (ctx.typeAssertionEntries[0]?.[1].typeArgs[0]) { + ctx.typeRelationshipAssertionEntries = + getTypeRelationshipAssertionsAtPosition(position) + if (ctx.typeRelationshipAssertionEntries[0][1].typeArgs[0]) { // if there is an expected type arg, check it immediately assertEquals(undefined, typeEqualityMapping, ctx) } @@ -66,4 +77,17 @@ export const attestInternal = ( return new ChainableAssertions(ctx) } -export const attest: AttestFn = attestInternal as never +attestInternal.instantiations = ( + args: Measure<"instantiations"> | undefined +) => { + const attestConfig = getConfig() + if (attestConfig.skipInlineInstantiations) return + + const calledFrom = caller() + const ctx = getBenchCtx([calledFrom.file]) + ctx.benchCallPosition = calledFrom + ctx.lastSnapCallPosition = calledFrom + instantiationDataHandler({ ...ctx, kind: "instantiations" }, args, false) +} + +export const attest: AttestFn = attestInternal as AttestFn diff --git a/ark/attest/assert/chainableAssertions.ts b/ark/attest/assert/chainableAssertions.ts index 32eda196e6..1a2e51193d 100644 --- a/ark/attest/assert/chainableAssertions.ts +++ b/ark/attest/assert/chainableAssertions.ts @@ -40,10 +40,13 @@ export class ChainableAssertions implements AssertionRecord { } private get actual() { - return this.ctx.actual instanceof TypeAssertionMapping ? - this.ctx.actual.fn(this.ctx.typeAssertionEntries![0][1], this.ctx)! - .actual - : this.ctx.actual + if (this.ctx.actual instanceof TypeAssertionMapping) { + return this.ctx.actual.fn( + this.ctx.typeRelationshipAssertionEntries![0][1], + this.ctx + )!.actual + } + return this.ctx.actual } private get serializedActual() { @@ -65,7 +68,9 @@ export class ChainableAssertions implements AssertionRecord { ctx: this.ctx, message: messageOnError ?? - `${this.serializedActual} failed to satisfy predicate${predicate.name ? ` ${predicate.name}` : ""}` + `${this.serializedActual} failed to satisfy predicate${ + predicate.name ? ` ${predicate.name}` : "" + }` }) } return this.actual as never diff --git a/ark/attest/bench/baseline.ts b/ark/attest/bench/baseline.ts index f2c4e04ea1..1c081625d3 100644 --- a/ark/attest/bench/baseline.ts +++ b/ark/attest/bench/baseline.ts @@ -1,9 +1,6 @@ -import { ensureDir } from "@arktype/fs" import { snapshot } from "@arktype/util" -import { rmSync } from "node:fs" import process from "node:process" import { queueSnapshotUpdate } from "../cache/snapshots.js" -import { getConfig } from "../config.js" import type { BenchAssertionContext, BenchContext } from "./bench.js" import { stringifyMeasure, @@ -12,8 +9,6 @@ import { type MeasureComparison } from "./measure.js" -let isFirstQueuedUpdate = true - export const queueBaselineUpdateIfNeeded = ( updated: Measure | MarkMeasure, baseline: Measure | MarkMeasure | undefined, @@ -28,13 +23,6 @@ export const queueBaselineUpdateIfNeeded = ( `Unable to update baseline for ${ctx.qualifiedName} ('lastSnapCallPosition' was unset).` ) } - if (isFirstQueuedUpdate) { - // remove any leftover cached snaps before the first is written - const { benchSnapCacheDir } = getConfig() - rmSync(benchSnapCacheDir, { recursive: true, force: true }) - ensureDir(benchSnapCacheDir) - isFirstQueuedUpdate = false - } queueSnapshotUpdate({ position: ctx.lastSnapCallPosition, serializedValue, @@ -81,4 +69,4 @@ const handleNegativeDelta = (formattedDelta: string, ctx: BenchContext) => { 1 )}! Consider setting a new baseline.` ) -} +} \ No newline at end of file diff --git a/ark/attest/bench/bench.ts b/ark/attest/bench/bench.ts index 4826c6dfcc..a64afdd192 100644 --- a/ark/attest/bench/bench.ts +++ b/ark/attest/bench/bench.ts @@ -1,13 +1,283 @@ import { caller, type SourcePosition } from "@arktype/fs" +import { performance } from "node:perf_hooks" import { ensureCacheDirs, getConfig, type ParsedAttestConfig } from "../config.js" import { chainableNoOpProxy } from "../utils.js" -import { BenchAssertions, type TimeAssertionName } from "./call.js" +import { await1K } from "./await1k.js" +import { compareToBaseline, queueBaselineUpdateIfNeeded } from "./baseline.js" +import { call1K } from "./call1k.js" +import { + createTimeComparison, + createTimeMeasure, + type MarkMeasure, + type Measure, + type TimeUnit +} from "./measure.js" import { createBenchTypeAssertion, type BenchTypeAssertions } from "./type.js" +export type StatName = keyof typeof stats + +export type TimeAssertionName = StatName | "mark" + +export const bench = ( + name: string, + fn: Fn, + options: BenchOptions = {} +): InitialBenchAssertions => { + const qualifiedPath = [...currentSuitePath, name] + const ctx = getBenchCtx( + qualifiedPath, + fn.constructor.name === "AsyncFunction", + options + ) + ctx.benchCallPosition = caller() + ensureCacheDirs() + if ( + typeof ctx.cfg.filter === "string" && + !qualifiedPath.includes(ctx.cfg.filter) + ) + return chainableNoOpProxy + else if ( + Array.isArray(ctx.cfg.filter) && + ctx.cfg.filter.some((segment, i) => segment !== qualifiedPath[i]) + ) + return chainableNoOpProxy + + const assertions = new BenchAssertions(fn, ctx) + Object.assign(assertions, createBenchTypeAssertion(ctx)) + return assertions as any +} + +export const stats = { + mean: (callTimes: number[]): number => { + const totalCallMs = callTimes.reduce((sum, duration) => sum + duration, 0) + return totalCallMs / callTimes.length + }, + median: (callTimes: number[]): number => { + const middleIndex = Math.floor(callTimes.length / 2) + const ms = + callTimes.length % 2 === 0 ? + (callTimes[middleIndex - 1] + callTimes[middleIndex]) / 2 + : callTimes[middleIndex] + return ms + } +} + +class ResultCollector { + results: number[] = [] + private benchStart = performance.now() + private bounds: Required + private lastInvocationStart: number + + constructor(private ctx: BenchContext) { + // By default, will run for either 5 seconds or 100_000 call sets (of 1000 calls), whichever comes first + this.bounds = { + ms: 5000, + count: 100_000, + ...ctx.options.until + } + this.lastInvocationStart = -1 + } + + start() { + this.ctx.options.hooks?.beforeCall?.() + this.lastInvocationStart = performance.now() + } + + stop() { + this.results.push((performance.now() - this.lastInvocationStart) / 1000) + this.ctx.options.hooks?.afterCall?.() + } + + done() { + const metMsTarget = performance.now() - this.benchStart >= this.bounds.ms + const metCountTarget = this.results.length >= this.bounds.count + return metMsTarget || metCountTarget + } +} + +const loopCalls = (fn: () => void, ctx: BenchContext) => { + const collector = new ResultCollector(ctx) + while (!collector.done()) { + collector.start() + // we use a function like this to make 1k explicit calls to the function + // to avoid certain optimizations V8 makes when looping + call1K(fn) + collector.stop() + } + return collector.results +} + +const loopAsyncCalls = async (fn: () => Promise, ctx: BenchContext) => { + const collector = new ResultCollector(ctx) + while (!collector.done()) { + collector.start() + await await1K(fn) + collector.stop() + } + return collector.results +} + +export class BenchAssertions< + Fn extends BenchableFunction, + NextAssertions = BenchTypeAssertions, + ReturnedAssertions = Fn extends () => Promise ? Promise + : NextAssertions +> { + private label: string + private lastCallTimes: number[] | undefined + constructor( + private fn: Fn, + private ctx: BenchContext + ) { + this.label = `Call: ${ctx.qualifiedName}` + } + + private applyCallTimeHooks() { + if (this.ctx.options.fakeCallMs !== undefined) { + const fakeMs = + this.ctx.options.fakeCallMs === "count" ? + this.lastCallTimes!.length + : this.ctx.options.fakeCallMs + this.lastCallTimes = this.lastCallTimes!.map(() => fakeMs) + } + } + + private callTimesSync() { + if (!this.lastCallTimes) { + this.lastCallTimes = loopCalls(this.fn as any, this.ctx) + this.lastCallTimes.sort() + } + this.applyCallTimeHooks() + return this.lastCallTimes + } + + private async callTimesAsync() { + if (!this.lastCallTimes) { + this.lastCallTimes = await loopAsyncCalls(this.fn as any, this.ctx) + this.lastCallTimes.sort() + } + this.applyCallTimeHooks() + return this.lastCallTimes + } + + private createAssertion( + name: Name, + baseline: Name extends "mark" ? + Record> | undefined + : Measure | undefined, + callTimes: number[] + ) { + if (name === "mark") return this.markAssertion(baseline as any, callTimes) + + const ms: number = stats[name as StatName](callTimes) + const comparison = createTimeComparison(ms, baseline as Measure) + console.group(`${this.label} (${name}):`) + compareToBaseline(comparison, this.ctx) + console.groupEnd() + queueBaselineUpdateIfNeeded(createTimeMeasure(ms), baseline, { + ...this.ctx, + kind: name + }) + return this.getNextAssertions() + } + + private markAssertion( + baseline: MarkMeasure | undefined, + callTimes: number[] + ) { + console.group(`${this.label}:`) + const markEntries: [StatName, Measure | undefined][] = ( + baseline ? + Object.entries(baseline) + // If nothing was passed, gather all available baselines by setting their values to undefined. + : Object.entries(stats).map(([kind]) => [kind, undefined])) as any + const markResults = Object.fromEntries( + markEntries.map(([kind, kindBaseline]) => { + console.group(kind) + const ms = stats[kind](callTimes) + const comparison = createTimeComparison(ms, kindBaseline) + compareToBaseline(comparison, this.ctx) + console.groupEnd() + return [kind, comparison.updated] + }) + ) + console.groupEnd() + queueBaselineUpdateIfNeeded(markResults, baseline, { + ...this.ctx, + kind: "mark" + }) + return this.getNextAssertions() + } + + private getNextAssertions(): NextAssertions { + return createBenchTypeAssertion(this.ctx) as any as NextAssertions + } + + private createStatMethod( + name: Name, + baseline: Name extends "mark" ? + Record> | undefined + : Measure | undefined + ) { + if (this.ctx.isAsync) { + return new Promise(resolve => { + this.callTimesAsync().then( + callTimes => { + resolve(this.createAssertion(name, baseline, callTimes)) + }, + e => { + this.addUnhandledBenchException(e) + resolve(chainableNoOpProxy) + } + ) + }) + } + let assertions = chainableNoOpProxy + try { + assertions = this.createAssertion(name, baseline, this.callTimesSync()) + } catch (e) { + this.addUnhandledBenchException(e) + } + return assertions + } + + private addUnhandledBenchException(reason: unknown) { + const message = `Bench ${ + this.ctx.qualifiedName + } threw during execution:\n${String(reason)}` + console.error(message) + unhandledExceptionMessages.push(message) + } + + median(baseline?: Measure): ReturnedAssertions { + this.ctx.lastSnapCallPosition = caller() + const assertions = this.createStatMethod( + "median", + baseline + ) as any as ReturnedAssertions + return assertions + } + + mean(baseline?: Measure): ReturnedAssertions { + this.ctx.lastSnapCallPosition = caller() + return this.createStatMethod("mean", baseline) as any as ReturnedAssertions + } + + mark(baseline?: MarkMeasure): ReturnedAssertions { + this.ctx.lastSnapCallPosition = caller() + return this.createStatMethod( + "mark", + baseline as any + ) as any as ReturnedAssertions + } +} + +const unhandledExceptionMessages: string[] = [] + export type UntilOptions = { ms?: number count?: number @@ -39,7 +309,7 @@ export type BenchContext = { } export type BenchAssertionContext = BenchContext & { - kind: TimeAssertionName | "types" + kind: TimeAssertionName | "types" | "instantiations" } export type BenchableFunction = () => unknown | Promise @@ -49,34 +319,27 @@ export type InitialBenchAssertions = const currentSuitePath: string[] = [] -export const bench = ( - name: string, - fn: Fn, +process.on("beforeExit", () => { + if (unhandledExceptionMessages.length) { + console.error( + `${unhandledExceptionMessages.length} unhandled exception(s) occurred during your benches (see details above).` + ) + process.exit(1) + } +}) + +export const getBenchCtx = ( + qualifiedPath: string[], + isAsync: boolean = false, options: BenchOptions = {} -): InitialBenchAssertions => { - const qualifiedPath = [...currentSuitePath, name] - const ctx: BenchContext = { +): BenchContext => { + return { qualifiedPath, qualifiedName: qualifiedPath.join("/"), options, cfg: getConfig(), benchCallPosition: caller(), lastSnapCallPosition: undefined, - isAsync: fn.constructor.name === "AsyncFunction" - } - ensureCacheDirs() - if ( - typeof ctx.cfg.filter === "string" && - !qualifiedPath.includes(ctx.cfg.filter) - ) - return chainableNoOpProxy - else if ( - Array.isArray(ctx.cfg.filter) && - ctx.cfg.filter.some((segment, i) => segment !== qualifiedPath[i]) - ) - return chainableNoOpProxy - - const assertions = new BenchAssertions(fn, ctx) - Object.assign(assertions, createBenchTypeAssertion(ctx)) - return assertions as any + isAsync + } as BenchContext } diff --git a/ark/attest/bench/call.ts b/ark/attest/bench/call.ts deleted file mode 100644 index 0d27c07e69..0000000000 --- a/ark/attest/bench/call.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { caller } from "@arktype/fs" -import { performance } from "node:perf_hooks" -import { chainableNoOpProxy } from "../utils.js" -import { await1K } from "./await1k.js" -import { compareToBaseline, queueBaselineUpdateIfNeeded } from "./baseline.js" -import type { BenchContext, BenchableFunction, UntilOptions } from "./bench.js" -import { call1K } from "./call1k.js" -import { - createTimeComparison, - createTimeMeasure, - type MarkMeasure, - type Measure, - type TimeUnit -} from "./measure.js" -import { createBenchTypeAssertion, type BenchTypeAssertions } from "./type.js" - -export type StatName = keyof typeof stats - -export type TimeAssertionName = StatName | "mark" - -export const stats = { - mean: (callTimes: number[]): number => { - const totalCallMs = callTimes.reduce((sum, duration) => sum + duration, 0) - return totalCallMs / callTimes.length - }, - median: (callTimes: number[]): number => { - const middleIndex = Math.floor(callTimes.length / 2) - const ms = - callTimes.length % 2 === 0 ? - (callTimes[middleIndex - 1] + callTimes[middleIndex]) / 2 - : callTimes[middleIndex] - return ms - } -} - -class ResultCollector { - results: number[] = [] - private benchStart = performance.now() - private bounds: Required - private lastInvocationStart: number - - constructor(private ctx: BenchContext) { - // By default, will run for either 5 seconds or 100_000 call sets (of 1000 calls), whichever comes first - this.bounds = { - ms: 5000, - count: 100_000, - ...ctx.options.until - } - this.lastInvocationStart = -1 - } - - start() { - this.ctx.options.hooks?.beforeCall?.() - this.lastInvocationStart = performance.now() - } - - stop() { - this.results.push((performance.now() - this.lastInvocationStart) / 1000) - this.ctx.options.hooks?.afterCall?.() - } - - done() { - const metMsTarget = performance.now() - this.benchStart >= this.bounds.ms - const metCountTarget = this.results.length >= this.bounds.count - return metMsTarget || metCountTarget - } -} - -const loopCalls = (fn: () => void, ctx: BenchContext) => { - const collector = new ResultCollector(ctx) - while (!collector.done()) { - collector.start() - // we use a function like this to make 1k explicit calls to the function - // to avoid certain optimizations V8 makes when looping - call1K(fn) - collector.stop() - } - return collector.results -} - -const loopAsyncCalls = async (fn: () => Promise, ctx: BenchContext) => { - const collector = new ResultCollector(ctx) - while (!collector.done()) { - collector.start() - await await1K(fn) - collector.stop() - } - return collector.results -} - -export class BenchAssertions< - Fn extends BenchableFunction, - NextAssertions = BenchTypeAssertions, - ReturnedAssertions = Fn extends () => Promise ? Promise - : NextAssertions -> { - private label: string - private lastCallTimes: number[] | undefined - constructor( - private fn: Fn, - private ctx: BenchContext - ) { - this.label = `Call: ${ctx.qualifiedName}` - } - - private applyCallTimeHooks() { - if (this.ctx.options.fakeCallMs !== undefined) { - const fakeMs = - this.ctx.options.fakeCallMs === "count" ? - this.lastCallTimes!.length - : this.ctx.options.fakeCallMs - this.lastCallTimes = this.lastCallTimes!.map(() => fakeMs) - } - } - - private callTimesSync() { - if (!this.lastCallTimes) { - this.lastCallTimes = loopCalls(this.fn as any, this.ctx) - this.lastCallTimes.sort() - } - this.applyCallTimeHooks() - return this.lastCallTimes - } - - private async callTimesAsync() { - if (!this.lastCallTimes) { - this.lastCallTimes = await loopAsyncCalls(this.fn as any, this.ctx) - this.lastCallTimes.sort() - } - this.applyCallTimeHooks() - return this.lastCallTimes - } - - private createAssertion( - name: Name, - baseline: Name extends "mark" ? - Record> | undefined - : Measure | undefined, - callTimes: number[] - ) { - if (name === "mark") return this.markAssertion(baseline as any, callTimes) - - const ms: number = stats[name as StatName](callTimes) - const comparison = createTimeComparison(ms, baseline as Measure) - console.group(`${this.label} (${name}):`) - compareToBaseline(comparison, this.ctx) - console.groupEnd() - queueBaselineUpdateIfNeeded(createTimeMeasure(ms), baseline, { - ...this.ctx, - kind: name - }) - return this.getNextAssertions() - } - - private markAssertion( - baseline: MarkMeasure | undefined, - callTimes: number[] - ) { - console.group(`${this.label}:`) - const markEntries: [StatName, Measure | undefined][] = ( - baseline ? - Object.entries(baseline) - // If nothing was passed, gather all available baselines by setting their values to undefined. - : Object.entries(stats).map(([kind]) => [kind, undefined])) as any - const markResults = Object.fromEntries( - markEntries.map(([kind, kindBaseline]) => { - console.group(kind) - const ms = stats[kind](callTimes) - const comparison = createTimeComparison(ms, kindBaseline) - compareToBaseline(comparison, this.ctx) - console.groupEnd() - return [kind, comparison.updated] - }) - ) - console.groupEnd() - queueBaselineUpdateIfNeeded(markResults, baseline, { - ...this.ctx, - kind: "mark" - }) - return this.getNextAssertions() - } - - private getNextAssertions(): NextAssertions { - return createBenchTypeAssertion(this.ctx) as any as NextAssertions - } - - private createStatMethod( - name: Name, - baseline: Name extends "mark" ? - Record> | undefined - : Measure | undefined - ) { - if (this.ctx.isAsync) { - return new Promise(resolve => { - this.callTimesAsync().then( - callTimes => { - resolve(this.createAssertion(name, baseline, callTimes)) - }, - e => { - this.addUnhandledBenchException(e) - resolve(chainableNoOpProxy) - } - ) - }) - } - let assertions = chainableNoOpProxy - try { - assertions = this.createAssertion(name, baseline, this.callTimesSync()) - } catch (e) { - this.addUnhandledBenchException(e) - } - return assertions - } - - private addUnhandledBenchException(reason: unknown) { - const message = `Bench ${ - this.ctx.qualifiedName - } threw during execution:\n${String(reason)}` - console.error(message) - unhandledExceptionMessages.push(message) - } - - median(baseline?: Measure): ReturnedAssertions { - this.ctx.lastSnapCallPosition = caller() - return this.createStatMethod("median", baseline) - } - - mean(baseline?: Measure): ReturnedAssertions { - this.ctx.lastSnapCallPosition = caller() - return this.createStatMethod("mean", baseline) - } - - mark(baseline?: MarkMeasure): ReturnedAssertions { - this.ctx.lastSnapCallPosition = caller() - return this.createStatMethod("mark", baseline as any) - } -} - -const unhandledExceptionMessages: string[] = [] - -process.on("beforeExit", () => { - if (unhandledExceptionMessages.length) { - console.error( - `${unhandledExceptionMessages.length} unhandled exception(s) occurred during your benches (see details above).` - ) - process.exit(1) - } -}) diff --git a/ark/attest/bench/measure.ts b/ark/attest/bench/measure.ts index 0fec3cc8d9..916771cb45 100644 --- a/ark/attest/bench/measure.ts +++ b/ark/attest/bench/measure.ts @@ -1,4 +1,4 @@ -import type { StatName } from "./call.js" +import type { StatName } from "./bench.js" type MeasureUnit = TimeUnit | TypeUnit @@ -89,4 +89,4 @@ export const createTimeComparison = ( updated: createTimeMeasure(ms), baseline: undefined } -} +} \ No newline at end of file diff --git a/ark/attest/bench/type.ts b/ark/attest/bench/type.ts index 45a127e793..e9ba737c39 100644 --- a/ark/attest/bench/type.ts +++ b/ark/attest/bench/type.ts @@ -1,20 +1,21 @@ -import { caller, filePath } from "@arktype/fs" -import { throwInternalError } from "@arktype/util" -import * as tsvfs from "@typescript/vfs" +import { caller } from "@arktype/fs" import ts from "typescript" +import { getTypeBenchAssertionsAtPosition } from "../cache/getCachedAssertions.js" import { TsServer, getAbsolutePosition, getAncestors, getDescendants, - getInternalTypeChecker, - getTsConfigInfoOrThrow, - getTsLibFiles, nearestCallExpressionChild } from "../cache/ts.js" -import { getExpressionsByName } from "../cache/writeAssertionCache.js" +import { + getCallExpressionsByName, + getInstantiationsContributedByNode +} from "../cache/utils.js" +import type { TypeRelationship } from "../cache/writeAssertionCache.js" +import { getConfig } from "../config.js" import { compareToBaseline, queueBaselineUpdateIfNeeded } from "./baseline.js" -import type { BenchContext } from "./bench.js" +import type { BenchAssertionContext, BenchContext } from "./bench.js" import { createTypeComparison, type Measure, @@ -26,133 +27,67 @@ export type BenchTypeAssertions = { types: (instantiations?: Measure) => void } -const getIsolatedEnv = () => { - const tsconfigInfo = getTsConfigInfoOrThrow() - const libFiles = getTsLibFiles(tsconfigInfo.parsed.options) - const projectRoot = process.cwd() - const system = tsvfs.createFSBackedSystem( - libFiles.defaultMapFromNodeModules, - projectRoot, - ts - ) - return tsvfs.createVirtualTypeScriptEnvironment( - system, - [], - ts, - tsconfigInfo.parsed.options - ) -} - -const createFile = ( - env: tsvfs.VirtualTypeScriptEnvironment, - fileName: string, - fileText: string -) => { - env.createFile(fileName, fileText) - return env.getSourceFile(fileName) -} +export const createBenchTypeAssertion = ( + ctx: BenchContext +): BenchTypeAssertions => ({ + types: (...args: [instantiations?: Measure | undefined]) => { + ctx.lastSnapCallPosition = caller() + instantiationDataHandler({ ...ctx, kind: "types" }, args[0]) + } +}) -const getProgram = (env?: tsvfs.VirtualTypeScriptEnvironment) => { - return env?.languageService.getProgram() -} -const getInstantiationsWithFile = (fileText: string, fileName: string) => { - const env = getIsolatedEnv() - const file = createFile(env, fileName, fileText) - getProgram(env)?.emit(file) - const instantiationCount = getInternalTypeChecker(env).getInstantiationCount() - return instantiationCount -} +export const getContributedInstantiations = (ctx: BenchContext): number => { + const testDeclarationAliases = getConfig().testDeclarationAliases + const instance = TsServer.instance + const file = instance.getSourceFileOrThrow(ctx.benchCallPosition.file) -const getFirstAncestorByKindOrThrow = (node: ts.Node, kind: ts.SyntaxKind) => - getAncestors(node).find(ancestor => ancestor.kind === kind) ?? - throwInternalError( - `Could not find an ancestor of kind ${ts.SyntaxKind[kind]}` + const node = nearestCallExpressionChild( + file, + getAbsolutePosition(file, ctx.benchCallPosition) ) -const getBaselineSourceFile = (originalFile: ts.SourceFile): string => { - const benchCalls = getExpressionsByName(originalFile, ["bench"]) - - const benchExpressions = benchCalls.map(node => - getFirstAncestorByKindOrThrow(node, ts.SyntaxKind.ExpressionStatement) + const firstMatchingNamedCall = getAncestors(node).find( + call => getCallExpressionsByName(call, testDeclarationAliases).length ) - let baselineSourceFileText = originalFile.getFullText() - - benchExpressions.forEach(benchExpression => { - baselineSourceFileText = baselineSourceFileText.replace( - benchExpression.getFullText(), - "" + if (!firstMatchingNamedCall) { + throw new Error( + `No call expressions matching the name(s) '${testDeclarationAliases.join()}' were found` ) - }) - - return baselineSourceFileText -} - -const instantiationsByPath: { [path: string]: number } = {} - -const getInstantiationsContributedByNode = ( - benchBlock: ts.FunctionExpression | ts.ArrowFunction -) => { - const originalFile = benchBlock.getSourceFile() - const originalPath = filePath(originalFile.fileName) - const fakePath = originalPath + ".nonexistent.ts" + } - const baselineFile = getBaselineSourceFile(originalFile) + const body = getDescendants(firstMatchingNamedCall).find( + node => ts.isArrowFunction(node) || ts.isFunctionExpression(node) + ) as ts.ArrowFunction | ts.FunctionExpression | undefined - const baselineFileWithBenchBlock = - baselineFile + `\nconst $attestIsolatedBench = ${benchBlock.getFullText()}` + if (!body) + throw new Error("Unable to retrieve contents of the call expression") - if (!instantiationsByPath[fakePath]) { - console.log(`⏳ attest: Analyzing type assertions...`) - const instantiationsWithoutNode = getInstantiationsWithFile( - baselineFile, - fakePath - ) + return getInstantiationsContributedByNode(file, body) +} - instantiationsByPath[fakePath] = instantiationsWithoutNode - console.log(`⏳ Cached type assertions \n`) +export type ArgAssertionData = { + type: string + relationships: { + args: TypeRelationship[] + typeArgs: TypeRelationship[] } +} - const instantiationsWithNode = getInstantiationsWithFile( - baselineFileWithBenchBlock, - fakePath +export const instantiationDataHandler = ( + ctx: BenchAssertionContext, + args?: Measure, + isBenchFunction = true +): void => { + const instantiationsContributed = + isBenchFunction ? + getContributedInstantiations(ctx) + : getTypeBenchAssertionsAtPosition(ctx.benchCallPosition)[0][1].count + + const comparison: MeasureComparison = createTypeComparison( + instantiationsContributed, + args ) - - return instantiationsWithNode - instantiationsByPath[fakePath] + compareToBaseline(comparison, ctx) + queueBaselineUpdateIfNeeded(comparison.updated, args, ctx) } - -export const createBenchTypeAssertion = ( - ctx: BenchContext -): BenchTypeAssertions => ({ - types: (...args: [instantiations?: Measure | undefined]) => { - ctx.lastSnapCallPosition = caller() - const instance = TsServer.instance - const file = instance.getSourceFileOrThrow(ctx.benchCallPosition.file) - - const benchNode = nearestCallExpressionChild( - file, - getAbsolutePosition(file, ctx.benchCallPosition) - ) - const benchFn = getExpressionsByName(benchNode, ["bench"]) - if (!benchFn) throw new Error("Unable to retrieve bench expression node.") - - const benchBody = getDescendants(benchFn[0]).find( - node => ts.isArrowFunction(node) || ts.isFunctionExpression(node) - ) as ts.ArrowFunction | ts.FunctionExpression | undefined - - if (!benchBody) throw new Error("Unable to retrieve bench body node.") - - const instantiationsContributed = - getInstantiationsContributedByNode(benchBody) - - const comparison: MeasureComparison = createTypeComparison( - instantiationsContributed, - args[0] - ) - compareToBaseline(comparison, ctx) - queueBaselineUpdateIfNeeded(comparison.updated, args[0], { - ...ctx, - kind: "types" - }) - } -}) diff --git a/ark/attest/cache/getCachedAssertions.ts b/ark/attest/cache/getCachedAssertions.ts index d71d50bb8a..9653232053 100644 --- a/ark/attest/cache/getCachedAssertions.ts +++ b/ark/attest/cache/getCachedAssertions.ts @@ -6,12 +6,15 @@ import { getFileKey } from "../utils.js" import type { AssertionsByFile, LinePositionRange, - TypeAssertionData + TypeAssertionData, + TypeBenchmarkingAssertionData, + TypeRelationshipAssertionData } from "./writeAssertionCache.js" export type VersionedAssertionsByFile = [ tsVersion: string, - assertions: AssertionsByFile + relationshipAssertions: AssertionsByFile, + benchAssertions: AssertionsByFile ] let assertionEntries: VersionedAssertionsByFile[] | undefined @@ -22,11 +25,28 @@ export const getCachedAssertionEntries = (): VersionedAssertionsByFile[] => { throwMissingAssertionDataError(config.assertionCacheDir) const assertionFiles = readdirSync(config.assertionCacheDir) - assertionEntries = assertionFiles.map(file => [ - // remove .json extension - file.slice(0, -5), - readJson(join(config.assertionCacheDir, file)) - ]) + const relationshipAssertions: AssertionsByFile = {} + const benchAssertions: AssertionsByFile = {} + + assertionEntries = assertionFiles.map(file => { + const data = readJson(join(config.assertionCacheDir, file)) + for (const fileName of Object.keys(data)) { + const relationshipAssertionData = data[fileName].filter( + (entry: TypeAssertionData) => "args" in entry + ) + const benchAssertionData = data[fileName].filter( + (entry: TypeAssertionData) => "count" in entry + ) + relationshipAssertions[fileName] = relationshipAssertionData + benchAssertions[fileName] = benchAssertionData + } + return [ + // remove .json extension + file.slice(0, -5), + relationshipAssertions, + benchAssertions + ] + }) } return assertionEntries! } @@ -51,37 +71,63 @@ const isPositionWithinRange = ( return true } -export type VersionedTypeAssertion = [ - tsVersion: string, - assertionData: TypeAssertionData -] +export type VersionedTypeAssertion< + data extends TypeAssertionData = TypeAssertionData +> = [tsVersion: string, assertionData: data] -export const getTypeAssertionsAtPosition = ( - position: SourcePosition -): VersionedTypeAssertion[] => { +type AssertionKind = "bench" | "type" + +const getTypeAssertionsAtPosition = ( + position: SourcePosition, + assertionType: AssertionKind +): VersionedTypeAssertion[] => { const fileKey = getFileKey(position.file) - return getCachedAssertionEntries().map(([version, data]) => { - if (!data[fileKey]) { - throw new Error( - `Found no assertion data for '${fileKey}' for TypeScript version ${version}.` - ) - } - const matchingAssertion = data[fileKey].find(assertion => { - /** - * Depending on the environment, a trace can refer to any of these points - * attest(...) - * ^ ^ ^ - * Because of this, it's safest to check if the call came from anywhere in the expected range. - * - */ - return isPositionWithinRange(position, assertion.location) - }) - if (!matchingAssertion) { - throw new Error( - `Found no assertion for TypeScript version ${version} at line ${position.line} char ${position.char} in '${fileKey}'. + return getCachedAssertionEntries().map( + ([version, typeRelationshipAssertions, BenchAssertionAssertions]) => { + const assertions = + assertionType === "type" ? + typeRelationshipAssertions + : BenchAssertionAssertions + if (!assertions[fileKey]) { + throw new Error( + `Found no assertion data for '${fileKey}' for TypeScript version ${version}.` + ) + } + const matchingAssertion = assertions[fileKey].find(assertion => { + /** + * Depending on the environment, a trace can refer to any of these points + * attest(...) + * ^ ^ ^ + * Because of this, it's safest to check if the call came from anywhere in the expected range. + * + */ + return isPositionWithinRange(position, assertion.location) + }) + if (!matchingAssertion) { + throw new Error( + `Found no assertion for TypeScript version ${version} at line ${position.line} char ${position.char} in '${fileKey}'. Are sourcemaps enabled and working properly?` - ) + ) + } + return [version, matchingAssertion] as VersionedTypeAssertion } - return [version, matchingAssertion] - }) + ) +} + +export const getTypeRelationshipAssertionsAtPosition = ( + position: SourcePosition +): VersionedTypeAssertion[] => { + return getTypeAssertionsAtPosition( + position, + "type" + ) +} + +export const getTypeBenchAssertionsAtPosition = ( + position: SourcePosition +): VersionedTypeAssertion[] => { + return getTypeAssertionsAtPosition( + position, + "bench" + ) } diff --git a/ark/attest/cache/snapshots.ts b/ark/attest/cache/snapshots.ts index a3cb9d5373..5be8c08dc0 100644 --- a/ark/attest/cache/snapshots.ts +++ b/ark/attest/cache/snapshots.ts @@ -8,8 +8,7 @@ import { writeJson, type SourcePosition } from "@arktype/fs" -import { randomUUID } from "node:crypto" -import { existsSync, readdirSync, rmSync } from "node:fs" +import { existsSync } from "node:fs" import { basename, dirname, isAbsolute, join } from "node:path" import type ts from "typescript" import { getConfig } from "../config.js" @@ -19,7 +18,7 @@ import { getAbsolutePosition, nearestCallExpressionChild } from "./ts.js" -import { getExpressionsByName } from "./writeAssertionCache.js" +import { getCallExpressionsByName } from "./utils.js" export type SnapshotArgs = { position: SourcePosition @@ -51,16 +50,9 @@ export const getSnapshotByName = ( * file by a cleanup process after all tests have completed. */ export const queueSnapshotUpdate = (args: SnapshotArgs): void => { - const isBench = args.baselinePath const config = getConfig() - writeJson( - join( - isBench ? config.benchSnapCacheDir : config.snapCacheDir, - `snap-${randomUUID()}.json` - ), - args - ) - if (isBench) writeSnapshotUpdatesOnExit() + writeSnapUpdate(config.defaultAssertionCachePath, args) + writeSnapshotUpdatesOnExit() } export type QueuedUpdate = { @@ -84,7 +76,7 @@ const findCallExpressionAncestor = ( const file = server.getSourceFileOrThrow(position.file) const absolutePosition = getAbsolutePosition(file, position) const startNode = nearestCallExpressionChild(file, absolutePosition) - const calls = getExpressionsByName(startNode, [functionName], true) + const calls = getCallExpressionsByName(startNode, [functionName], true) if (calls.length) return startNode throw new Error( @@ -118,47 +110,51 @@ export const writeSnapshotUpdatesOnExit = (): void => { snapshotsWillBeWritten = true } -/** - * This will fail if you have a sub process that writes cached snapshots and then deletes the snapshot cache that the root - * process is using - */ const writeCachedInlineSnapshotUpdates = () => { const config = getConfig() const updates: QueuedUpdate[] = [] - if (existsSync(config.snapCacheDir)) - updates.push(...getQueuedUpdates(config.snapCacheDir)) - if (existsSync(config.benchSnapCacheDir)) - updates.push(...getQueuedUpdates(config.benchSnapCacheDir)) + if (existsSync(config.assertionCacheDir)) + updates.push(...getQueuedUpdates(config.defaultAssertionCachePath)) writeUpdates(updates) - rmSync(config.snapCacheDir, { recursive: true, force: true }) - rmSync(config.benchSnapCacheDir, { recursive: true, force: true }) + writeSnapUpdate(config.defaultAssertionCachePath) } -const getQueuedUpdates = (dir: string) => { - const queuedUpdates: QueuedUpdate[] = [] - for (const updateFile of readdirSync(dir)) { - if (/snap.*\.json$/.test(updateFile)) { - let snapshotData: SnapshotArgs | undefined - try { - snapshotData = readJson(join(dir, updateFile)) - } catch { - // If we can't read the snapshot, log an error and move onto the next update - console.error( - `Unable to read snapshot data from expected location ${updateFile}.` - ) - } - if (snapshotData) { - try { - queuedUpdates.push(snapshotArgsToQueuedUpdate(snapshotData)) - } catch (error) { - // If writeInlineSnapshotToFile throws an error, log it and move on to the next update - console.error(String(error)) - } - } +const writeSnapUpdate = (path: string, update?: SnapshotArgs) => { + const assertions = + existsSync(path) ? readJson(path) : { updates: [] as SnapshotArgs[] } + + assertions.updates = + update !== undefined ? [...(assertions.updates ?? []), update] : [] + + writeJson(path, assertions) +} +const updateQueue = (queue: QueuedUpdate[], path: string) => { + let snapshotData: SnapshotArgs[] | undefined + try { + snapshotData = readJson(path).updates + } catch { + // If we can't read the snapshot, log an error and move onto the next update + console.error( + `Unable to read snapshot data from expected location ${path}.` + ) + } + if (snapshotData) { + try { + snapshotData.forEach(snapshot => + queue.push(snapshotArgsToQueuedUpdate(snapshot)) + ) + } catch (error) { + // If writeInlineSnapshotToFile throws an error, log it and move on to the next update + console.error(String(error)) } } +} + +const getQueuedUpdates = (path: string) => { + const queuedUpdates: QueuedUpdate[] = [] + updateQueue(queuedUpdates, path) return queuedUpdates } @@ -201,10 +197,13 @@ export const writeUpdates = (queuedUpdates: QueuedUpdate[]): void => { ) ) } - runPrettierIfAvailable(queuedUpdates) + runFormatterIfAvailable(queuedUpdates) } -const runPrettierIfAvailable = (queuedUpdates: QueuedUpdate[]) => { +const runFormatterIfAvailable = (queuedUpdates: QueuedUpdate[]) => { + const { formatter, shouldFormat } = getConfig() + if (!shouldFormat) return + try { const updatedPaths = [ ...new Set( @@ -213,9 +212,9 @@ const runPrettierIfAvailable = (queuedUpdates: QueuedUpdate[]) => { ) ) ] - shell(`npm exec --no -- prettier --write ${updatedPaths.join(" ")}`) + shell(`${formatter} ${updatedPaths.join(" ")}`) } catch { - // If prettier is unavailable, do nothing. + // If formatter is unavailable or skipped, do nothing. } } diff --git a/ark/attest/cache/ts.ts b/ark/attest/cache/ts.ts index b5eb8bb07b..1d06474064 100644 --- a/ark/attest/cache/ts.ts +++ b/ark/attest/cache/ts.ts @@ -1,4 +1,5 @@ import { fromCwd, type SourcePosition } from "@arktype/fs" +import { throwInternalError } from "@arktype/util" import * as tsvfs from "@typescript/vfs" import { readFileSync } from "node:fs" import { dirname, join } from "node:path" @@ -98,13 +99,12 @@ export type TsconfigInfo = { } export const getTsConfigInfoOrThrow = (): TsconfigInfo => { - const config = getConfig() + const config = getConfig().tsconfig const configFilePath = - config.tsconfig ?? - ts.findConfigFile(fromCwd(), ts.sys.fileExists, "tsconfig.json") + config ?? ts.findConfigFile(fromCwd(), ts.sys.fileExists, "tsconfig.json") if (!configFilePath) { throw new Error( - `File ${config.tsconfig ?? join(fromCwd(), "tsconfig.json")} must exist.` + `File ${config ?? join(fromCwd(), "tsconfig.json")} must exist.` ) } @@ -128,10 +128,8 @@ export const getTsConfigInfoOrThrow = (): TsconfigInfo => { {}, configFilePath ) - // ensure type.toString is as precise as possible configParseResult.options.noErrorTruncation = true - if (configParseResult.errors.length > 0) { throw new Error( ts.formatDiagnostics(configParseResult.errors, { @@ -148,7 +146,7 @@ export const getTsConfigInfoOrThrow = (): TsconfigInfo => { } } -export type TsLibFiles = { +type TsLibFiles = { defaultMapFromNodeModules: Map resolvedPaths: string[] } @@ -182,7 +180,8 @@ export interface InternalTypeChecker extends ts.TypeChecker { export const getInternalTypeChecker = ( env?: tsvfs.VirtualTypeScriptEnvironment -): InternalTypeChecker => getProgram(env).getTypeChecker() as never +): InternalTypeChecker => + getProgram(env).getTypeChecker() as InternalTypeChecker export interface StringifiableType extends ts.Type { toString(): string @@ -224,9 +223,19 @@ const getDescendantsRecurse = (node: ts.Node): ts.Node[] => [ export const getAncestors = (node: ts.Node): ts.Node[] => { const ancestors: ts.Node[] = [] - while (node.parent) { - ancestors.push(node) - node = node.parent + let baseNode = node.parent + while (baseNode.parent !== undefined) { + ancestors.push(baseNode) + baseNode = baseNode.parent } return ancestors } + +export const getFirstAncestorByKindOrThrow = ( + node: ts.Node, + kind: ts.SyntaxKind +): ts.Node => + getAncestors(node).find(ancestor => ancestor.kind === kind) ?? + throwInternalError( + `Could not find an ancestor of kind ${ts.SyntaxKind[kind]}` + ) diff --git a/ark/attest/cache/utils.ts b/ark/attest/cache/utils.ts new file mode 100644 index 0000000000..5aba5362e3 --- /dev/null +++ b/ark/attest/cache/utils.ts @@ -0,0 +1,198 @@ +import { filePath } from "@arktype/fs" +import { throwInternalError } from "@arktype/util" +import * as tsvfs from "@typescript/vfs" +import ts from "typescript" +import { getConfig } from "../config.js" +import { getFileKey } from "../utils.js" +import { + getDescendants, + getFirstAncestorByKindOrThrow, + getProgram, + getTsConfigInfoOrThrow, + getTsLibFiles +} from "./ts.js" +import type { + AssertionsByFile, + LinePositionRange +} from "./writeAssertionCache.js" + +export const getCallLocationFromCallExpression = ( + callExpression: ts.CallExpression +): LinePositionRange => { + const start = ts.getLineAndCharacterOfPosition( + callExpression.getSourceFile(), + callExpression.getStart() + ) + const end = ts.getLineAndCharacterOfPosition( + callExpression.getSourceFile(), + callExpression.getEnd() + ) + // Add 1 to everything, since trace positions are 1-based and TS positions are 0-based. + const location: LinePositionRange = { + start: { + line: start.line + 1, + char: start.character + 1 + }, + end: { + line: end.line + 1, + char: end.character + 1 + } + } + return location +} + +export const gatherInlineInstantiationData = ( + file: ts.SourceFile, + fileAssertions: AssertionsByFile, + attestAliasInstantiationMethodCalls: string[] +): void => { + const expressions = getCallExpressionsByName( + file, + attestAliasInstantiationMethodCalls + ) + if (!expressions.length) return + + const enclosingFunctions = expressions.map(expression => { + const attestInstantiationsExpression = getFirstAncestorByKindOrThrow( + expression, + ts.SyntaxKind.ExpressionStatement + ) + return { + ancestor: getFirstAncestorByKindOrThrow( + attestInstantiationsExpression, + ts.SyntaxKind.ExpressionStatement + ), + position: getCallLocationFromCallExpression(expression) + } + }) + const instantiationInfo = enclosingFunctions.map(enclosingFunction => { + const body = getDescendants(enclosingFunction.ancestor).find( + node => ts.isArrowFunction(node) || ts.isFunctionExpression(node) + ) as ts.ArrowFunction | ts.FunctionExpression | undefined + if (!body) throw new Error("Unable to find file contents") + + return { + location: enclosingFunction.position, + count: getInstantiationsContributedByNode(file, body) + } + }) + const assertions = fileAssertions[getFileKey(file.fileName)] ?? [] + fileAssertions[getFileKey(file.fileName)] = [ + ...assertions, + ...instantiationInfo + ] +} + +export const getCallExpressionsByName = ( + startNode: ts.Node, + names: string[], + isSnapCall = false +): ts.CallExpression[] => { + const calls: ts.CallExpression[] = [] + getDescendants(startNode).forEach(descendant => { + if (ts.isCallExpression(descendant)) { + if (names.includes(descendant.expression.getText()) || !names.length) + calls.push(descendant) + } else if (isSnapCall) { + if (ts.isIdentifier(descendant)) { + if (names.includes(descendant.getText()) || !names.length) + calls.push(descendant as any as ts.CallExpression) + } + } + }) + return calls +} + +const instantiationsByPath: { [path: string]: number } = {} + +export const getInstantiationsContributedByNode = ( + file: ts.SourceFile, + benchBlock: ts.FunctionExpression | ts.ArrowFunction +): number => { + const originalPath = filePath(file.fileName) + const fakePath = originalPath + ".nonexistent.ts" + + const baselineFile = getBaselineSourceFile(file) + + const baselineFileWithBenchBlock = + baselineFile + `\nconst $attestIsolatedBench = ${benchBlock.getFullText()}` + + if (!instantiationsByPath[fakePath]) { + console.log(`⏳ attest: Analyzing type assertions...`) + const instantiationsWithoutNode = getInstantiationsWithFile( + baselineFile, + fakePath + ) + + instantiationsByPath[fakePath] = instantiationsWithoutNode + console.log(`⏳ Cached type assertions \n`) + } + + const instantiationsWithNode = getInstantiationsWithFile( + baselineFileWithBenchBlock, + fakePath + ) + + return instantiationsWithNode - instantiationsByPath[fakePath] +} + +export const createOrUpdateFile = ( + env: tsvfs.VirtualTypeScriptEnvironment, + fileName: string, + fileText: string +): ts.SourceFile | undefined => { + env.sys.fileExists(fileName) ? + env.updateFile(fileName, fileText) + : env.createFile(fileName, fileText) + return env.getSourceFile(fileName) +} + +const getInstantiationsWithFile = (fileText: string, fileName: string) => { + const env = getIsolatedEnv() + const file = createOrUpdateFile(env, fileName, fileText) + const program = getProgram(env) + program.emit(file) + const count = program.getInstantiationCount() + if (count === undefined) + throwInternalError(`Unable to gather instantiation count for ${fileText}`) + + return count +} + +let virtualEnv: tsvfs.VirtualTypeScriptEnvironment | undefined = undefined + +export const getIsolatedEnv = (): tsvfs.VirtualTypeScriptEnvironment => { + if (virtualEnv !== undefined) return virtualEnv + + const tsconfigInfo = getTsConfigInfoOrThrow() + const libFiles = getTsLibFiles(tsconfigInfo.parsed.options) + const projectRoot = process.cwd() + const system = tsvfs.createFSBackedSystem( + libFiles.defaultMapFromNodeModules, + projectRoot, + ts + ) + virtualEnv = tsvfs.createVirtualTypeScriptEnvironment( + system, + [], + ts, + tsconfigInfo.parsed.options + ) + return virtualEnv +} + +const getBaselineSourceFile = (originalFile: ts.SourceFile): string => { + const functionNames = getConfig().testDeclarationAliases + + const calls = getCallExpressionsByName(originalFile, functionNames) + + let baselineSourceFileText = originalFile.getFullText() + + calls.forEach(call => { + baselineSourceFileText = baselineSourceFileText.replace( + call.getFullText(), + "" + ) + }) + return baselineSourceFileText +} \ No newline at end of file diff --git a/ark/attest/cache/writeAssertionCache.ts b/ark/attest/cache/writeAssertionCache.ts index 2897a107a1..9147c95746 100644 --- a/ark/attest/cache/writeAssertionCache.ts +++ b/ark/attest/cache/writeAssertionCache.ts @@ -1,6 +1,7 @@ import type { LinePosition } from "@arktype/fs" import { flatMorph } from "@arktype/util" import ts from "typescript" + import { getConfig } from "../config.js" import { getFileKey } from "../utils.js" import { @@ -11,6 +12,11 @@ import { type ArgumentTypes, type StringifiableType } from "./ts.js" +import { + gatherInlineInstantiationData, + getCallExpressionsByName, + getCallLocationFromCallExpression +} from "./utils.js" export type AssertionsByFile = Record @@ -20,6 +26,9 @@ export const analyzeProjectAssertions = (): AssertionsByFile => { const filePaths = instance.rootFiles const diagnosticsByFile = getDiagnosticsByFile() const assertionsByFile: AssertionsByFile = {} + const attestAliasInstantiationMethodCalls = config.attestAliases.map( + alias => `${alias}.instantiations` + ) for (const path of filePaths) { const file = instance.getSourceFileOrThrow(path) const assertionsInFile = getAssertionsInFile( @@ -29,6 +38,13 @@ export const analyzeProjectAssertions = (): AssertionsByFile => { ) if (assertionsInFile.length) assertionsByFile[getFileKey(file.fileName)] = assertionsInFile + if (!config.skipInlineInstantiations) { + gatherInlineInstantiationData( + file, + assertionsByFile, + attestAliasInstantiationMethodCalls + ) + } } return assertionsByFile } @@ -38,65 +54,16 @@ export const getAssertionsInFile = ( diagnosticsByFile: DiagnosticsByFile, attestAliases: string[] ): TypeAssertionData[] => { - const assertCalls = getExpressionsByName(file, attestAliases) + const assertCalls = getCallExpressionsByName(file, attestAliases) return assertCalls.map(call => analyzeAssertCall(call, diagnosticsByFile)) } -export const getAssertCallLocation = ( - assertCall: ts.CallExpression -): LinePositionRange => { - const start = ts.getLineAndCharacterOfPosition( - assertCall.getSourceFile(), - assertCall.getStart() - ) - const end = ts.getLineAndCharacterOfPosition( - assertCall.getSourceFile(), - assertCall.getEnd() - ) - // Add 1 to everything, since trace positions are 1-based and TS positions are 0-based. - return { - start: { - line: start.line + 1, - char: start.character + 1 - }, - end: { - line: end.line + 1, - char: end.character + 1 - } - } -} - -export const getExpressionsByName = ( - startNode: ts.Node, - names: string[], - isSnapCall = false -): ts.CallExpression[] => { - /* - * We get might get some extraneous calls to other "attest" functions, - * but they won't be referenced at runtime so shouldn't matter. - */ - const calls: ts.CallExpression[] = [] - const visit = (node: ts.Node) => { - if (ts.isCallExpression(node)) { - if (names.includes(node.expression.getText())) calls.push(node) - } else if (isSnapCall) { - if (ts.isIdentifier(node)) { - if (names.includes(node.getText())) - calls.push(node as any as ts.CallExpression) - } - } - ts.forEachChild(node, visit) - } - visit(startNode) - return calls -} - export const analyzeAssertCall = ( assertCall: ts.CallExpression, diagnosticsByFile: DiagnosticsByFile ): TypeAssertionData => { const types = extractArgumentTypesFromCall(assertCall) - const location = getAssertCallLocation(assertCall) + const location = getCallLocationFromCallExpression(assertCall) const args = types.args.map(arg => serializeArg(arg, types)) const typeArgs = types.typeArgs.map(typeArg => serializeArg(typeArg, types)) const errors = checkDiagnosticMessages(assertCall, diagnosticsByFile) @@ -216,11 +183,6 @@ const concatenateChainedErrors = ( ) .join("\n") -export type LinePositionRange = { - start: LinePosition - end: LinePosition -} - export type ArgAssertionData = { type: string relationships: { @@ -228,7 +190,8 @@ export type ArgAssertionData = { typeArgs: TypeRelationship[] } } -export type TypeAssertionData = { + +export type TypeRelationshipAssertionData = { location: LinePositionRange args: ArgAssertionData[] typeArgs: ArgAssertionData[] @@ -236,6 +199,20 @@ export type TypeAssertionData = { completions: Completions } +export type TypeBenchmarkingAssertionData = { + location: LinePositionRange + count: number +} + +export type TypeAssertionData = + | TypeRelationshipAssertionData + | TypeBenchmarkingAssertionData + +export type LinePositionRange = { + start: LinePosition + end: LinePosition +} + export type TypeRelationship = "subtype" | "supertype" | "equality" | "none" export const compareTsTypes = ( diff --git a/ark/attest/config.ts b/ark/attest/config.ts index e29044606e..1d0b971ade 100644 --- a/ark/attest/config.ts +++ b/ark/attest/config.ts @@ -32,10 +32,14 @@ type BaseAttestConfig = { */ tsVersions: TsVersionAliases | TsVersionData[] skipTypes: boolean + skipInlineInstantiations: boolean attestAliases: string[] benchPercentThreshold: number benchErrorOnThresholdExceeded: boolean filter: string | undefined + testDeclarationAliases: string[] + formatter: string + shouldFormat: boolean } export type AttestConfig = Partial @@ -49,10 +53,14 @@ export const getDefaultAttestConfig = (): BaseAttestConfig => { attestAliases: ["attest", "attestInternal"], updateSnapshots: false, skipTypes: false, + skipInlineInstantiations: false, tsVersions: "typescript", benchPercentThreshold: 20, benchErrorOnThresholdExceeded: false, - filter: undefined + filter: undefined, + testDeclarationAliases: ["bench", "it"], + formatter: `npm exec --no -- prettier --write`, + shouldFormat: true } } @@ -99,24 +107,21 @@ const addEnvConfig = (config: BaseAttestConfig) => { export interface ParsedAttestConfig extends Readonly { cacheDir: string - snapCacheDir: string - benchSnapCacheDir: string assertionCacheDir: string + defaultAssertionCachePath: string tsVersions: TsVersionData[] } const parseConfig = (): ParsedAttestConfig => { const baseConfig = addEnvConfig(getDefaultAttestConfig()) const cacheDir = resolve(".attest") - const snapCacheDir = join(cacheDir, "snaps") - const benchSnapCacheDir = join(cacheDir, "benchSnaps") const assertionCacheDir = join(cacheDir, "assertions") + const defaultAssertionCachePath = join(assertionCacheDir, "typescript.json") return Object.assign(baseConfig, { cacheDir, - snapCacheDir, - benchSnapCacheDir, assertionCacheDir, + defaultAssertionCachePath, tsVersions: baseConfig.skipTypes ? [] : isTsVersionAliases(baseConfig.tsVersions) ? @@ -153,7 +158,5 @@ export const getConfig = (): ParsedAttestConfig => cachedConfig export const ensureCacheDirs = (): void => { ensureDir(cachedConfig.cacheDir) - ensureDir(cachedConfig.snapCacheDir) - ensureDir(cachedConfig.benchSnapCacheDir) ensureDir(cachedConfig.assertionCacheDir) } diff --git a/ark/attest/fixtures.ts b/ark/attest/fixtures.ts index 832c17ede2..8600b82598 100644 --- a/ark/attest/fixtures.ts +++ b/ark/attest/fixtures.ts @@ -17,7 +17,7 @@ export const setup = (options: Partial = {}): void => { config.tsVersions.length === 1 && config.tsVersions[0].alias === "typescript" ) - writeAssertionData(join(config.assertionCacheDir, "typescript.json")) + writeAssertionData(config.defaultAssertionCachePath) else { forTypeScriptVersions(config.tsVersions, version => shell( diff --git a/ark/attest/main.ts b/ark/attest/main.ts index 1f23315ca7..b1596dffae 100644 --- a/ark/attest/main.ts +++ b/ark/attest/main.ts @@ -1,7 +1,10 @@ export { caller, type CallerOfOptions } from "@arktype/fs" export { attest } from "./assert/attest.js" export { bench } from "./bench/bench.js" -export { getTypeAssertionsAtPosition } from "./cache/getCachedAssertions.js" +export { + getTypeBenchAssertionsAtPosition, + getTypeRelationshipAssertionsAtPosition +} from "./cache/getCachedAssertions.js" export type { ArgAssertionData, LinePositionRange,