fix(attest): improve bench extraction, add docs on baseline expression (

#1062)

ark/attest/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @ark/attest
 
+## 0.9.4
+
+Improve benchmark source extraction, add notes on baseline expressions
+
 ## 0.9.2
 
 Fix a bug preventing consecutive benchmark runs from populating snapshots inline

ark/attest/README.md

@@ -168,6 +168,31 @@ bench(
 	.types([337, "instantiations"])
 ```
 
+If you're benchmarking an API, you'll need to include a "baseline expression" so that instantiations created when your API is initially invoked don't add noise to the individual tests.
+
+Here's an example of what that looks like:
+
+```ts
+import { bench } from "@ark/attest"
+import { type } from "arktype"
+
+// baseline expression
+type("boolean")
+
+bench("single-quoted", () => {
+	const _ = type("'nineteen characters'")
+	// would be 2697 without baseline
+}).types([610, "instantiations"])
+
+bench("keyword", () => {
+	const _ = type("string")
+	// would be 2507 without baseline
+}).types([356, "instantiations"])
+```
+
+> [!WARNING]  
+> Be sure your baseline expression is not identical to an expression you are using in any of your benchmarks. If it is, the individual benchmarks will reuse its cached types, leading to reduced (or 0) instantiations.
+
 If you'd like to fail in CI above a threshold, you can add flags like the following (default value is 20%, but it will not throw unless `--benchErrorOnThresholdExceeded` is set):
 
 ```

ark/attest/bench/bench.ts

@@ -58,7 +58,7 @@ export const bench = <Fn extends BenchableFunction>(
 
 	const assertions = new BenchAssertions(fn, ctx)
 	Object.assign(assertions, createBenchTypeAssertion(ctx))
-	return assertions as any
+	return assertions as never
 }
 
 export const stats = {
@@ -158,7 +158,7 @@ export class BenchAssertions<
 
 	private callTimesSync() {
 		if (!this.lastCallTimes) {
-			this.lastCallTimes = loopCalls(this.fn as any, this.ctx)
+			this.lastCallTimes = loopCalls(this.fn as never, this.ctx)
 			this.lastCallTimes.sort()
 		}
 		this.applyCallTimeHooks()
@@ -167,7 +167,7 @@ export class BenchAssertions<
 
 	private async callTimesAsync() {
 		if (!this.lastCallTimes) {
-			this.lastCallTimes = await loopAsyncCalls(this.fn as any, this.ctx)
+			this.lastCallTimes = await loopAsyncCalls(this.fn as never, this.ctx)
 			this.lastCallTimes.sort()
 		}
 		this.applyCallTimeHooks()
@@ -181,7 +181,7 @@ export class BenchAssertions<
 		:	Measure<TimeUnit> | undefined,
 		callTimes: number[]
 	) {
-		if (name === "mark") return this.markAssertion(baseline as any, callTimes)
+		if (name === "mark") return this.markAssertion(baseline as never, callTimes)
 
 		const ms: number = stats[name as StatName](callTimes)
 		const comparison = createTimeComparison(ms, baseline as Measure<TimeUnit>)
@@ -204,7 +204,7 @@ export class BenchAssertions<
 			baseline ?
 				Object.entries(baseline)
 				// If nothing was passed, gather all available baselines by setting their values to undefined.
-			:	Object.entries(stats).map(([kind]) => [kind, undefined])) as any
+			:	Object.entries(stats).map(([kind]) => [kind, undefined])) as never
 		const markResults = Object.fromEntries(
 			markEntries.map(([kind, kindBaseline]) => {
 				console.group(kind)
@@ -224,7 +224,7 @@ export class BenchAssertions<
 	}
 
 	private getNextAssertions(): NextAssertions {
-		return createBenchTypeAssertion(this.ctx) as any as NextAssertions
+		return createBenchTypeAssertion(this.ctx) as never as NextAssertions
 	}
 
 	private createStatMethod<Name extends TimeAssertionName>(
@@ -268,21 +268,24 @@ export class BenchAssertions<
 		const assertions = this.createStatMethod(
 			"median",
 			baseline
-		) as any as ReturnedAssertions
+		) as never as ReturnedAssertions
 		return assertions
 	}
 
 	mean(baseline?: Measure<TimeUnit>): ReturnedAssertions {
 		this.ctx.lastSnapCallPosition = caller()
-		return this.createStatMethod("mean", baseline) as any as ReturnedAssertions
+		return this.createStatMethod(
+			"mean",
+			baseline
+		) as never as ReturnedAssertions
 	}
 
 	mark(baseline?: MarkMeasure): ReturnedAssertions {
 		this.ctx.lastSnapCallPosition = caller()
 		return this.createStatMethod(
 			"mark",
-			baseline as any
-		) as any as ReturnedAssertions
+			baseline as never
+		) as never as ReturnedAssertions
 	}
 }
 

ark/attest/cache/ts.ts

@@ -9,6 +9,7 @@ import { getConfig } from "../config.js"
 export class TsServer {
 	rootFiles!: string[]
 	virtualEnv!: tsvfs.VirtualTypeScriptEnvironment
+	program!: ts.Program
 
 	private static _instance: TsServer | null = null
 	static get instance(): TsServer {
@@ -40,6 +41,8 @@ export class TsServer {
 			this.tsConfigInfo.parsed.options
 		)
 
+		this.program = this.virtualEnv.languageService.getProgram()!
+
 		TsServer._instance = this
 	}
 

ark/attest/cache/utils.ts

@@ -1,5 +1,4 @@
 import { filePath } from "@ark/fs"
-import { throwInternalError } from "@ark/util"
 import * as tsvfs from "@typescript/vfs"
 import ts from "typescript"
 import { getConfig } from "../config.js"
@@ -145,15 +144,25 @@ export const createOrUpdateFile = (
 	return env.getSourceFile(fileName)
 }
 
+declare module "typescript" {
+	interface SourceFile {
+		imports: ts.StringLiteral[]
+	}
+
+	interface Program {
+		getResolvedModuleFromModuleSpecifier(
+			moduleSpecifier: ts.StringLiteralLike,
+			sourceFile?: ts.SourceFile
+		): ts.ResolvedModuleWithFailedLookupLocations
+	}
+}
+
 const getInstantiationsWithFile = (fileText: string, fileName: string) => {
 	const env = getIsolatedEnv()
 	const file = createOrUpdateFile(env, fileName, fileText)
 	const program = getProgram(env)
 	program.emit(file)
 	const count = program.getInstantiationCount()
-	if (count === undefined)
-		throwInternalError(`Unable to gather instantiation count for ${fileText}`)
-
 	return count
 }
 

ark/attest/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ark/attest",
-	"version": "0.9.3",
+	"version": "0.9.4",
 	"author": {
 		"name": "David Blass",
 		"email": "david@arktype.io",

ark/type/__tests__/cyclic.bench.ts

@@ -1,19 +1,21 @@
 import { bench } from "@ark/attest"
-import { scope } from "arktype"
+import { scope, type } from "arktype"
 import { cyclic10, cyclic100, cyclic500 } from "./generated/cyclic.js"
 
+type("never")
+
 bench("cyclic 10 intersection", () => {
 	const s = scope(cyclic10).type("user&user2").infer
-}).types([30633, "instantiations"])
+}).types([28410, "instantiations"])
 
 bench("cyclic(10)", () => {
 	const types = scope(cyclic10).export()
-}).types([7538, "instantiations"])
+}).types([6558, "instantiations"])
 
 bench("cyclic(100)", () => {
 	const types = scope(cyclic100).export()
-}).types([38109, "instantiations"])
+}).types([38209, "instantiations"])
 
 bench("cyclic(500)", () => {
 	const types = scope(cyclic500).export()
-}).types([169672, "instantiations"])
+}).types([174572, "instantiations"])

ark/type/__tests__/object.bench.ts

@@ -1,22 +1,24 @@
 import { bench } from "@ark/attest"
 import { type } from "arktype"
 
+type("never")
+
 bench("dictionary", () => {
 	const dict = type({
 		a: "string[]",
 		b: "number[]",
 		c: { nested: "boolean[]" }
 	})
-}).types([5212, "instantiations"])
+}).types([3175, "instantiations"])
 
 bench("dictionary with optional keys", () => {
 	const dict = type({
 		"a?": "string[]",
 		"b?": "number[]",
 		"c?": { "nested?": "boolean[]" }
 	})
-}).types([5051, "instantiations"])
+}).types([3018, "instantiations"])
 
 bench("tuple", () => {
 	const tuple = type(["string[]", "number[]", ["boolean[]"]])
-}).types([11888, "instantiations"])
+}).types([10381, "instantiations"])

ark/type/__tests__/operand.bench.ts

@@ -1,33 +1,35 @@
 import { bench } from "@ark/attest"
 import { type } from "arktype"
 
+type("never")
+
 bench("single-quoted", () => {
 	const _ = type("'nineteen characters'")
-}).types([2697, "instantiations"])
+}).types([610, "instantiations"])
 
 bench("double-quoted", () => {
 	const _ = type('"nineteen characters"')
-}).types([2697, "instantiations"])
+}).types([610, "instantiations"])
 
 bench("regex literal", () => {
 	const _ = type("/nineteen characters/")
-}).types([2741, "instantiations"])
+}).types([654, "instantiations"])
 
 bench("keyword", () => {
 	const _ = type("string")
-}).types([2507, "instantiations"])
+}).types([357, "instantiations"])
 
 bench("number", () => {
 	const _ = type("-98765.4321")
-}).types([2589, "instantiations"])
+}).types([432, "instantiations"])
 
 bench("bigint", () => {
 	const _ = type("-987654321n")
-}).types([2611, "instantiations"])
+}).types([450, "instantiations"])
 
 bench("instantiations", () => {
 	const t = type({ foo: "string" })
-}).types([3341, "instantiations"])
+}).types([1207, "instantiations"])
 
 bench("union", () => {
 	// Union is automatically discriminated using shallow or deep keys
@@ -42,4 +44,4 @@ bench("union", () => {
 		.or({
 			kind: "'pleb'"
 		})
-}).types([7565, "instantiations"])
+}).types([5445, "instantiations"])