1import os from "node:os"
2import process from "node:process"
3import { analyzePairs } from "./stats.mts"
4import {
5 aggregateMeasurements,
6 calibrateFixture,
7 configPath,
8 coverageSummary,
9 currentGitState,
10 formatNs,
11 hashFile,
12 libraryVersions,
13 loadRegistry,
14 makeRunId,
15 measureFixture,
16 parseArgs,
17 printTable,
18 relativeToRepo,
19 reportPath,
20 resolveDefaults,
21 runPath,
22 scenarioBatchSize,
23 selectFixtures,
24 statsPath,
25 utilsPath,
26 workerPath,
27 writeJson
28} from "./utils.mts"
29
30const usage = `Usage: pnpm runtimeperf [suite[/fixture]|scenario] [options]
31
32Options:
33 --rounds <n>
34 --time <ms>
35 --warmup-time <ms>
36 --tier <0-3>
37 --family <name>
38 --implementation <effect|effect-jit|effect-aot|fast-check-v4|valibot|zod4|zod4-jitless|zod4-validate|zod4-compiled>
39`
40
41const rotate = (items, offset) => items.map((_, index) => items[(index + offset) % items.length])
42
43const main = () => {
44 const options = parseArgs(process.argv.slice(2))
45 if (options.help) {
46 process.stdout.write(usage)
47 return
48 }
49 const { config, fixtures } = loadRegistry()
50 const selected = selectFixtures(fixtures, options)
51 const defaults = resolveDefaults(config, options)
52 const runId = makeRunId()
53 const groups = Map.groupBy(selected, (fixture) => fixture.scenario)
54 const results = []
55 const executionOrder = []
56
57 for (const [scenario, group] of groups) {
58 const calibrations = new Map(group.map((fixture) => [fixture, calibrateFixture(fixture, defaults)]))
59 const batchSize = scenarioBatchSize(group, calibrations)
60 const byTarget = new Map(group.map((fixture) => [fixture.target, []]))
61
62 for (let round = 0; round < defaults.rounds; round++) {
63 for (const fixture of rotate(group, round % group.length)) {
64 const measurement = measureFixture(fixture, defaults, batchSize)
65 byTarget.get(fixture.target).push(measurement)
66 executionOrder.push({ scenario, round: round + 1, target: fixture.target })
67 }
68 }
69
70 for (const fixture of group) {
71 const calibration = calibrations.get(fixture)
72 const measurements = byTarget.get(fixture.target)
73 results.push({
74 fixture,
75 batchSize,
76 calibration,
77 measurements,
78 aggregate: aggregateMeasurements(measurements)
79 })
80 }
81 }
82
83 const crossLibrary = []
84 for (const [scenario, group] of Map.groupBy(results, (result) => result.fixture.scenario)) {
85 const effect = group.find((result) => result.fixture.implementation === "effect")
86 if (!effect) continue
87 for (const candidate of group) {
88 if (candidate.fixture.implementation === "effect") continue
89 crossLibrary.push({
90 scenario,
91 implementation: candidate.fixture.implementation,
92 comparison: analyzePairs(
93 effect.measurements.map((item) => item.nsPerOp),
94 candidate.measurements.map((item) => item.nsPerOp),
95 {
96 iterations: defaults.bootstrapIterations,
97 seed: defaults.bootstrapSeed
98 }
99 )
100 })
101 }
102 }
103
104 const report = {
105 schemaVersion: 1,
106 kind: "single",
107 runId,
108 target: options.target ?? null,
109 filters: {
110 tier: options.tier ?? null,
111 family: options.family ?? null,
112 implementation: options.implementation ?? null
113 },
114 config: defaults,
115 environment: {
116 node: process.version,
117 v8: process.versions.v8,
118 platform: process.platform,
119 arch: process.arch,
120 cpu: os.cpus()[0]?.model ?? "unknown"
121 },
122 libraries: libraryVersions(),
123 artifactMode: "repository",
124 git: currentGitState(),
125 coverage: coverageSummary(selected),
126 hashes: {
127 config: hashFile(configPath),
128 run: hashFile(runPath),
129 stats: hashFile(statsPath),
130 utils: hashFile(utilsPath),
131 worker: hashFile(workerPath),
132 fixtures: Object.fromEntries(
133 [...new Set(selected.map((fixture) => fixture.fixturePath))]
134 .map((path) => [relativeToRepo(path), hashFile(path)])
135 )
136 },
137 executionOrder,
138 results,
139 crossLibrary
140 }
141 const path = reportPath(runId, options.target, "single")
142 writeJson(path, report)
143 const comparisons = new Map(crossLibrary.map((item) => [`${item.scenario}/${item.implementation}`, item]))
144 printTable(
145 ["scenario", "family", "implementation", "ns/op", "mad", "vs Effect"],
146 results.map((result) => {
147 const comparison = comparisons.get(`${result.fixture.scenario}/${result.fixture.implementation}`)
148 return [
149 result.fixture.scenario,
150 result.fixture.family,
151 result.fixture.implementation,
152 formatNs(result.aggregate.median),
153 formatNs(result.aggregate.mad),
154 comparison ? `${comparison.comparison.ratio.toFixed(3)}x` : "-"
155 ]
156 })
157 )
158 process.stdout.write(`\nReport: ${relativeToRepo(path)}\n`)
159}
160
161try {
162 main()
163} catch (error) {
164 process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : error}\n`)
165 process.exitCode = 1
166}