1import { readFileSync } from "node:fs"
2import { resolve } from "node:path"
3
4type Sample = {
5 readonly aotBuild?: {
6 readonly generateCpu: { readonly cpuMicros: number }
7 readonly maxRssBytes: number
8 readonly schemaCpu: { readonly cpuMicros: number }
9 readonly sourceBytes: number
10 }
11 readonly case: string
12 readonly count: number
13 readonly cpu: {
14 readonly aotModule?: { readonly cpuMicros: number }
15 readonly firstCall: { readonly cpuMicros: number }
16 readonly prepare: { readonly cpuMicros: number }
17 }
18 readonly implementation: string
19 readonly memory: {
20 readonly compilerPerSchema: Memory
21 readonly module: Memory
22 readonly retainedLibraryPerSchema: Memory
23 }
24 readonly round: number
25}
26
27type Memory = {
28 readonly bytecodeBytes: number
29 readonly codeBytes: number
30 readonly externalSourceBytes: number
31 readonly heapBytes: number
32 readonly maxRssBytes: number
33}
34
35type Report = {
36 readonly environment: Record<string, string>
37 readonly measurement: {
38 readonly cases: ReadonlyArray<string>
39 readonly counts: ReadonlyArray<number>
40 readonly implementations: ReadonlyArray<string>
41 readonly rounds: number
42 }
43 readonly results: ReadonlyArray<Sample>
44}
45
46const path = resolve(process.argv[2] ?? "tmp/schema-compiler-resources.json")
47const report = JSON.parse(readFileSync(path, "utf8")) as Report
48const lowCount = Math.min(...report.measurement.counts)
49const highCount = Math.max(...report.measurement.counts)
50if (lowCount === highCount) throw new Error("the report needs at least two schema counts")
51
52const median = (values: ReadonlyArray<number>) => {
53 const sorted = values.toSorted((a, b) => a - b)
54 const middle = Math.floor(sorted.length / 2)
55 return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]
56}
57
58const samples = (implementation: string, caseName: string) => {
59 const pairs = []
60 for (let round = 0; round < report.measurement.rounds; round++) {
61 const low = report.results.find((sample) =>
62 sample.round === round && sample.implementation === implementation && sample.case === caseName &&
63 sample.count === lowCount
64 )
65 const high = report.results.find((sample) =>
66 sample.round === round && sample.implementation === implementation && sample.case === caseName &&
67 sample.count === highCount
68 )
69 if (low === undefined || high === undefined) throw new Error(`missing samples for ${implementation}/${caseName}`)
70 pairs.push({ high, low })
71 }
72 return pairs
73}
74
75const perSchemaSlope = (
76 implementation: string,
77 caseName: string,
78 value: (sample: Sample) => number
79) => median(samples(implementation, caseName).map(({ high, low }) =>
80 (value(high) * highCount - value(low) * lowCount) / (highCount - lowCount)))
81
82const totalSlope = (
83 implementation: string,
84 caseName: string,
85 value: (sample: Sample) => number
86) => median(samples(implementation, caseName).map(({ high, low }) =>
87 (value(high) - value(low)) / (highCount - lowCount)))
88
89const heap = (implementation: string, caseName: string) =>
90 perSchemaSlope(implementation, caseName, (sample) => sample.memory.retainedLibraryPerSchema.heapBytes) / 1024
91const code = (implementation: string, caseName: string) =>
92 perSchemaSlope(
93 implementation,
94 caseName,
95 (sample) => {
96 const memory = sample.memory.retainedLibraryPerSchema
97 return memory.bytecodeBytes + memory.codeBytes + memory.externalSourceBytes
98 }
99 ) / 1024
100const peakRss = (implementation: string, caseName: string) =>
101 perSchemaSlope(implementation, caseName, (sample) => sample.memory.retainedLibraryPerSchema.maxRssBytes) / 1024
102const startupCpu = (implementation: string, caseName: string) =>
103 totalSlope(
104 implementation,
105 caseName,
106 (sample) =>
107 sample.cpu.prepare.cpuMicros + sample.cpu.firstCall.cpuMicros +
108 (implementation === "effect-aot" ? sample.cpu.aotModule?.cpuMicros ?? 0 : 0)
109 )
110
111const fixedDelta = (
112 left: string,
113 right: string,
114 value: (sample: Sample) => number
115) => {
116 const differences = []
117 for (const caseName of report.measurement.cases) {
118 for (let round = 0; round < report.measurement.rounds; round++) {
119 for (const count of report.measurement.counts) {
120 const find = (implementation: string) => report.results.find((sample) =>
121 sample.implementation === implementation && sample.case === caseName && sample.round === round &&
122 sample.count === count
123 )!
124 differences.push(value(find(left)) - value(find(right)))
125 }
126 }
127 }
128 return median(differences)
129}
130
131const names: Record<string, string> = {
132 "array-decode": "Array of 32 Structs",
133 "default-make": "Construct 32 defaults",
134 "struct-decode": "Struct decode",
135 "struct-invalid": "Struct invalid decode",
136 "struct-is": "Struct guard",
137 "transform-decode": "32 transformations",
138 "union-decode": "Discriminated Union (8)"
139}
140
141const table = (
142 implementations: ReadonlyArray<string>,
143 value: (implementation: string, caseName: string) => number,
144 unit: string,
145 digits: number
146) => {
147 const labels: Record<string, string> = {
148 "effect-aot": "Effect AOT",
149 "effect-interpreted": "Effect interpreted",
150 "effect-jit": "Effect JIT",
151 "valibot": "Valibot",
152 "zod-compiled": "Zod compile",
153 "zod-jitless": "Zod jitless"
154 }
155 const lines = [
156 `| Case | ${implementations.map((implementation) => labels[implementation]).join(" | ")} |`,
157 `|---|${implementations.map(() => "---:").join("|")}|`
158 ]
159 for (const caseName of report.measurement.cases) {
160 lines.push(
161 `| ${names[caseName]} | ${
162 implementations.map((implementation) => `${value(implementation, caseName).toFixed(digits)} ${unit}`).join(" | ")
163 } |`
164 )
165 }
166 return lines.join("\n")
167}
168
169const interpreted = ["effect-interpreted", "valibot", "zod-jitless"]
170const compiled = ["effect-jit", "effect-aot", "zod-compiled"]
171const environment = report.environment
172
173process.stdout.write(`# Schema compiler resource comparison
174
175Incremental costs are median per-schema slopes between ${lowCount} and ${highCount} distinct schemas across ${
176 report.measurement.rounds
177} fresh-process rounds. Lower is better. Fixed module imports and fixture inputs are excluded.
178
179Environment: Node ${environment.node}, V8 ${environment.v8}, ${environment.cpu}, ${environment.platform} ${
180 environment.arch
181}; Valibot ${environment.valibot}, Zod ${environment.zod}.
182
183Importing the JIT compiler retains ${
184 (fixedDelta("effect-jit", "effect-interpreted", (sample) => sample.memory.module.heapBytes) / 1024).toFixed(1)
185} KiB of additional fixed JavaScript heap and ${
186 (fixedDelta("effect-jit", "effect-interpreted", (sample) =>
187 sample.memory.module.codeBytes + sample.memory.module.bytecodeBytes + sample.memory.module.externalSourceBytes) / 1024)
188 .toFixed(1)
189} KiB of V8 code, bytecode and source in this source-tree setup.
190
191## Retained JavaScript heap
192
193### Interpreted
194
195${table(interpreted, heap, "KiB/schema", 2)}
196
197### Compiled
198
199${table(compiled, heap, "KiB/schema", 2)}
200
201## Retained V8 code, bytecode and source
202
203${table(compiled, code, "KiB/schema", 2)}
204
205## Runtime preparation CPU
206
207This includes adapter creation and the first call. AOT also includes importing and installing the generated module. AOT build-time generation is excluded.
208
209### Interpreted
210
211${table(interpreted, startupCpu, "µs/schema", 1)}
212
213### Compiled
214
215${table(compiled, startupCpu, "µs/schema", 1)}
216
217## Peak runtime RSS growth
218
219${table(compiled, peakRss, "KiB/schema", 1)}
220
221## AOT build cost
222
223| Case | CPU | Generated source | Peak RSS growth |
224|---|---:|---:|---:|
225${report.measurement.cases.map((caseName) => {
226 const cpu = totalSlope(
227 "effect-aot",
228 caseName,
229 (sample) => (sample.aotBuild?.schemaCpu.cpuMicros ?? 0) + (sample.aotBuild?.generateCpu.cpuMicros ?? 0)
230 )
231 const source = totalSlope("effect-aot", caseName, (sample) => sample.aotBuild?.sourceBytes ?? 0) / 1024
232 const rss = totalSlope("effect-aot", caseName, (sample) => sample.aotBuild?.maxRssBytes ?? 0) / 1024
233 return `| ${names[caseName]} | ${cpu.toFixed(1)} µs/schema | ${source.toFixed(2)} KiB/schema | ${rss.toFixed(1)} KiB/schema |`
234}).join("\n")}
235`)