1import assert from "node:assert/strict"
2import { describe, it } from "node:test"
3import { aggregate, analyzePairs, bootstrapMedianLogRatio, median, percentile } from "../stats.mts"
4
5describe("runtimeperf stats", () => {
6 it("calculates medians and percentiles", () => {
7 assert.equal(median([3, 1, 2]), 2)
8 assert.equal(median([4, 1, 3, 2]), 2.5)
9 assert.equal(percentile([1, 2, 3, 4, 5], 0.5), 3)
10 })
11
12 it("aggregates raw observations", () => {
13 assert.deepEqual(aggregate([10, 12, 14]), {
14 median: 12,
15 min: 10,
16 max: 14,
17 mad: 2
18 })
19 })
20
21 it("classifies improvements, regressions and parity", () => {
22 const options = { iterations: 1_000, seed: 1 }
23 assert.equal(analyzePairs([100, 101, 99], [80, 81, 79], options).status, "improvement")
24 assert.equal(analyzePairs([100, 101, 99], [120, 121, 119], options).status, "regression")
25 assert.equal(analyzePairs([100, 101, 99], [100, 101, 99], options).status, "inconclusive")
26 })
27
28 it("keeps exact percentage thresholds inconclusive", () => {
29 const options = { iterations: 100, seed: 1 }
30 assert.equal(analyzePairs([100, 100, 100], [98, 98, 98], options).status, "inconclusive")
31 assert.equal(analyzePairs([100, 100, 100], [105, 105, 105], options).status, "inconclusive")
32 })
33
34 it("is deterministic for a fixed seed", () => {
35 const first = bootstrapMedianLogRatio([0.8, 0.9, 1], { iterations: 1_000, seed: 42 })
36 const second = bootstrapMedianLogRatio([0.8, 0.9, 1], { iterations: 1_000, seed: 42 })
37 assert.deepEqual(first, second)
38 })
39
40 it("rejects invalid input", () => {
41 assert.throws(() => median([]), /non-empty/)
42 assert.throws(() => percentile([1], 2), /between 0 and 1/)
43 assert.throws(() => bootstrapMedianLogRatio([0]), /finite positive/)
44 assert.throws(() => analyzePairs([1], [1, 2]), /same number/)
45 })
46})