authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-30 21:44:49-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-30 21:44:49-07:00
log6f8e2534f2d935e6befa7353a003c4b59fe66f6f
treec3fb203ada9f1c487ec24df02f1a5a903a42ec86
parent67b175a4c916ac0d5dbac809da22271db3f542d7
signature Signed by PGP key B5690EEEBB952194

perf(ci): run PR checks side by side and balance server shards (#14025)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

7 files changed, 340 insertions(+), 21 deletions(-)

.github/workflows/ci.yml+157-21
...@@ -14,8 +14,11 @@ concurrency:...@@ -14,8 +14,11 @@ concurrency:
14 cancel-in-progress: ${{ github.event_name == 'pull_request' }}14 cancel-in-progress: ${{ github.event_name == 'pull_request' }}
1515
16jobs:16jobs:
17 check:17 # Lint, typecheck, and the desktop build each get a runner so they run side by
18 name: Check18 # side instead of one after another. The `Check` job at the end of this file
19 # passes only when every other job does.
20 lint:
21 name: Lint
19 runs-on: blacksmith-8vcpu-ubuntu-240422 runs-on: blacksmith-8vcpu-ubuntu-2404
20 timeout-minutes: 1023 timeout-minutes: 10
21 steps:24 steps:
...@@ -52,15 +55,73 @@ jobs:...@@ -52,15 +55,73 @@ jobs:
52 - name: Check55 - name: Check
53 run: vp check56 run: vp check
5457
58 typecheck:
59 name: Typecheck
60 runs-on: blacksmith-8vcpu-ubuntu-2404
61 timeout-minutes: 10
62 steps:
63 - name: Checkout
64 uses: actions/checkout@v6
65 with:
66 sparse-checkout: |
67 /*
68 !/.repos/
69 sparse-checkout-cone-mode: false
70
71 - name: Setup Vite+
72 uses: voidzero-dev/setup-vp@v1
73 with:
74 node-version-file: package.json
75 cache: true
76 run-install: true
77
78 - name: Ensure Electron runtime is installed
79 run: vp run --filter @t3tools/desktop ensure:electron
80
55 - name: Typecheck81 - name: Typecheck
56 run: vpr typecheck82 run: vpr typecheck
5783
84 build:
85 name: Build
86 runs-on: blacksmith-8vcpu-ubuntu-2404
87 timeout-minutes: 10
88 steps:
89 - name: Checkout
90 uses: actions/checkout@v6
91 with:
92 sparse-checkout: |
93 /*
94 !/.repos/
95 sparse-checkout-cone-mode: false
96
58 - uses: ./.github/actions/setup-apt-mirrors97 - uses: ./.github/actions/setup-apt-mirrors
5998
60 - name: Install browser secret helper build libraries99 # Runs in the background while Vite+ installs; the step before the build waits for it.
100 - name: Start installing browser secret helper build libraries
61 run: |101 run: |
62 sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources102 sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources
63 sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config103 (
104 set +e
105 sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config
106 echo "$?" > "$RUNNER_TEMP/apt-status"
107 ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null &
108
109 - name: Setup Vite+
110 uses: voidzero-dev/setup-vp@v1
111 with:
112 node-version-file: package.json
113 cache: true
114 run-install: true
115
116 - name: Ensure Electron runtime is installed
117 run: vp run --filter @t3tools/desktop ensure:electron
118
119 - name: Finish installing browser secret helper build libraries
120 timeout-minutes: 5
121 run: |
122 until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done
123 cat "$RUNNER_TEMP/apt.log"
124 exit "$(cat "$RUNNER_TEMP/apt-status")"
64125
65 - name: Build desktop pipeline126 - name: Build desktop pipeline
66 run: vp run build:desktop127 run: vp run build:desktop
...@@ -68,11 +129,12 @@ jobs:...@@ -68,11 +129,12 @@ jobs:
68 - name: Verify preload bundle output129 - name: Verify preload bundle output
69 run: node apps/desktop/scripts/verify-preload-bundle.mjs130 run: node apps/desktop/scripts/verify-preload-bundle.mjs
70131
71 # Everything except `t3` (apps/server). `--parallel` drops the package132 # Everything except `t3` (apps/server) and the web app, which has its own job
72 # dependency ordering that `vp run` applies by default: these `test` tasks133 # below. `--parallel` drops the package dependency ordering that `vp run`
73 # declare no `dependsOn` and resolve workspace deps from source, so ordering134 # applies by default: these `test` tasks declare no `dependsOn` and resolve
74 # only bought us idle runners between dependency layers. The concurrency135 # workspace deps from source, so ordering only bought us idle runners between
75 # limit stays at the default 4 so peak load per runner is unchanged.136 # dependency layers. The concurrency limit stays at the default 4 so peak load
137 # per runner is unchanged.
76 test:138 test:
77 name: Test139 name: Test
78 runs-on: blacksmith-8vcpu-ubuntu-2404140 runs-on: blacksmith-8vcpu-ubuntu-2404
...@@ -86,6 +148,19 @@ jobs:...@@ -86,6 +148,19 @@ jobs:
86 !/.repos/148 !/.repos/
87 sparse-checkout-cone-mode: false149 sparse-checkout-cone-mode: false
88150
151 - uses: ./.github/actions/setup-apt-mirrors
152
153 # The desktop tests compile the browser secret helper. The install runs in
154 # the background while Vite+ installs; the step before the tests waits for it.
155 - name: Start installing browser secret helper build libraries
156 run: |
157 sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources
158 (
159 set +e
160 sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config
161 echo "$?" > "$RUNNER_TEMP/apt-status"
162 ) > "$RUNNER_TEMP/apt.log" 2>&1 < /dev/null &
163
89 - name: Setup Vite+164 - name: Setup Vite+
90 uses: voidzero-dev/setup-vp@v1165 uses: voidzero-dev/setup-vp@v1
91 with:166 with:
...@@ -96,34 +171,61 @@ jobs:...@@ -96,34 +171,61 @@ jobs:
96 - name: Ensure Electron runtime is installed171 - name: Ensure Electron runtime is installed
97 run: vp run --filter @t3tools/desktop ensure:electron172 run: vp run --filter @t3tools/desktop ensure:electron
98173
99 - uses: ./.github/actions/setup-apt-mirrors
100
101 - name: Install browser secret helper build libraries
102 run: |
103 sudo sed -i 's|http://|https://|g' /etc/apt/blacksmith-ubuntu-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources
104 sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config
105
106 - name: Test preview artifact validation174 - name: Test preview artifact validation
107 run: python3 -B .github/scripts/stage-preview-bundle.test.py175 run: python3 -B .github/scripts/stage-preview-bundle.test.py
108176
109 - name: Test nightly release checks177 - name: Test nightly release checks
110 run: node --test .github/scripts/check-nightly-release.test.cjs178 run: node --test .github/scripts/check-nightly-release.test.cjs
111179
180 - name: Finish installing browser secret helper build libraries
181 timeout-minutes: 5
182 run: |
183 until test -s "$RUNNER_TEMP/apt-status"; do sleep 1; done
184 cat "$RUNNER_TEMP/apt.log"
185 exit "$(cat "$RUNNER_TEMP/apt-status")"
186
187 - name: Test
188 run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web' test
189
190 # The web suite is the largest outside the server. In the shared Test job it
191 # started last and ran alone for about 40s after everything else finished.
192 test_web:
193 name: Test Web
194 runs-on: blacksmith-8vcpu-ubuntu-2404
195 timeout-minutes: 10
196 steps:
197 - name: Checkout
198 uses: actions/checkout@v6
199 with:
200 sparse-checkout: |
201 /*
202 !/.repos/
203 sparse-checkout-cone-mode: false
204
205 - name: Setup Vite+
206 uses: voidzero-dev/setup-vp@v1
207 with:
208 node-version-file: package.json
209 cache: true
210 run-install: true
211
112 - name: Test212 - name: Test
113 run: vp run --parallel --concurrency-limit 4 --filter '!t3' --filter '!@t3tools/monorepo' test213 run: vp run --filter @t3tools/web test
114214
115 # apps/server sets `fileParallelism: false`, so its 239 files run strictly215 # apps/server sets `fileParallelism: false`, so its test files run strictly
116 # one at a time. Sharding spreads them over separate runners instead of216 # one at a time. Sharding spreads them over separate runners instead of
117 # separate workers, so no two server test files ever share a machine and the217 # separate workers, so no two server test files ever share a machine and the
118 # isolation that flag buys is preserved exactly.218 # isolation that flag buys is preserved exactly. One file at a time keeps
219 # about 1.5 cores busy, so 4 vCPU runners lose nothing against 8. The shards
220 # split files by recorded duration (apps/server/src/testUtils/weightedShardSequencer.ts).
119 test_server:221 test_server:
120 name: Test Server ${{ matrix.shard }}222 name: Test Server ${{ matrix.shard }}
121 runs-on: blacksmith-8vcpu-ubuntu-2404223 runs-on: blacksmith-4vcpu-ubuntu-2404
122 timeout-minutes: 10224 timeout-minutes: 10
123 strategy:225 strategy:
124 fail-fast: false226 fail-fast: false
125 matrix:227 matrix:
126 shard: [1, 2, 3]228 shard: [1, 2, 3, 4, 5, 6]
127 steps:229 steps:
128 - name: Checkout230 - name: Checkout
129 uses: actions/checkout@v6231 uses: actions/checkout@v6
...@@ -347,3 +449,37 @@ jobs:...@@ -347,3 +449,37 @@ jobs:
347449
348 - name: Exercise release-only workflow steps450 - name: Exercise release-only workflow steps
349 run: node scripts/release-smoke.ts451 run: node scripts/release-smoke.ts
452
453 # Branch protection requires this job by its name, `Check`. It fails when any
454 # other job fails, is cancelled, or is skipped without a reason, so splitting
455 # or resharding jobs needs no ruleset change. Add every new job to `needs`.
456 # `always()` rather than `!cancelled()`: a skipped job reports success to
457 # branch protection, so a cancelled run must still produce a failing check.
458 check:
459 name: Check
460 if: ${{ always() }}
461 needs:
462 [
463 lint,
464 typecheck,
465 build,
466 test,
467 test_web,
468 test_server,
469 rust,
470 mobile_native_changes,
471 mobile_native_static_analysis,
472 release_smoke,
473 ]
474 runs-on: blacksmith-2vcpu-ubuntu-2404
475 timeout-minutes: 5
476 steps:
477 - name: Require every job to pass
478 env:
479 RESULTS: ${{ toJSON(needs) }}
480 run: |
481 echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"'
482 # Only the macOS lint skips on purpose, when no native mobile code changed.
483 echo "$RESULTS" | jq -e 'all(to_entries[];
484 .value.result == "success"
485 or (.key == "mobile_native_static_analysis" and .value.result == "skipped"))' > /dev/null
apps/server/scripts/update-test-shard-weights.ts created+54
...@@ -0,0 +1,54 @@
1// @effect-diagnostics nodeBuiltinImport:off - a one-off maintenance script with no
2// Effect runtime; it only spawns vitest and reads and writes two files.
3// Run with: node apps/server/scripts/update-test-shard-weights.ts
4// Runs the whole server suite once and records how long each test file takes,
5// so CI can split the suite into shards of equal duration. See
6// src/testUtils/weightedShardSequencer.ts. Rerun it when a shard in CI runs
7// much longer than the others.
8import * as NodeChildProcess from "node:child_process";
9import * as NodeFS from "node:fs";
10import * as NodeOS from "node:os";
11import * as NodePath from "node:path";
12import * as NodeURL from "node:url";
13
14// Files faster than this are all treated alike, which keeps the weights file short.
15const MIN_RECORDED_SECONDS = 0.5;
16
17interface VitestJsonReport {
18 readonly testResults: ReadonlyArray<{
19 readonly name: string;
20 readonly startTime: number;
21 readonly endTime: number;
22 }>;
23}
24
25const serverDir = NodeURL.fileURLToPath(new URL("..", import.meta.url));
26const weightsPath = NodePath.join(serverDir, "src/testUtils/shardWeights.json");
27const reportDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-server-test-report-"));
28const reportPath = NodePath.join(reportDir, "report.json");
29
30// `node --run` puts the package's own `vp` on PATH, on every platform. Failing
31// tests still report their duration, so the exit code is not checked.
32NodeChildProcess.spawnSync(
33 process.execPath,
34 ["--run", "test", "--", "--reporter=json", `--outputFile=${reportPath}`],
35 { cwd: serverDir, stdio: "inherit" },
36);
37
38const report: VitestJsonReport = JSON.parse(NodeFS.readFileSync(reportPath, "utf8"));
39NodeFS.rmSync(reportDir, { recursive: true, force: true });
40
41const weights = Object.fromEntries(
42 report.testResults
43 .map(
44 (file) =>
45 [
46 NodePath.relative(serverDir, file.name).replaceAll("\\", "/"),
47 Math.round((file.endTime - file.startTime) / 100) / 10,
48 ] as const,
49 )
50 .filter(([, seconds]) => seconds >= MIN_RECORDED_SECONDS)
51 .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
52);
53
54NodeFS.writeFileSync(weightsPath, `${JSON.stringify(weights, null, 2)}\n`);
apps/server/src/testUtils/shardWeights.json created+40
...@@ -0,0 +1,40 @@
1{
2 "integration/orchestrationEngine.integration.test.ts": 1.9,
3 "integration/providerService.integration.test.ts": 0.5,
4 "src/bin.test.ts": 0.8,
5 "src/checkpointing/CheckpointStore.test.ts": 0.6,
6 "src/device/sshDeviceScript.test.ts": 2.9,
7 "src/git/GitManager.test.ts": 18.7,
8 "src/orchestration/Layers/CheckpointReactor.test.ts": 7.1,
9 "src/orchestration/Layers/OrchestrationEngine.test.ts": 1.4,
10 "src/orchestration/Layers/ProjectionPipeline.test.ts": 0.5,
11 "src/orchestration/Layers/ProviderCommandReactor.test.ts": 3.8,
12 "src/orchestration/Layers/ProviderRuntimeIngestion.test.ts": 2.4,
13 "src/persistence/Layers/OrchestrationEventStore.test.ts": 2.2,
14 "src/process/externalLauncher.test.ts": 2.2,
15 "src/project/AgentSessionScanner.test.ts": 4.3,
16 "src/provider/CodexChatGptAuth.test.ts": 5.9,
17 "src/provider/Drivers/AntigravityDriver.test.ts": 2,
18 "src/provider/Layers/AntigravityAdapter.test.ts": 0.5,
19 "src/provider/Layers/ClaudeAdapter.test.ts": 0.7,
20 "src/provider/Layers/CodexCollabRuntime.integration.test.ts": 3.4,
21 "src/provider/Layers/CursorAdapter.test.ts": 8,
22 "src/provider/Layers/CursorProvider.test.ts": 1.4,
23 "src/provider/Layers/GrokAdapter.test.ts": 9,
24 "src/provider/Layers/GrokProvider.test.ts": 0.7,
25 "src/provider/Layers/ProviderService.test.ts": 0.5,
26 "src/provider/acp/AcpJsonRpcConnection.test.ts": 6.2,
27 "src/provider/acp/XAiAcpExtension.test.ts": 0.6,
28 "src/provider/opencodeRuntime.environment.test.ts": 1.1,
29 "src/pullRequest/PullRequestService.test.ts": 3.1,
30 "src/server.test.ts": 10.2,
31 "src/serverSettings.test.ts": 0.5,
32 "src/serviceLauncher.test.ts": 6.2,
33 "src/terminal/Manager.test.ts": 1.6,
34 "src/textGeneration/CursorTextGeneration.test.ts": 0.8,
35 "src/textGeneration/GrokTextGeneration.test.ts": 1.2,
36 "src/usage/usageTranscriptStreaming.test.ts": 0.8,
37 "src/vcs/GitVcsDriver.test.ts": 3.7,
38 "src/vcs/GitVcsDriverCore.test.ts": 6.5,
39 "src/workspace/WorkspaceEntries.test.ts": 1.9
40}
apps/server/src/testUtils/weightedShardSequencer.test.ts created+42
...@@ -0,0 +1,42 @@
1import { describe, expect, it } from "vite-plus/test";
2import type { TestSpecification, Vitest } from "vite-plus/test/node";
3
4import shardWeights from "./shardWeights.json" with { type: "json" };
5import { WeightedShardSequencer } from "./weightedShardSequencer.ts";
6
7const root = "/repo/apps/server";
8const recordedSeconds: Readonly<Record<string, number>> = shardWeights;
9const recorded = Object.keys(recordedSeconds);
10const files = [...recorded, ...Array.from({ length: 300 }, (_, i) => `src/fast${i}.test.ts`)];
11const specs = files.map((file) => ({ moduleId: `${root}/${file}` }) as TestSpecification);
12
13const shardModuleIds = (count: number) =>
14 Promise.all(
15 Array.from({ length: count }, async (_, i) => {
16 const ctx = { config: { root, shard: { index: i + 1, count } } } as unknown as Vitest;
17 const shard = await new WeightedShardSequencer(ctx).shard(specs);
18 return shard.map((spec) => spec.moduleId);
19 }),
20 );
21
22describe("WeightedShardSequencer", () => {
23 it("runs every file in exactly one shard", async () => {
24 const shards = await shardModuleIds(6);
25
26 expect(shards.flat().toSorted()).toEqual(specs.map((spec) => spec.moduleId).toSorted());
27 });
28
29 it("puts each of the slowest files in a different shard", async () => {
30 const slowest = new Set(
31 recorded
32 .toSorted((a, b) => (recordedSeconds[b] ?? 0) - (recordedSeconds[a] ?? 0))
33 .slice(0, 6)
34 .map((file) => `${root}/${file}`),
35 );
36 const shards = await shardModuleIds(6);
37
38 for (const shard of shards) {
39 expect(shard.filter((moduleId) => slowest.has(moduleId))).toHaveLength(1);
40 }
41 });
42});
apps/server/src/testUtils/weightedShardSequencer.ts created+43
...@@ -0,0 +1,43 @@
1// @effect-diagnostics nodeBuiltinImport:off - vitest loads this sequencer from the config,
2// outside any Effect runtime, and only needs path.relative.
3import * as NodePath from "node:path";
4
5import { BaseSequencer, type TestSpecification } from "vite-plus/test/node";
6
7import shardWeights from "./shardWeights.json" with { type: "json" };
8
9// What a file costs beyond its recorded test time: CI spends about 0.2s per
10// server test file on imports and setup, which the recorded times leave out.
11const FILE_OVERHEAD_SECONDS = 0.25;
12
13const recordedSeconds: Readonly<Record<string, number>> = shardWeights;
14
15/**
16 * Splits server test files across `--shard` runs by recorded duration. Vitest's
17 * default split hashes file paths into equal-count shards, which can put the
18 * slowest files on one runner. Here the longest file goes to the lightest shard
19 * first. Every shard derives the same split from the same file list, so each
20 * file still runs in exactly one shard. Files without a recorded time count as
21 * fast files. Record fresh times with `node scripts/update-test-shard-weights.ts`.
22 */
23export class WeightedShardSequencer extends BaseSequencer {
24 override async shard(files: TestSpecification[]) {
25 const { index, count } = this.ctx.config.shard ?? { index: 1, count: 1 };
26 const weighted = files
27 .map((spec) => {
28 const key = NodePath.relative(this.ctx.config.root, spec.moduleId).replaceAll("\\", "/");
29 return { spec, key, seconds: (recordedSeconds[key] ?? 0) + FILE_OVERHEAD_SECONDS };
30 })
31 .toSorted((a, b) => b.seconds - a.seconds || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
32
33 const loads = Array.from({ length: count }, () => 0);
34 const picked: TestSpecification[] = [];
35 for (const file of weighted) {
36 const least = Math.min(...loads);
37 const lightest = loads.indexOf(least);
38 loads[lightest] = least + file.seconds;
39 if (lightest === index - 1) picked.push(file.spec);
40 }
41 return picked;
42 }
43}
apps/server/vite.config.ts+3
...@@ -4,6 +4,7 @@ import { defineConfig, mergeConfig } from "vite-plus";...@@ -4,6 +4,7 @@ import { defineConfig, mergeConfig } from "vite-plus";
4import baseConfig from "../../vite.config.ts";4import baseConfig from "../../vite.config.ts";
5import { loadRepoEnv } from "../../scripts/lib/public-config.ts";5import { loadRepoEnv } from "../../scripts/lib/public-config.ts";
6import packageJson from "./package.json" with { type: "json" };6import packageJson from "./package.json" with { type: "json" };
7import { WeightedShardSequencer } from "./src/testUtils/weightedShardSequencer.ts";
78
8// The bundle used to inline only workspace packages, leaving every third-party9// The bundle used to inline only workspace packages, leaving every third-party
9// runtime dep external. External deps must exist on the real filesystem (the WSL10// runtime dep external. External deps must exist on the real filesystem (the WSL
...@@ -133,6 +134,8 @@ export default mergeConfig(...@@ -133,6 +134,8 @@ export default mergeConfig(
133 // The server suite exercises sqlite, git, temp worktrees, and orchestration134 // The server suite exercises sqlite, git, temp worktrees, and orchestration
134 // runtimes heavily. Running files in parallel introduces load-sensitive flakes.135 // runtimes heavily. Running files in parallel introduces load-sensitive flakes.
135 fileParallelism: false,136 fileParallelism: false,
137 // CI runs the suite as `--shard` runs of equal recorded duration.
138 sequence: { sequencer: WeightedShardSequencer },
136 // Appended to the root setup, which mergeConfig concatenates.139 // Appended to the root setup, which mergeConfig concatenates.
137 setupFiles: ["./src/testUtils/gitConfig.setup.ts"],140 setupFiles: ["./src/testUtils/gitConfig.setup.ts"],
138 // Server integration tests exercise sqlite, git, and orchestration together.141 // Server integration tests exercise sqlite, git, and orchestration together.
knip.jsonc+1
...@@ -25,6 +25,7 @@...@@ -25,6 +25,7 @@
25 "scripts/cli.ts",25 "scripts/cli.ts",
26 "scripts/evaluate-thread-titles.ts",26 "scripts/evaluate-thread-titles.ts",
27 "scripts/measure-pr-preview.ts",27 "scripts/measure-pr-preview.ts",
28 "scripts/update-test-shard-weights.ts",
28 "src/provider/testFixtures/*.mjs",29 "src/provider/testFixtures/*.mjs",
29 ],30 ],
30 // Keep the transitive Effect runtime pinned for standalone npm installs.31 // Keep the transitive Effect runtime pinned for standalone npm installs.