authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 17:05:21-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 17:05:21-07:00
logab3a3cd162d90fd9f01ac62efdb9d4ad55c33639
tree96fad38975a7f8775bd473bdc82ce024464702dd
parent742c229957b69a0cc5a1379dff6c66b595854057
signature Signed by PGP key B5690EEEBB952194

perf(server): commands on busy threads stop decoding thousands of finished tool nodes (#19675)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

3 files changed, 147 insertions(+), 4 deletions(-)

apps/server/src/orchestration-v2/Orchestrator.ts+7-1
......@@ -940,6 +940,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
940940
941941 // Command decisions need control records, not historical assistant/tool output.
942942 // Handoff preparation explicitly reads its history after choosing a strategy.
943 // This runs inside the write transaction for every command, so finished leaf
944 // nodes of finished runs stay unread: busy threads hold thousands of them.
943945 const readCommandProjection = (threadId: ThreadId) =>
944946 projectionStore
945947 .getThreadRecords(
......@@ -958,7 +960,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
958960 "checkpointScopes",
959961 "contextTransfers",
960962 ],
961 { turnItemTypes: ["user_message", "error"], messageRoles: ["user"] },
963 {
964 turnItemTypes: ["user_message", "error"],
965 messageRoles: ["user"],
966 omitSettledLeafNodes: true,
967 },
962968 )
963969 .pipe(
964970 Effect.map((records): OrchestrationV2ThreadProjection => ({
apps/server/src/orchestration-v2/ProjectionStore.test.ts+87
......@@ -382,6 +382,89 @@ const threadWorktreePathsReadActiveThreads = Effect.gen(function* () {
382382 ]);
383383});
384384
385const commandReadsSkipSettledLeafNodes = Effect.gen(function* () {
386 const store = yield* ProjectionStore.ProjectionStoreV2;
387 const now = yield* DateTime.now;
388 const threadId = yield* addRolledBackRecoveryCandidate("settled-leaf-nodes");
389 const settledRun = (yield* store.getThreadProjection(threadId)).runs[0]!;
390 const activeRun = {
391 ...settledRun,
392 id: RunId.make("run:settled-leaf-nodes:active"),
393 ordinal: 2,
394 userMessageId: MessageId.make("message:settled-leaf-nodes:active"),
395 rootNodeId: NodeId.make("node:settled-leaf-nodes:active-root"),
396 status: "running" as const,
397 completedAt: null,
398 };
399 yield* store.apply({
400 id: EventId.make("event:settled-leaf-nodes:active-run"),
401 type: "run.created",
402 threadId,
403 runId: activeRun.id,
404 nodeId: activeRun.rootNodeId,
405 driver,
406 providerInstanceId,
407 occurredAt: now,
408 payload: activeRun,
409 });
410 const nodes = [
411 ["settled-root", settledRun, "root_turn", "completed"],
412 ["settled-tool", settledRun, "tool_call", "completed"],
413 ["settled-message", settledRun, "assistant_message", "failed"],
414 ["settled-reasoning", settledRun, "reasoning", "cancelled"],
415 ["settled-subagent", settledRun, "subagent", "completed"],
416 ["settled-run-live-tool", settledRun, "tool_call", "running"],
417 ["active-run-done-tool", activeRun, "tool_call", "completed"],
418 ] as const;
419 for (const [name, run, kind, status] of nodes) {
420 const id = NodeId.make(`node:settled-leaf-nodes:${name}`);
421 yield* store.apply({
422 id: EventId.make(`event:settled-leaf-nodes:${name}`),
423 type: "node.updated",
424 threadId,
425 runId: run.id,
426 nodeId: id,
427 driver,
428 occurredAt: now,
429 payload: {
430 id,
431 threadId,
432 runId: run.id,
433 rootNodeId: kind === "root_turn" ? id : run.rootNodeId!,
434 parentNodeId: kind === "root_turn" ? null : run.rootNodeId,
435 kind,
436 status,
437 countsForRun: kind === "root_turn",
438 providerThreadId: null,
439 providerTurnId: null,
440 nativeItemRef: null,
441 runtimeRequestId: null,
442 checkpointScopeId: null,
443 startedAt: now,
444 completedAt: status === "running" ? null : now,
445 },
446 });
447 }
448 const nodeNames = (records: { readonly nodes: ReadonlyArray<{ readonly id: string }> }) =>
449 records.nodes
450 .map((node) => node.id.replace("node:settled-leaf-nodes:", ""))
451 .filter((name) => nodes.some(([candidate]) => candidate === name))
452 .toSorted();
453
454 assert.deepEqual(
455 nodeNames(yield* store.getThreadRecords(threadId, ["nodes"], { omitSettledLeafNodes: true })),
456 ["active-run-done-tool", "settled-root", "settled-run-live-tool", "settled-subagent"],
457 );
458 assert.deepEqual(
459 nodeNames(yield* store.getThreadRecords(threadId, ["nodes"])),
460 nodes.map(([name]) => name).toSorted(),
461 );
462});
463
464it.effect("memory command reads skip finished leaf nodes of finished runs", () =>
465 commandReadsSkipSettledLeafNodes.pipe(Effect.provide(ProjectionStore.layerMemory)),
466);
467
385468it.effect("memory worktree path lookup reads only active threads", () =>
386469 threadWorktreePathsReadActiveThreads.pipe(Effect.provide(ProjectionStore.layerMemory)),
387470);
......@@ -657,6 +740,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
657740 assert.deepEqual(recovery?.restartCancelledBackgroundWork, work);
658741 }),
659742 );
743 it.effect(
744 "command reads skip finished leaf nodes of finished runs",
745 () => commandReadsSkipSettledLeafNodes,
746 );
660747 it.effect("limits turn-start history to the requested runs, including an empty selection", () =>
661748 Effect.gen(function* () {
662749 const store = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ProjectionStore.ts+53-3
......@@ -304,7 +304,39 @@ export interface ProjectionRecordFilter {
304304 readonly runIds?: ReadonlyArray<RunId>;
305305 readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
306306 readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
307 // Skips finished tool call, assistant message and reasoning nodes of finished
308 // runs. Command decisions only read root, subagent and active-run nodes.
309 readonly omitSettledLeafNodes?: boolean;
307310}
311
312type ProjectionNode = OrchestrationV2ThreadProjection["nodes"][number];
313type ProjectionRun = OrchestrationV2ThreadProjection["runs"][number];
314const settledLeafNodeKinds: ReadonlySet<ProjectionNode["kind"]> = new Set([
315 "tool_call",
316 "assistant_message",
317 "reasoning",
318]);
319const settledNodeStatuses: ReadonlySet<ProjectionNode["status"]> = new Set([
320 "completed",
321 "interrupted",
322 "failed",
323 "cancelled",
324 "rolled_back",
325]);
326const activeRunStatuses: ReadonlySet<ProjectionRun["status"]> = new Set([
327 "preparing",
328 "queued",
329 "starting",
330 "running",
331 "waiting",
332]);
333
334// In-memory twin of the omitSettledLeafNodes SQL clause in readCanonicalProjection.
335const isSettledLeafNode = (node: ProjectionNode, runs: ReadonlyArray<ProjectionRun>) =>
336 settledLeafNodeKinds.has(node.kind) &&
337 settledNodeStatuses.has(node.status) &&
338 node.runId !== null &&
339 !runs.some((run) => run.id === node.runId && activeRunStatuses.has(run.status));
308340export type ProjectionRecordField = Exclude<
309341 keyof OrchestrationV2ThreadProjection,
310342 "thread" | "updatedAt" | "visibleTurnItems"
......@@ -3069,9 +3101,23 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
30693101 : window === undefined
30703102 ? sql<PayloadRow>`
30713103 SELECT payload_json
3072 FROM orchestration_v2_projection_nodes
3073 WHERE thread_id = ${threadId}
3074 ORDER BY COALESCE(started_at, ''), node_id ASC
3104 FROM orchestration_v2_projection_nodes AS node
3105 WHERE node.thread_id = ${threadId}
3106 ${
3107 filter?.omitSettledLeafNodes === true
3108 ? sql`AND NOT (
3109 node.kind IN ('tool_call', 'assistant_message', 'reasoning')
3110 AND node.status IN ('completed', 'interrupted', 'failed', 'cancelled', 'rolled_back')
3111 AND node.run_id IS NOT NULL
3112 AND NOT EXISTS (
3113 SELECT 1 FROM orchestration_v2_projection_runs AS run
3114 WHERE run.run_id = node.run_id
3115 AND run.status IN ('preparing', 'queued', 'starting', 'running', 'waiting')
3116 )
3117 )`
3118 : sql``
3119 }
3120 ORDER BY COALESCE(node.started_at, ''), node.node_id ASC
30753121 `
30763122 : sql<PayloadRow & { readonly position: number }>`
30773123 -- Keep retained IDs outermost so SQLite looks up each node by its
......@@ -6423,6 +6469,10 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
64236469 filter.turnItemStatuses.includes(row.status)) &&
64246470 (filter?.turnItemRunId === undefined || filter.turnItemRunId === row.runId),
64256471 ),
6472 nodes:
6473 filter?.omitSettledLeafNodes === true
6474 ? projection.nodes.filter((node) => !isSettledLeafNode(node, projection.runs))
6475 : projection.nodes,
64266476 };
64276477 return Object.fromEntries([
64286478 ["thread", projection.thread],