authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-05 17:15:34-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 17:15:34-07:00
logfc4ecc835e6d54eb411c333d737f2c4881884338
tree3a30ace2981e1c2d27c8101b0d0922811f4f61e1
parent6302d66f7773ee43d198155d2cfc22d9d66dbbe0
signature Signed by PGP key B5690EEEBB952194

feat(server): log how long PR watches stay quiet before they end (#16262)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

2 files changed, 235 insertions(+), 14 deletions(-)

apps/server/src/orchestration-v2/PullRequestWatchReactor.ts+137-14
......@@ -91,6 +91,44 @@ interface LastRead {
9191
9292const watchKey = ({ thread, watch }: WatchTarget) => `${thread.id} ${watch.startedAt}`;
9393
94/**
95 * Why a watch ended. `stopped` is anything outside this reactor: the agent unwatched, the user
96 * pressed Stop, or the thread settled, archived, or was deleted between passes.
97 */
98type WatchEndReason =
99 | "merged"
100 | "closed"
101 | "unreadable"
102 | "comment-limit"
103 | "settled"
104 | "subagent"
105 | "stopped";
106
107/**
108 * What one watch did while this server ran, logged once when it ends so we can see how long
109 * watches stay quiet. Kept in memory: a watch older than the server process only has partial
110 * numbers, and one that ends while the server is down, or starts and ends between two passes,
111 * is not logged.
112 */
113interface WatchLife {
114 readonly threadId: string;
115 readonly pullRequest: string;
116 readonly startedAt: number;
117 /** The head commit the last successful read saw. */
118 readonly headSha: string | null;
119 /** When a pass last saw the head commit move, or the start. */
120 readonly pushedAt: number;
121 /** Longest time between pushes, not counting the time since the last one. */
122 readonly longestQuietMs: number;
123 readonly wakes: number;
124 readonly reads: number;
125}
126
127const lifeKey = ({ thread, link, watch }: WatchTarget) =>
128 `${thread.id} ${threadPullRequestKeyOf(link)} ${watch.startedAt}`;
129
130const minutes = (ms: number) => Math.max(0, Math.round(ms / 60_000));
131
94132const snapshotFingerprint = ({ link }: WatchTarget) => {
95133 const snapshot = link.snapshot;
96134 return snapshot === null
......@@ -136,7 +174,8 @@ function watchesEqual(left: ThreadPullRequestWatch, right: ThreadPullRequestWatc
136174 * of a project that watch it, and skips one whose sync snapshot has not moved while nothing is
137175 * in flight.
138176 * Settling or archiving a thread ends its watches, and a merged or closed pull request ends
139 * its watch.
177 * its watch. Each ended watch is logged once with why it ended and how long it went without
178 * a push, for debugging watches that live too long.
140179 */
141180export class PullRequestWatchReactor extends Context.Service<
142181 PullRequestWatchReactor,
......@@ -153,6 +192,51 @@ export const make = Effect.gen(function* () {
153192 const projections = yield* ProjectionStore.ProjectionStoreV2;
154193 const pullRequests = yield* PullRequestService.PullRequestService;
155194 const crypto = yield* Crypto.Crypto;
195 const bootedAt = yield* Clock.currentTimeMillis;
196 const lives = new Map<string, WatchLife>();
197
198 const lifeOf = (target: WatchTarget): WatchLife => {
199 const existing = lives.get(lifeKey(target));
200 if (existing !== undefined) return existing;
201 const startedAt = Date.parse(target.watch.startedAt);
202 const life: WatchLife = {
203 threadId: target.thread.id,
204 pullRequest: threadPullRequestKeyOf(target.link),
205 startedAt,
206 headSha: target.watch.headSha,
207 pushedAt: startedAt,
208 longestQuietMs: 0,
209 wakes: 0,
210 reads: 0,
211 };
212 lives.set(lifeKey(target), life);
213 return life;
214 };
215 const woke = (target: WatchTarget) =>
216 Effect.sync(() => {
217 const life = lifeOf(target);
218 lives.set(lifeKey(target), { ...life, wakes: life.wakes + 1 });
219 });
220
221 const reportEnd = (key: string, life: WatchLife, reason: WatchEndReason) =>
222 Effect.gen(function* () {
223 lives.delete(key);
224 const now = yield* Clock.currentTimeMillis;
225 const quietMs = now - life.pushedAt;
226 yield* Effect.logInfo("pull request watch ended", {
227 threadId: life.threadId,
228 pullRequest: life.pullRequest,
229 reason,
230 minutes: minutes(now - life.startedAt),
231 quietMinutes: minutes(quietMs),
232 longestQuietMinutes: minutes(Math.max(life.longestQuietMs, quietMs)),
233 wakes: life.wakes,
234 reads: life.reads,
235 partial: life.startedAt < bootedAt,
236 });
237 });
238 const ended = (target: WatchTarget, reason: WatchEndReason) =>
239 Effect.suspend(() => reportEnd(lifeKey(target), lifeOf(target), reason));
156240
157241 // Reads in a row that failed, per pull request. Kept in memory: a restart only delays the stop.
158242 const readFailures = new Map<string, number>();
......@@ -205,7 +289,11 @@ export const make = Effect.gen(function* () {
205289 outcome: "failed",
206290 summary: `#${target.link.number}: stopped watching, could not read it`,
207291 },
208 }).pipe(Effect.catch(() => record(target, null)));
292 }).pipe(
293 Effect.tap(() => woke(target)),
294 Effect.catch(() => record(target, null)),
295 Effect.tap(() => ended(target, "unreadable")),
296 );
209297
210298 const readRemarks = Effect.fn("PullRequestWatchReactor.readRemarks")(
211299 function* (key: string, reference: PullRequestRef, activity: PullRequestActivity) {
......@@ -269,16 +357,22 @@ export const make = Effect.gen(function* () {
269357 outcome: "updated",
270358 summary: `#${target.link.number}: closed, stopped watching`,
271359 },
272 });
360 }).pipe(
361 Effect.tap(() => woke(target)),
362 Effect.tap(() => ended(target, "closed")),
363 );
273364
274 /** Watches that end without a host read; the rest are read once per pull request. */
275 const endsWithoutRead = ({ thread, link }: WatchTarget) =>
365 /** Why a watch ends without a host read; the rest are read once per pull request. */
366 const endsWithoutRead = ({ thread, link }: WatchTarget): WatchEndReason | undefined =>
276367 // A merged pull request cannot reopen. Settling and archiving end watches, and a subagent
277368 // cannot start one; a watch left from before those rules ends here.
278 link.snapshot?.state === "merged" ||
279 thread.settledOverride === "settled" ||
280 thread.settledAt !== null ||
281 thread.lineage.relationshipToParent === "subagent";
369 link.snapshot?.state === "merged"
370 ? "merged"
371 : thread.settledOverride === "settled" || thread.settledAt !== null
372 ? "settled"
373 : thread.lineage.relationshipToParent === "subagent"
374 ? "subagent"
375 : undefined;
282376
283377 /**
284378 * Runs one thread's step for each thread in a group, so one refusal does not skip the rest.
......@@ -332,10 +426,26 @@ export const make = Effect.gen(function* () {
332426 }
333427 readFailures.delete(group.key);
334428 const [detail, activity] = read.value;
429 const headSha = detail.headSha ?? null;
430 for (const target of group.targets) {
431 const life = lifeOf(target);
432 // The first read only learns the head, so it is not a push.
433 const pushed = life.headSha !== null && headSha !== life.headSha;
434 lives.set(lifeKey(target), {
435 ...life,
436 headSha,
437 reads: life.reads + 1,
438 ...(pushed
439 ? { pushedAt: now, longestQuietMs: Math.max(life.longestQuietMs, now - life.pushedAt) }
440 : {}),
441 });
442 }
335443 if (detail.state !== "open") {
336444 lastReads.delete(group.key);
337445 return yield* eachTarget(group, (target) =>
338 detail.state === "closed" ? closed(target) : record(target, null),
446 detail.state === "closed"
447 ? closed(target)
448 : record(target, null).pipe(Effect.tap(() => ended(target, "merged"))),
339449 );
340450 }
341451
......@@ -364,6 +474,9 @@ export const make = Effect.gen(function* () {
364474 headSha: report.next.headSha,
365475 report,
366476 }),
477 ).pipe(
478 Effect.tap(() => woke(target)),
479 Effect.tap(() => (report.exhausted ? ended(target, "comment-limit") : Effect.void)),
367480 );
368481 }
369482 return watchesEqual(report.next, target.watch) ? Effect.void : record(target, report.next);
......@@ -377,11 +490,20 @@ export const make = Effect.gen(function* () {
377490 link.watch === undefined ? [] : [{ thread, link, watch: link.watch }],
378491 ),
379492 );
493 // A watch seen last pass and gone now was ended outside this reactor.
494 const present = new Set(targets.map(lifeKey));
495 yield* Effect.forEach(
496 [...lives].filter(([key]) => !present.has(key)),
497 ([key, life]) => reportEnd(key, life, "stopped"),
498 { discard: true },
499 );
500 for (const target of targets) lifeOf(target);
380501 const byPullRequest = new Map<string, Array<WatchTarget>>();
381 const ending: Array<WatchTarget> = [];
502 const ending: Array<readonly [WatchTarget, WatchEndReason]> = [];
382503 for (const target of targets) {
383 if (endsWithoutRead(target)) {
384 ending.push(target);
504 const reason = endsWithoutRead(target);
505 if (reason !== undefined) {
506 ending.push([target, reason]);
385507 continue;
386508 }
387509 // Grouped per project too: each project reads through its own checkout, so one that cannot
......@@ -399,8 +521,9 @@ export const make = Effect.gen(function* () {
399521 }
400522 yield* Effect.forEach(
401523 ending,
402 (target) =>
524 ([target, reason]) =>
403525 record(target, null).pipe(
526 Effect.tap(() => ended(target, reason)),
404527 Effect.catchCause(
405528 logFailure("pull request watch stop failed", {
406529 threadId: target.thread.id,
apps/server/src/orchestration-v2/runtimeLayer.test.ts+98
......@@ -32,6 +32,7 @@ import * as DateTime from "effect/DateTime";
3232import * as Deferred from "effect/Deferred";
3333import * as Fiber from "effect/Fiber";
3434import * as Layer from "effect/Layer";
35import * as Logger from "effect/Logger";
3536import * as Path from "effect/Path";
3637import * as Option from "effect/Option";
3738import * as Queue from "effect/Queue";
......@@ -2669,6 +2670,103 @@ it.layer(TestLayer)("OrchestrationV2LayerLive lifecycle", (it) => {
26692670 }),
26702671 );
26712672
2673 it.effect("reports how long an ended watch was quiet", () =>
2674 Effect.gen(function* () {
2675 const orchestrator = yield* Orchestrator.OrchestratorV2;
2676 const threadId = ThreadId.make("runtime-pull-request-watch-ended");
2677 const projectId = ProjectId.make("pr-watch-ended-project");
2678 yield* seedProject({
2679 projectId,
2680 title: "Watch ended",
2681 workspaceRoot: "/workspace/watch-ended",
2682 defaultModelSelection: null,
2683 createdAt: "2026-10-01T00:00:00.000Z",
2684 });
2685 yield* orchestrator.dispatch({
2686 type: "thread.create",
2687 createdBy: "user",
2688 creationSource: "web",
2689 commandId: CommandId.make("pr-watch-ended-create"),
2690 threadId,
2691 projectId,
2692 title: "Watch ended",
2693 modelSelection,
2694 runtimeMode: "full-access",
2695 interactionMode: "default",
2696 branch: null,
2697 worktreePath: null,
2698 });
2699 const key = { host: "github.com", repository: "pingdotgg/t3code", number: 11 };
2700 const watching = (on: boolean, id: string) =>
2701 orchestrator.dispatch({
2702 type: "thread.pull-request.watch",
2703 commandId: CommandId.make(`pr-watch-ended-${id}`),
2704 threadId,
2705 ...key,
2706 watching: on,
2707 link: { url: "https://github.com/pingdotgg/t3code/pull/11", source: "agent" },
2708 });
2709 yield* watching(true, "start");
2710
2711 let headSha = "aaaaaaa";
2712 const ended: Array<unknown> = [];
2713 const capture = Logger.make(({ message }) => {
2714 const [text, fields] = Array.isArray(message) ? message : [message];
2715 if (text === "pull request watch ended") ended.push(fields);
2716 });
2717 const reactor = yield* PullRequestWatchReactor.make.pipe(
2718 Effect.provide(
2719 Layer.mergeAll(
2720 NodeServices.layer,
2721 Layer.mock(PullRequestService.PullRequestService)({
2722 detail: () =>
2723 Effect.sync(() => ({
2724 ...watchedPullRequestDetail({ projectId, number: key.number, at: "2026-10-02" }),
2725 headSha,
2726 checks: [],
2727 })),
2728 activity: () =>
2729 Effect.succeed({
2730 comments: [],
2731 commentCount: 0,
2732 commentsTruncated: false,
2733 reviewThreads: [],
2734 commits: [],
2735 }),
2736 }),
2737 ),
2738 ),
2739 );
2740 const sweep = reactor.sweep.pipe(Effect.provide(Logger.layer([capture])));
2741
2742 // The first read learns the head. A push 6 hours later, then 2 quiet hours.
2743 yield* sweep;
2744 yield* TestClock.adjust("6 hours");
2745 headSha = "bbbbbbb";
2746 yield* sweep;
2747 yield* TestClock.adjust("2 hours");
2748 yield* sweep;
2749 yield* watching(false, "stop");
2750 // The end is reported once.
2751 yield* sweep;
2752 yield* sweep;
2753
2754 assert.deepEqual(ended, [
2755 {
2756 threadId,
2757 pullRequest: "github.com/pingdotgg/t3code#11",
2758 reason: "stopped",
2759 minutes: 480,
2760 quietMinutes: 120,
2761 longestQuietMinutes: 360,
2762 wakes: 0,
2763 reads: 3,
2764 partial: false,
2765 },
2766 ]);
2767 }),
2768 );
2769
26722770 it.effect.each([
26732771 "single page",
26742772 "paginated",