authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-11 19:46:05-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-11 19:46:05-07:00
logb9ad3daf8d489604e0eca9d32a7e08af7c4e9fb7
treebbd60aa1e15c1c4031e68fade5043a3c5aa02df9
parent12b34e83af63da05368e97b7cd0bb7719b17a62d
signature Signed by PGP key B5690EEEBB952194

fix(server): cap automatic thread continuations and turn them on by default (#18461)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

10 files changed, 270 insertions(+), 38 deletions(-)

apps/server/src/orchestration-v2/FoundationPersistence.test.ts+2
...@@ -2779,6 +2779,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {...@@ -2779,6 +2779,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2779 { id: sessionId, driver: "codex", providerInstanceId, status: "running" },2779 { id: sessionId, driver: "codex", providerInstanceId, status: "running" },
2780 ],2780 ],
2781 providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }],2781 providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }],
2782 runtimeRequests: [],
2783 turnItems: [],
2782 } as unknown as OrchestrationV2ThreadProjection;2784 } as unknown as OrchestrationV2ThreadProjection;
2783 const recovery = yield* ProviderRuntimeRecovery.make.pipe(2785 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
2784 Effect.provide(2786 Effect.provide(
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts+1-1
...@@ -502,7 +502,7 @@ it.effect("includes shared sessions and provider-owned background rosters in rec...@@ -502,7 +502,7 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
502 preparedState.providerSessions.map((session) => session.id),502 preparedState.providerSessions.map((session) => session.id),
503 [preparedSessionId],503 [preparedSessionId],
504 );504 );
505 assert.equal(restartContinuationRun(preparedState)?.id, preparedRunId);505 assert.equal(restartContinuationRun(preparedState, yield* DateTime.now)?.id, preparedRunId);
506 assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);506 assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
507 const sql = yield* SqlClient.SqlClient;507 const sql = yield* SqlClient.SqlClient;
508 yield* sql`508 yield* sql`
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+2-2
...@@ -561,7 +561,7 @@ export const make = Effect.gen(function* () {...@@ -561,7 +561,7 @@ export const make = Effect.gen(function* () {
561 }561 }
562 const continuationRun =562 const continuationRun =
563 continueAfterRestart && trigger === "startup"563 continueAfterRestart && trigger === "startup"
564 ? restartContinuationRun(projection)564 ? restartContinuationRun(projection, now)
565 : undefined;565 : undefined;
566 const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun566 const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun
567 ? [567 ? [
...@@ -697,7 +697,7 @@ export const make = Effect.gen(function* () {...@@ -697,7 +697,7 @@ export const make = Effect.gen(function* () {
697 .continueThreadsAfterServerUpdate697 .continueThreadsAfterServerUpdate
698 )698 )
699 return;699 return;
700 const run = restartContinuationRun(projection);700 const run = restartContinuationRun(projection, yield* DateTime.now);
701 if (!run) return;701 if (!run) return;
702 const commandId = CommandId.make(`command:restart-prepare:${run.id}`);702 const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
703 yield* eventSink.writeWithEffects({703 yield* eventSink.writeWithEffects({
apps/server/src/orchestration-v2/RestartContinuation.test.ts+151-18
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1import { assert, it } from "@effect/vitest";1import { assert, it } from "@effect/vitest";
2import {2import {
3 MessageId,3 MessageId,
4 NodeId,
4 ProjectId,5 ProjectId,
5 ProviderDriverKind,6 ProviderDriverKind,
6 ProviderInstanceId,7 ProviderInstanceId,
...@@ -9,6 +10,7 @@ import {...@@ -9,6 +10,7 @@ import {
9 ProviderTurnId,10 ProviderTurnId,
10 RunAttemptId,11 RunAttemptId,
11 RunId,12 RunId,
13 RuntimeRequestId,
12 ThreadId,14 ThreadId,
13 type OrchestrationV2Command,15 type OrchestrationV2Command,
14 type OrchestrationV2ThreadProjection,16 type OrchestrationV2ThreadProjection,
...@@ -35,6 +37,7 @@ const sessionId = ProviderSessionId.make("session:restart");...@@ -35,6 +37,7 @@ const sessionId = ProviderSessionId.make("session:restart");
35const attemptId = RunAttemptId.make("attempt:restart");37const attemptId = RunAttemptId.make("attempt:restart");
36// "No project" threads belong to the environment's Scratch project.38// "No project" threads belong to the environment's Scratch project.
37const scratchProjectId = ProjectId.make("project:scratch");39const scratchProjectId = ProjectId.make("project:scratch");
40const now = DateTime.makeUnsafe("2026-10-03T12:00:00.000Z");
3841
39function makeProjection() {42function makeProjection() {
40 return {43 return {
...@@ -90,7 +93,7 @@ function makeProjection() {...@@ -90,7 +93,7 @@ function makeProjection() {
9093
91it("requires matching saved native state for an unfinished root run", () => {94it("requires matching saved native state for an unfinished root run", () => {
92 const projection = makeProjection();95 const projection = makeProjection();
93 assert.equal(restartContinuationRun(projection)?.id, runId);96 assert.equal(restartContinuationRun(projection, now)?.id, runId);
94 for (const invalid of [97 for (const invalid of [
95 { ...projection, thread: { ...projection.thread, archivedAt: {} } },98 { ...projection, thread: { ...projection.thread, archivedAt: {} } },
96 { ...projection, thread: { ...projection.thread, deletedAt: {} } },99 { ...projection, thread: { ...projection.thread, deletedAt: {} } },
...@@ -123,7 +126,7 @@ it("requires matching saved native state for an unfinished root run", () => {...@@ -123,7 +126,7 @@ it("requires matching saved native state for an unfinished root run", () => {
123 "interrupted",126 "interrupted",
124 ].map((status) => ({ ...projection, runs: [{ ...projection.runs[0]!, status }] })),127 ].map((status) => ({ ...projection, runs: [{ ...projection.runs[0]!, status }] })),
125 ])128 ])
126 assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection));129 assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection, now));
127});130});
128131
129it("continues a live turn whose session the adapter never marked running", () => {132it("continues a live turn whose session the adapter never marked running", () => {
...@@ -135,9 +138,9 @@ it("continues a live turn whose session the adapter never marked running", () =>...@@ -135,9 +138,9 @@ it("continues a live turn whose session the adapter never marked running", () =>
135 providerSessions: [{ ...projection.providerSessions[0]!, status }],138 providerSessions: [{ ...projection.providerSessions[0]!, status }],
136 }) as OrchestrationV2ThreadProjection;139 }) as OrchestrationV2ThreadProjection;
137 for (const status of ["starting", "ready", "running", "waiting"])140 for (const status of ["starting", "ready", "running", "waiting"])
138 assert.equal(restartContinuationRun(withSessionStatus(status))?.id, runId, status);141 assert.equal(restartContinuationRun(withSessionStatus(status), now)?.id, runId, status);
139 for (const status of ["stopped", "error"])142 for (const status of ["stopped", "error"])
140 assert.isUndefined(restartContinuationRun(withSessionStatus(status)), status);143 assert.isUndefined(restartContinuationRun(withSessionStatus(status), now), status);
141});144});
142145
143it("recovers an admitted continuation after another crash before provider start", () => {146it("recovers an admitted continuation after another crash before provider start", () => {
...@@ -155,7 +158,7 @@ it("recovers an admitted continuation after another crash before provider start"...@@ -155,7 +158,7 @@ it("recovers an admitted continuation after another crash before provider start"
155 providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],158 providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
156 providerTurns: [],159 providerTurns: [],
157 };160 };
158 assert.equal(restartContinuationRun(starting)?.id, runId);161 assert.equal(restartContinuationRun(starting, now)?.id, runId);
159});162});
160163
161it("does not continue settled root runs with restart-cancelled background work", () => {164it("does not continue settled root runs with restart-cancelled background work", () => {
...@@ -176,11 +179,14 @@ it("does not continue settled root runs with restart-cancelled background work",...@@ -176,11 +179,14 @@ it("does not continue settled root runs with restart-cancelled background work",
176 for (const projectId of [scratchProjectId, projection.thread.projectId])179 for (const projectId of [scratchProjectId, projection.thread.projectId])
177 for (const status of ["completed", "waiting"] as const)180 for (const status of ["completed", "waiting"] as const)
178 assert.isUndefined(181 assert.isUndefined(
179 restartContinuationRun({182 restartContinuationRun(
180 ...settled,183 {
181 thread: { ...settled.thread, projectId },184 ...settled,
182 runs: [{ ...settled.runs[0]!, status }],185 thread: { ...settled.thread, projectId },
183 }),186 runs: [{ ...settled.runs[0]!, status }],
187 },
188 now,
189 ),
184 );190 );
185});191});
186192
...@@ -663,10 +669,13 @@ it.effect("continues a cut run past queued follow-ups, which stay held", () =>...@@ -663,10 +669,13 @@ it.effect("continues a cut run past queued follow-ups, which stay held", () =>
663 Effect.gen(function* () {669 Effect.gen(function* () {
664 const live = makeProjection();670 const live = makeProjection();
665 assert.equal(671 assert.equal(
666 restartContinuationRun({672 restartContinuationRun(
667 ...live,673 {
668 runs: [...live.runs, queuedFollowUp],674 ...live,
669 } as unknown as OrchestrationV2ThreadProjection)?.id,675 runs: [...live.runs, queuedFollowUp],
676 } as unknown as OrchestrationV2ThreadProjection,
677 now,
678 )?.id,
670 runId,679 runId,
671 );680 );
672 const cut = cutMidTurn();681 const cut = cutMidTurn();
...@@ -692,10 +701,13 @@ it.effect("continues a resumed queued run that ran after an earlier continuation...@@ -692,10 +701,13 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
692 };701 };
693 const live = makeProjection();702 const live = makeProjection();
694 assert.equal(703 assert.equal(
695 restartContinuationRun({704 restartContinuationRun(
696 ...live,705 {
697 runs: [...live.runs, finishedContinuation],706 ...live,
698 } as unknown as OrchestrationV2ThreadProjection)?.id,707 runs: [...live.runs, finishedContinuation],
708 } as unknown as OrchestrationV2ThreadProjection,
709 now,
710 )?.id,
699 runId,711 runId,
700 );712 );
701 const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });713 const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });
...@@ -835,3 +847,124 @@ it.effect("prepares later threads' continuations when one thread fails", () =>...@@ -835,3 +847,124 @@ it.effect("prepares later threads' continuations when one thread fails", () =>
835 );847 );
836 }),848 }),
837);849);
850
851it.effect("stops continuing after three automatic restart continuations in a row", () =>
852 Effect.gen(function* () {
853 const first = RunId.make("run:first-continuation");
854 const second = RunId.make("run:second-continuation");
855 const cut = cutMidTurn({ restartContinuationOfRunId: second });
856 const withTwoLinks = {
857 ...cut,
858 runs: [
859 ...cut.runs,
860 { ...cut.runs[0]!, id: second, restartContinuationOfRunId: first },
861 { ...cut.runs[0]!, id: first, restartContinuationOfRunId: undefined },
862 ],
863 } as unknown as OrchestrationV2ThreadProjection;
864 assert.lengthOf(yield* continuationTexts(withTwoLinks), 1);
865
866 const withThreeLinks = {
867 ...withTwoLinks,
868 runs: withTwoLinks.runs.map((run) =>
869 run.id === first ? { ...run, restartContinuationOfRunId: RunId.make("run:root") } : run,
870 ),
871 } as unknown as OrchestrationV2ThreadProjection;
872 assert.lengthOf(yield* continuationTexts(withThreeLinks), 0);
873 }),
874);
875
876it.effect.each([
877 ["user_input", "pending", "live", false],
878 ["user_input", "expired", "not_resumable", false],
879 ["user_input", "cancelled", "not_resumable", false],
880 ["command", "pending", "live", false],
881 ["command", "expired", "not_resumable", false],
882 ["user_input", "resolved", "live", true],
883 ["command", "cancelled", "live", true],
884 ["user_input", "pending", "message", true],
885 ["dynamic_tool_call", "pending", "live", true],
886 ["auth_refresh", "pending", "live", true],
887] as const)(
888 "a %s request that is %s with %s capability allows continuation: %s",
889 ([kind, status, capability, shouldContinue]) =>
890 Effect.gen(function* () {
891 const base = makeProjection();
892 const runtimeRequests = [
893 {
894 id: RuntimeRequestId.make("request:restart"),
895 nodeId: NodeId.make("node:restart-question"),
896 providerTurnId: base.providerTurns[0]!.id,
897 kind,
898 status,
899 responseCapability:
900 capability === "live"
901 ? { type: capability, providerSessionId: sessionId }
902 : capability === "not_resumable"
903 ? { type: capability, reason: "The server restarted." }
904 : { type: capability },
905 },
906 ];
907 const live = { ...base, runtimeRequests } as unknown as OrchestrationV2ThreadProjection;
908 assert.equal(restartContinuationRun(live, now) !== undefined, shouldContinue);
909 const cut = {
910 ...cutMidTurn(),
911 runtimeRequests,
912 } as unknown as OrchestrationV2ThreadProjection;
913 assert.lengthOf(yield* continuationTexts(cut), shouldContinue ? 1 : 0);
914 }),
915);
916
917it.effect("does not wake a snoozed or settled thread, but continues once a snooze elapsed", () =>
918 Effect.gen(function* () {
919 const live = makeProjection();
920 for (const held of [
921 { snoozedUntil: DateTime.add(now, { days: 1 }) },
922 { settledOverride: "settled" as const },
923 ]) {
924 const thread = { ...live.thread, ...held };
925 assert.isUndefined(restartContinuationRun({ ...live, thread }, now));
926 assert.deepEqual(yield* continuationTexts({ ...cutMidTurn(), thread }), []);
927 }
928 const thread = { ...live.thread, snoozedUntil: DateTime.subtract(now, { minutes: 1 }) };
929 assert.equal(restartContinuationRun({ ...live, thread }, now)?.id, runId);
930 }),
931);
932
933it.effect("does not continue a run waiting on a secret the restart closed", () =>
934 Effect.gen(function* () {
935 const completedAt = DateTime.makeUnsafe("2026-10-03T10:05:00.000Z");
936 const secret = (secretStatus: string, at: DateTime.Utc | null) =>
937 ({ type: "secret_request", runId, secretStatus, completedAt: at }) as never;
938 assert.isUndefined(
939 restartContinuationRun({ ...makeProjection(), turnItems: [secret("pending", null)] }, now),
940 );
941 const cut = cutMidTurn({ completedAt });
942 assert.deepEqual(
943 yield* continuationTexts({ ...cut, turnItems: [secret("cancelled", completedAt)] }),
944 [],
945 );
946 // A secret the user dismissed earlier in the run does not hold it.
947 assert.lengthOf(
948 yield* continuationTexts({
949 ...cut,
950 turnItems: [secret("cancelled", DateTime.subtract(completedAt, { seconds: 1 }))],
951 }),
952 1,
953 );
954 }),
955);
956
957it.effect.each(["completed", "failed", "interrupted"] as const)(
958 "does not continue when the provider turn %s before the app run settled",
959 (status) =>
960 Effect.gen(function* () {
961 const cut = cutMidTurn();
962 assert.deepEqual(
963 yield* continuationTexts({
964 ...cut,
965 providerTurns: [{ ...cut.providerTurns[0]!, status }],
966 }),
967 [],
968 );
969 }),
970);
apps/server/src/orchestration-v2/RestartContinuation.ts+89-7
...@@ -2,11 +2,13 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";...@@ -2,11 +2,13 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2import { resolveProjectSettings } from "@t3tools/shared/projectSettings";2import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3import {3import {
4 CommandId,4 CommandId,
5 latestProviderTurnForAttempt,
5 MessageId,6 MessageId,
6 type OrchestrationV2Run,7 type OrchestrationV2Run,
7 type RunId,8 type RunId,
8 type ThreadId,9 type ThreadId,
9} from "@t3tools/contracts";10} from "@t3tools/contracts";
11import * as DateTime from "effect/DateTime";
10import * as Effect from "effect/Effect";12import * as Effect from "effect/Effect";
11import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";13import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
1214
...@@ -21,6 +23,64 @@ import {...@@ -21,6 +23,64 @@ import {
2123
22const CONTINUE_PROMPT = "Continue where you left off.";24const CONTINUE_PROMPT = "Continue where you left off.";
2325
26/** Consecutive automatic continuations allowed before a person has to step in. */
27const MAX_CONTINUATION_CHAIN = 3;
28
29/**
30 * Whether a restart must leave this run for the user: the thread is put away,
31 * the provider turn already ended, or the user owes an answer. Checked when
32 * intent is recorded and again when the continuation is delivered.
33 */
34function restartContinuationBlocked(
35 projection: Pick<
36 ProjectionRuntimeRecoveryState,
37 "thread" | "runtimeRequests" | "providerTurns" | "turnItems"
38 >,
39 source: OrchestrationV2Run,
40 now: DateTime.Utc,
41): boolean {
42 const { thread } = projection;
43 if (
44 thread.archivedAt !== null ||
45 thread.deletedAt !== null ||
46 thread.settledOverride === "settled" ||
47 (thread.snoozedUntil != null && DateTime.isGreaterThan(thread.snoozedUntil, now))
48 )
49 return true;
50 const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId);
51 if (
52 turn !== undefined &&
53 (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")
54 )
55 return true;
56 const waitingRequest = projection.runtimeRequests.some(
57 (request) =>
58 // Tool calls and auth refreshes do not wait on a person, and a
59 // message-capable question stays answerable after the restart.
60 request.kind !== "dynamic_tool_call" &&
61 request.kind !== "auth_refresh" &&
62 request.responseCapability.type !== "message" &&
63 (request.status === "pending" ||
64 // Recovery closes live callbacks on the cut turn; it does not answer them.
65 ((request.status === "expired" || request.status === "cancelled") &&
66 request.responseCapability.type === "not_resumable" &&
67 turn !== undefined &&
68 turn.id === request.providerTurnId)),
69 );
70 if (waitingRequest) return true;
71 return projection.turnItems.some(
72 (item) =>
73 item.type === "secret_request" &&
74 item.runId === source.id &&
75 (item.secretStatus === "pending" ||
76 // Recovery cancels an open secret card and its run at the same instant.
77 (item.secretStatus === "cancelled" &&
78 item.completedAt !== null &&
79 source.completedAt !== null &&
80 DateTime.Equivalence(item.completedAt, source.completedAt))),
81 );
82}
83
24/**84/**
25 * Resume only an unfinished root turn. Leftover background work is cleaned up85 * Resume only an unfinished root turn. Leftover background work is cleaned up
26 * separately and reported on the next user turn; it must not wake a settled run.86 * separately and reported on the next user turn; it must not wake a settled run.
...@@ -28,10 +88,16 @@ const CONTINUE_PROMPT = "Continue where you left off.";...@@ -28,10 +88,16 @@ const CONTINUE_PROMPT = "Continue where you left off.";
28export function restartContinuationRun(88export function restartContinuationRun(
29 projection: Pick<89 projection: Pick<
30 ProjectionRuntimeRecoveryState,90 ProjectionRuntimeRecoveryState,
31 "thread" | "runs" | "providerThreads" | "providerSessions" | "providerTurns"91 | "thread"
92 | "runs"
93 | "providerThreads"
94 | "providerSessions"
95 | "providerTurns"
96 | "runtimeRequests"
97 | "turnItems"
32 >,98 >,
99 now: DateTime.Utc,
33): OrchestrationV2Run | undefined {100): OrchestrationV2Run | undefined {
34 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return;
35 // Queued runs never started; recovery holds them behind the cut run.101 // Queued runs never started; recovery holds them behind the cut run.
36 const run = projection.runs.reduce<OrchestrationV2Run | undefined>(102 const run = projection.runs.reduce<OrchestrationV2Run | undefined>(
37 (latest, candidate) =>103 (latest, candidate) =>
...@@ -44,6 +110,7 @@ export function restartContinuationRun(...@@ -44,6 +110,7 @@ export function restartContinuationRun(
44 const preparedContinuation =110 const preparedContinuation =
45 run.status === "starting" && run.restartContinuationOfRunId !== undefined;111 run.status === "starting" && run.restartContinuationOfRunId !== undefined;
46 if (run.status !== "running" && !preparedContinuation) return;112 if (run.status !== "running" && !preparedContinuation) return;
113 if (restartContinuationBlocked(projection, run, now)) return;
47 const liveTurnRequired = !preparedContinuation;114 const liveTurnRequired = !preparedContinuation;
48 if (projection.thread.providerInstanceId !== run.providerInstanceId) return;115 if (projection.thread.providerInstanceId !== run.providerInstanceId) return;
49 const providerThread = projection.providerThreads.find(116 const providerThread = projection.providerThreads.find(
...@@ -96,16 +163,14 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta...@@ -96,16 +163,14 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
96 const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);163 const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);
97 const projection = yield* threads.getThreadRecords(164 const projection = yield* threads.getThreadRecords(
98 input.threadId,165 input.threadId,
99 ["messages", "runs", "providerTurns", "attempts"],166 ["messages", "runs", "providerTurns", "attempts", "runtimeRequests", "turnItems"],
100 { messageIds: [messageId] },167 { messageIds: [messageId], turnItemTypes: ["secret_request"] },
101 );168 );
102 if (169 if (
103 !resolveProjectSettings(enabled, projection.thread.projectId).settings170 !resolveProjectSettings(enabled, projection.thread.projectId).settings
104 .continueThreadsAfterServerUpdate171 .continueThreadsAfterServerUpdate
105 )172 )
106 return;173 return;
107 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return;
108
109 if (projection.messages.some((message) => message.id === messageId)) return;174 if (projection.messages.some((message) => message.id === messageId)) return;
110 const source = projection.runs.find((run) => run.id === input.sourceRunId);175 const source = projection.runs.find((run) => run.id === input.sourceRunId);
111 // Pending effects from older versions may target settled background work,176 // Pending effects from older versions may target settled background work,
...@@ -113,7 +178,8 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta...@@ -113,7 +178,8 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
113 if (178 if (
114 !source ||179 !source ||
115 source.status !== "cancelled" ||180 source.status !== "cancelled" ||
116 isRestartNoteSource(source, projection.providerTurns)181 isRestartNoteSource(source, projection.providerTurns) ||
182 restartContinuationBlocked(projection, source, yield* DateTime.now)
117 )183 )
118 return;184 return;
119 // A user submission after reconciliation takes precedence over an automatic185 // A user submission after reconciliation takes precedence over an automatic
...@@ -125,6 +191,22 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta...@@ -125,6 +191,22 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
125 )191 )
126 return;192 return;
127 if (projection.thread.providerInstanceId !== source.providerInstanceId) return;193 if (projection.thread.providerInstanceId !== source.providerInstanceId) return;
194 // A turn that keeps crashing the server would otherwise restart it forever.
195 let chainLength = 0;
196 for (
197 let run: OrchestrationV2Run | undefined = source;
198 run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN;
199 run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId)
200 ) {
201 chainLength += 1;
202 }
203 if (chainLength >= MAX_CONTINUATION_CHAIN) {
204 yield* Effect.logWarning("Not continuing a run restarted too many times in a row", {
205 threadId: input.threadId,
206 sourceRunId: input.sourceRunId,
207 });
208 return;
209 }
128 const sourceRecords = yield* threads.getThreadRecords(210 const sourceRecords = yield* threads.getThreadRecords(
129 input.threadId,211 input.threadId,
130 ["messages", "turnItems"],212 ["messages", "turnItems"],
apps/server/src/serverSettings.test.ts+5-1
...@@ -1118,7 +1118,8 @@ it.layer(NodeServices.layer)("server settings", (it) => {...@@ -1118,7 +1118,8 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1118 const serverConfig = yield* ServerConfig.ServerConfig;1118 const serverConfig = yield* ServerConfig.ServerConfig;
1119 const fileSystem = yield* FileSystem.FileSystem;1119 const fileSystem = yield* FileSystem.FileSystem;
1120 const serverSettings = yield* ServerSettingsModule.ServerSettingsService;1120 const serverSettings = yield* ServerSettingsModule.ServerSettingsService;
1121 const raw = '{"addProjectBaseDirectory":42,"providers":{"codex":{"binaryPath":"/x"}}}';1121 const raw =
1122 '{"addProjectBaseDirectory":42,"continueThreadsAfterServerUpdate":"false","providers":{"codex":{"binaryPath":"/x"}}}';
1122 yield* fileSystem.writeFileString(serverConfig.settingsPath, raw);1123 yield* fileSystem.writeFileString(serverConfig.settingsPath, raw);
11231124
1124 const settings = yield* serverSettings.getSettings;1125 const settings = yield* serverSettings.getSettings;
...@@ -1128,6 +1129,8 @@ it.layer(NodeServices.layer)("server settings", (it) => {...@@ -1128,6 +1129,8 @@ it.layer(NodeServices.layer)("server settings", (it) => {
1128 driver: ProviderDriverKind.make("codex"),1129 driver: ProviderDriverKind.make("codex"),
1129 config: { binaryPath: "/x" },1130 config: { binaryPath: "/x" },
1130 });1131 });
1132 // An unreadable file may hold an opt-out, so threads are not resumed on a guess.
1133 assert.isFalse(settings.continueThreadsAfterServerUpdate);
1131 assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw);1134 assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), raw);
1132 }).pipe(Effect.provide(layerServerSettings())),1135 }).pipe(Effect.provide(layerServerSettings())),
1133 );1136 );
...@@ -1969,6 +1972,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {...@@ -1969,6 +1972,7 @@ it.layer(NodeServices.layer)("server settings", (it) => {
19691972
1970 const settings = yield* serverSettings.getSettings;1973 const settings = yield* serverSettings.getSettings;
1971 assert.isFalse(settings.projectSettingsFolded);1974 assert.isFalse(settings.projectSettingsFolded);
1975 assert.isFalse(settings.continueThreadsAfterServerUpdate);
1972 assert.deepEqual(settings.projectSettingsOverrides, {});1976 assert.deepEqual(settings.projectSettingsOverrides, {});
1973 // The user's file is still there to repair; nothing was written over it.1977 // The user's file is still there to repair; nothing was written over it.
1974 assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), broken);1978 assert.equal(yield* fileSystem.readFileString(serverConfig.settingsPath), broken);
apps/server/src/serverSettings.ts+2
...@@ -834,6 +834,8 @@ const make = Effect.gen(function* () {...@@ -834,6 +834,8 @@ const make = Effect.gen(function* () {
834 if (decoded._tag === "Failure" || legacySettings._tag === "Failure") {834 if (decoded._tag === "Failure" || legacySettings._tag === "Failure") {
835 const failure = decoded._tag === "Failure" ? decoded : legacySettings;835 const failure = decoded._tag === "Failure" ? decoded : legacySettings;
836 settingsFileTrusted = false;836 settingsFileTrusted = false;
837 // The file may hold an opt-out we can no longer read; never resume threads on a guess.
838 settings = { ...DEFAULT_SERVER_SETTINGS, continueThreadsAfterServerUpdate: false };
837 if (failure._tag === "Failure") {839 if (failure._tag === "Failure") {
838 yield* Effect.logWarning("failed to parse settings.json, using defaults", {840 yield* Effect.logWarning("failed to parse settings.json, using defaults", {
839 path: settingsPath,841 path: settingsPath,
docs/internals/server-updates.md+10-3
...@@ -58,7 +58,9 @@ stopped backends and replays the failure for the same token....@@ -58,7 +58,9 @@ stopped backends and replays the failure for the same token.
5858
59## Recovering interrupted threads59## Recovering interrupted threads
6060
61Restart continuation is an environment-owned preference, off by default. The61Restart continuation is an environment-owned preference, on by default. A
62`settings.json` that fails to decode turns it off, since the file may hold an
63opt-out. The
62[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)64[v2 recovery service](../../apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts)
63requires matching durable run, provider thread, session, and native resume identity.65requires matching durable run, provider thread, session, and native resume identity.
64Queued runs never started, so recovery holds them and continues the run they wait66Queued runs never started, so recovery holds them and continues the run they wait
...@@ -73,8 +75,13 @@ closing providers, then reconciles after ingestion has stopped so a late complet...@@ -73,8 +75,13 @@ closing providers, then reconciles after ingestion has stopped so a late complet
73cannot be overwritten by a stale cancellation.75cannot be overwritten by a stale cancellation.
7476
75The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)77The [continuation handler](../../apps/server/src/orchestration-v2/RestartContinuation.ts)
76rechecks the preference, archive state, provider selection, newer user work, a stop78rechecks the preference, provider selection, newer user work, a stop the user requested,
77the user requested, and maintenance turns such as `/compact` before dispatching. Stable79and maintenance turns such as `/compact` before dispatching. Intent and delivery both
80refuse a thread that is archived, settled, or still snoozed, a provider turn that already
81ended, and a run waiting on the user: an approval, question, or secret that was pending,
82or that recovery closed on the cut turn. Message-capable questions stay answerable, so
83they do not block. Delivery also stops after three consecutive continuations in one
84`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable
78command and message IDs prevent duplicate submissions after an outbox retry. Codex85command and message IDs prevent duplicate submissions after an outbox retry. Codex
79resumes without adding provider prompt text unless the turn lost background work;86resumes without adding provider prompt text unless the turn lost background work;
80other adapters receive the continuation message through their normal turn path.87other adapters receive the continuation message through their normal turn path.
docs/user/updating.md+5-4
...@@ -10,16 +10,17 @@ notice....@@ -10,16 +10,17 @@ notice.
10Server updates restart the connection and can interrupt active agents and10Server updates restart the connection and can interrupt active agents and
11terminal commands. Saved threads, settings, and project files remain.11terminal commands. Saved threads, settings, and project files remain.
1212
13**Settings → General → Continue threads after restarts** is off by default.13**Settings → General → Continue threads after restarts** is on by default.
14Enable it to resume supported active threads after an update, crash, or machine14It resumes supported active threads after an update, crash, or machine
15restart. Changes are saved to connected environments that support this setting;15restart. Changes are saved to connected environments that support this setting;
16update older servers first. If a supported environment was offline or has a16update older servers first. If a supported environment was offline or has a
17different value, use **Apply to all** in Settings after it connects.17different value, use **Apply to all** in Settings after it connects.
18T3 Code must start again on that machine;18T3 Code must start again on that machine;
19the setting does not enable automatic startup. Terminal commands may still be19the setting does not enable automatic startup. Terminal commands may still be
20interrupted, and threads without saved provider resume state need a new message.20interrupted, and threads without saved provider resume state need a new message.
21If you previously enabled continuation for updates, enable this setting once21A thread is not resumed while it waits for your approval, answer, or secret, or
22to allow recovery without a connected client.22while it is archived, snoozed, or settled. After three automatic resumes in a row,
23the thread waits for your next message.
2324
24Updates from the previous orchestration system preserve conversation transcripts but cannot carry25Updates from the previous orchestration system preserve conversation transcripts but cannot carry
25every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md)26every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md)
packages/contracts/src/settings.ts+3-2
...@@ -1107,9 +1107,10 @@ export const ServerSettings = Schema.Struct({...@@ -1107,9 +1107,10 @@ export const ServerSettings = Schema.Struct({
1107 Schema.withDecodingDefault(Effect.succeed("paragraph" as const)),1107 Schema.withDecodingDefault(Effect.succeed("paragraph" as const)),
1108 ),1108 ),
1109 enableProviderUpdateChecks: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),1109 enableProviderUpdateChecks: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
1110 // Retain the update-era key; recovery now needs an environment-owned opt-in.1110 // Retain the update-era key. On by default so restarts resume the work they
1111 // interrupted; recovery stops after three automatic continuations in a row.
1111 continueThreadsAfterServerUpdate: Schema.Boolean.pipe(1112 continueThreadsAfterServerUpdate: Schema.Boolean.pipe(
1112 Schema.withDecodingDefault(Effect.succeed(false)),1113 Schema.withDecodingDefault(Effect.succeed(true)),
1113 ),1114 ),
1114 /**1115 /**
1115 * Whether agents may drive the in-app preview browser. Turning this off1116 * Whether agents may drive the in-app preview browser. Turning this off