authorgravatar for 91240327+shivamhwp@users.noreply.github.comshivam <91240327+shivamhwp@users.noreply.github.com> 2026-08-29 02:46:37+05:30
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-29 02:46:37+05:30
loga40aef4ccbc10b7bcd7bc817151db8d5a9b1c257
treec652816f82524c36207701ce54e7f2c87363abb5
parent48c176b3cfda7a714c10f18a12d398528ff4c2f9
signature Signed by PGP key B5690EEEBB952194

fix(server): a draft can retry its first send after a failed bootstrap (#8226)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

16 files changed, 690 insertions(+), 25 deletions(-)

apps/server/integration/OrchestrationEngineHarness.integration.ts+1-1
...@@ -373,7 +373,7 @@ export const makeOrchestrationIntegrationHarness = (...@@ -373,7 +373,7 @@ export const makeOrchestrationIntegrationHarness = (
373 Layer.provideMerge(373 Layer.provideMerge(
374 Layer.succeed(ThreadDeletionReactor, {374 Layer.succeed(ThreadDeletionReactor, {
375 start: () => Effect.void,375 start: () => Effect.void,
376 drain: Effect.void,376 drainThrough: () => Effect.void,
377 }),377 }),
378 ),378 ),
379 Layer.provideMerge(379 Layer.provideMerge(
apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+3
...@@ -113,6 +113,7 @@ describe("OrchestrationEngine", () => {...@@ -113,6 +113,7 @@ describe("OrchestrationEngine", () => {
113 detail: "historical replay should not be used during bootstrap",113 detail: "historical replay should not be used during bootstrap",
114 }),114 }),
115 ),115 ),
116 hasEventAfter: () => Effect.succeed(false),
116 };117 };
117118
118 const projectionSnapshot = {119 const projectionSnapshot = {
...@@ -812,6 +813,7 @@ describe("OrchestrationEngine", () => {...@@ -812,6 +813,7 @@ describe("OrchestrationEngine", () => {
812 readAll() {813 readAll() {
813 return Stream.fromIterable(events);814 return Stream.fromIterable(events);
814 },815 },
816 hasEventAfter: () => Effect.succeed(false),
815 };817 };
816818
817 const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {819 const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
...@@ -1048,6 +1050,7 @@ describe("OrchestrationEngine", () => {...@@ -1048,6 +1050,7 @@ describe("OrchestrationEngine", () => {
1048 readAll() {1050 readAll() {
1049 return Stream.fromIterable(events);1051 return Stream.fromIterable(events);
1050 },1052 },
1053 hasEventAfter: () => Effect.succeed(false),
1051 };1054 };
10521055
1053 let shouldFailProjection = true;1056 let shouldFailProjection = true;
apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts+1-1
...@@ -61,7 +61,7 @@ describe("OrchestrationReactor", () => {...@@ -61,7 +61,7 @@ describe("OrchestrationReactor", () => {
61 started.push("thread-deletion-reactor");61 started.push("thread-deletion-reactor");
62 return Effect.void;62 return Effect.void;
63 },63 },
64 drain: Effect.void,64 drainThrough: () => Effect.void,
65 }),65 }),
66 ),66 ),
67 Layer.provideMerge(67 Layer.provideMerge(
apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts+251-1
...@@ -9,6 +9,7 @@ import {...@@ -9,6 +9,7 @@ import {
9 TurnId,9 TurnId,
10 ProviderInstanceId,10 ProviderInstanceId,
11} from "@t3tools/contracts";11} from "@t3tools/contracts";
12import * as Option from "effect/Option";
12import * as NodeServices from "@effect/platform-node/NodeServices";13import * as NodeServices from "@effect/platform-node/NodeServices";
13import { assert, it } from "@effect/vitest";14import { assert, it } from "@effect/vitest";
14import * as Effect from "effect/Effect";15import * as Effect from "effect/Effect";
...@@ -31,6 +32,7 @@ import {...@@ -31,6 +32,7 @@ import {
31 OrchestrationProjectionPipelineLive,32 OrchestrationProjectionPipelineLive,
32} from "./ProjectionPipeline.ts";33} from "./ProjectionPipeline.ts";
33import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts";34import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts";
35import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts";
34import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts";36import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts";
35import * as ThreadPlanProgress from "../ThreadPlanProgress.ts";37import * as ThreadPlanProgress from "../ThreadPlanProgress.ts";
36import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";38import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";
...@@ -1277,6 +1279,111 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-atta...@@ -1277,6 +1279,111 @@ it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-atta
1277 },1279 },
1278);1280);
12791281
1282it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-projection-attachments-replay-")))(
1283 "OrchestrationProjectionPipeline",
1284 (it) => {
1285 it.effect("replaying a superseded thread.deleted keeps the re-created thread's files", () =>
1286 Effect.gen(function* () {
1287 const fileSystem = yield* FileSystem.FileSystem;
1288 const path = yield* Path.Path;
1289 const projectionPipeline = yield* OrchestrationProjectionPipeline;
1290 const eventStore = yield* OrchestrationEventStore;
1291 const { attachmentsDir } = yield* ServerConfig;
1292 const now = "2026-01-01T00:00:00.000Z";
1293 const projectId = ProjectId.make("project-replay");
1294 const retriedThreadId = ThreadId.make("thread-replay-retried");
1295 const goneThreadId = ThreadId.make("thread-replay-gone");
1296 const retriedAttachmentPath = path.join(
1297 attachmentsDir,
1298 "thread-replay-retried-00000000-0000-4000-8000-000000000001.png",
1299 );
1300 const goneAttachmentPath = path.join(
1301 attachmentsDir,
1302 "thread-replay-gone-00000000-0000-4000-8000-000000000002.png",
1303 );
1304 const threadCreated = (threadId: ThreadId, suffix: string) =>
1305 eventStore.append({
1306 type: "thread.created",
1307 eventId: EventId.make(`evt-replay-create-${suffix}`),
1308 aggregateKind: "thread",
1309 aggregateId: threadId,
1310 occurredAt: now,
1311 commandId: CommandId.make(`cmd-replay-create-${suffix}`),
1312 causationEventId: null,
1313 correlationId: CorrelationId.make(`cmd-replay-create-${suffix}`),
1314 metadata: {},
1315 payload: {
1316 threadId,
1317 projectId,
1318 title: `Thread ${suffix}`,
1319 modelSelection: {
1320 instanceId: ProviderInstanceId.make("codex"),
1321 model: "gpt-5-codex",
1322 },
1323 runtimeMode: "full-access",
1324 branch: null,
1325 worktreePath: null,
1326 createdAt: now,
1327 updatedAt: now,
1328 },
1329 });
1330 const threadDeleted = (threadId: ThreadId, suffix: string) =>
1331 eventStore.append({
1332 type: "thread.deleted",
1333 eventId: EventId.make(`evt-replay-delete-${suffix}`),
1334 aggregateKind: "thread",
1335 aggregateId: threadId,
1336 occurredAt: now,
1337 commandId: CommandId.make(`cmd-replay-delete-${suffix}`),
1338 causationEventId: null,
1339 correlationId: CorrelationId.make(`cmd-replay-delete-${suffix}`),
1340 metadata: {},
1341 payload: { threadId, deletedAt: now },
1342 });
1343
1344 yield* eventStore.append({
1345 type: "project.created",
1346 eventId: EventId.make("evt-replay-project"),
1347 aggregateKind: "project",
1348 aggregateId: projectId,
1349 occurredAt: now,
1350 commandId: CommandId.make("cmd-replay-project"),
1351 causationEventId: null,
1352 correlationId: CorrelationId.make("cmd-replay-project"),
1353 metadata: {},
1354 payload: {
1355 projectId,
1356 title: "Replay",
1357 workspaceRoot: "/tmp/project-replay",
1358 defaultModelSelection: null,
1359 scripts: [],
1360 createdAt: now,
1361 updatedAt: now,
1362 },
1363 });
1364 // A failed first send: create, roll back, then the draft retries the id.
1365 yield* threadCreated(retriedThreadId, "retried-1");
1366 yield* threadDeleted(retriedThreadId, "retried");
1367 yield* threadCreated(retriedThreadId, "retried-2");
1368 // A thread that was deleted for good.
1369 yield* threadCreated(goneThreadId, "gone");
1370 yield* threadDeleted(goneThreadId, "gone");
1371
1372 // Files on disk are not event-sourced: by the time anything replays,
1373 // the retried thread's attachments already belong to its second life.
1374 yield* fileSystem.makeDirectory(attachmentsDir, { recursive: true });
1375 yield* fileSystem.writeFileString(retriedAttachmentPath, "second incarnation");
1376 yield* fileSystem.writeFileString(goneAttachmentPath, "gone");
1377
1378 yield* projectionPipeline.bootstrap;
1379
1380 assert.isTrue(yield* exists(retriedAttachmentPath));
1381 assert.isFalse(yield* exists(goneAttachmentPath));
1382 }),
1383 );
1384 },
1385);
1386
1280it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => {1387it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => {
1281 it.effect("replays a bootstrap backlog larger than the event store default limit", () =>1388 it.effect("replays a bootstrap backlog larger than the event store default limit", () =>
1282 Effect.gen(function* () {1389 Effect.gen(function* () {
...@@ -2850,7 +2957,7 @@ it.effect("restores pending turn-start metadata across projection pipeline resta...@@ -2850,7 +2957,7 @@ it.effect("restores pending turn-start metadata across projection pipeline resta
28502957
2851const engineLayer = it.layer(2958const engineLayer = it.layer(
2852 OrchestrationEngineLive.pipe(2959 OrchestrationEngineLive.pipe(
2853 Layer.provide(OrchestrationProjectionSnapshotQueryLive),2960 Layer.provideMerge(OrchestrationProjectionSnapshotQueryLive),
2854 Layer.provide(ThreadBackgroundLiveness.layer),2961 Layer.provide(ThreadBackgroundLiveness.layer),
2855 Layer.provide(ThreadPlanProgress.layer),2962 Layer.provide(ThreadPlanProgress.layer),
2856 Layer.provide(OrchestrationProjectionPipelineLive),2963 Layer.provide(OrchestrationProjectionPipelineLive),
...@@ -2967,4 +3074,147 @@ engineLayer("OrchestrationProjectionPipeline via engine dispatch", (it) => {...@@ -2967,4 +3074,147 @@ engineLayer("OrchestrationProjectionPipeline via engine dispatch", (it) => {
2967 ]);3074 ]);
2968 }),3075 }),
2969 );3076 );
3077
3078 it.effect("re-creating a deleted thread id starts from an empty projection", () =>
3079 Effect.gen(function* () {
3080 const engine = yield* OrchestrationEngineService;
3081 const snapshotQuery = yield* ProjectionSnapshotQuery;
3082 const sql = yield* SqlClient.SqlClient;
3083 const createdAt = "2026-01-01T00:00:00.000Z";
3084 const projectId = ProjectId.make("project-retry");
3085 const threadId = ThreadId.make("thread-retry");
3086 const modelSelection = {
3087 instanceId: ProviderInstanceId.make("codex"),
3088 model: "gpt-5-codex",
3089 };
3090 const createThread = (commandId: string, title: string) =>
3091 engine.dispatch({
3092 type: "thread.create",
3093 commandId: CommandId.make(commandId),
3094 threadId,
3095 projectId,
3096 title,
3097 modelSelection,
3098 runtimeMode: "full-access",
3099 interactionMode: "default",
3100 branch: null,
3101 worktreePath: null,
3102 createdAt,
3103 });
3104 const countRowsForThread = (table: string) =>
3105 sql<{ readonly count: number }>`
3106 SELECT COUNT(*) AS count FROM ${sql(table)} WHERE thread_id = ${threadId}
3107 `.pipe(Effect.map((rows) => rows[0]?.count ?? 0));
3108 const perThreadTables = [
3109 "projection_thread_messages",
3110 "projection_thread_activities",
3111 "projection_thread_sessions",
3112 "projection_turns",
3113 "projection_thread_proposed_plans",
3114 "projection_pending_approvals",
3115 ];
3116
3117 yield* engine.dispatch({
3118 type: "project.create",
3119 commandId: CommandId.make("cmd-retry-project"),
3120 projectId,
3121 title: "Retry Project",
3122 workspaceRoot: "/tmp/project-retry",
3123 defaultModelSelection: modelSelection,
3124 createdAt,
3125 });
3126
3127 // First attempt: the thread gets a turn, a message, an activity, and a
3128 // running session before its bootstrap fails and the server rolls back.
3129 yield* createThread("cmd-retry-create-1", "First attempt");
3130 yield* engine.dispatch({
3131 type: "thread.turn.start",
3132 commandId: CommandId.make("cmd-retry-turn-1"),
3133 threadId,
3134 message: {
3135 messageId: MessageId.make("message-retry-1"),
3136 role: "user",
3137 text: "first attempt",
3138 attachments: [],
3139 },
3140 runtimeMode: "full-access",
3141 interactionMode: "default",
3142 createdAt,
3143 });
3144 yield* engine.dispatch({
3145 type: "thread.activity.append",
3146 commandId: CommandId.make("cmd-retry-activity-1"),
3147 threadId,
3148 activity: {
3149 id: EventId.make("activity-retry-1"),
3150 tone: "info",
3151 kind: "approval.requested",
3152 summary: "approval requested",
3153 payload: { requestId: "request-retry-1" },
3154 turnId: null,
3155 createdAt,
3156 },
3157 createdAt,
3158 });
3159 yield* engine.dispatch({
3160 type: "thread.proposed-plan.upsert",
3161 commandId: CommandId.make("cmd-retry-plan-1"),
3162 threadId,
3163 proposedPlan: {
3164 id: "plan-retry-1",
3165 turnId: null,
3166 planMarkdown: "# Plan",
3167 implementedAt: null,
3168 implementationThreadId: null,
3169 createdAt,
3170 updatedAt: createdAt,
3171 },
3172 createdAt,
3173 });
3174 yield* engine.dispatch({
3175 type: "thread.session.set",
3176 commandId: CommandId.make("cmd-retry-session-1"),
3177 threadId,
3178 session: {
3179 threadId,
3180 status: "running",
3181 providerName: "codex",
3182 runtimeMode: "full-access",
3183 activeTurnId: TurnId.make("turn-retry-1"),
3184 lastError: null,
3185 updatedAt: createdAt,
3186 },
3187 createdAt,
3188 });
3189 for (const table of perThreadTables) {
3190 assert.isAbove(yield* countRowsForThread(table), 0, `${table} should be populated`);
3191 }
3192 const populatedShell = Option.getOrThrow(yield* snapshotQuery.getThreadShellById(threadId));
3193 assert.isTrue(populatedShell.hasPendingApprovals);
3194 assert.isTrue(populatedShell.hasActionableProposedPlan);
3195
3196 yield* engine.dispatch({
3197 type: "thread.delete",
3198 commandId: CommandId.make("cmd-retry-delete"),
3199 threadId,
3200 });
3201 assert.isTrue(Option.isNone(yield* snapshotQuery.getThreadShellById(threadId)));
3202
3203 // Retry from the same draft reuses the thread id.
3204 yield* createThread("cmd-retry-create-2", "Second attempt");
3205
3206 const shell = Option.getOrThrow(yield* snapshotQuery.getThreadShellById(threadId));
3207 assert.strictEqual(shell.title, "Second attempt");
3208 assert.isFalse(shell.hasPendingApprovals);
3209 assert.isFalse(shell.hasActionableProposedPlan);
3210 for (const table of perThreadTables) {
3211 assert.strictEqual(yield* countRowsForThread(table), 0, `${table} should be empty`);
3212 }
3213 const detail = Option.getOrThrow(yield* snapshotQuery.getThreadDetailById(threadId));
3214 assert.deepEqual(detail.messages, []);
3215 assert.deepEqual(detail.activities, []);
3216 assert.isNull(detail.latestTurn);
3217 assert.isNull(detail.session);
3218 }),
3219 );
2970});3220});
apps/server/src/orchestration/Layers/ProjectionPipeline.ts+51-1
...@@ -871,7 +871,18 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -871,7 +871,18 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
871 }871 }
872872
873 case "thread.deleted": {873 case "thread.deleted": {
874 attachmentSideEffects.deletedThreadIds.add(event.payload.threadId);874 // A draft retry can re-create this id later in the log. During
875 // replay the attachment files on disk already belong to that later
876 // incarnation, so only an unsuperseded deletion removes them.
877 const recreatedLater = yield* eventStore.hasEventAfter({
878 aggregateKind: "thread",
879 aggregateId: event.payload.threadId,
880 type: "thread.created",
881 sequenceExclusive: event.sequence,
882 });
883 if (!recreatedLater) {
884 attachmentSideEffects.deletedThreadIds.add(event.payload.threadId);
885 }
875 const existingRow = yield* projectionThreadRepository.getById({886 const existingRow = yield* projectionThreadRepository.getById({
876 threadId: event.payload.threadId,887 threadId: event.payload.threadId,
877 });888 });
...@@ -987,6 +998,15 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -987,6 +998,15 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
987 "applyThreadMessagesProjection",998 "applyThreadMessagesProjection",
988 )(function* (event, attachmentSideEffects) {999 )(function* (event, attachmentSideEffects) {
989 switch (event.type) {1000 switch (event.type) {
1001 // A draft retry re-creates a soft-deleted thread id. Every projector
1002 // drops its own rows for the old incarnation here so replay from any
1003 // per-projector cursor rebuilds the new thread without stale history.
1004 case "thread.created":
1005 yield* projectionThreadMessageRepository.deleteByThreadId({
1006 threadId: event.payload.threadId,
1007 });
1008 return;
1009
990 case "thread.message-sent": {1010 case "thread.message-sent": {
991 const existingMessage = yield* projectionThreadMessageRepository.getByMessageId({1011 const existingMessage = yield* projectionThreadMessageRepository.getByMessageId({
992 messageId: event.payload.messageId,1012 messageId: event.payload.messageId,
...@@ -1066,6 +1086,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -1066,6 +1086,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
1066 "applyThreadProposedPlansProjection",1086 "applyThreadProposedPlansProjection",
1067 )(function* (event, _attachmentSideEffects) {1087 )(function* (event, _attachmentSideEffects) {
1068 switch (event.type) {1088 switch (event.type) {
1089 case "thread.created":
1090 yield* projectionThreadProposedPlanRepository.deleteByThreadId({
1091 threadId: event.payload.threadId,
1092 });
1093 return;
1094
1069 case "thread.proposed-plan-upserted":1095 case "thread.proposed-plan-upserted":
1070 yield* projectionThreadProposedPlanRepository.upsert({1096 yield* projectionThreadProposedPlanRepository.upsert({
1071 planId: event.payload.proposedPlan.id,1097 planId: event.payload.proposedPlan.id,
...@@ -1117,6 +1143,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -1117,6 +1143,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
1117 "applyThreadActivitiesProjection",1143 "applyThreadActivitiesProjection",
1118 )(function* (event, _attachmentSideEffects) {1144 )(function* (event, _attachmentSideEffects) {
1119 switch (event.type) {1145 switch (event.type) {
1146 case "thread.created":
1147 yield* projectionThreadActivityRepository.deleteByThreadId({
1148 threadId: event.payload.threadId,
1149 });
1150 return;
1151
1120 case "thread.activity-appended":1152 case "thread.activity-appended":
1121 yield* projectionThreadActivityRepository.upsert({1153 yield* projectionThreadActivityRepository.upsert({
1122 activityId: event.payload.activity.id,1154 activityId: event.payload.activity.id,
...@@ -1168,6 +1200,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -1168,6 +1200,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
1168 const applyThreadSessionsProjection: ProjectorDefinition["apply"] = Effect.fn(1200 const applyThreadSessionsProjection: ProjectorDefinition["apply"] = Effect.fn(
1169 "applyThreadSessionsProjection",1201 "applyThreadSessionsProjection",
1170 )(function* (event, _attachmentSideEffects) {1202 )(function* (event, _attachmentSideEffects) {
1203 if (event.type === "thread.created") {
1204 yield* projectionThreadSessionRepository.deleteByThreadId({
1205 threadId: event.payload.threadId,
1206 });
1207 return;
1208 }
1171 if (event.type !== "thread.session-set") {1209 if (event.type !== "thread.session-set") {
1172 return;1210 return;
1173 }1211 }
...@@ -1187,6 +1225,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -1187,6 +1225,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
1187 "applyThreadTurnsProjection",1225 "applyThreadTurnsProjection",
1188 )(function* (event, _attachmentSideEffects) {1226 )(function* (event, _attachmentSideEffects) {
1189 switch (event.type) {1227 switch (event.type) {
1228 case "thread.created":
1229 yield* projectionTurnRepository.deleteByThreadId({
1230 threadId: event.payload.threadId,
1231 });
1232 return;
1233
1190 case "thread.turn-start-requested": {1234 case "thread.turn-start-requested": {
1191 yield* projectionTurnRepository.replacePendingTurnStart({1235 yield* projectionTurnRepository.replacePendingTurnStart({
1192 threadId: event.payload.threadId,1236 threadId: event.payload.threadId,
...@@ -1524,6 +1568,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti...@@ -1524,6 +1568,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
1524 "applyPendingApprovalsProjection",1568 "applyPendingApprovalsProjection",
1525 )(function* (event, _attachmentSideEffects) {1569 )(function* (event, _attachmentSideEffects) {
1526 switch (event.type) {1570 switch (event.type) {
1571 case "thread.created":
1572 yield* projectionPendingApprovalRepository.deleteByThreadId({
1573 threadId: event.payload.threadId,
1574 });
1575 return;
1576
1527 case "thread.activity-appended": {1577 case "thread.activity-appended": {
1528 const requestId =1578 const requestId =
1529 extractActivityRequestId(event.payload.activity.payload) ??1579 extractActivityRequestId(event.payload.activity.payload) ??
apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts+103-2
...@@ -1,10 +1,35 @@...@@ -1,10 +1,35 @@
1import { ThreadId } from "@t3tools/contracts";1import {
2 CommandId,
3 CorrelationId,
4 EventId,
5 type OrchestrationEvent,
6 ThreadId,
7} from "@t3tools/contracts";
8import { it as effectIt } from "@effect/vitest";
2import * as Cause from "effect/Cause";9import * as Cause from "effect/Cause";
10import * as Deferred from "effect/Deferred";
3import * as Effect from "effect/Effect";11import * as Effect from "effect/Effect";
4import * as Exit from "effect/Exit";12import * as Exit from "effect/Exit";
13import * as Fiber from "effect/Fiber";
14import * as Layer from "effect/Layer";
15import * as Ref from "effect/Ref";
16import * as Stream from "effect/Stream";
5import { describe, expect, it } from "vite-plus/test";17import { describe, expect, it } from "vite-plus/test";
618
7import { logCleanupCauseUnlessInterrupted } from "./ThreadDeletionReactor.ts";19import {
20 ProviderService,
21 type ProviderServiceShape,
22} from "../../provider/Services/ProviderService.ts";
23import * as TerminalManager from "../../terminal/Manager.ts";
24import {
25 OrchestrationEngineService,
26 type OrchestrationEngineShape,
27} from "../Services/OrchestrationEngine.ts";
28import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts";
29import {
30 logCleanupCauseUnlessInterrupted,
31 ThreadDeletionReactorLive,
32} from "./ThreadDeletionReactor.ts";
833
9describe("logCleanupCauseUnlessInterrupted", () => {34describe("logCleanupCauseUnlessInterrupted", () => {
10 const threadId = ThreadId.make("thread-deletion-reactor-test");35 const threadId = ThreadId.make("thread-deletion-reactor-test");
...@@ -36,3 +61,79 @@ describe("logCleanupCauseUnlessInterrupted", () => {...@@ -36,3 +61,79 @@ describe("logCleanupCauseUnlessInterrupted", () => {
36 }61 }
37 });62 });
38});63});
64
65describe("ThreadDeletionReactor drain", () => {
66 const now = "2026-01-01T00:00:00.000Z";
67 const threadId = ThreadId.make("thread-deletion-reactor-drain");
68 const deletedEvent = (sequence: number): OrchestrationEvent => ({
69 sequence,
70 eventId: EventId.make(`evt-deleted-${sequence}`),
71 aggregateKind: "thread",
72 aggregateId: threadId,
73 type: "thread.deleted",
74 occurredAt: now,
75 commandId: CommandId.make(`cmd-deleted-${sequence}`),
76 causationEventId: null,
77 correlationId: CorrelationId.make(`cmd-deleted-${sequence}`),
78 metadata: {},
79 payload: { threadId, deletedAt: now },
80 });
81
82 effectIt.effect("waits for a published deletion the subscriber has not consumed yet", () =>
83 Effect.gen(function* () {
84 const stops: Array<number> = [];
85 const firstCleanupDone = yield* Deferred.make<void>();
86 // The engine has already committed and published sequence 2, but the
87 // subscriber has not received it yet: the stream releases it on demand.
88 const releaseSecondEvent = yield* Deferred.make<void>();
89 const latestSequence = yield* Ref.make(0);
90 const engine = {
91 latestSequence: Ref.get(latestSequence),
92 streamDomainEvents: Stream.concat(
93 Stream.make(deletedEvent(1)),
94 Stream.fromEffect(Deferred.await(releaseSecondEvent)).pipe(
95 Stream.map(() => deletedEvent(2)),
96 ),
97 ),
98 } as unknown as OrchestrationEngineShape;
99 const providerService = {
100 stopSession: () =>
101 Effect.gen(function* () {
102 stops.push(stops.length + 1);
103 if (stops.length === 1) {
104 yield* Deferred.succeed(firstCleanupDone, undefined);
105 }
106 }),
107 } as unknown as ProviderServiceShape;
108 const terminalManager = {
109 close: () => Effect.void,
110 } as unknown as TerminalManager.TerminalManager["Service"];
111 const layer = ThreadDeletionReactorLive.pipe(
112 Layer.provide(Layer.succeed(ProviderService, providerService)),
113 Layer.provide(Layer.succeed(TerminalManager.TerminalManager, terminalManager)),
114 Layer.provide(Layer.succeed(OrchestrationEngineService, engine)),
115 );
116
117 yield* Effect.scoped(
118 Effect.gen(function* () {
119 const reactor = yield* ThreadDeletionReactor;
120 yield* reactor.start();
121 yield* Deferred.await(firstCleanupDone);
122
123 // Sequence 1 is fully cleaned and the worker queue is idle. Sequence
124 // 2 is committed and published but still in flight to the subscriber.
125 yield* Ref.set(latestSequence, 2);
126 const drained = yield* Effect.forkChild(reactor.drainThrough(2));
127 yield* Effect.yieldNow;
128 yield* Effect.yieldNow;
129 expect(stops).toEqual([1]);
130 expect(drained.pollUnsafe()).toBeUndefined();
131
132 yield* Deferred.succeed(releaseSecondEvent, undefined);
133 yield* Fiber.join(drained);
134 expect(stops).toEqual([1, 2]);
135 }),
136 ).pipe(Effect.provide(layer));
137 }),
138 );
139});
apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts+31-7
...@@ -4,6 +4,7 @@ import * as Cause from "effect/Cause";...@@ -4,6 +4,7 @@ import * as Cause from "effect/Cause";
4import * as Effect from "effect/Effect";4import * as Effect from "effect/Effect";
5import * as Layer from "effect/Layer";5import * as Layer from "effect/Layer";
6import * as Stream from "effect/Stream";6import * as Stream from "effect/Stream";
7import * as SubscriptionRef from "effect/SubscriptionRef";
78
8import { ProviderService } from "../../provider/Services/ProviderService.ts";9import { ProviderService } from "../../provider/Services/ProviderService.ts";
9import * as TerminalManager from "../../terminal/Manager.ts";10import * as TerminalManager from "../../terminal/Manager.ts";
...@@ -80,20 +81,43 @@ const make = Effect.gen(function* () {...@@ -80,20 +81,43 @@ const make = Effect.gen(function* () {
8081
81 const worker = yield* makeDrainableWorker(processThreadDeletedSafely);82 const worker = yield* makeDrainableWorker(processThreadDeletedSafely);
8283
84 // Highest event sequence the subscriber has handed to the worker. Waiting
85 // through a successful thread.created sequence covers every deletion that
86 // was ahead of that create in the engine queue; the worker drain then covers
87 // the in-flight cleanup.
88 const seenSequence = yield* SubscriptionRef.make(0);
89 const noteSeen = (sequence: number) =>
90 SubscriptionRef.update(seenSequence, (seen) => Math.max(seen, sequence));
91
83 const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () {92 const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () {
84 yield* forkParked(93 yield* forkParked(
85 Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => {94 Stream.runForEach(
86 if (event.type !== "thread.deleted") {95 orchestrationEngine.streamDomainEvents.pipe(
87 return Effect.void;96 // Events that landed before the subscription are not replayed, so
88 }97 // start the watermark at the current head instead of zero.
89 return worker.enqueue(event);98 Stream.onStart(orchestrationEngine.latestSequence.pipe(Effect.flatMap(noteSeen))),
90 }),99 ),
100 (event) =>
101 (event.type === "thread.deleted" ? worker.enqueue(event) : Effect.void).pipe(
102 Effect.andThen(noteSeen(event.sequence)),
103 ),
104 ),
105 );
106 });
107
108 const drainThrough: ThreadDeletionReactorShape["drainThrough"] = Effect.fn(
109 "ThreadDeletionReactor.drainThrough",
110 )(function* (target) {
111 yield* SubscriptionRef.changes(seenSequence).pipe(
112 Stream.filter((seen) => seen >= target),
113 Stream.runHead,
91 );114 );
115 yield* worker.drain;
92 });116 });
93117
94 return {118 return {
95 start,119 start,
96 drain: worker.drain,120 drainThrough,
97 } satisfies ThreadDeletionReactorShape;121 } satisfies ThreadDeletionReactorShape;
98});122});
99123
apps/server/src/orchestration/Services/ThreadDeletionReactor.ts+5-3
...@@ -23,10 +23,12 @@ export interface ThreadDeletionReactorShape {...@@ -23,10 +23,12 @@ export interface ThreadDeletionReactorShape {
23 readonly start: () => Effect.Effect<void, never, Scope.Scope>;23 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
2424
25 /**25 /**
26 * Resolves when the internal processing queue is empty and idle.26 * Resolves once every thread.deleted at or before the supplied event
27 * Intended for test use to replace timing-sensitive sleeps.27 * sequence has been handed to the worker and the worker is empty and idle.
28 * A successful thread.create sequence is the fence callers use before the
29 * new incarnation can own runtime resources.
28 */30 */
29 readonly drain: Effect.Effect<void>;31 readonly drainThrough: (sequence: number) => Effect.Effect<void>;
30}32}
3133
32/**34/**
apps/server/src/orchestration/commandInvariants.test.ts+30
...@@ -199,4 +199,34 @@ describe("commandInvariants", () => {...@@ -199,4 +199,34 @@ describe("commandInvariants", () => {
199 ),199 ),
200 ).rejects.toThrow("already exists");200 ).rejects.toThrow("already exists");
201 });201 });
202
203 it("lets a draft retry re-create a thread id after its first attempt was deleted", async () => {
204 const threadId = ThreadId.make("thread-1");
205 const firstAttempt = readModel.threads.find((thread) => thread.id === threadId)!;
206 const afterRollback: OrchestrationReadModel = {
207 ...readModel,
208 threads: readModel.threads.map((thread) =>
209 thread.id === threadId ? { ...thread, deletedAt: now, updatedAt: now } : thread,
210 ),
211 };
212 const retry: OrchestrationCommand = {
213 type: "thread.create",
214 commandId: CommandId.make("cmd-retry"),
215 threadId,
216 projectId: firstAttempt.projectId,
217 title: firstAttempt.title,
218 modelSelection: firstAttempt.modelSelection,
219 interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
220 runtimeMode: "approval-required",
221 branch: null,
222 worktreePath: null,
223 createdAt: now,
224 };
225
226 await expect(
227 Effect.runPromise(
228 requireThreadAbsent({ readModel: afterRollback, command: retry, threadId }),
229 ),
230 ).resolves.toBeUndefined();
231 });
202});232});
apps/server/src/orchestration/commandInvariants.ts+5-1
...@@ -156,7 +156,11 @@ export function requireThreadAbsent(input: {...@@ -156,7 +156,11 @@ export function requireThreadAbsent(input: {
156 readonly command: OrchestrationCommand;156 readonly command: OrchestrationCommand;
157 readonly threadId: ThreadId;157 readonly threadId: ThreadId;
158}): Effect.Effect<void, OrchestrationCommandInvariantError> {158}): Effect.Effect<void, OrchestrationCommandInvariantError> {
159 if (!findThreadById(input.readModel, input.threadId)) {159 // Thread deletion is a soft delete and a draft keeps its client-minted id
160 // across retries, so only a live row blocks creation. Projectors reset the
161 // thread's rows when the id is created again.
162 const existing = findThreadById(input.readModel, input.threadId);
163 if (existing === undefined || existing.deletedAt !== null) {
160 return Effect.void;164 return Effect.void;
161 }165 }
162 return Effect.fail(166 return Effect.fail(
apps/server/src/persistence/Layers/OrchestrationEventStore.ts+35
...@@ -15,6 +15,7 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";...@@ -15,6 +15,7 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";
15import * as SqlSchema from "effect/unstable/sql/SqlSchema";15import * as SqlSchema from "effect/unstable/sql/SqlSchema";
16import * as Effect from "effect/Effect";16import * as Effect from "effect/Effect";
17import * as Layer from "effect/Layer";17import * as Layer from "effect/Layer";
18import * as Option from "effect/Option";
18import * as Schema from "effect/Schema";19import * as Schema from "effect/Schema";
19import * as Stream from "effect/Stream";20import * as Stream from "effect/Stream";
2021
...@@ -60,6 +61,13 @@ const OrchestrationEventPersistedRowSchema = Schema.Struct({...@@ -60,6 +61,13 @@ const OrchestrationEventPersistedRowSchema = Schema.Struct({
60 metadata: EventMetadataFromJsonString,61 metadata: EventMetadataFromJsonString,
61});62});
6263
64const HasEventAfterRequestSchema = Schema.Struct({
65 aggregateKind: Schema.String,
66 aggregateId: Schema.String,
67 type: Schema.String,
68 sequenceExclusive: NonNegativeInt,
69});
70
63const ReadFromSequenceRequestSchema = Schema.Struct({71const ReadFromSequenceRequestSchema = Schema.Struct({
64 sequenceExclusive: NonNegativeInt,72 sequenceExclusive: NonNegativeInt,
65 limit: Schema.Number,73 limit: Schema.Number,
...@@ -260,10 +268,37 @@ const makeEventStore = Effect.gen(function* () {...@@ -260,10 +268,37 @@ const makeEventStore = Effect.gen(function* () {
260 return readPage(sequenceExclusive, normalizedLimit);268 return readPage(sequenceExclusive, normalizedLimit);
261 };269 };
262270
271 const findEventAfter = SqlSchema.findOneOption({
272 Request: HasEventAfterRequestSchema,
273 Result: Schema.Struct({ sequence: Schema.Number }),
274 execute: (request) =>
275 sql`
276 SELECT sequence
277 FROM orchestration_events
278 WHERE aggregate_kind = ${request.aggregateKind}
279 AND stream_id = ${request.aggregateId}
280 AND event_type = ${request.type}
281 AND sequence > ${request.sequenceExclusive}
282 LIMIT 1
283 `,
284 });
285
286 const hasEventAfter: OrchestrationEventStoreShape["hasEventAfter"] = (input) =>
287 findEventAfter(input).pipe(
288 Effect.map(Option.isSome),
289 Effect.mapError(
290 toPersistenceSqlOrDecodeError(
291 "OrchestrationEventStore.hasEventAfter:query",
292 "OrchestrationEventStore.hasEventAfter:decodeRow",
293 ),
294 ),
295 );
296
263 return {297 return {
264 append,298 append,
265 readFromSequence,299 readFromSequence,
266 readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER),300 readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER),
301 hasEventAfter,
267 } satisfies OrchestrationEventStoreShape;302 } satisfies OrchestrationEventStoreShape;
268});303});
269304
apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts+17
...@@ -95,6 +95,15 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {...@@ -95,6 +95,15 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {
95 `,95 `,
96 });96 });
9797
98 const deleteProjectionPendingApprovalRowsByThread = SqlSchema.void({
99 Request: ListProjectionPendingApprovalsInput,
100 execute: ({ threadId }) =>
101 sql`
102 DELETE FROM projection_pending_approvals
103 WHERE thread_id = ${threadId}
104 `,
105 });
106
98 const upsert: ProjectionPendingApprovalRepositoryShape["upsert"] = (row) =>107 const upsert: ProjectionPendingApprovalRepositoryShape["upsert"] = (row) =>
99 upsertProjectionPendingApprovalRow(row).pipe(108 upsertProjectionPendingApprovalRow(row).pipe(
100 Effect.mapError(toPersistenceSqlError("ProjectionPendingApprovalRepository.upsert:query")),109 Effect.mapError(toPersistenceSqlError("ProjectionPendingApprovalRepository.upsert:query")),
...@@ -123,11 +132,19 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {...@@ -123,11 +132,19 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {
123 ),132 ),
124 );133 );
125134
135 const deleteByThreadId: ProjectionPendingApprovalRepositoryShape["deleteByThreadId"] = (input) =>
136 deleteProjectionPendingApprovalRowsByThread(input).pipe(
137 Effect.mapError(
138 toPersistenceSqlError("ProjectionPendingApprovalRepository.deleteByThreadId:query"),
139 ),
140 );
141
126 return {142 return {
127 upsert,143 upsert,
128 listByThreadId,144 listByThreadId,
129 getByRequestId,145 getByRequestId,
130 deleteByRequestId,146 deleteByRequestId,
147 deleteByThreadId,
131 } satisfies ProjectionPendingApprovalRepositoryShape;148 } satisfies ProjectionPendingApprovalRepositoryShape;
132});149});
133150
apps/server/src/persistence/Services/OrchestrationEventStore.ts+13
...@@ -52,6 +52,19 @@ export interface OrchestrationEventStoreShape {...@@ -52,6 +52,19 @@ export interface OrchestrationEventStoreShape {
52 * @returns Stream containing all stored events.52 * @returns Stream containing all stored events.
53 */53 */
54 readonly readAll: () => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;54 readonly readAll: () => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;
55
56 /**
57 * Check whether an aggregate has an event of the given type after a sequence.
58 *
59 * Used during replay to tell whether a later event supersedes the one being
60 * applied, without streaming the rest of the log.
61 */
62 readonly hasEventAfter: (input: {
63 readonly aggregateKind: OrchestrationEvent["aggregateKind"];
64 readonly aggregateId: string;
65 readonly type: OrchestrationEvent["type"];
66 readonly sequenceExclusive: number;
67 }) => Effect.Effect<boolean, OrchestrationEventStoreError>;
55}68}
5669
57/**70/**
apps/server/src/persistence/Services/ProjectionPendingApprovals.ts+7
...@@ -82,6 +82,13 @@ export interface ProjectionPendingApprovalRepositoryShape {...@@ -82,6 +82,13 @@ export interface ProjectionPendingApprovalRepositoryShape {
82 readonly deleteByRequestId: (82 readonly deleteByRequestId: (
83 input: DeleteProjectionPendingApprovalInput,83 input: DeleteProjectionPendingApprovalInput,
84 ) => Effect.Effect<void, ProjectionRepositoryError>;84 ) => Effect.Effect<void, ProjectionRepositoryError>;
85
86 /**
87 * Delete every pending approval row for a thread.
88 */
89 readonly deleteByThreadId: (
90 input: ListProjectionPendingApprovalsInput,
91 ) => Effect.Effect<void, ProjectionRepositoryError>;
85}92}
8693
87/**94/**
apps/server/src/server.test.ts+121-7
...@@ -116,6 +116,7 @@ import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";...@@ -116,6 +116,7 @@ import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
116import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";116import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";
117import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts";117import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts";
118import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";118import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
119import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts";
119import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts";120import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts";
120import { PersistenceSqlError } from "./persistence/Errors.ts";121import { PersistenceSqlError } from "./persistence/Errors.ts";
121import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts";122import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts";
...@@ -411,6 +412,7 @@ const buildAppUnderTest = (options?: {...@@ -411,6 +412,7 @@ const buildAppUnderTest = (options?: {
411 >;412 >;
412 terminalManager?: Partial<TerminalManager.TerminalManager["Service"]>;413 terminalManager?: Partial<TerminalManager.TerminalManager["Service"]>;
413 orchestrationEngine?: Partial<OrchestrationEngine.OrchestrationEngineService["Service"]>;414 orchestrationEngine?: Partial<OrchestrationEngine.OrchestrationEngineService["Service"]>;
415 threadDeletionReactor?: Partial<ThreadDeletionReactor["Service"]>;
414 analyticsService?: Partial<AnalyticsService.AnalyticsService["Service"]>;416 analyticsService?: Partial<AnalyticsService.AnalyticsService["Service"]>;
415 projectionSnapshotQuery?: Partial<ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"]>;417 projectionSnapshotQuery?: Partial<ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"]>;
416 checkpointDiffQuery?: Partial<CheckpointDiffQuery.CheckpointDiffQuery["Service"]>;418 checkpointDiffQuery?: Partial<CheckpointDiffQuery.CheckpointDiffQuery["Service"]>;
...@@ -787,13 +789,20 @@ const buildAppUnderTest = (options?: {...@@ -787,13 +789,20 @@ const buildAppUnderTest = (options?: {
787 ),789 ),
788 ),790 ),
789 Layer.provide(791 Layer.provide(
790 Layer.mock(OrchestrationEngine.OrchestrationEngineService)({792 Layer.mergeAll(
791 readEvents: () => Stream.empty,793 Layer.mock(OrchestrationEngine.OrchestrationEngineService)({
792 dispatch: () => Effect.succeed({ sequence: 0 }),794 readEvents: () => Stream.empty,
793 streamDomainEvents: Stream.empty,795 dispatch: () => Effect.succeed({ sequence: 0 }),
794 latestSequence: Effect.succeed(0),796 streamDomainEvents: Stream.empty,
795 ...options?.layers?.orchestrationEngine,797 latestSequence: Effect.succeed(0),
796 }),798 ...options?.layers?.orchestrationEngine,
799 }),
800 Layer.mock(ThreadDeletionReactor)({
801 start: () => Effect.void,
802 drainThrough: () => Effect.void,
803 ...options?.layers?.threadDeletionReactor,
804 }),
805 ),
797 ),806 ),
798 Layer.provide(807 Layer.provide(
799 Layer.mock(ProjectionSnapshotQuery.ProjectionSnapshotQuery)({808 Layer.mock(ProjectionSnapshotQuery.ProjectionSnapshotQuery)({
...@@ -8476,6 +8485,111 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -8476,6 +8485,111 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
8476 }).pipe(Effect.provide(NodeHttpServer.layerTest)),8485 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
8477 );8486 );
84788487
8488 it.effect("drains deletion cleanup through the re-created thread event", () =>
8489 Effect.gen(function* () {
8490 // A draft retry reuses the thread id its failed bootstrap deleted. The
8491 // deletion reactor stops sessions and closes terminals by that id, so
8492 // both thread.create paths use the created event as a fence, then drain
8493 // cleanup before handing the new incarnation to resource-owning work.
8494 const trace: Array<string> = [];
8495 const drainRequested = yield* Deferred.make<void>();
8496 const cleanupDone = yield* Deferred.make<void>();
8497 yield* buildAppUnderTest({
8498 layers: {
8499 threadDeletionReactor: {
8500 drainThrough: (sequence) =>
8501 Effect.gen(function* () {
8502 trace.push(`drain:${sequence}`);
8503 yield* Deferred.succeed(drainRequested, undefined);
8504 yield* Deferred.await(cleanupDone);
8505 }),
8506 },
8507 orchestrationEngine: {
8508 dispatch: (command) =>
8509 Effect.sync(() => {
8510 trace.push(command.type);
8511 return { sequence: trace.length };
8512 }),
8513 readEvents: () => Stream.empty,
8514 },
8515 },
8516 });
8517
8518 const createdAt = "2026-01-01T00:00:00.000Z";
8519 const threadId = ThreadId.make("thread-retry-after-delete");
8520 const wsUrl = yield* getWsServerUrl("/ws");
8521
8522 yield* Effect.scoped(
8523 withWsRpcClient(wsUrl, (client) =>
8524 Effect.gen(function* () {
8525 const directCreate = yield* Effect.forkChild(
8526 client[ORCHESTRATION_WS_METHODS.dispatchCommand]({
8527 type: "thread.create",
8528 commandId: CommandId.make("cmd-retry-create"),
8529 threadId,
8530 projectId: defaultProjectId,
8531 title: "Retry",
8532 modelSelection: defaultModelSelection,
8533 runtimeMode: "full-access",
8534 interactionMode: "default",
8535 branch: null,
8536 worktreePath: null,
8537 createdAt,
8538 }),
8539 );
8540 yield* Deferred.await(drainRequested);
8541 assert.deepEqual(trace, ["thread.create", "drain:1"]);
8542 yield* Deferred.succeed(cleanupDone, undefined);
8543 yield* Fiber.join(directCreate);
8544 }),
8545 ),
8546 );
8547 assert.deepEqual(trace, ["thread.create", "drain:1"]);
8548
8549 // Cleanup is already released; the bootstrap path must still drain
8550 // between creating the thread and starting its turn.
8551 trace.length = 0;
8552 yield* Effect.scoped(
8553 withWsRpcClient(wsUrl, (client) =>
8554 Effect.gen(function* () {
8555 const bootstrapCreate = yield* Effect.forkChild(
8556 client[ORCHESTRATION_WS_METHODS.dispatchCommand]({
8557 type: "thread.turn.start",
8558 commandId: CommandId.make("cmd-retry-bootstrap"),
8559 threadId,
8560 message: {
8561 messageId: MessageId.make("msg-retry-bootstrap"),
8562 role: "user",
8563 text: "hello",
8564 attachments: [],
8565 },
8566 modelSelection: defaultModelSelection,
8567 runtimeMode: "full-access",
8568 interactionMode: "default",
8569 bootstrap: {
8570 createThread: {
8571 projectId: defaultProjectId,
8572 title: "Retry",
8573 modelSelection: defaultModelSelection,
8574 runtimeMode: "full-access",
8575 interactionMode: "default",
8576 branch: null,
8577 worktreePath: null,
8578 createdAt,
8579 },
8580 runSetupScript: false,
8581 },
8582 createdAt,
8583 }),
8584 );
8585 yield* Fiber.join(bootstrapCreate);
8586 }),
8587 ),
8588 );
8589 assert.deepEqual(trace, ["thread.create", "drain:1", "thread.turn.start"]);
8590 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
8591 );
8592
8479 it.effect("does not report a deleted bootstrap thread when cleanup fails", () =>8593 it.effect("does not report a deleted bootstrap thread when cleanup fails", () =>
8480 Effect.gen(function* () {8594 Effect.gen(function* () {
8481 const dispatchedCommands: Array<OrchestrationCommand> = [];8595 const dispatchedCommands: Array<OrchestrationCommand> = [];
apps/server/src/ws.ts+16-1
...@@ -85,6 +85,7 @@ import {...@@ -85,6 +85,7 @@ import {
85} from "./orchestration/Normalizer.ts";85} from "./orchestration/Normalizer.ts";
86import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";86import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";
87import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";87import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
88import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts";
88import {89import {
89 observeRpcEffect as instrumentRpcEffect,90 observeRpcEffect as instrumentRpcEffect,
90 observeRpcStream as instrumentRpcStream,91 observeRpcStream as instrumentRpcStream,
...@@ -460,6 +461,7 @@ const makeWsRpcLayer = (...@@ -460,6 +461,7 @@ const makeWsRpcLayer = (
460 const crypto = yield* Crypto.Crypto;461 const crypto = yield* Crypto.Crypto;
461 const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;462 const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
462 const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;463 const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;
464 const threadDeletionReactor = yield* ThreadDeletionReactor;
463 const analytics = yield* AnalyticsService.AnalyticsService;465 const analytics = yield* AnalyticsService.AnalyticsService;
464 // Every command dispatched on this connection carries the connecting466 // Every command dispatched on this connection carries the connecting
465 // client's origin, including server-generated bootstrap sub-commands:467 // client's origin, including server-generated bootstrap sub-commands:
...@@ -1026,7 +1028,7 @@ const makeWsRpcLayer = (...@@ -1026,7 +1028,7 @@ const makeWsRpcLayer = (
10261028
1027 const bootstrapProgram = Effect.gen(function* () {1029 const bootstrapProgram = Effect.gen(function* () {
1028 if (bootstrap?.createThread) {1030 if (bootstrap?.createThread) {
1029 yield* dispatchFromClient({1031 const created = yield* dispatchFromClient({
1030 type: "thread.create",1032 type: "thread.create",
1031 commandId: yield* serverCommandId("bootstrap-thread-create"),1033 commandId: yield* serverCommandId("bootstrap-thread-create"),
1032 threadId: command.threadId,1034 threadId: command.threadId,
...@@ -1039,6 +1041,11 @@ const makeWsRpcLayer = (...@@ -1039,6 +1041,11 @@ const makeWsRpcLayer = (
1039 worktreePath: bootstrap.createThread.worktreePath,1041 worktreePath: bootstrap.createThread.worktreePath,
1040 createdAt: bootstrap.createThread.createdAt,1042 createdAt: bootstrap.createThread.createdAt,
1041 });1043 });
1044 // The successful create is a fence in the engine command queue:
1045 // every delete for the prior incarnation committed before it.
1046 // Drain through that event before setup or turn start can own
1047 // terminals and provider sessions under the reused thread id.
1048 yield* threadDeletionReactor.drainThrough(created.sequence);
1042 createdThread = true;1049 createdThread = true;
1043 }1050 }
10441051
...@@ -1126,6 +1133,14 @@ const makeWsRpcLayer = (...@@ -1126,6 +1133,14 @@ const makeWsRpcLayer = (
1126 normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap1133 normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap
1127 ? dispatchBootstrapTurnStart(normalizedCommand)1134 ? dispatchBootstrapTurnStart(normalizedCommand)
1128 : dispatchFromClient(normalizedCommand).pipe(1135 : dispatchFromClient(normalizedCommand).pipe(
1136 Effect.tap(({ sequence }) =>
1137 // Returning from thread.create is the handoff point at which
1138 // clients may start resources for the new incarnation. Use
1139 // its event sequence as the exact deletion-cleanup fence.
1140 normalizedCommand.type === "thread.create"
1141 ? threadDeletionReactor.drainThrough(sequence)
1142 : Effect.void,
1143 ),
1129 Effect.mapError((cause) =>1144 Effect.mapError((cause) =>
1130 toDispatchCommandError(cause, "Failed to dispatch orchestration command"),1145 toDispatchCommandError(cause, "Failed to dispatch orchestration command"),
1131 ),1146 ),