diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index 5d8025c5fe12f3f86be3a2281ef0029a08576674..44899e872e83763d3facf75c339ab7074859ff60 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -316,7 +316,7 @@ describe("OrchestratorMcpService", () => { }), ); - it.effect("does not dispose delivery when the child interrupt fails", () => + it.effect("does not dispose delivery when the child stop fails", () => Effect.gen(function* () { const parentThreadId = ThreadId.make("thread:mcp-cancel-failed-parent"); const childThreadId = ThreadId.make("thread:mcp-cancel-failed-child"); @@ -355,7 +355,7 @@ describe("OrchestratorMcpService", () => { Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection), dispatch: (command) => Ref.update(dispatched, (commands) => [...commands, command]).pipe( - Effect.andThen(Effect.fail(new Error("simulated interrupt failure") as never)), + Effect.andThen(Effect.fail(new Error("simulated stop failure") as never)), ), }), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), @@ -386,13 +386,13 @@ describe("OrchestratorMcpService", () => { assert.equal(error.code, "task_not_cancellable"); assert.deepEqual( (yield* Ref.get(dispatched)).map((command) => (command as { type: string }).type), - ["run.interrupt"], + ["thread.stop"], ); }).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies)))); }), ); - it.effect("returns cancel requested when post-interrupt disposal fails", () => + it.effect("returns cancel requested when post-stop disposal fails", () => Effect.gen(function* () { const parentThreadId = ThreadId.make("thread:mcp-cancel-dispose-failed-parent"); const childThreadId = ThreadId.make("thread:mcp-cancel-dispose-failed-child"); @@ -437,6 +437,7 @@ describe("OrchestratorMcpService", () => { : Effect.succeed({} as never), ), ), + stopDelegatedTasks: () => Effect.void, }), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ @@ -467,7 +468,7 @@ describe("OrchestratorMcpService", () => { assert.equal(result.status, "cancel_requested"); assert.deepEqual( (yield* Ref.get(dispatched)).map((command) => (command as { type: string }).type), - ["run.interrupt", "delegated_task.completion-delivery.dispose"], + ["thread.stop", "delegated_task.completion-delivery.dispose"], ); }).pipe(Effect.provide(OrchestratorMcpService.layer.pipe(Layer.provide(dependencies)))); }), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 097ce03562646818e516de7ef68c916311ee047d..9287dda3564247b3aae2dcdd98b23531cea6c9cb 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -1662,43 +1662,60 @@ const make = Effect.gen(function* () { ), ), ); - // Published task results stay terminal. Later child-thread messages do not - // reopen the task, so cancelling it must not interrupt those separate runs. + // Cancelling stops the child thread the way Stop does, including work that came + // after a published result: follow-up runs, pull request watch wakes, and the + // tasks it delegated. The published result stays terminal. + const stopChild = Effect.gen(function* () { + const commandId = stableCommandId({ scope, requestKey: key, operation: "cancel-task" }); + const reason = input.reason; + yield* threadManagement + .dispatch({ + type: "thread.stop", + commandId, + threadId: current.childThreadId, + ...(reason === undefined ? {} : { reason }), + }) + .pipe( + Effect.mapError((error) => + failure( + "task_not_cancellable", + `Unable to stop delegated task ${input.taskId}: ${errorMessage(error)}`, + ), + ), + ); + // A retry with the same clientRequestId repeats only the stops that failed. + yield* threadManagement + .stopDelegatedTasks({ threadId: current.childThreadId, commandId, reason }) + .pipe( + Effect.mapError((error) => + failure( + "task_not_cancellable", + `Stopped delegated task ${input.taskId}, but not every task it delegated: ${errorMessage(error)}`, + ), + ), + ); + }); if (isTerminalTaskStatus(current.status)) { + yield* stopChild; yield* disposeCompletionDelivery; return { taskId: input.taskId, status: current.status, } satisfies OrchestratorMcpTaskCancelResult; } + // A task waiting on its own delegates has work to stop below it. One with neither + // a running turn nor delegates (such as one awaiting a restart) keeps its delivery. const child = yield* loadProjection(current.childThreadId); - const activeRun = ThreadManagementService.latestActiveRun(child); - if (activeRun === undefined) { + if ( + current.workState !== "waiting_for_children" && + ThreadManagementService.latestActiveRun(child) === undefined + ) { return yield* failure( "task_not_cancellable", `Delegated task ${input.taskId} has no interruptible child run.`, ); } - yield* threadManagement - .dispatch({ - type: "run.interrupt", - commandId: stableCommandId({ - scope, - requestKey: key, - operation: "cancel-task", - }), - threadId: current.childThreadId, - runId: activeRun.id, - ...(input.reason === undefined ? {} : { reason: input.reason }), - }) - .pipe( - Effect.mapError((error) => - failure( - "task_not_cancellable", - `Unable to interrupt delegated task ${input.taskId}: ${errorMessage(error)}`, - ), - ), - ); + yield* stopChild; yield* disposeCompletionDelivery.pipe( Effect.catchCause((cause) => Effect.logWarning("orchestrator-mcp.cancel-task.delivery-dispose-failed", { diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index 2b91ae43db2ae8f18f13546ce9034bc750a062d7..cd4a0df423a73920f113e422993517d8b16fe803 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -1663,7 +1663,7 @@ describe("orchestrator MCP toolkit", () => { ).toBe(true); const completedTaskCancelCall = yield* invoke("task_cancel", { taskId: delegated.taskId, - reason: "Must not interrupt a later unrelated child run.", + reason: "Stop the child's later work too.", clientRequestId: "cancel-completed-delegated-task-1", }); const completedTaskCancel = yield* decodeTaskCancelResult( @@ -1681,36 +1681,19 @@ describe("orchestrator MCP toolkit", () => { result: delegatedResult, completionDelivery: { state: "disposed" }, }); - expect( - (yield* orchestrator.getThreadProjection(delegated.childThreadId)).runs.find( - (run) => run.id === activeChildFollowup.runId, - )?.status, - ).toBe("running"); - const activeChildCleanupCall = yield* invoke("t3_thread_interrupt", { - threadId: delegated.childThreadId, - runId: activeChildFollowup.runId, - reason: "Clean up the active follow-up after verifying task cancellation isolation.", - clientRequestId: "interrupt-delegated-child-followup-1", - }); - const activeChildCleanup = yield* decodeThreadInterruptResult( - activeChildCleanupCall.structuredContent, - ).pipe(Effect.orDie); - expect(activeChildCleanup).toMatchObject({ - runId: activeChildFollowup.runId, - status: "interrupt_requested", - }); + // Cancelling a finished task still stops the child thread's later work. yield* waitForProjection(orchestrator, delegated.childThreadId, (projection) => projection.runs.some( (run) => run.id === activeChildFollowup.runId && run.status === "interrupted", ), ); - const delegatedStatusAfterCleanupCall = yield* invoke("task_status", { + const delegatedStatusAfterCancelCall = yield* invoke("task_status", { taskId: delegated.taskId, }); - const delegatedStatusAfterCleanup = yield* decodeDelegateTaskResult( - delegatedStatusAfterCleanupCall.structuredContent, + const delegatedStatusAfterCancel = yield* decodeDelegateTaskResult( + delegatedStatusAfterCancelCall.structuredContent, ).pipe(Effect.orDie); - expect(delegatedStatusAfterCleanup).toMatchObject({ + expect(delegatedStatusAfterCancel).toMatchObject({ childRunId: delegated.childRunId, status: "completed", summary: delegatedResult, diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts index 0e5b57e6d8a19ce2e4ac6dfdbf968c01a67e79c2..e95639d308dc707cc5383108d39741f2825c394c 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.test.ts @@ -26,7 +26,7 @@ describe("orchestrator MCP tool guidance", () => { ); assert.include( OrchestratorToolkit.tools.task_cancel.description ?? "", - "without interrupting later child-thread runs", + "This includes later child-thread runs, even after the task is terminal", ); }); diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 8daa419cc1319c726de6686226cdcc5760b26e54..80fec3103162ee3d74ba979238a64a6059c1e9b7 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -71,7 +71,7 @@ export const DelegateTaskTool = Tool.make("delegate_task", { const TaskStatusTool = Tool.make("task_status", { description: - "Needs an agent running inside a T3 thread. Read a T3-owned delegated task created by this parent thread. childRunId identifies the original delegated run. workState distinguishes working, waiting_for_children, and result_available; a completed turn with live nested work is not a completed task. summary is the final task result, including provider errors on failure, and remains stable after publication. hasPendingChildRuns reports later queued or executing turns in the backing child thread, even after the task is terminal; it does not reopen the task or extend task_cancel to those turns. latestTerminal* provides later non-monitor turn results. Reading a terminal result acknowledges its automatic parent delivery.", + "Needs an agent running inside a T3 thread. Read a T3-owned delegated task created by this parent thread. childRunId identifies the original delegated run. workState distinguishes working, waiting_for_children, and result_available; a completed turn with live nested work is not a completed task. summary is the final task result, including provider errors on failure, and remains stable after publication. hasPendingChildRuns reports later queued or executing turns in the backing child thread, even after the task is terminal; it does not reopen the task, and task_cancel stops those turns too. latestTerminal* provides later non-monitor turn results. Reading a terminal result acknowledges its automatic parent delivery.", parameters: OrchestratorMcpTaskStatusInput, success: OrchestratorMcpDelegateTaskResult, failure: OrchestratorMcpFailure, @@ -85,7 +85,7 @@ const TaskStatusTool = Tool.make("task_status", { const TaskCancelTool = Tool.make("task_cancel", { description: - "Needs an agent running inside a T3 thread. Request interruption of an active T3-owned delegated task and dispose its automatic parent delivery. For a terminal task, return its existing status and dispose delivery without interrupting later child-thread runs, even when task_status reports hasPendingChildRuns=true. Published task results remain available. Use t3_thread_interrupt for a later active run.", + "Needs an agent running inside a T3 thread. Stop a T3-owned delegated task and dispose its automatic parent delivery. Its child thread stops like a user Stop: the running turn is interrupted, queued turns are held, pull request watches end, and the tasks it delegated stop too. This includes later child-thread runs, even after the task is terminal. A terminal task returns its existing status, and published task results remain available.", parameters: OrchestratorMcpTaskCancelInput, success: OrchestratorMcpTaskCancelResult, failure: OrchestratorMcpFailure, diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index fa839896c88b7c3c036cbcdea3aa618438e68aa4..7ae9f27736361075619322fd84f3ab4626a84d66 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -289,7 +289,7 @@ const ListThreadPullRequestsTool = Tool.make("list_thread_pull_requests", { const WatchPullRequestTool = Tool.make("watch_pull_request", { description: - "Have T3 Code watch an open pull request for this thread, linking it first if needed. T3 Code checks it every minute and wakes you with a message when a check fails, the required checks pass, someone else comments or reviews, or the branch starts to conflict with its base. Use this to monitor or babysit a pull request instead of polling, sleeping, or running a watcher. Only comments posted after this call wake you, so handle the existing ones first, then end your turn. A wake is news, not a merge decision: check readiness yourself before merging. Watching ends when the pull request merges or closes, when its thread settles, when T3 Code cannot read it for 15 minutes, or when you call unwatch_pull_request. Unsettle the thread before starting a new watch.", + "Have T3 Code watch an open pull request for this thread, linking it first if needed. T3 Code checks it every minute and wakes you with a message when a check fails, the required checks pass, someone else comments or reviews, or the branch starts to conflict with its base. Use this to monitor or babysit a pull request instead of polling, sleeping, or running a watcher. Only comments posted after this call wake you, so handle the existing ones first, then end your turn. A wake is news, not a merge decision: check readiness yourself before merging. Watching ends when the pull request merges or closes, when its thread settles, when T3 Code cannot read it for 15 minutes, when the user stops this thread, or when you call unwatch_pull_request. Unsettle the thread before starting a new watch.", parameters: PullRequestTargetInput, success: WatchPullRequestResult, failure: PullRequestToolError, diff --git a/apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts b/apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts index 3c183da6268c6273e4b08dad5bf77f3139264461..bf87f16f5ecaf78f7c9d813f849e8d8545f90512 100644 --- a/apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts +++ b/apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts @@ -39,9 +39,17 @@ const instanceId = ProviderInstanceId.make("codex"); const modelSelection = { instanceId, model: "test-model" }; // Codex turns leave commands running, then the thread moves to another -// provider thread (a provider switch). Stop on the newer, settled run must -// reach both provider threads and end all of the Codex work. -const stopEarlierBackgroundWork = (failedStart: boolean) => +// provider thread (a provider switch). Stop must end all of the Codex work, +// including when it selects an older resumed run on the other provider thread. +const stopEarlierBackgroundWork = ({ + failedStart = false, + stopWithQueue, + olderStart = false, +}: { + readonly failedStart?: boolean; + readonly stopWithQueue?: "thread.stop" | "run.interrupt"; + readonly olderStart?: boolean; +}) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("background-work-stop"); @@ -377,10 +385,11 @@ const stopEarlierBackgroundWork = (failedStart: boolean) => // the thread moves on to another provider thread, which also has a // live session. const watcherId = TurnItemId.make("turn-item:watcher"); + const otherProviderThreadId = ProviderThreadId.make("provider-thread:other"); const watcherRun = settledRun({ ordinal: 2, - providerThreadId: codexProviderThread.id, - runningItem: { id: watcherId, kind: "command" }, + providerThreadId: olderStart ? otherProviderThreadId : codexProviderThread.id, + ...(olderStart ? {} : { runningItem: { id: watcherId, kind: "command" as const } }), }); const reviewerId = TurnItemId.make("turn-item:reviewer"); const reviewerRun = settledRun({ @@ -388,7 +397,6 @@ const stopEarlierBackgroundWork = (failedStart: boolean) => providerThreadId: codexProviderThread.id, runningItem: { id: reviewerId, kind: "subagent" }, }); - const otherProviderThreadId = ProviderThreadId.make("provider-thread:other"); const latestRun = settledRun({ ordinal: 4, providerThreadId: otherProviderThreadId }); yield* sink.write({ events: [ @@ -440,30 +448,126 @@ const stopEarlierBackgroundWork = (failedStart: boolean) => }); } - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("stop-background-work"), - threadId, - runId: failedStart ? failedRun.runId : latestRun.runId, - }); + if (stopWithQueue !== undefined) { + const owner = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === latestRun.runId, + )!; + // A message queues while checkpointing finishes, then the user holds the queue. + yield* sink.write({ + events: [ + { + id: EventId.make("latest-run-waiting"), + type: "run.updated", + threadId, + runId: owner.id, + occurredAt: now, + payload: { ...owner, status: "waiting", completedAt: null }, + }, + ], + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("queue-follow-up"), + threadId, + messageId: MessageId.make("message:queue-follow-up"), + text: "Follow up after the background work", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + assert.equal(queued.status, "queued"); + yield* sink.write({ + events: [ + { + id: EventId.make("queue-held"), + type: "run.updated", + threadId, + runId: queued.id, + occurredAt: now, + payload: { ...queued, queueHeld: true }, + }, + { + id: EventId.make("latest-run-settled"), + type: "run.updated", + threadId, + runId: owner.id, + occurredAt: now, + payload: owner, + }, + ], + }); + } + + if (olderStart) { + const older = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === watcherRun.runId, + )!; + // A resumed queue can start a lower-ordinal run after later runs have ended. + yield* sink.write({ + events: [ + { + id: EventId.make("older-run-starting"), + type: "run.updated", + threadId, + runId: older.id, + occurredAt: now, + payload: { ...older, status: "starting", startedAt: null, completedAt: null }, + }, + ], + }); + } + + yield* orchestrator.dispatch( + stopWithQueue === "thread.stop" + ? { + type: "thread.stop", + commandId: CommandId.make("stop-background-work"), + threadId, + } + : { + type: "run.interrupt", + commandId: CommandId.make("stop-background-work"), + threadId, + runId: olderStart + ? watcherRun.runId + : failedStart + ? failedRun.runId + : latestRun.runId, + ...(stopWithQueue === undefined ? {} : { holdQueue: true }), + }, + ); yield* worker.drain(); - // Stop reaches both provider threads. The Codex one is interrupted at - // its latest pending work, the subagent's parent turn, so its settle - // covers all three Codex runs. + // The Codex interrupt targets its latest pending work, the subagent's parent turn, + // so its settlement covers the Codex background work. assert.sameDeepMembers( interrupts.map((interrupt) => [interrupt.providerThread.id, interrupt.providerTurnId]), [ - [otherProviderThreadId, latestRun.providerTurnId], + ...(olderStart ? [] : [[otherProviderThreadId, latestRun.providerTurnId]]), [codexProviderThread.id, reviewerRun.providerTurnId], ], ); const after = yield* orchestrator.getThreadProjection(threadId); + if (olderStart) { + assert.equal( + after.runs.find((run) => run.id === watcherRun.runId)?.status, + "interrupted", + ); + } + if (stopWithQueue !== undefined) { + assert.equal(after.runs.at(-1)?.status, "queued"); + assert.equal(after.runs.at(-1)?.queueHeld, true); + assert.lengthOf(started, 1); + } assert.deepEqual( - [devServerId, watcherId, reviewerId].map( + [devServerId, ...(olderStart ? [] : [watcherId]), reviewerId].map( (id) => after.turnItems.find((item) => item.id === id)?.status, ), - ["interrupted", "interrupted", "interrupted"], + olderStart + ? ["interrupted", "interrupted"] + : ["interrupted", "interrupted", "interrupted"], ); }).pipe( Effect.provide( @@ -478,10 +582,20 @@ const stopEarlierBackgroundWork = (failedStart: boolean) => ); it.effect("Stop reaches background work an earlier provider thread still runs", () => - stopEarlierBackgroundWork(false), + stopEarlierBackgroundWork({}), ); it.effect( "Stop reaches earlier background work after the newest run fails before provider start", - () => stopEarlierBackgroundWork(true), + () => stopEarlierBackgroundWork({ failedStart: true }), +); + +it.effect.each(["thread.stop", "run.interrupt"] as const)( + "%s stops background work when a later message is queued", + (stopType) => stopEarlierBackgroundWork({ stopWithQueue: stopType }), +); + +it.effect.each(["thread.stop", "run.interrupt"] as const)( + "%s reaches later background work when an older run is starting", + (stopType) => stopEarlierBackgroundWork({ stopWithQueue: stopType, olderStart: true }), ); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index db0ea8366a04208f2695426c61cb5edea860d083..a7b035629dda8e321d3c6384adfc7ff117278662 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -102,6 +102,11 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ Schema.Struct({ type: Schema.Literal("regenerate") }), ]), }), + /** Follows a Stop: sends `thread.stop` to every delegated task under the stopped thread. */ + Schema.Struct({ + type: Schema.Literal("delegated-tasks.stop"), + reason: Schema.optional(Schema.String), + }), ]); export type OrchestrationEffectRequestV2 = typeof OrchestrationEffectRequestV2.Type; @@ -113,6 +118,7 @@ export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [ "terminal.cleanup", "attachment.cleanup", "thread-title.generate", + "delegated-tasks.stop", ] as const satisfies ReadonlyArray; export const PROCESS_BOUND_EFFECT_TYPES = [ diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 149494739182fc771fee52d529c5d122a12217c9..282815cd0c11c28ac1d76ecaa8dd319e88baa016 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -459,6 +459,23 @@ export const executorLayer: Layer.Layer< }), ), ); + case "delegated-tasks.stop": + return threads + .stopDelegatedTasks({ + threadId: effect.threadId, + commandId: effect.commandId, + reason: effect.request.reason, + }) + .pipe( + Effect.mapError( + (cause) => + new OrchestrationEffectExecutionError({ + effectId: effect.id, + effectType: effect.request.type, + cause, + }), + ), + ); } }, }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 913abee220486b8924893426162bb0c8d1c60dfd..e0a557b414a82fb30506d787944850e590ce99a7 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -59,6 +59,7 @@ import { import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Layer from "effect/Layer"; @@ -418,6 +419,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId { case "checkpoint.rollback": case "checkpoint.rollback.fail": case "thread.background-work.settle": + case "thread.stop": case "provider.switch": return command.threadId; case "delegated_task.request": @@ -1038,7 +1040,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); const emitQueuedRunCancellation = (input: { - readonly command: OrchestrationV2Command; + readonly command: OrchestrationV2ServerCommand; readonly events: Ref.Ref>; readonly projection: Pick; readonly run: OrchestrationV2Run; @@ -2012,7 +2014,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); const disposeDelegatedCompletionCohort = (input: { - readonly command: OrchestrationV2Command; + readonly command: OrchestrationV2ServerCommand; readonly events: Ref.Ref>; readonly projection: Pick< OrchestrationV2ThreadProjection, @@ -2379,6 +2381,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Thread ${command.threadId} is settled and cannot watch pull requests.`, }); } + // Only the agent's watch_pull_request links as "agent". A call that raced a Stop may land + // after its run ended, so the latest run that executed decides. A user can still watch. + if ( + command.type === "thread.pull-request.watch" && + command.watching && + command.link?.source === "agent" + ) { + const { runs } = yield* projectionStore + .getThreadRecords(command.threadId, ["runs"]) + .pipe(mapDispatchError(command)); + const latest = latestExecutedRun(runs); + if (latest !== null && (yield* stopReachedRun(command, command.threadId, latest.id))) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Thread ${command.threadId} was stopped.`, + }); + } + } if ( command.type === "thread.metadata.update" && command.expectedWorktreePath !== undefined && @@ -4402,7 +4423,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // Held queued runs never started; they wait behind the continuation. projection.runs.some( (run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source), - ) + ) || + (yield* stopReachedRun(command, command.threadId, source.id)) ) { // Preserve the current row so stale automatic deliveries receive an // accepted receipt without changing work or repeatedly retrying. @@ -6362,6 +6384,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Parent run ${command.parentRunId} is not active.`, }); } + if (yield* stopReachedRun(command, command.parentThreadId, parentRun.id)) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Parent run ${command.parentRunId} is stopping.`, + }); + } const parentNode = parentProjection.nodes.find( (candidate) => candidate.id === command.parentNodeId, ); @@ -7960,6 +7989,159 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } }); + /** + * Whether Stop reached a run, as its interrupt request records. Its agent can call tools + * until the provider stops it, and a slow call can land after that, but nothing it starts + * may outlive the Stop. A restart continuation of that run must not start either. + */ + const stopReachedRun = ( + command: OrchestrationV2ServerCommand, + threadId: ThreadId, + runId: RunId, + ) => + projectionStore + .getThreadRecords(threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: [runId], + }) + .pipe( + Effect.map(({ turnItems }) => turnItems.length > 0), + mapDispatchError(command), + ); + + /** Records Stop for a run without replacing a marker already written by its interrupt. */ + const markStoppedRun = Effect.fnUntraced(function* (input: { + readonly command: Extract< + OrchestrationV2ServerCommand, + { readonly type: "run.interrupt" | "thread.stop" } + >; + readonly events: Ref.Ref>; + readonly thread: OrchestrationV2AppThread; + readonly run: OrchestrationV2Run; + readonly now: DateTime.Utc; + }) { + const { command, events, thread, run, now } = input; + if (run.rootNodeId === null || run.providerThreadId === null) return; + const pendingItems = (yield* Ref.get(events)).flatMap((event) => + event.type === "turn-item.updated" ? [event.payload] : [], + ); + if ( + pendingItems.some((item) => item.runId === run.id && item.type === "run_interrupt_request") || + (yield* stopReachedRun(command, thread.id, run.id)) + ) { + return; + } + yield* emit( + events, + command, + )({ + type: "turn-item.updated", + threadId: thread.id, + runId: run.id, + nodeId: run.rootNodeId, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { + id: idAllocator.derive.runSignalTurnItem({ runId: run.id, signal: "interrupt-request" }), + threadId: thread.id, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: run.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: yield* nextTurnItemOrdinal({ thread, turnItems: pendingItems }), + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "run_interrupt_request", + message: command.reason ?? "Interrupt requested", + }, + }); + }); + + /** + * What Stop holds besides the run it interrupts: queued runs wait for the user, the + * thread's pull request watches end, and every wake its delegated tasks still owe is + * dropped, since Stop stops those tasks too. Nothing automatic starts the thread again. + * `cohortRunIds` are dropped even when they owe nothing, like a plain interrupt's. + */ + const holdStoppedThread = (input: { + readonly command: Extract< + OrchestrationV2ServerCommand, + { readonly type: "run.interrupt" | "thread.stop" } + >; + readonly events: Ref.Ref>; + readonly projection: Pick; + readonly cohortRunIds: ReadonlyArray; + readonly now: DateTime.Utc; + }) => + Effect.gen(function* () { + const emitEvent = emit(input.events, input.command); + const { thread, runs, subagents } = input.projection; + for (const run of runs) { + // A resumed queue can have a lower ordinal than a pending continuation source. + // Stop must reach those sources even when interrupting the resumed run succeeds. + if ( + run.status === "cancelled" && + (yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command))) + ) { + yield* markStoppedRun({ ...input, thread, run }); + } + if (run.status !== "queued" || run.queueHeld === true) continue; + yield* emitEvent({ + type: "run.updated", + threadId: thread.id, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: input.now, + payload: { ...run, queueHeld: true }, + }); + } + const pullRequests = thread.pullRequests ?? []; + if (pullRequests.some((link) => link.watch !== undefined)) { + yield* emitEvent({ + type: "thread.pull-request-synced", + threadId: thread.id, + providerInstanceId: thread.providerInstanceId, + occurredAt: input.now, + payload: { + ...thread, + pullRequests: pullRequests.map((link) => withPullRequestWatch(link, undefined)), + }, + }); + } + const cohortRunIds = new Set(input.cohortRunIds); + for (const run of runs) { + if (run.delegatedCompletion?.delivery != null) cohortRunIds.add(run.id); + } + for (const task of subagents) { + const delivery = task.completionDelivery?.state; + if ( + task.origin === "app_owned" && + task.runId !== null && + delivery !== "acknowledged" && + delivery !== "delivered" && + delivery !== "disposed" + ) { + cohortRunIds.add(task.runId); + } + } + // Each disposal reads the events the ones before it wrote. + for (const parentRunId of cohortRunIds) { + yield* disposeDelegatedCompletionCohort({ + command: input.command, + events: input.events, + projection: yield* getProjectionWithPendingEvents(thread.id, input.events), + parentRunId, + disposition: "stopped", + now: input.now, + }); + } + }); + const dispatchBackgroundWorkSettle = ( command: Extract< OrchestrationV2InternalCommand, @@ -8045,7 +8227,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? undefined : projection.providerThreads.find((candidate) => candidate.id === run?.providerThreadId); const hasBackgroundWork = - run?.id === projection.runs.at(-1)?.id && + run?.id === projection.runs.findLast((candidate) => candidate.status !== "queued")?.id && derivePendingBackgroundWork({ latestRun: run, providerThreads: projection.providerThreads, @@ -8078,34 +8260,29 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (candidate) => candidate.id === run.userMessageId, ); const completionCohortRunId = completionMessage?.delegatedCompletion?.parentRunId ?? run.id; - const stopCompletionCohort = () => - Effect.gen(function* () { - yield* disposeDelegatedCompletionCohort({ - command, - events, - projection: yield* getProjectionWithPendingEvents(command.threadId, events), - parentRunId: completionCohortRunId, - disposition: "stopped", - now, - }); - }); + // Stop holds the rest of the thread too. A plain interrupt only drops the wakes owed by + // this run's delegated tasks, or on a wake run, by the cohort that woke it. + const stopRemainingWork = () => + command.holdQueue === true + ? holdStoppedThread({ + command, + events, + projection, + cohortRunIds: [completionCohortRunId], + now, + }) + : Effect.gen(function* () { + yield* disposeDelegatedCompletionCohort({ + command, + events, + projection: yield* getProjectionWithPendingEvents(command.threadId, events), + parentRunId: completionCohortRunId, + disposition: "stopped", + now, + }); + }); const emitEvent = emit(events, command); - const holdQueuedRuns = Effect.forEach( - projection.runs.filter( - (candidate) => candidate.status === "queued" && !candidate.queueHeld, - ), - (queuedRun) => - emitEvent({ - type: "run.updated", - threadId: command.threadId, - runId: queuedRun.id, - providerInstanceId: queuedRun.providerInstanceId, - occurredAt: now, - payload: { ...queuedRun, queueHeld: true }, - }), - { discard: true }, - ); const interruptRequestItem: OrchestrationV2TurnItem = { id: idAllocator.derive.runSignalTurnItem({ runId: run.id, @@ -8128,141 +8305,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio message: command.reason ?? "Interrupt requested", }; - if ( - providerTurn === undefined && - (run.status === "preparing" || run.status === "starting" || run.status === "running") - ) { - const attempt = projection.attempts.find( - (candidate) => candidate.id === run.activeAttemptId, - ); - if (attempt === undefined) { - return yield* new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: `Run ${command.runId} has no active attempt to interrupt.`, - }); - } - const interruptResultItem: OrchestrationV2TurnItem = { - id: idAllocator.derive.runSignalTurnItem({ - runId: run.id, - signal: "interrupt-result", - }), - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerThreadId: providerThread.id, - providerTurnId: null, - nativeItemRef: null, - parentItemId: interruptRequestItem.id, - ordinal: interruptRequestItem.ordinal + 1, - status: "interrupted", - title: "Interrupted", - startedAt: now, - completedAt: now, - updatedAt: now, - type: "run_interrupt_result", - message: "Run interrupted before provider start", - }; - yield* emitEvent({ - type: "turn-item.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: interruptRequestItem, - }); - yield* emitEvent({ - type: "turn-item.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: interruptResultItem, - }); - const preparationItem = projection.turnItems.find( - ( - candidate, - ): candidate is Extract< - OrchestrationV2TurnItem, - { readonly type: "command_execution" } - > => - candidate.runId === run.id && - candidate.type === "command_execution" && - candidate.input === WORKSPACE_PREPARATION_INPUT && - candidate.status === "running", - ); - if (preparationItem !== undefined) { - yield* emitEvent({ - type: "turn-item.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { - ...preparationItem, - status: "interrupted", - title: "Workspace preparation interrupted", - output: command.reason ?? "Interrupted before provider start", - completedAt: now, - updatedAt: now, - }, - }); - } - yield* emitEvent({ - type: "run-attempt.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { ...attempt, status: "interrupted", completedAt: now }, - }); - yield* emitEvent({ - type: "node.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { ...rootNode, status: "interrupted", completedAt: now }, - }); - yield* emitEvent({ - type: "run.updated", - threadId: command.threadId, - runId: run.id, - nodeId: rootNode.id, - providerInstanceId: run.providerInstanceId, - occurredAt: now, - payload: { ...run, status: "interrupted", completedAt: now }, - }); - if (command.holdQueue === true) yield* holdQueuedRuns; - yield* stopCompletionCohort(); - return { - effectTypes: ["provider-turn.start", "provider-turn.restart"], - reason: `Run ${run.id} was interrupted before its provider turn started.`, - } satisfies { - readonly effectTypes: ReadonlyArray; - readonly reason: string; - }; - } - - if (providerTurn === undefined) { - return yield* new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: `Run ${command.runId} is not interruptible.`, - }); - } // Background work can outlive a provider switch, such as a Codex dev // server left running when the thread moved to Claude. Stop also reaches // each other live provider thread that owns pending work; that // interrupt's settle follow-up ends what its provider leaves behind. // Work on a dead session is settled with this run's below. const otherProviderInterrupts: Array = []; - if (hasBackgroundWork) { + if (hasBackgroundWork || command.holdQueue === true) { const runOrdinals = new Map( projection.runs.map((candidate) => [candidate.id, candidate.ordinal]), ); @@ -8313,6 +8362,135 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); } } + + if ( + providerTurn === undefined && + (run.status === "preparing" || run.status === "starting" || run.status === "running") + ) { + const attempt = projection.attempts.find( + (candidate) => candidate.id === run.activeAttemptId, + ); + if (attempt === undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Run ${command.runId} has no active attempt to interrupt.`, + }); + } + const interruptResultItem: OrchestrationV2TurnItem = { + id: idAllocator.derive.runSignalTurnItem({ + runId: run.id, + signal: "interrupt-result", + }), + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerThreadId: providerThread.id, + providerTurnId: null, + nativeItemRef: null, + parentItemId: interruptRequestItem.id, + ordinal: interruptRequestItem.ordinal + 1, + status: "interrupted", + title: "Interrupted", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "run_interrupt_result", + message: "Run interrupted before provider start", + }; + yield* emitEvent({ + type: "turn-item.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: interruptRequestItem, + }); + yield* emitEvent({ + type: "turn-item.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: interruptResultItem, + }); + const preparationItem = projection.turnItems.find( + ( + candidate, + ): candidate is Extract< + OrchestrationV2TurnItem, + { readonly type: "command_execution" } + > => + candidate.runId === run.id && + candidate.type === "command_execution" && + candidate.input === WORKSPACE_PREPARATION_INPUT && + candidate.status === "running", + ); + if (preparationItem !== undefined) { + yield* emitEvent({ + type: "turn-item.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { + ...preparationItem, + status: "interrupted", + title: "Workspace preparation interrupted", + output: command.reason ?? "Interrupted before provider start", + completedAt: now, + updatedAt: now, + }, + }); + } + yield* emitEvent({ + type: "run-attempt.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...attempt, status: "interrupted", completedAt: now }, + }); + yield* emitEvent({ + type: "node.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...rootNode, status: "interrupted", completedAt: now }, + }); + yield* emitEvent({ + type: "run.updated", + threadId: command.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, status: "interrupted", completedAt: now }, + }); + yield* stopRemainingWork(); + yield* Ref.update(effects, (existing) => [...existing, ...otherProviderInterrupts]); + return { + effectTypes: ["provider-turn.start", "provider-turn.restart"], + reason: `Run ${run.id} was interrupted before its provider turn started.`, + } satisfies { + readonly effectTypes: ReadonlyArray; + readonly reason: string; + }; + } + + if (providerTurn === undefined) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: `Run ${command.runId} is not interruptible.`, + }); + } // Stop on a settled thread's background work. Its process may be gone // (released, restarted) and only the projection still shows the work; // the settle follow-up ends whatever no provider reports ending. @@ -8334,8 +8512,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: interruptRequestItem, }); - if (command.holdQueue === true) yield* holdQueuedRuns; - yield* stopCompletionCohort(); + yield* stopRemainingWork(); yield* settleBackgroundWork({ command, events, @@ -8391,8 +8568,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: interruptRequestItem, }); - if (command.holdQueue === true) yield* holdQueuedRuns; - yield* stopCompletionCohort(); + yield* stopRemainingWork(); yield* Ref.update(effects, (existing) => [ ...existing, { @@ -8411,6 +8587,91 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return undefined; }); + /** + * Stop for a thread whatever it is doing, the way the Stop button stops the run it + * shows: the running turn (or one whose background work is still pending) is interrupted, + * and the rest of the thread is held. A turn that cannot be interrupted still leaves the + * thread held. Delegated tasks under the thread are stopped by the sender. + */ + const dispatchThreadStop = ( + command: Extract, + events: Ref.Ref>, + effects: Ref.Ref>, + ) => + Effect.gen(function* () { + const projection = yield* projectionStore + .getThreadRecords(command.threadId, ["runs", "providerThreads", "turnItems", "subagents"], { + turnItemTypes: ["command_execution", "dynamic_tool", "subagent"], + turnItemStatuses: ["pending", "running", "waiting"], + }) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), + ), + ); + if (projection.thread.deletedAt !== null) return undefined; + // Queued messages never started and cannot own background work. + const latestRun = projection.runs.findLast((run) => run.status !== "queued"); + const target = + projection.runs.findLast( + (run) => + run.status === "preparing" || run.status === "starting" || run.status === "running", + ) ?? + (derivePendingBackgroundWork({ + latestRun, + providerThreads: projection.providerThreads, + turnItems: projection.turnItems, + activeProviderThreadId: projection.thread.activeProviderThreadId, + runs: projection.runs, + }).length > 0 + ? latestRun + : undefined); + if (target !== undefined) { + const interruptEvents = yield* Ref.make>([]); + const interruptEffects = yield* Ref.make>([]); + const interrupted = yield* Effect.exit( + dispatchRunInterrupt( + { + type: "run.interrupt", + commandId: command.commandId, + threadId: command.threadId, + runId: target.id, + holdQueue: true, + ...(command.reason === undefined ? {} : { reason: command.reason }), + }, + interruptEvents, + interruptEffects, + ), + ); + // A failed interrupt may have written part of its events, so they are kept apart. + if (Exit.isSuccess(interrupted)) { + const written = yield* Ref.get(interruptEvents); + const enqueued = yield* Ref.get(interruptEffects); + yield* Ref.update(events, (existing) => [...existing, ...written]); + yield* Ref.update(effects, (existing) => [...existing, ...enqueued]); + return interrupted.value; + } + yield* Effect.logWarning("thread.stop could not interrupt the running turn", { + threadId: command.threadId, + runId: target.id, + cause: interrupted.cause, + }); + } + const now = yield* DateTime.now; + // Record that Stop reached the turn it could not interrupt and the thread's last executed + // run, so a late tool call from their agent starts nothing. + const executed = latestExecutedRun(projection.runs); + const reached = new Map(); + if (target !== undefined) reached.set(target.id, target); + if (executed !== null && !hasLiveRun({ runs: [executed] })) + reached.set(executed.id, executed); + for (const run of reached.values()) { + yield* markStoppedRun({ command, events, thread: projection.thread, run, now }); + } + yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now }); + return undefined; + }); + const dispatchCheckpointRollback = ( command: Extract, events: Ref.Ref>, @@ -9591,6 +9852,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio break; case "run.interrupt": cancelUnsettledEffects = yield* dispatchRunInterrupt(command, events, effects); + // Stop also stops every delegated task under the thread once it commits. + if (command.holdQueue === true) { + yield* Ref.update(effects, (existing) => [ + ...existing, + { + id: `effect:${command.commandId}:delegated-tasks.stop`, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "delegated-tasks.stop", + ...(command.reason === undefined ? {} : { reason: command.reason }), + }, + } satisfies PendingOrchestrationEffectV2, + ]); + } + break; + case "thread.stop": + cancelUnsettledEffects = yield* dispatchThreadStop(command, events, effects); break; case "queued-message.promote-to-steer": yield* dispatchQueuedMessagePromoteToSteer(command, events, effects); @@ -9763,9 +10042,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const plan = yield* dispatchOnce(command).pipe( Effect.flatMap((planned) => - // A settle that finds the provider already ended everything has - // nothing to record, which is its expected outcome, not a failure. - planned.events.length > 0 || command.type === "thread.background-work.settle" + // A settle that finds the provider already ended everything, or a + // stop that finds nothing running, has nothing to record. That is + // its expected outcome, not a failure. + planned.events.length > 0 || + command.type === "thread.background-work.settle" || + command.type === "thread.stop" ? Effect.succeed(planned) : Effect.fail( new OrchestratorDispatchError({ diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index f0468364a25c43ed67d28dd3dcea65637131aa6c..2909ea2435aa48123a82e0cc4d52da2b6cdf5e5c 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -5,7 +5,7 @@ import type { } from "./ProjectionStore.ts"; import { type ChatAttachment, - type CommandId, + CommandId, MessageId, type ModelSelection, type OrchestrationV2Actor, @@ -323,6 +323,16 @@ export interface ThreadManagementServiceShape { readonly interruptThread: ( input: ThreadManagementInterruptInput, ) => Effect.Effect; + /** + * Sends `thread.stop` to every delegated task under a thread, depth first. Their command IDs + * derive from `commandId`, so a retry repeats nothing that already stopped. A task that + * cannot be stopped does not keep the others running: all are tried, then it fails. + */ + readonly stopDelegatedTasks: (input: { + readonly threadId: ThreadId; + readonly commandId: CommandId; + readonly reason?: string | undefined; + }) => Effect.Effect; readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"]; readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"]; readonly delegatedTaskResultPending: Orchestrator.OrchestratorV2["Service"]["delegatedTaskResultPending"]; @@ -721,6 +731,32 @@ const make = Effect.gen(function* () { return { type: "interrupt_requested", run: interruptibleRun, dispatch } as const; }); + const stopDelegatedTasks: ThreadManagementServiceShape["stopDelegatedTasks"] = (input) => + Effect.gen(function* () { + const { subagents } = yield* orchestrator.getThreadRecords(input.threadId, ["subagents"]); + const failures: Array = []; + for (const task of subagents) { + if (task.origin !== "app_owned" || task.childThreadId === null) continue; + const threadId = task.childThreadId; + yield* dispatch({ + type: "thread.stop", + commandId: CommandId.make(`${input.commandId}:stop:${threadId}`), + threadId, + ...(input.reason === undefined ? {} : { reason: input.reason }), + }).pipe( + Effect.andThen(stopDelegatedTasks({ ...input, threadId })), + Effect.catch((error) => + Effect.logWarning("Unable to stop a delegated task", { + parentThreadId: input.threadId, + threadId, + error, + }).pipe(Effect.andThen(Effect.sync(() => failures.push(error)))), + ), + ); + } + if (failures[0] !== undefined) return yield* Effect.fail(failures[0]); + }); + return ThreadManagementService.of({ ensureLegacyTranscript, dispatch, @@ -753,6 +789,7 @@ const make = Effect.gen(function* () { sendToThread, waitForThread, interruptThread, + stopDelegatedTasks, getThreadEventSequence: orchestrator.getThreadEventSequence, recoverDelegatedTask: orchestrator.recoverDelegatedTask, delegatedTaskResultPending: orchestrator.delegatedTaskResultPending, diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..07783107598512825d91d1c5b4a15addb73cca30 --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -0,0 +1,605 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + EventId, + MessageId, + NodeId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderTurnId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; + +import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import { OrchestrationEffectRequestV2 } from "./EffectOutbox.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ThreadManagementService from "./ThreadManagementService.ts"; +import { makeOrchestratorV2ReplayLayerWithRegistry } from "./testkit/ProviderReplayHarness.ts"; + +const instanceId = ProviderInstanceId.make("codex"); +const modelSelection = { instanceId, model: "gpt-5.1-codex" }; +const adapter = { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("Runs here never reach a provider"), +} as ProviderAdapterV2Shape; +const database = SqlitePersistenceMemory; +// No effect worker: runs stay unstarted, so Stop ends them without a provider. +const testLayer = ThreadManagementService.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + database, + ProjectionStore.layer.pipe(Layer.provide(database)), + makeOrchestratorV2ReplayLayerWithRegistry( + { name: "thread-stop" }, + ProviderAdapterRegistry.makeLayer([adapter]), + { databaseLayer: database, runEffectWorker: false }, + ), + ), + ), +); + +const encodeEffectRequest = Schema.encodeSync(Schema.fromJsonString(OrchestrationEffectRequestV2)); + +const pullRequest = (number: number) => ({ + host: "github.com", + repository: "pingdotgg/t3code", + number, +}); + +const createWatchingThread = (threadId: ThreadId, number: number) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("project:thread-stop"), + title: threadId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* watch(threadId, number); + }); + +const watch = (threadId: ThreadId, number: number) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`watch:${threadId}:${number}`), + threadId, + ...pullRequest(number), + watching: true, + link: { url: `https://github.com/pingdotgg/t3code/pull/${number}`, source: "agent" }, + }); + }); + +const send = (threadId: ThreadId, text: string, type: "start_immediately" | "queue_after_active") => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`send:${threadId}:${text}`), + threadId, + messageId: MessageId.make(`message:${threadId}:${text}`), + text, + attachments: [], + dispatchMode: { type }, + createdBy: "user", + creationSource: "web", + }); + }); + +/** Delegates `task` from the parent's latest run and returns the child thread. */ +const delegate = (parentThreadId: ThreadId, task: string) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "delegated_task.request", + commandId: CommandId.make(`delegate:${task}`), + parentThreadId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "always", + createdBy: "agent", + creationSource: "mcp", + }); + const projection = yield* orchestrator.getThreadProjection(parentThreadId); + return projection.subagents.find((candidate) => candidate.prompt === task)!.childThreadId!; + }); + +const threadState = (threadId: ThreadId) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projection = yield* orchestrator.getThreadProjection(threadId); + return { + runs: projection.runs.map((run) => `${run.status}${run.queueHeld === true ? ":held" : ""}`), + watched: (projection.thread.pullRequests ?? []) + .filter((link) => link.watch !== undefined) + .map((link) => link.number), + }; + }); + +it.effect("Stop ends watches, holds queues, and stops the delegated tasks under the thread", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; + const parentThreadId = ThreadId.make("thread:stop-parent"); + yield* createWatchingThread(parentThreadId, 1); + yield* send(parentThreadId, "first", "start_immediately"); + const childThreadId = yield* delegate(parentThreadId, "child task"); + yield* watch(childThreadId, 2); + const grandchildThreadId = yield* delegate(childThreadId, "grandchild task"); + yield* send(childThreadId, "child follow-up", "queue_after_active"); + + // The run that delegated the child ends before the run the user stops, so + // only a Stop that covers earlier runs keeps the stopped child from waking it. + const now = yield* DateTime.now; + const firstRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; + yield* projections.apply({ + id: EventId.make("event:stop-parent:first-run-completed"), + type: "run.updated", + threadId: parentThreadId, + runId: firstRun.id, + occurredAt: now, + payload: { ...firstRun, status: "completed", completedAt: now }, + }); + yield* send(parentThreadId, "second", "start_immediately"); + const secondRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!; + const childWatch = (yield* orchestrator.getThreadProjection(childThreadId)).thread + .pullRequests?.[0]?.watch; + assert.isDefined(childWatch); + + const stopCommandId = CommandId.make("stop-parent"); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: stopCommandId, + threadId: parentThreadId, + runId: secondRun.id, + holdQueue: true, + }); + + assert.deepEqual(yield* threadState(parentThreadId), { + runs: ["completed", "interrupted"], + watched: [], + }); + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + assert.equal(parent.subagents[0]?.completionDelivery?.state, "disposed"); + const effects = yield* sql<{ readonly effect_type: string }>` + SELECT effect_type FROM orchestration_v2_effect_outbox WHERE command_id = ${stopCommandId} + `; + assert.include( + effects.map((row) => row.effect_type), + "delegated-tasks.stop", + ); + + // What the delegated-tasks.stop effect runs once the Stop commits. + yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId }); + assert.deepEqual(yield* threadState(childThreadId), { + runs: ["interrupted", "queued:held"], + watched: [], + }); + assert.deepEqual((yield* threadState(grandchildThreadId)).runs, ["interrupted"]); + + // A watch read that raced the Stop cannot wake the stopped child. + const lateWake = yield* Effect.exit( + orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make("late-watch-wake"), + threadId: childThreadId, + ...pullRequest(2), + startedAt: childWatch!.startedAt, + watch: null, + wake: { + messageId: MessageId.make("message:late-watch-wake"), + text: "Checks passed.", + notification: { source: { kind: "monitor" }, outcome: "completed", summary: "#2" }, + }, + }), + ); + assert.isTrue(Exit.isFailure(lateWake)); + assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted", "queued:held"]); + + // A retried effect stops nothing twice. + yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId }); + assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted", "queued:held"]); + }).pipe(Effect.provide(testLayer)), +); + +it.effect( + "thread.stop ends an idle thread's watches and accepts a thread with nothing to stop", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("thread:stop-idle"); + yield* createWatchingThread(threadId, 3); + + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-idle"), + threadId, + }); + assert.deepEqual(yield* threadState(threadId), { runs: [], watched: [] }); + + const again = yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-idle-again"), + threadId, + }); + assert.lengthOf(again.storedEvents, 0); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("a run Stop reached cannot delegate or start a watch, even after it ends", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:stop-barrier"); + yield* createWatchingThread(threadId, 4); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const run = { ...(yield* orchestrator.getThreadProjection(threadId)).runs[0]!, startedAt: now }; + yield* projections.apply({ + id: EventId.make("event:stop-barrier:running"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running" }, + }); + // A queued message the user cancelled does not count as the latest run. + yield* send(threadId, "queued", "queue_after_active"); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("cancel-queued"), + threadId, + runId: queued.id, + }); + // Stop reached the run, but its provider has not stopped it yet. + yield* projections.apply({ + id: EventId.make("event:stop-barrier:interrupt-request"), + type: "turn-item.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:stop-barrier:interrupt-request"), + threadId, + runId: run.id, + nodeId: run.rootNodeId!, + providerThreadId: run.providerThreadId!, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1_000, + status: "completed", + title: "Interrupt requested", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "run_interrupt_request", + message: "Stop", + }, + }); + + assert.isTrue(Exit.isFailure(yield* Effect.exit(delegate(threadId, "late task")))); + assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 5)))); + + // A slow watch_pull_request call can land after the stopped run ended. + yield* projections.apply({ + id: EventId.make("event:stop-barrier:interrupted"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "interrupted", completedAt: now }, + }); + assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 5)))); + assert.deepEqual(yield* threadState(threadId), { + runs: ["interrupted", "cancelled"], + watched: [4], + }); + + // The user can still stop and restart a watch by hand. + for (const watching of [false, true]) { + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`manual-watch:${watching}`), + threadId, + ...pullRequest(4), + watching, + }); + } + assert.deepEqual((yield* threadState(threadId)).watched, [4]); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("thread.stop keeps a restart continuation of the stopped run from starting", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const threadId = ThreadId.make("thread:stop-restart"); + yield* createWatchingThread(threadId, 8); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + // A server restart cut the run before its provider started; its continuation is pending. + yield* projections.apply({ + id: EventId.make("event:stop-restart:cancelled"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "cancelled", startedAt: null, completedAt: now }, + }); + const at = DateTime.formatIso(now); + yield* sql` + INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, status, + attempt_count, available_at, created_at, updated_at + ) VALUES ( + ${`effect:restart-continuation:${run.id}`}, 'command:restart', ${threadId}, + 'provider-runtime.continue', + ${encodeEffectRequest({ type: "provider-runtime.continue", sourceRunId: run.id })}, + 'pending', 0, ${at}, ${at}, ${at} + ) + `; + + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-restart"), + threadId, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart-continuation"), + threadId, + messageId: MessageId.make("message:restart-continuation"), + text: "Continue.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "agent", + creationSource: "server", + restartContinuationOfRunId: run.id, + }); + assert.deepEqual(yield* threadState(threadId), { runs: ["cancelled"], watched: [] }); + }).pipe(Effect.provide(testLayer)), +); + +it.effect.each(["thread.stop", "run.interrupt"] as const)( + "a successful %s blocks a newer restart continuation when completion timestamps tie", + (stopType) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const threadId = ThreadId.make("thread:stop-resumed-queue"); + yield* createWatchingThread(threadId, 12); + yield* send(threadId, "first", "start_immediately"); + yield* send(threadId, "queued", "queue_after_active"); + const now = yield* DateTime.now; + const [first, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs; + assert.isDefined(first); + assert.isDefined(queued); + for (const run of [first!, queued!]) { + yield* projections.apply({ + id: EventId.make(`event:stop-resumed-queue:hold:${run.id}`), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: + run.id === first!.id + ? { ...run, status: "completed", completedAt: now } + : { ...run, queueHeld: true }, + }); + } + // A later turn runs ahead of the held queue, then recovery cancels it. + yield* send(threadId, "restart source", "start_immediately"); + const source = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + yield* projections.apply({ + id: EventId.make("event:stop-resumed-queue:restart-cancelled"), + type: "run.updated", + threadId, + runId: source.id, + occurredAt: now, + payload: { ...source, status: "cancelled", startedAt: now, completedAt: now }, + }); + const at = DateTime.formatIso(now); + yield* sql` + INSERT INTO orchestration_v2_effect_outbox ( + effect_id, command_id, thread_id, effect_type, payload_json, status, + attempt_count, available_at, created_at, updated_at + ) VALUES ( + ${`effect:restart-continuation:${source.id}`}, 'command:restart', ${threadId}, + 'provider-runtime.continue', + ${encodeEffectRequest({ type: "provider-runtime.continue", sourceRunId: source.id })}, + 'pending', 0, ${at}, ${at}, ${at} + ) + `; + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("resume-older-queue"), + threadId, + }); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === queued!.id, + )?.status, + "starting", + ); + yield* orchestrator.dispatch( + stopType === "thread.stop" + ? { type: "thread.stop", commandId: CommandId.make("stop-resumed-queue"), threadId } + : { + type: "run.interrupt", + commandId: CommandId.make("stop-resumed-queue"), + threadId, + runId: queued!.id, + holdQueue: true, + }, + ); + const stopped = yield* orchestrator.getThreadProjection(threadId); + const interrupted = stopped.runs.find((run) => run.id === queued!.id)!; + assert.equal(interrupted.status, "interrupted"); + assert.equal(DateTime.toEpochMillis(interrupted.completedAt!), DateTime.toEpochMillis(now)); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart-after-stop"), + threadId, + messageId: MessageId.make("message:restart-after-stop"), + text: "Continue.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "agent", + creationSource: "server", + restartContinuationOfRunId: source.id, + }); + assert.deepEqual(yield* threadState(threadId), { + runs: ["completed", "interrupted", "cancelled"], + watched: [], + }); + }).pipe(Effect.provide(testLayer.pipe(Layer.provideMerge(TestClock.layer())))), +); + +it.effect("a delegated task that cannot be stopped fails the walk after its siblings stop", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const parentThreadId = ThreadId.make("thread:stop-partial"); + yield* createWatchingThread(parentThreadId, 6); + yield* send(parentThreadId, "work", "start_immediately"); + const childThreadId = yield* delegate(parentThreadId, "stoppable task"); + const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; + // A task whose thread is gone, listed before the one that can stop. + yield* projections.apply({ + id: EventId.make("event:stop-partial:missing-task"), + type: "subagent.updated", + threadId: parentThreadId, + runId: task.runId!, + nodeId: NodeId.make("node:stop-partial:missing"), + providerInstanceId: instanceId, + occurredAt: yield* DateTime.now, + payload: { + ...task, + id: NodeId.make("node:stop-partial:missing"), + childThreadId: ThreadId.make("thread:stop-partial:missing"), + startedAt: DateTime.subtract(task.startedAt!, { hours: 1 }), + }, + }); + + const walked = yield* Effect.exit( + threads.stopDelegatedTasks({ + threadId: parentThreadId, + commandId: CommandId.make("stop-partial"), + }), + ); + assert.isTrue(Exit.isFailure(walked)); + assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted"]); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("thread.stop on a finished thread refuses a late agent watch", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:stop-finished"); + yield* createWatchingThread(threadId, 9); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + yield* projections.apply({ + id: EventId.make("event:stop-finished:completed"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", startedAt: now, completedAt: now }, + }); + + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-finished"), + threadId, + }); + assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 10)))); + assert.deepEqual(yield* threadState(threadId), { runs: ["completed"], watched: [] }); + }).pipe(Effect.provide(testLayer)), +); + +it.effect("thread.stop marks a turn it cannot interrupt so a late agent watch is refused", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:stop-lost-session"); + yield* createWatchingThread(threadId, 11); + yield* send(threadId, "work", "start_immediately"); + const now = yield* DateTime.now; + const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + // The turn is running, but its provider session is gone, so the interrupt fails. + yield* projections.apply({ + id: EventId.make("event:stop-lost-session:running"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running", startedAt: now }, + }); + yield* projections.apply({ + id: EventId.make("event:stop-lost-session:turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("provider-turn:stop-lost-session"), + providerThreadId: run.providerThreadId!, + nodeId: run.rootNodeId!, + runAttemptId: run.activeAttemptId!, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-lost-session"), + threadId, + }); + assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(threadId, 12)))); + assert.deepEqual(yield* threadState(threadId), { runs: ["running"], watched: [] }); + }).pipe(Effect.provide(testLayer)), +); diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 9c69a2cc524c73f61db957a3607a81aa38da67d2..79f100fb46e944681b5169d5a4298ca43bbff14c 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -206,6 +206,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( sendToThread: unused, waitForThread: unused, interruptThread: unused, + stopDelegatedTasks: unused, getThreadEventSequence: unused, recoverDelegatedTask: unused, delegatedTaskResultPending: unused, diff --git a/docs/orchestration-v2/feature-lifecycles.md b/docs/orchestration-v2/feature-lifecycles.md index 164455b4e22f54b53ed3b60114075dfc36275f13..4e1e1a0568ae5682104e57d1b6baf005e2047b7e 100644 --- a/docs/orchestration-v2/feature-lifecycles.md +++ b/docs/orchestration-v2/feature-lifecycles.md @@ -251,6 +251,14 @@ If the provider does not support interruption: - best case: stop provider session and mark run interrupted or cancelled by policy. - worst case: mark interrupt unsupported and leave run active until provider exits. +The Stop button sends `run.interrupt` with `holdQueue`. A Stop also holds the queue, +ends the thread's pull request watches, and drops wakes its delegated tasks still +owe, so nothing automatic restarts the thread. It then stops every delegated task +under the thread: a `delegated-tasks.stop` effect sends the internal `thread.stop` +command to each child thread, depth first. This runs after commit, not inside the +parent's command, because a command holds only its own thread's lock. `task_cancel` +sends the same `thread.stop` to its child. + ## Resumption Resumption has three related meanings: diff --git a/docs/orchestration-v2/orchestrator-mcp-server.md b/docs/orchestration-v2/orchestrator-mcp-server.md index eab33d76f5b0fe7ceb958c47776549cec4f3bb64..cf23884731c609990c1e85b73b39d06822fc4562 100644 --- a/docs/orchestration-v2/orchestrator-mcp-server.md +++ b/docs/orchestration-v2/orchestrator-mcp-server.md @@ -279,13 +279,13 @@ the published task result. ### `task_cancel` -Interrupts the currently active task run through the normal V2 `run.interrupt` -command and disposes automatic parent delivery. Native background work between -turns currently has no interruptible run. For a terminal task, it returns the -existing status and disposes delivery without interrupting later child-thread runs, -even when `task_status` reports `hasPendingChildRuns: true`. Published task results -remain available. It accepts an optional cancellation reason. Use -`t3_thread_interrupt` to stop a later active run. +Stops the child thread with the internal `thread.stop` command, then stops every +task the child delegated, and disposes automatic parent delivery. Like a user Stop, +`thread.stop` interrupts the running turn, holds queued turns, and ends pull request +watches. A nonterminal task with no interruptible run is rejected. A terminal task +returns its existing status, and its child thread still stops, including later +runs and watch wakes. Published task results remain available. It accepts an +optional cancellation reason. ### `create_threads` diff --git a/docs/user/source-control.md b/docs/user/source-control.md index fb00b56677c311c1ebc1af1be5e834f56fb25468..e8ab0a48885a77f3303dee15802b79db13867a4c 100644 --- a/docs/user/source-control.md +++ b/docs/user/source-control.md @@ -189,9 +189,10 @@ Ask the agent to watch, monitor, or babysit a pull request and it calls `watch_p the thread is active, the server checks the pull request every minute and wakes the agent when a check fails, the required checks pass, someone else comments or reviews, or the branch starts to conflict. Comments from your own account do not wake it. Watching ends when the pull request merges or closes, -after 10 wakes in a row that bring only comments, or when the server cannot read the pull request for -15 minutes. Settling a thread also ends all its watches. Unsettle the thread before starting a new -watch. To start or stop it yourself, use the row menu in the **Linked pull requests** panel. +after 10 wakes in a row that bring only comments, when the server cannot read the pull request for +15 minutes, or when you press Stop on the thread. Settling a thread also ends all its watches. +Unsettle the thread before starting a new watch. To start or stop it yourself, use the row menu in +the **Linked pull requests** panel. Cross-repository links use a project on the same host. Azure DevOps reviews require a project checked out from the matching organization and repository. diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index eb7c4e6a9ffdbe79b55944d1fe9665f16984c31a..aed09c042794ae7762311f60763c141ca6af86ac 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -208,6 +208,7 @@ snooze limit stops by default. Providers without a reset time offer manual retry and the normal snooze choices. On web and desktop, use **Agents** to follow work delegated to subagents. +Stop on a thread also stops the subagents it delegated to. Subagent threads started by the agent can't take messages; message the parent thread instead. When such a subagent needs an approval or an answer, the parent diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 9498fdb00002369887f0973e4c608def89341f15..8107e1ed1c20ff5fe1243115162e0d91c0bc5492 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -2830,6 +2830,10 @@ export const OrchestrationV2Command = Schema.Union([ threadId: ThreadId, runId: RunId, reason: Schema.optional(Schema.String), + /** + * Set by the Stop button. Stop also holds the queue, ends the thread's pull request + * watches, and stops every delegated task under the thread. + */ holdQueue: Schema.optional(Schema.Boolean), }), Schema.Struct({ @@ -3013,6 +3017,17 @@ const OrchestrationV2InternalCommand = Schema.Union([ providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, }), + /** + * Stop for one thread, whatever it is doing: interrupts its running turn, holds its queue, + * ends its pull request watches, and drops pending delegated-task wakes. Nothing to stop is + * an accepted no-op. Delegated tasks under the thread get their own `thread.stop`. + */ + Schema.Struct({ + type: Schema.Literal("thread.stop"), + commandId: CommandId, + threadId: ThreadId, + reason: Schema.optional(Schema.String), + }), ]); export type OrchestrationV2InternalCommand = typeof OrchestrationV2InternalCommand.Type;