1import { assert, describe, it } from "@effect/vitest";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import type { ProviderReplayTranscript } from "@t3tools/contracts";
4import * as CodexReplay from "effect-codex-app-server/replay";
5import * as Effect from "effect/Effect";
6import * as Schema from "effect/Schema";
7
8import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
9import {
10 THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER,
11 THREAD_FORK_NATIVE_CONTINUE_RECALL,
12 THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER,
13 THREAD_MERGE_BACK_FORK_MARKER,
14 THREAD_MERGE_BACK_RECALL,
15 THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER,
16 THREAD_MERGE_BACK_SIBLINGS_RECALL,
17 THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER,
18 THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER,
19 THREAD_MERGE_BACK_SOURCE_MARKER,
20} from "./fixtures/shared.ts";
21import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
22
23const PROVIDER_THREAD_RESUME_FIRST_FINAL = "provider thread resume fixture first turn complete";
24const PROVIDER_THREAD_RESUME_SECOND_FINAL = "provider thread resume fixture second turn complete";
25
26type ProtocolReplayEntry = Extract<
27 ProviderReplayTranscript["entries"][number],
28 { readonly type: "expect_outbound" | "emit_inbound" }
29>;
30
31interface CodexReplayFixtureRegistration {
32 readonly registrationScenario: string;
33 readonly recordedScenario: string;
34 readonly transcriptFile: URL;
35}
36
37const CODEX_REPLAY_FIXTURE_REGISTRATIONS = ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
38 fixture.providers
39 .filter((provider) => provider.driver === "codex")
40 .map((provider) => ({
41 registrationScenario: fixture.name,
42 recordedScenario: provider.recordedScenario ?? fixture.name,
43 transcriptFile: provider.transcriptFile,
44 })),
45).concat([
46 {
47 registrationScenario: "provider_thread_resume",
48 recordedScenario: "provider_thread_resume",
49 transcriptFile: new URL(
50 "./fixtures/provider_thread_resume/codex_transcript.ndjson",
51 import.meta.url,
52 ),
53 },
54 {
55 registrationScenario: "thread_fork_native_continue",
56 recordedScenario: "thread_fork_native_continue",
57 transcriptFile: new URL(
58 "./fixtures/thread_fork_native_continue/codex_transcript.ndjson",
59 import.meta.url,
60 ),
61 },
62 {
63 registrationScenario: "thread_fork_native_siblings",
64 recordedScenario: "thread_fork_native_siblings",
65 transcriptFile: new URL(
66 "./fixtures/thread_fork_native_siblings/codex_transcript.ndjson",
67 import.meta.url,
68 ),
69 },
70 {
71 registrationScenario: "thread_merge_back_continue",
72 recordedScenario: "thread_merge_back_continue",
73 transcriptFile: new URL(
74 "./fixtures/thread_merge_back_continue/codex_transcript.ndjson",
75 import.meta.url,
76 ),
77 },
78 {
79 registrationScenario: "thread_merge_back_siblings",
80 recordedScenario: "thread_merge_back_siblings",
81 transcriptFile: new URL(
82 "./fixtures/thread_merge_back_siblings/codex_transcript.ndjson",
83 import.meta.url,
84 ),
85 },
86]);
87
88function uniqueCanonicalTranscripts(
89 registrations: ReadonlyArray<CodexReplayFixtureRegistration>,
90): ReadonlyArray<CodexReplayFixtureRegistration> {
91 const canonicalTranscripts = new Map<string, CodexReplayFixtureRegistration>();
92 for (const registration of registrations) {
93 const canonicalUrl = registration.transcriptFile.href;
94 const existing = canonicalTranscripts.get(canonicalUrl);
95 if (existing !== undefined && existing.recordedScenario !== registration.recordedScenario) {
96 throw new Error(
97 `Codex replay transcript ${canonicalUrl} has conflicting recorded scenarios ${existing.recordedScenario} (${existing.registrationScenario}) and ${registration.recordedScenario} (${registration.registrationScenario}).`,
98 );
99 }
100 canonicalTranscripts.set(canonicalUrl, existing ?? registration);
101 }
102 return Array.from(canonicalTranscripts.values());
103}
104
105const CODEX_REPLAY_TRANSCRIPTS = uniqueCanonicalTranscripts(CODEX_REPLAY_FIXTURE_REGISTRATIONS);
106
107const scenarioExpectations = {
108 simple: {
109 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
110 incoming: ["thread/started", "turn/started", "turn/completed"],
111 turnStartCount: 1,
112 turnCompletedCount: 1,
113 approvalRequestCount: 0,
114 },
115 tool_call_read_only_on_request: {
116 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
117 incoming: ["item/commandExecution/requestApproval", "serverRequest/resolved", "turn/completed"],
118 turnStartCount: 1,
119 turnCompletedCount: 1,
120 approvalRequestCount: 1,
121 },
122 tool_call_workspace_never: {
123 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
124 incoming: ["turn/completed"],
125 turnStartCount: 1,
126 turnCompletedCount: 1,
127 approvalRequestCount: 0,
128 },
129 tool_call_restricted_granular: {
130 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
131 incoming: ["item/fileChange/requestApproval", "serverRequest/resolved", "turn/completed"],
132 turnStartCount: 1,
133 turnCompletedCount: 1,
134 approvalRequestCount: 1,
135 },
136 subagent: {
137 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
138 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
139 turnStartCount: 1,
140 turnCompletedCount: 3,
141 approvalRequestCount: 0,
142 },
143 subagent_continue: {
144 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
145 incoming: ["turn/started", "item/completed", "turn/completed"],
146 turnStartCount: 2,
147 turnCompletedCount: 4,
148 approvalRequestCount: 0,
149 },
150 subagent_v2: {
151 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
152 incoming: ["turn/started", "item/completed", "item/agentMessage/delta", "turn/completed"],
153 turnStartCount: 1,
154 turnCompletedCount: 2,
155 approvalRequestCount: 0,
156 },
157 subagent_v2_approval: {
158 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
159 incoming: [
160 "item/commandExecution/requestApproval",
161 "serverRequest/resolved",
162 "item/completed",
163 "turn/completed",
164 ],
165 turnStartCount: 1,
166 turnCompletedCount: 2,
167 approvalRequestCount: 1,
168 },
169 subagent_v2_nested: {
170 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
171 incoming: ["turn/started", "item/completed", "item/agentMessage/delta", "turn/completed"],
172 turnStartCount: 1,
173 turnCompletedCount: 4,
174 approvalRequestCount: 0,
175 },
176 subagent_v2_nested_approval: {
177 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
178 incoming: [
179 "item/commandExecution/requestApproval",
180 "serverRequest/resolved",
181 "item/completed",
182 "turn/completed",
183 ],
184 turnStartCount: 1,
185 turnCompletedCount: 3,
186 approvalRequestCount: 1,
187 },
188 multi_turn: {
189 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
190 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
191 turnStartCount: 2,
192 turnCompletedCount: 2,
193 approvalRequestCount: 0,
194 },
195 plan_questions: {
196 outgoing: [
197 "initialize",
198 "initialized",
199 "thread/start",
200 "turn/start",
201 "item/tool/requestUserInput",
202 ],
203 incoming: [
204 "turn/started",
205 "turn/completed",
206 "item/tool/requestUserInput",
207 "serverRequest/resolved",
208 "item/agentMessage/delta",
209 ],
210 turnStartCount: 1,
211 turnCompletedCount: 1,
212 approvalRequestCount: 0,
213 },
214 proposed_plan: {
215 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
216 incoming: ["turn/started", "turn/completed", "item/plan/delta"],
217 turnStartCount: 1,
218 turnCompletedCount: 1,
219 approvalRequestCount: 0,
220 },
221 provider_thread_resume: {
222 outgoing: ["initialize", "initialized", "thread/start", "thread/resume", "turn/start"],
223 incoming: ["thread/started", "turn/started", "turn/completed", "item/agentMessage/delta"],
224 turnStartCount: 2,
225 turnCompletedCount: 2,
226 approvalRequestCount: 0,
227 },
228 queued_turn: {
229 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
230 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
231 turnStartCount: 2,
232 turnCompletedCount: 2,
233 approvalRequestCount: 0,
234 },
235 message_steering: {
236 outgoing: ["initialize", "initialized", "thread/start", "turn/start", "turn/steer"],
237 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
238 turnStartCount: 1,
239 turnCompletedCount: 1,
240 approvalRequestCount: 0,
241 },
242 turn_interrupt: {
243 outgoing: ["initialize", "initialized", "thread/start", "turn/start", "turn/interrupt"],
244 incoming: ["turn/started", "turn/completed"],
245 turnStartCount: 1,
246 turnCompletedCount: 1,
247 approvalRequestCount: 0,
248 },
249 turn_interrupt_mid_tool: {
250 outgoing: [
251 "initialize",
252 "initialized",
253 "thread/start",
254 "turn/start",
255 "turn/interrupt",
256 "thread/backgroundTerminals/terminate",
257 ],
258 incoming: ["turn/started", "item/started", "turn/completed"],
259 turnStartCount: 1,
260 turnCompletedCount: 1,
261 approvalRequestCount: 0,
262 },
263 thread_rollback: {
264 outgoing: [
265 "initialize",
266 "initialized",
267 "thread/start",
268 "turn/start",
269 "thread/read",
270 "thread/turns/list",
271 "thread/revert",
272 ],
273 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
274 turnStartCount: 3,
275 turnCompletedCount: 3,
276 approvalRequestCount: 0,
277 },
278 thread_rollback_after_restart: {
279 outgoing: [
280 "initialize",
281 "initialized",
282 "thread/start",
283 "turn/start",
284 "thread/read",
285 "thread/resume",
286 "thread/turns/list",
287 "thread/revert",
288 ],
289 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta", "thread/reverted"],
290 turnStartCount: 3,
291 turnCompletedCount: 3,
292 approvalRequestCount: 0,
293 },
294 thread_rollback_to_stopped_turn: {
295 outgoing: [
296 "initialize",
297 "initialized",
298 "thread/start",
299 "turn/start",
300 "turn/interrupt",
301 "thread/backgroundTerminals/terminate",
302 "thread/read",
303 "thread/turns/list",
304 "thread/revert",
305 ],
306 incoming: ["turn/started", "item/started", "turn/completed", "thread/reverted"],
307 turnStartCount: 4,
308 turnCompletedCount: 4,
309 approvalRequestCount: 0,
310 },
311 thread_fork_native_continue: {
312 outgoing: ["initialize", "initialized", "thread/start", "thread/fork", "turn/start"],
313 incoming: ["thread/started", "turn/started", "turn/completed", "item/agentMessage/delta"],
314 turnStartCount: 3,
315 turnCompletedCount: 3,
316 approvalRequestCount: 0,
317 },
318 thread_fork_native_siblings: {
319 outgoing: ["initialize", "initialized", "thread/start", "thread/fork", "turn/start"],
320 incoming: ["thread/started", "turn/started", "turn/completed", "item/agentMessage/delta"],
321 turnStartCount: 3,
322 turnCompletedCount: 3,
323 approvalRequestCount: 0,
324 },
325 thread_merge_back_continue: {
326 outgoing: ["initialize", "initialized", "thread/start", "thread/fork", "turn/start"],
327 incoming: ["thread/started", "turn/started", "turn/completed", "item/agentMessage/delta"],
328 turnStartCount: 4,
329 turnCompletedCount: 4,
330 approvalRequestCount: 0,
331 },
332 thread_merge_back_siblings: {
333 outgoing: ["initialize", "initialized", "thread/start", "thread/fork", "turn/start"],
334 incoming: ["thread/started", "turn/started", "turn/completed", "item/agentMessage/delta"],
335 turnStartCount: 6,
336 turnCompletedCount: 6,
337 approvalRequestCount: 0,
338 },
339 todo_list: {
340 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
341 incoming: ["turn/started", "turn/completed", "turn/plan/updated", "item/agentMessage/delta"],
342 turnStartCount: 1,
343 turnCompletedCount: 1,
344 approvalRequestCount: 0,
345 },
346 web_search: {
347 outgoing: ["initialize", "initialized", "thread/start", "turn/start"],
348 incoming: ["turn/started", "turn/completed", "item/agentMessage/delta"],
349 turnStartCount: 1,
350 turnCompletedCount: 1,
351 approvalRequestCount: 0,
352 },
353} as const;
354
355const decodeCodexTranscript = Schema.decodeUnknownEffect(
356 CodexReplay.CodexAppServerReplayTranscript,
357);
358const readTranscript = Effect.fn("readCodexReplayFixture")(function* (file: URL) {
359 return yield* readProviderReplayTranscript(file);
360}, Effect.provide(NodeServices.layer));
361
362function labels(
363 transcript: ProviderReplayTranscript,
364 type: "expect_outbound" | "emit_inbound",
365): ReadonlyArray<string> {
366 return transcript.entries.flatMap((entry) => {
367 if (entry.type !== type || entry.label === undefined) {
368 return [];
369 }
370 return [entry.label];
371 });
372}
373
374function countLabel(
375 transcript: ProviderReplayTranscript,
376 type: "expect_outbound" | "emit_inbound",
377 label: string,
378) {
379 return labels(transcript, type).filter((entryLabel) => entryLabel === label).length;
380}
381
382function countApprovalRequests(transcript: ProviderReplayTranscript) {
383 return labels(transcript, "emit_inbound").filter((label) => label.endsWith("/requestApproval"))
384 .length;
385}
386
387function isRecord(value: unknown): value is Record<string, unknown> {
388 return typeof value === "object" && value !== null && !Array.isArray(value);
389}
390
391function readPath(value: unknown, path: ReadonlyArray<string | number>): unknown {
392 let current = value;
393 for (const segment of path) {
394 if (typeof segment === "number") {
395 if (!Array.isArray(current)) {
396 throw new Error(`Expected array while reading ${path.join(".")}.`);
397 }
398 current = current[segment];
399 continue;
400 }
401
402 if (!isRecord(current)) {
403 throw new Error(`Expected object while reading ${path.join(".")}.`);
404 }
405 current = current[segment];
406 }
407 return current;
408}
409
410function readString(value: unknown, path: ReadonlyArray<string | number>): string {
411 const current = readPath(value, path);
412 if (typeof current !== "string") {
413 throw new Error(`Expected string at ${path.join(".")}.`);
414 }
415 return current;
416}
417
418function readArray(value: unknown, path: ReadonlyArray<string | number>): ReadonlyArray<unknown> {
419 const current = readPath(value, path);
420 if (!Array.isArray(current)) {
421 throw new Error(`Expected array at ${path.join(".")}.`);
422 }
423 return current;
424}
425
426function findProtocolEntry(
427 transcript: ProviderReplayTranscript,
428 type: "expect_outbound" | "emit_inbound",
429 label: string,
430 occurrence = 0,
431): ProtocolReplayEntry {
432 const matches = transcript.entries.filter(
433 (entry): entry is ProtocolReplayEntry => entry.type === type && entry.label === label,
434 );
435 const entry = matches[occurrence];
436 if (!entry) {
437 throw new Error(`Missing ${type} ${label} occurrence ${occurrence}.`);
438 }
439 return entry;
440}
441
442function agentMessageTexts(transcript: ProviderReplayTranscript): ReadonlyArray<string> {
443 return transcript.entries.flatMap((entry) => {
444 if (entry.type !== "emit_inbound" || entry.label !== "item/completed") {
445 return [];
446 }
447
448 const item = readPath(entry.frame, ["params", "item"]);
449 if (!isRecord(item) || item.type !== "agentMessage" || typeof item.text !== "string") {
450 return [];
451 }
452 return [item.text];
453 });
454}
455
456function assertScenarioExpectations(transcript: ProviderReplayTranscript) {
457 const expectation =
458 scenarioExpectations[transcript.scenario as keyof typeof scenarioExpectations];
459 const outgoingLabels = labels(transcript, "expect_outbound");
460 const incomingLabels = labels(transcript, "emit_inbound");
461
462 assert.isDefined(expectation, `missing scenario expectation for ${transcript.scenario}`);
463 for (const label of expectation.outgoing) {
464 assert.include(outgoingLabels, label, `${transcript.scenario} missing outgoing ${label}`);
465 }
466 for (const label of expectation.incoming) {
467 assert.include(incomingLabels, label, `${transcript.scenario} missing incoming ${label}`);
468 }
469
470 assert.equal(countLabel(transcript, "expect_outbound", "turn/start"), expectation.turnStartCount);
471 assert.equal(
472 countLabel(transcript, "emit_inbound", "turn/completed"),
473 expectation.turnCompletedCount,
474 );
475 assert.equal(countApprovalRequests(transcript), expectation.approvalRequestCount);
476}
477
478function assertProviderThreadResumeSemantics(transcript: ProviderReplayTranscript) {
479 if (transcript.scenario !== "provider_thread_resume") {
480 return;
481 }
482
483 const startThreadId = readString(
484 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
485 ["result", "thread", "id"],
486 );
487 const resumeRequestedThreadId = readString(
488 findProtocolEntry(transcript, "expect_outbound", "thread/resume").frame,
489 ["params", "threadId"],
490 );
491 const resumeRequest = findProtocolEntry(transcript, "expect_outbound", "thread/resume").frame;
492 const resumedThreadFrame = findProtocolEntry(transcript, "emit_inbound", "thread/resume").frame;
493 const resumedThreadId = readString(resumedThreadFrame, ["result", "thread", "id"]);
494 const secondTurnThreadId = readString(
495 findProtocolEntry(transcript, "expect_outbound", "turn/start", 1).frame,
496 ["params", "threadId"],
497 );
498 const texts = agentMessageTexts(transcript);
499 const secondFinalText = texts[1] ?? "";
500
501 assert.equal(
502 resumeRequestedThreadId,
503 startThreadId,
504 "thread/resume must request the provider thread created by thread/start",
505 );
506 assert.equal(
507 resumedThreadId,
508 startThreadId,
509 "thread/resume must return the same provider thread id",
510 );
511 assert.equal(
512 secondTurnThreadId,
513 startThreadId,
514 "turn after resume must run on the resumed provider thread",
515 );
516 // The adapter resumes with excludeTurns, so history reaches the model, not the response.
517 assert.equal(readPath(resumeRequest, ["params", "excludeTurns"]), true);
518 assert.lengthOf(readArray(resumedThreadFrame, ["result", "thread", "turns"]), 0);
519 assert.equal(texts[0], PROVIDER_THREAD_RESUME_FIRST_FINAL);
520 assert.include(
521 secondFinalText,
522 PROVIDER_THREAD_RESUME_FIRST_FINAL,
523 "second turn must demonstrate access to resumed conversation history",
524 );
525 assert.include(
526 secondFinalText,
527 PROVIDER_THREAD_RESUME_SECOND_FINAL,
528 "second turn must include its own completion marker",
529 );
530}
531
532function assertContinuedForkSemantics(transcript: ProviderReplayTranscript) {
533 if (transcript.scenario !== "thread_fork_native_continue") {
534 return;
535 }
536
537 const sourceThreadId = readString(
538 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
539 ["result", "thread", "id"],
540 );
541 const forkRequestThreadId = readString(
542 findProtocolEntry(transcript, "expect_outbound", "thread/fork").frame,
543 ["params", "threadId"],
544 );
545 const forkThreadId = readString(
546 findProtocolEntry(transcript, "emit_inbound", "thread/fork").frame,
547 ["result", "thread", "id"],
548 );
549 const firstForkTurnThreadId = readString(
550 findProtocolEntry(transcript, "expect_outbound", "turn/start", 1).frame,
551 ["params", "threadId"],
552 );
553 const secondForkTurnThreadId = readString(
554 findProtocolEntry(transcript, "expect_outbound", "turn/start", 2).frame,
555 ["params", "threadId"],
556 );
557 const recallPrompt = readString(
558 findProtocolEntry(transcript, "expect_outbound", "turn/start", 2).frame,
559 ["params", "input", 0, "text"],
560 );
561
562 assert.equal(forkRequestThreadId, sourceThreadId);
563 assert.notEqual(forkThreadId, sourceThreadId);
564 assert.equal(firstForkTurnThreadId, forkThreadId);
565 assert.equal(secondForkTurnThreadId, forkThreadId);
566 assert.notInclude(recallPrompt, THREAD_FORK_NATIVE_CONTINUE_SOURCE_MARKER);
567 assert.notInclude(recallPrompt, THREAD_FORK_NATIVE_CONTINUE_FORK_MARKER);
568 const texts = agentMessageTexts(transcript);
569 for (const expected of [
570 "source marker stored",
571 "fork marker stored",
572 THREAD_FORK_NATIVE_CONTINUE_RECALL,
573 ]) {
574 assert.include(texts, expected);
575 }
576 assert.equal(
577 texts.at(-1),
578 THREAD_FORK_NATIVE_CONTINUE_RECALL,
579 "the second fork-local turn must recall both the inherited source marker and fork-local marker",
580 );
581}
582
583function assertSiblingForkSemantics(transcript: ProviderReplayTranscript) {
584 if (transcript.scenario !== "thread_fork_native_siblings") {
585 return;
586 }
587
588 const sourceThreadId = readString(
589 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
590 ["result", "thread", "id"],
591 );
592 const firstForkFrame = findProtocolEntry(transcript, "emit_inbound", "thread/fork", 0).frame;
593 const secondForkFrame = findProtocolEntry(transcript, "emit_inbound", "thread/fork", 1).frame;
594 const firstForkId = readString(firstForkFrame, ["result", "thread", "id"]);
595 const secondForkId = readString(secondForkFrame, ["result", "thread", "id"]);
596 const firstForkParentId = readString(firstForkFrame, ["result", "thread", "forkedFromId"]);
597 const secondForkParentId = readString(secondForkFrame, ["result", "thread", "forkedFromId"]);
598 const firstForkTurnThreadId = readString(
599 findProtocolEntry(transcript, "expect_outbound", "turn/start", 1).frame,
600 ["params", "threadId"],
601 );
602 const secondForkTurnThreadId = readString(
603 findProtocolEntry(transcript, "expect_outbound", "turn/start", 2).frame,
604 ["params", "threadId"],
605 );
606 const texts = agentMessageTexts(transcript);
607 const firstRecall = texts[1]?.replace(/\s*\|\s*/u, "|") ?? "";
608 const secondRecall = texts[2]?.replace(/\s*\|\s*/u, "|") ?? "";
609
610 assert.equal(firstForkParentId, sourceThreadId);
611 assert.equal(secondForkParentId, sourceThreadId);
612 assert.notEqual(firstForkId, secondForkId);
613 assert.equal(firstForkTurnThreadId, firstForkId);
614 assert.equal(secondForkTurnThreadId, secondForkId);
615 assert.equal(firstRecall, "sibling-source-8R3D|sibling-first-5L2P");
616 assert.notInclude(firstRecall, "sibling-second-9N6C");
617 assert.equal(secondRecall, "sibling-source-8R3D|sibling-second-9N6C");
618 assert.notInclude(secondRecall, "sibling-first-5L2P");
619}
620
621function assertMergeBackSemantics(transcript: ProviderReplayTranscript) {
622 if (transcript.scenario !== "thread_merge_back_continue") {
623 return;
624 }
625
626 const sourceThreadId = readString(
627 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
628 ["result", "thread", "id"],
629 );
630 const forkThreadId = readString(
631 findProtocolEntry(transcript, "emit_inbound", "thread/fork").frame,
632 ["result", "thread", "id"],
633 );
634 const mergeThreadId = readString(
635 findProtocolEntry(transcript, "expect_outbound", "turn/start", 2).frame,
636 ["params", "threadId"],
637 );
638 const recallFrame = findProtocolEntry(transcript, "expect_outbound", "turn/start", 3).frame;
639 const recallThreadId = readString(recallFrame, ["params", "threadId"]);
640 const recallPrompt = readString(recallFrame, ["params", "input", 0, "text"]);
641 const texts = agentMessageTexts(transcript);
642
643 assert.notEqual(forkThreadId, sourceThreadId);
644 assert.equal(mergeThreadId, sourceThreadId);
645 assert.equal(recallThreadId, sourceThreadId);
646 assert.notInclude(recallPrompt, THREAD_MERGE_BACK_SOURCE_MARKER);
647 assert.notInclude(recallPrompt, THREAD_MERGE_BACK_FORK_MARKER);
648 assert.equal(texts.at(-1)?.replace(/\s*\|\s*/u, "|"), THREAD_MERGE_BACK_RECALL);
649}
650
651function assertSiblingMergeBackSemantics(transcript: ProviderReplayTranscript) {
652 if (transcript.scenario !== "thread_merge_back_siblings") {
653 return;
654 }
655
656 const sourceThreadId = readString(
657 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
658 ["result", "thread", "id"],
659 );
660 const firstForkId = readString(
661 findProtocolEntry(transcript, "emit_inbound", "thread/fork", 0).frame,
662 ["result", "thread", "id"],
663 );
664 const secondForkId = readString(
665 findProtocolEntry(transcript, "emit_inbound", "thread/fork", 1).frame,
666 ["result", "thread", "id"],
667 );
668 const firstMergeThreadId = readString(
669 findProtocolEntry(transcript, "expect_outbound", "turn/start", 3).frame,
670 ["params", "threadId"],
671 );
672 const secondMergeThreadId = readString(
673 findProtocolEntry(transcript, "expect_outbound", "turn/start", 4).frame,
674 ["params", "threadId"],
675 );
676 const recallFrame = findProtocolEntry(transcript, "expect_outbound", "turn/start", 5).frame;
677 const recallPrompt = readString(recallFrame, ["params", "input", 0, "text"]);
678 const finalText = agentMessageTexts(transcript)
679 .at(-1)
680 ?.replace(/\s*\|\s*/gu, "|");
681
682 assert.notEqual(firstForkId, secondForkId);
683 assert.equal(firstMergeThreadId, sourceThreadId);
684 assert.equal(secondMergeThreadId, sourceThreadId);
685 assert.notInclude(recallPrompt, THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER);
686 assert.notInclude(recallPrompt, THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER);
687 assert.notInclude(recallPrompt, THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER);
688 assert.equal(finalText, THREAD_MERGE_BACK_SIBLINGS_RECALL);
689}
690
691/**
692 * Codex announces a v2 child with the parent's `subAgentActivity(started)`
693 * before the child's first `turn/started`; the adapter registers children
694 * from that activity.
695 */
696function assertSubagentActivityPrecedesChildTurns(transcript: ProviderReplayTranscript) {
697 if (transcript.scenario !== "subagent_v2" && transcript.scenario !== "subagent_v2_nested") {
698 return;
699 }
700
701 const announcedChildren = new Set<string>();
702 const startedChildren = new Set<string>();
703 const rootThreadId = readString(
704 findProtocolEntry(transcript, "emit_inbound", "thread/start").frame,
705 ["result", "thread", "id"],
706 );
707 for (const entry of transcript.entries) {
708 if (entry.type !== "emit_inbound") continue;
709 const method = readPath(entry.frame, ["method"]);
710 if (method === "item/started") {
711 const item = readPath(entry.frame, ["params", "item"]);
712 if (isRecord(item) && item.type === "subAgentActivity" && item.kind === "started") {
713 announcedChildren.add(readString(item, ["agentThreadId"]));
714 }
715 }
716 if (method === "turn/started") {
717 const threadId = readString(entry.frame, ["params", "threadId"]);
718 if (threadId === rootThreadId) continue;
719 assert.isTrue(
720 announcedChildren.has(threadId),
721 `${transcript.scenario}: child ${threadId} started a turn before its subAgentActivity`,
722 );
723 startedChildren.add(threadId);
724 }
725 }
726 assert.equal(
727 startedChildren.size,
728 transcript.scenario === "subagent_v2" ? 1 : 3,
729 `${transcript.scenario}: unexpected number of child turns`,
730 );
731}
732
733describe("Codex replay fixtures", () => {
734 it.effect("loads each canonical Codex fixture as an app-server replay transcript", () =>
735 Effect.gen(function* () {
736 for (const fixture of CODEX_REPLAY_TRANSCRIPTS) {
737 const transcript = yield* readTranscript(fixture.transcriptFile);
738 const codexTranscript = yield* decodeCodexTranscript(transcript);
739 const first = transcript.entries[0];
740
741 assert.equal(codexTranscript.provider, "codex");
742 assert.equal(codexTranscript.protocol, "codex.app-server");
743 assert.equal(codexTranscript.scenario, fixture.recordedScenario);
744 assert.deepEqual(codexTranscript.entries.at(-1), {
745 type: "runtime_exit",
746 status: "success",
747 });
748 assert.equal(first?.type, "expect_outbound");
749 if (first?.type !== "expect_outbound") {
750 throw new Error(
751 `Expected ${fixture.recordedScenario} to start with initialize outbound frame.`,
752 );
753 }
754 assert.equal(first.label, "initialize");
755
756 assertScenarioExpectations(transcript);
757 assertProviderThreadResumeSemantics(transcript);
758 assertContinuedForkSemantics(transcript);
759 assertSiblingForkSemantics(transcript);
760 assertMergeBackSemantics(transcript);
761 assertSiblingMergeBackSemantics(transcript);
762 assertSubagentActivityPrecedesChildTurns(transcript);
763 }
764 }),
765 );
766
767 it("covers the expected replay suite exactly", () => {
768 assert.deepEqual(
769 CODEX_REPLAY_TRANSCRIPTS.map((fixture) => fixture.recordedScenario).toSorted(),
770 Object.keys(scenarioExpectations).toSorted(),
771 );
772 });
773});