authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-17 16:41:06-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-17 16:41:06-07:00
log67993623afb14c017bb0b959478dc892f9c427da
treed758d6a42459270400f4b5abcc16b709cfcad95f
parentb4620d595554654c259ec7c0afb3d3d5532959a8
signature Signed by PGP key B5690EEEBB952194

fix(server): capture checkpoints when baseline lookup fails (#12307)


2 files changed, 104 insertions(+), 5 deletions(-)

apps/server/src/orchestration/Layers/CheckpointReactor.test.ts+90-1
...@@ -5,6 +5,8 @@ import * as NodePath from "node:path";...@@ -5,6 +5,8 @@ import * as NodePath from "node:path";
5import * as NodeChildProcess from "node:child_process";5import * as NodeChildProcess from "node:child_process";
66
7import {7import {
8 VcsProcessTimeoutError,
9 VcsProcessSpawnError,
8 ProviderDriverKind,10 ProviderDriverKind,
9 ProviderRuntimeEvent,11 ProviderRuntimeEvent,
10 ProviderSession,12 ProviderSession,
...@@ -290,6 +292,9 @@ describe("CheckpointReactor", () => {...@@ -290,6 +292,9 @@ describe("CheckpointReactor", () => {
290 });292 });
291293
292 async function createHarness(options?: {294 async function createHarness(options?: {
295 readonly checkpointLookupFailure?: (
296 cwd: string,
297 ) => VcsProcessTimeoutError | VcsProcessSpawnError | undefined;
293 readonly hasSession?: boolean;298 readonly hasSession?: boolean;
294 readonly seedFilesystemCheckpoints?: boolean;299 readonly seedFilesystemCheckpoints?: boolean;
295 readonly initializeGit?: boolean;300 readonly initializeGit?: boolean;
...@@ -370,7 +375,20 @@ describe("CheckpointReactor", () => {...@@ -370,7 +375,20 @@ describe("CheckpointReactor", () => {
370 Layer.provideMerge(Layer.succeed(ProviderService, provider.service)),375 Layer.provideMerge(Layer.succeed(ProviderService, provider.service)),
371 Layer.provideMerge(Layer.mock(PullRequestService)({ refreshAfterTurn })),376 Layer.provideMerge(Layer.mock(PullRequestService)({ refreshAfterTurn })),
372 Layer.provideMerge(vcsStatusBroadcasterLayer),377 Layer.provideMerge(vcsStatusBroadcasterLayer),
373 Layer.provideMerge(CheckpointStore.layer.pipe(Layer.provide(VcsDriverRegistry.layer))),378 Layer.provideMerge(
379 Layer.effect(
380 CheckpointStore.CheckpointStore,
381 CheckpointStore.make.pipe(
382 Effect.map((store) => ({
383 ...store,
384 hasCheckpointRef: (input) => {
385 const failure = options?.checkpointLookupFailure?.(input.cwd);
386 return failure ? Effect.fail(failure) : store.hasCheckpointRef(input);
387 },
388 })),
389 ),
390 ).pipe(Layer.provide(VcsDriverRegistry.layer)),
391 ),
374 Layer.provideMerge(392 Layer.provideMerge(
375 WorkspaceEntries.layer.pipe(393 WorkspaceEntries.layer.pipe(
376 Layer.provide(WorkspacePaths.layer),394 Layer.provide(WorkspacePaths.layer),
...@@ -575,6 +593,77 @@ describe("CheckpointReactor", () => {...@@ -575,6 +593,77 @@ describe("CheckpointReactor", () => {
575 }),593 }),
576 );594 );
577595
596 effectIt.effect.each(["timeout", "spawn"] as const)(
597 "captures and finalizes a turn when previous checkpoint lookup fails (%s)",
598 (failureKind) =>
599 Effect.gen(function* () {
600 let failLookup = false;
601 const harness = yield* Effect.promise(() =>
602 createHarness({
603 seedFilesystemCheckpoints: false,
604 checkpointLookupFailure: (cwd) =>
605 !failLookup
606 ? undefined
607 : failureKind === "timeout"
608 ? new VcsProcessTimeoutError({
609 operation: "test.refLookup",
610 command: "git",
611 cwd,
612 timeoutMs: 30000,
613 })
614 : new VcsProcessSpawnError({
615 operation: "test.refLookup",
616 command: "git",
617 cwd,
618 cause: new Error("transient lookup spawn failure"),
619 }),
620 }),
621 );
622 const threadId = ThreadId.make("thread-1");
623 const turnId = asTurnId("turn-ref-timeout");
624 harness.provider.emit({
625 type: "turn.started",
626 eventId: EventId.make("evt-ref-start"),
627 provider: ProviderDriverKind.make("codex"),
628 createdAt: "2026-01-01T00:00:00.000Z",
629 threadId,
630 turnId,
631 });
632 expect(yield* harness.nextReceipt).toMatchObject({ type: "checkpoint.baseline.captured" });
633 NodeFS.writeFileSync(NodePath.join(harness.cwd, "README.md"), "new snapshot\n");
634 failLookup = true;
635 harness.provider.emit({
636 type: "turn.completed",
637 eventId: EventId.make("evt-ref-complete"),
638 provider: ProviderDriverKind.make("codex"),
639 createdAt: "2026-01-01T00:00:01.000Z",
640 threadId,
641 turnId,
642 payload: { state: "completed" },
643 });
644 yield* Effect.promise(harness.drain);
645 const ref = checkpointRefForThreadTurn(threadId, 1);
646 expect(gitShowFileAtRef(harness.cwd, ref, "README.md")).toBe("new snapshot\n");
647 expect(yield* harness.nextReceipt).toMatchObject({
648 type: "checkpoint.diff.finalized",
649 turnId,
650 });
651 expect(yield* harness.nextReceipt).toMatchObject({
652 type: "turn.processing.quiesced",
653 turnId,
654 });
655 const model = yield* Effect.promise(harness.readModel);
656 expect(model.threads[0]?.checkpoints[0]).toMatchObject({
657 checkpointRef: ref,
658 status: "ready",
659 files: [],
660 });
661 expect(
662 model.threads[0]?.activities.some((a) => a.kind === "checkpoint.capture.failed"),
663 ).toBe(false);
664 }),
665 );
666
578 effectIt.effect("captures baseline and large turn summaries before completion receipts", () =>667 effectIt.effect("captures baseline and large turn summaries before completion receipts", () =>
579 Effect.gen(function* () {668 Effect.gen(function* () {
580 const harness = yield* Effect.promise(() =>669 const harness = yield* Effect.promise(() =>
apps/server/src/orchestration/Layers/CheckpointReactor.ts+14-4
...@@ -241,10 +241,20 @@ const make = Effect.gen(function* () {...@@ -241,10 +241,20 @@ const make = Effect.gen(function* () {
241 const fromCheckpointRef = checkpointRefForThreadTurn(input.threadId, fromTurnCount);241 const fromCheckpointRef = checkpointRefForThreadTurn(input.threadId, fromTurnCount);
242 const targetCheckpointRef = checkpointRefForThreadTurn(input.threadId, input.turnCount);242 const targetCheckpointRef = checkpointRefForThreadTurn(input.threadId, input.turnCount);
243243
244 const fromCheckpointExists = yield* checkpointStore.hasCheckpointRef({244 const fromCheckpointExists = yield* checkpointStore
245 cwd: input.cwd,245 .hasCheckpointRef({
246 checkpointRef: fromCheckpointRef,246 cwd: input.cwd,
247 });247 checkpointRef: fromCheckpointRef,
248 })
249 .pipe(
250 Effect.catch((error) =>
251 Effect.logWarning("checkpoint capture previous ref lookup failed", {
252 threadId: input.threadId,
253 checkpointRef: fromCheckpointRef,
254 category: error._tag,
255 }).pipe(Effect.as(false)),
256 ),
257 );
248 if (!fromCheckpointExists) {258 if (!fromCheckpointExists) {
249 yield* Effect.logWarning("checkpoint capture missing pre-turn baseline", {259 yield* Effect.logWarning("checkpoint capture missing pre-turn baseline", {
250 threadId: input.threadId,260 threadId: input.threadId,