authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-07 06:16:11-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-08-07 03:16:11-07:00
log7963cc70f3c9873768a5eaad7ca0a20a23d67aed
treef445773459cf2c3486edb09ec51551b278890e57
parentb792ed9f7b9c7c6386a14a1697c4393567812546
signature Signed by PGP key B5690EEEBB952194

feat(server): record runtime mode per turn and on mode changes (#5593)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

2 files changed, 129 insertions(+), 29 deletions(-)

apps/server/integration/providerService.integration.test.ts+109-29
...@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";...@@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";9import * as Path from "effect/Path";
10import * as Queue from "effect/Queue";10import * as Queue from "effect/Queue";
11import * as Ref from "effect/Ref";
11import * as Stream from "effect/Stream";12import * as Stream from "effect/Stream";
1213
13import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";14import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts";
...@@ -54,35 +55,58 @@ interface IntegrationFixture {...@@ -54,35 +55,58 @@ interface IntegrationFixture {
54 readonly layer: Layer.Layer<ProviderService, unknown, never>;55 readonly layer: Layer.Layer<ProviderService, unknown, never>;
55}56}
5657
57const makeIntegrationFixture = Effect.gen(function* () {58interface RecordedAnalyticsEvent {
58 const cwd = yield* makeWorkspaceDirectory;59 readonly event: string;
59 const harness = yield* makeTestProviderAdapterHarness();60 readonly properties: Readonly<Record<string, unknown>> | undefined;
6061}
61 const registry = makeAdapterRegistryMock({
62 [ProviderDriverKind.make("codex")]: harness.adapter,
63 });
6462
65 const directoryLayer = ProviderSessionDirectoryLive.pipe(63/**
66 Layer.provide(ProviderSessionRuntime.layer),64 * Analytics layer that keeps captured events in memory so tests can assert on
65 * telemetry payloads. `AnalyticsService.layerTest` discards them.
66 */
67const makeRecordingAnalytics = Effect.gen(function* () {
68 const recorded = yield* Ref.make<ReadonlyArray<RecordedAnalyticsEvent>>([]);
69 const layer = Layer.succeed(
70 AnalyticsService,
71 AnalyticsService.of({
72 record: (event, properties) =>
73 Ref.update(recorded, (current) => [...current, { event, properties }]),
74 flush: Effect.void,
75 }),
67 );76 );
6877 return { layer, get: Ref.get(recorded) } as const;
69 const shared = Layer.mergeAll(
70 directoryLayer,
71 Layer.succeed(ProviderAdapterRegistry, registry),
72 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
73 AnalyticsService.layerTest,
74 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
75 ).pipe(Layer.provide(SqlitePersistenceMemory));
76
77 const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
78
79 return {
80 cwd,
81 harness,
82 layer,
83 } satisfies IntegrationFixture;
84});78});
8579
80const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer<AnalyticsService> }) =>
81 Effect.gen(function* () {
82 const cwd = yield* makeWorkspaceDirectory;
83 const harness = yield* makeTestProviderAdapterHarness();
84
85 const registry = makeAdapterRegistryMock({
86 [ProviderDriverKind.make("codex")]: harness.adapter,
87 });
88
89 const directoryLayer = ProviderSessionDirectoryLive.pipe(
90 Layer.provide(ProviderSessionRuntime.layer),
91 );
92
93 const shared = Layer.mergeAll(
94 directoryLayer,
95 Layer.succeed(ProviderAdapterRegistry, registry),
96 ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS),
97 options?.analytics ?? AnalyticsService.layerTest,
98 Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers),
99 ).pipe(Layer.provide(SqlitePersistenceMemory));
100
101 const layer = makeProviderServiceLive().pipe(Layer.provide(shared));
102
103 return {
104 cwd,
105 harness,
106 layer,
107 } satisfies IntegrationFixture;
108 });
109
86const collectEventsDuring = <A, E, R>(110const collectEventsDuring = <A, E, R>(
87 stream: Stream.Stream<ProviderRuntimeEvent>,111 stream: Stream.Stream<ProviderRuntimeEvent>,
88 count: number,112 count: number,
...@@ -126,7 +150,7 @@ const runTurn = (input: {...@@ -126,7 +150,7 @@ const runTurn = (input: {
126150
127it.live("replays typed runtime fixture events", () =>151it.live("replays typed runtime fixture events", () =>
128 Effect.gen(function* () {152 Effect.gen(function* () {
129 const fixture = yield* makeIntegrationFixture;153 const fixture = yield* makeIntegrationFixture();
130154
131 yield* Effect.gen(function* () {155 yield* Effect.gen(function* () {
132 const provider = yield* ProviderService;156 const provider = yield* ProviderService;
...@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>...@@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () =>
161185
162it.live("replays file-changing fixture turn events", () =>186it.live("replays file-changing fixture turn events", () =>
163 Effect.gen(function* () {187 Effect.gen(function* () {
164 const fixture = yield* makeIntegrationFixture;188 const fixture = yield* makeIntegrationFixture();
165 const { join } = yield* Path.Path;189 const { join } = yield* Path.Path;
166 const { writeFileString } = yield* FileSystem.FileSystem;190 const { writeFileString } = yield* FileSystem.FileSystem;
167191
...@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>...@@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () =>
198222
199it.live("runs multi-turn tool/approval flow", () =>223it.live("runs multi-turn tool/approval flow", () =>
200 Effect.gen(function* () {224 Effect.gen(function* () {
201 const fixture = yield* makeIntegrationFixture;225 const fixture = yield* makeIntegrationFixture();
202 const { join } = yield* Path.Path;226 const { join } = yield* Path.Path;
203 const { writeFileString } = yield* FileSystem.FileSystem;227 const { writeFileString } = yield* FileSystem.FileSystem;
204228
...@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>...@@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () =>
250274
251it.live("rolls back provider conversation state only", () =>275it.live("rolls back provider conversation state only", () =>
252 Effect.gen(function* () {276 Effect.gen(function* () {
253 const fixture = yield* makeIntegrationFixture;277 const fixture = yield* makeIntegrationFixture();
254 const { join } = yield* Path.Path;278 const { join } = yield* Path.Path;
255 const { writeFileString, readFileString } = yield* FileSystem.FileSystem;279 const { writeFileString, readFileString } = yield* FileSystem.FileSystem;
256280
...@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>...@@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () =>
302 }).pipe(Effect.provide(fixture.layer));326 }).pipe(Effect.provide(fixture.layer));
303 }).pipe(Effect.provide(NodeServices.layer)),327 }).pipe(Effect.provide(NodeServices.layer)),
304);328);
329
330it.live("reports runtime mode per turn and on mode transitions", () =>
331 Effect.gen(function* () {
332 const analytics = yield* makeRecordingAnalytics;
333 const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer });
334 const threadId = ThreadId.make("thread-integration-runtime-mode");
335
336 yield* Effect.gen(function* () {
337 const provider = yield* ProviderService;
338 const startSession = (runtimeMode: "approval-required" | "full-access") =>
339 provider.startSession(threadId, {
340 threadId,
341 provider: ProviderDriverKind.make("codex"),
342 providerInstanceId: codexInstanceId,
343 cwd: fixture.cwd,
344 runtimeMode,
345 });
346
347 yield* startSession("approval-required");
348 yield* runTurn({
349 provider,
350 harness: fixture.harness,
351 threadId,
352 userText: "supervised turn",
353 response: { events: codexTurnTextFixture },
354 });
355
356 // Toggling the mode restarts the session, which is the only place the
357 // transition is observable.
358 yield* startSession("full-access");
359 yield* runTurn({
360 provider,
361 harness: fixture.harness,
362 threadId,
363 userText: "full access turn",
364 response: { events: codexTurnTextFixture },
365 });
366
367 const recorded = yield* analytics.get;
368
369 assert.deepEqual(
370 recorded
371 .filter((entry) => entry.event === "provider.turn.sent")
372 .map((entry) => entry.properties?.runtimeMode),
373 ["approval-required", "full-access"],
374 );
375
376 assert.deepEqual(
377 recorded
378 .filter((entry) => entry.event === "provider.runtime_mode.changed")
379 .map((entry) => [entry.properties?.from, entry.properties?.to]),
380 [["approval-required", "full-access"]],
381 );
382 }).pipe(Effect.provide(fixture.layer));
383 }).pipe(Effect.provide(NodeServices.layer)),
384);
apps/server/src/provider/Layers/ProviderService.ts+20
...@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
459 adapter,459 adapter,
460 instanceId,460 instanceId,
461 threadId: input.threadId,461 threadId: input.threadId,
462 runtimeMode: binding.runtimeMode,
462 isActive: true,463 isActive: true,
463 } as const;464 } as const;
464 }465 }
...@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
468 adapter,469 adapter,
469 instanceId,470 instanceId,
470 threadId: input.threadId,471 threadId: input.threadId,
472 runtimeMode: binding.runtimeMode,
471 isActive: false,473 isActive: false,
472 } as const;474 } as const;
473 }475 }
...@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
480 adapter: recovered.adapter,482 adapter: recovered.adapter,
481 instanceId,483 instanceId,
482 threadId: input.threadId,484 threadId: input.threadId,
485 runtimeMode: recovered.session.runtimeMode,
483 isActive: true,486 isActive: true,
484 } as const;487 } as const;
485 });488 });
...@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
629 input.modelSelection.model.trim().length > 0,632 input.modelSelection.model.trim().length > 0,
630 });633 });
631634
635 // Changing runtime mode restarts the session, so the transition is only
636 // observable here, by diffing against the mode the previous session for
637 // this thread was bound to. Recording it separately is what makes the
638 // "started supervised, switched to full access" funnel answerable.
639 const previousRuntimeMode = persistedBinding?.runtimeMode;
640 if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) {
641 yield* analytics.record("provider.runtime_mode.changed", {
642 provider: sessionWithInstance.provider,
643 from: previousRuntimeMode,
644 to: input.runtimeMode,
645 });
646 }
647
632 return sessionWithInstance;648 return sessionWithInstance;
633 }).pipe(649 }).pipe(
634 withMetrics({650 withMetrics({
...@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
703 provider: routed.adapter.provider,719 provider: routed.adapter.provider,
704 model: input.modelSelection?.model,720 model: input.modelSelection?.model,
705 interactionMode: input.interactionMode,721 interactionMode: input.interactionMode,
722 // Session-start events alone skew runtime mode toward users who toggle
723 // often, since every toggle restarts the session. Recording it per turn
724 // gives a usage-weighted view and lets it cross with interactionMode.
725 runtimeMode: routed.runtimeMode,
706 attachmentCount: input.attachments.length,726 attachmentCount: input.attachments.length,
707 hasInput: typeof input.input === "string" && input.input.trim().length > 0,727 hasInput: typeof input.input === "string" && input.input.trim().length > 0,
708 });728 });