authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-01 19:16:09-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-01 19:16:09-07:00
logc37fd136edf546fd58daeac146548436c1fa38c2
treef7e56ca1dcdede8c798700a8486599bc662ca61a
parentb2f25d390a8546e42eb6186115b673a53b8c38dc
signature Signed by PGP key B5690EEEBB952194

test(server): measure shell, second client, and reconnect transfer (#9157)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

6 files changed, 436 insertions(+), 65 deletions(-)

apps/server/integration/NetworkTransferMeasurement.integration.ts+59-1
......@@ -5,8 +5,10 @@ import * as NodeZlib from "node:zlib";
55import * as NodeSocket from "@effect/platform-node/NodeSocket";
66import { WsRpcGroup } from "@t3tools/contracts";
77import * as Effect from "effect/Effect";
8import * as Exit from "effect/Exit";
89import * as Layer from "effect/Layer";
910import * as Schema from "effect/Schema";
11import * as Scope from "effect/Scope";
1012import { RpcClient, RpcSerialization } from "effect/unstable/rpc";
1113import * as Socket from "effect/unstable/socket/Socket";
1214
......@@ -101,6 +103,8 @@ export interface WebSocketTransferRecorder {
101103 ) => globalThis.WebSocket;
102104 readonly totals: () => WebSocketTransferTotals;
103105 readonly negotiatedExtensions: () => string;
106 /** Resolves once the upgrade completes, so totals taken after it exclude the upgrade response. */
107 readonly awaitOpen: Effect.Effect<void>;
104108}
105109
106110interface NodeWebSocketWithTransport extends NodeSocket.NodeWS.WebSocket {
......@@ -118,8 +122,15 @@ function rawDataBytes(data: NodeSocket.NodeWS.RawData): number {
118122
119123export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
120124 let socket: NodeWebSocketWithTransport | null = null;
125 // Held separately from the WebSocket so wire totals survive a close, which
126 // is when a reconnect measurement reads them.
127 let transport: NodeWebSocketWithTransport["_socket"] | null = null;
121128 let decodedBytes = 0;
122129 let messages = 0;
130 let resolveOpen: () => void = () => {};
131 const opened = new Promise<void>((resolve) => {
132 resolveOpen = resolve;
133 });
123134
124135 return {
125136 connect: (url, protocols, cookie) => {
......@@ -128,6 +139,10 @@ export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
128139 perMessageDeflate: true,
129140 }) as NodeWebSocketWithTransport;
130141 socket = nextSocket;
142 nextSocket.once("open", () => {
143 transport = nextSocket._socket ?? null;
144 resolveOpen();
145 });
131146 nextSocket.on("message", (data) => {
132147 const bytes = rawDataBytes(data);
133148 decodedBytes += bytes;
......@@ -136,11 +151,17 @@ export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
136151 return nextSocket as unknown as globalThis.WebSocket;
137152 },
138153 totals: () => ({
139 wireBytes: socket?._socket?.bytesRead ?? 0,
154 wireBytes: transport?.bytesRead ?? socket?._socket?.bytesRead ?? 0,
140155 decodedBytes,
141156 messages,
142157 }),
143158 negotiatedExtensions: () => socket?.extensions ?? "",
159 awaitOpen: Effect.promise(() => opened).pipe(
160 Effect.timeoutOrElse({
161 duration: "10 seconds",
162 orElse: () => Effect.die(new Error("Timed out waiting for the WebSocket to open")),
163 }),
164 ),
144165 };
145166}
146167
......@@ -175,3 +196,40 @@ export function countingWsRpcProtocolLayer(input: {
175196
176197export const makeCountingWsRpcClient = RpcClient.make(WsRpcGroup);
177198export type CountingWsRpcClient = Effect.Success<typeof makeCountingWsRpcClient>;
199
200export interface MeasuredWsClient {
201 readonly client: CountingWsRpcClient;
202 readonly recorder: WebSocketTransferRecorder;
203 /** Fork subscription consumers here so they stop before the socket closes. */
204 readonly scope: Scope.Scope;
205 /** Closes the socket now. The enclosing scope closes it otherwise. */
206 readonly close: Effect.Effect<void>;
207}
208
209/**
210 * Opens one WebSocket RPC client on a child of the current scope. Several
211 * clients can share one test scope and still disconnect independently, which
212 * a reconnect measurement needs.
213 */
214export const openMeasuredWsClient = Effect.fn("TransferBudget.openMeasuredWsClient")(
215 function* (input: { readonly url: string; readonly cookie: string }) {
216 const recorder = makeWebSocketTransferRecorder();
217 const parent = yield* Effect.scope;
218 const scope = yield* Scope.fork(parent);
219 const protocol = yield* Layer.buildWithScope(
220 countingWsRpcProtocolLayer({ url: input.url, cookie: input.cookie, recorder }),
221 scope,
222 );
223 const client = yield* makeCountingWsRpcClient.pipe(
224 Effect.provide(protocol),
225 Scope.provide(scope),
226 );
227 yield* recorder.awaitOpen;
228 return {
229 client,
230 recorder,
231 scope,
232 close: Scope.close(scope, Exit.void),
233 } satisfies MeasuredWsClient;
234 },
235);
apps/server/integration/OrchestrationEngineHarness.integration.ts+6
......@@ -22,6 +22,7 @@ import * as Schedule from "effect/Schedule";
2222import * as Schema from "effect/Schema";
2323import * as Scope from "effect/Scope";
2424import * as Stream from "effect/Stream";
25import * as Tracer from "effect/Tracer";
2526
2627import * as CheckpointStore from "../src/checkpointing/CheckpointStore.ts";
2728import { TextGeneration, type TextGenerationShape } from "../src/textGeneration/TextGeneration.ts";
......@@ -231,6 +232,8 @@ export interface OrchestrationIntegrationHarness {
231232interface MakeOrchestrationIntegrationHarnessOptions {
232233 readonly provider?: ProviderDriverKind;
233234 readonly realCodex?: boolean;
235 /** Tracer for every fiber the harness runtime runs, including reactors. */
236 readonly tracer?: Tracer.Tracer;
234237}
235238
236239export const makeOrchestrationIntegrationHarness = (
......@@ -399,6 +402,9 @@ export const makeOrchestrationIntegrationHarness = (
399402 Layer.provideMerge(ServerSettingsService.layerTest()),
400403 Layer.provideMerge(ServerConfig.layerTest(workspaceDir, rootDir)),
401404 Layer.provideMerge(NodeServices.layer),
405 Layer.provideMerge(
406 options?.tracer ? Layer.succeed(Tracer.Tracer, options.tracer) : Layer.empty,
407 ),
402408 );
403409
404410 const runtime = ManagedRuntime.make(layer);
apps/server/integration/SqlStatementCounter.integration.ts created+24
......@@ -0,0 +1,24 @@
1import * as Tracer from "effect/Tracer";
2
3export interface SqlStatementCounter {
4 readonly tracer: Tracer.Tracer;
5 /** Statements executed so far. Read before and after a phase and subtract. */
6 readonly count: () => number;
7}
8
9/**
10 * Counts `sql.execute` spans, which the Effect SQL client opens once per
11 * statement. Spans behave exactly as with the native tracer. Install the same
12 * counter on every runtime under test, otherwise statements run by background
13 * reactors and statements run by request handlers land in different counters.
14 */
15export function makeSqlStatementCounter(): SqlStatementCounter {
16 let statements = 0;
17 const tracer = Tracer.make({
18 span: (options) => {
19 if (options.name === "sql.execute") statements += 1;
20 return new Tracer.NativeSpan(options);
21 },
22 });
23 return { tracer, count: () => statements };
24}
apps/server/integration/TransferBudgetReport.integration.ts+82
......@@ -12,10 +12,28 @@ import {
1212 TRANSFER_MEASURED_TOOLS,
1313} from "./fixtures/transferBudget.ts";
1414
15/** Catch-up delivered to a resubscribing client, and which path the server chose. */
16export interface WebSocketCatchUpMeasurement extends WebSocketTransferTotals {
17 readonly mode: "replay" | "snapshot";
18}
19
1520export interface TransferBudgetRun {
1621 readonly provider: ProviderDriverKind;
1722 readonly threadSnapshot: HttpTransferMeasurement;
23 /** One socket holding only the thread subscription. This is the capped measurement. */
1824 readonly measuredTurnWebSocket: WebSocketTransferTotals;
25 readonly shellSnapshot: HttpTransferMeasurement;
26 /** One socket holding only the shell (sidebar) subscription during the same turn. */
27 readonly measuredTurnShellWebSocket: WebSocketTransferTotals;
28 /** A second client: one socket holding both the thread and shell subscriptions. */
29 readonly measuredTurnSecondClientWebSocket: WebSocketTransferTotals;
30 /** The second client resubscribes after the turn from the cursor it held before it. */
31 readonly reconnectThread: WebSocketCatchUpMeasurement;
32 readonly reconnectShell: WebSocketCatchUpMeasurement;
33 /** `sql.execute` spans opened server-wide during the measured turn. */
34 readonly measuredTurnSqlStatements: number;
35 /** `sql.execute` spans opened while serving both reconnect catch-ups. */
36 readonly reconnectSqlStatements: number;
1937}
2038
2139interface ProviderTransferBudget {
......@@ -47,6 +65,15 @@ function totalWireBytes(run: TransferBudgetRun): number {
4765 return run.threadSnapshot.wireBytes + run.measuredTurnWebSocket.wireBytes;
4866}
4967
68/** Bytes the server wrote to every measured socket during the turn. */
69function serverEgressWireBytes(run: TransferBudgetRun): number {
70 return (
71 run.measuredTurnWebSocket.wireBytes +
72 run.measuredTurnShellWebSocket.wireBytes +
73 run.measuredTurnSecondClientWebSocket.wireBytes
74 );
75}
76
5077function observedTransfer(run: TransferBudgetRun) {
5178 return {
5279 totalWireBytes: totalWireBytes(run),
......@@ -105,6 +132,33 @@ function row(
105132 return `| ${provider} | ${phase} | ${metric} | ${format(observed)} | ${format(maximum)} | ${status} |`;
106133}
107134
135// Shell, second-client, reconnect, and SQL rows are reported without a cap.
136// Shell delivery coalesces on a 50 ms window, so message counts and bytes move
137// with scheduler timing between runs, and the reconnect and SQL figures follow
138// the same batches. The rows exist so CI shows the numbers next to the capped
139// thread measurement.
140function infoRow(
141 provider: ProviderDriverKind,
142 phase: string,
143 metric: string,
144 observed: number,
145 format: (value: number) => string = formatBytes,
146): string {
147 return `| ${provider} | ${phase} | ${metric} | ${format(observed)} | none | INFO |`;
148}
149
150function webSocketRows(
151 provider: ProviderDriverKind,
152 phase: string,
153 totals: WebSocketTransferTotals,
154): string[] {
155 return [
156 infoRow(provider, phase, "WebSocket wire", totals.wireBytes),
157 infoRow(provider, phase, "WebSocket decoded", totals.decodedBytes),
158 infoRow(provider, phase, "WebSocket messages", totals.messages, String),
159 ];
160}
161
108162export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>): string[] {
109163 const violations: string[] = [];
110164 for (const run of runs) {
......@@ -146,6 +200,7 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
146200 "# T3 Code thread transfer budget",
147201 "",
148202 "Wire values are thread data bytes read from local HTTP and WebSocket sockets. HTTP includes response headers; WebSocket measurement starts after the resumed thread subscription synchronizes. TCP/IP, TLS framing, and the WebSocket upgrade are excluded. WebSocket permessage-deflate is negotiated.",
203 "The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and resubscribes from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are `sql.execute` spans counted across the orchestration runtime and the WebSocket handlers.",
149204 `Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Payload sizes are calibrated from heavy local Codex and Claude histories and contain no user data.`,
150205 "",
151206 "| Provider | Total thread wire | Budget | Result |",
......@@ -198,6 +253,32 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
198253 budget.measuredTurnWebSocketMessages,
199254 String,
200255 ),
256 infoRow(run.provider, "shell snapshot", "HTTP wire", run.shellSnapshot.wireBytes),
257 ...webSocketRows(run.provider, "measured turn, shell", run.measuredTurnShellWebSocket),
258 ...webSocketRows(
259 run.provider,
260 "measured turn, second client",
261 run.measuredTurnSecondClientWebSocket,
262 ),
263 infoRow(run.provider, "measured turn", "server egress wire", serverEgressWireBytes(run)),
264 infoRow(
265 run.provider,
266 "measured turn",
267 "SQL statements",
268 run.measuredTurnSqlStatements,
269 String,
270 ),
271 ...webSocketRows(
272 run.provider,
273 `reconnect, thread (${run.reconnectThread.mode})`,
274 run.reconnectThread,
275 ),
276 ...webSocketRows(
277 run.provider,
278 `reconnect, shell (${run.reconnectShell.mode})`,
279 run.reconnectShell,
280 ),
281 infoRow(run.provider, "reconnect", "SQL statements", run.reconnectSqlStatements, String),
201282 );
202283 }
203284
......@@ -205,6 +286,7 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
205286 for (const run of runs) {
206287 lines.push(
207288 `- ${run.provider}: thread snapshot ${formatBytes(run.threadSnapshot.decodedBodyBytes)} decoded to ${formatBytes(run.threadSnapshot.encodedBodyBytes)} gzip.`,
289 `- ${run.provider}: shell snapshot ${formatBytes(run.shellSnapshot.decodedBodyBytes)} decoded to ${formatBytes(run.shellSnapshot.encodedBodyBytes)} gzip.`,
208290 );
209291 }
210292
apps/server/integration/TransferBudgetScenario.integration.ts+84
......@@ -5,13 +5,20 @@ import {
55 DEFAULT_MODEL_BY_PROVIDER,
66 DEFAULT_PROVIDER_INTERACTION_MODE,
77 MessageId,
8 ORCHESTRATION_WS_METHODS,
9 type OrchestrationShellStreamItem,
10 type OrchestrationThreadStreamItem,
811 ProjectId,
912 ProviderDriverKind,
1013 ThreadId,
1114} from "@t3tools/contracts";
1215import * as Effect from "effect/Effect";
16import * as Queue from "effect/Queue";
17import * as Scope from "effect/Scope";
18import * as Stream from "effect/Stream";
1319
1420import type { TurnProcessingQuiescedReceipt } from "../src/orchestration/Services/RuntimeReceiptBus.ts";
21import type { MeasuredWsClient } from "./NetworkTransferMeasurement.integration.ts";
1522import type { OrchestrationIntegrationHarness } from "./OrchestrationEngineHarness.integration.ts";
1623import {
1724 expectedRecordedAssistantText,
......@@ -125,4 +132,81 @@ export function expectedMeasuredAssistantText(provider: ProviderDriverKind): str
125132 return expectedRecordedAssistantText(provider, TRANSFER_MEASURED_TURN_INDEX);
126133}
127134
135/** Takes from the queue until one value matches, and returns everything taken. */
136export const collectQueueUntil = Effect.fn("TransferBudget.collectQueueUntil")(function* <A>(
137 queue: Queue.Queue<A>,
138 predicate: (value: A) => boolean,
139 waitDescription: string,
140) {
141 return yield* Effect.gen(function* () {
142 const values: A[] = [];
143 while (true) {
144 const value = yield* Queue.take(queue);
145 values.push(value);
146 if (predicate(value)) return values;
147 }
148 }).pipe(
149 Effect.timeoutOrElse({
150 duration: "10 seconds",
151 orElse: () => Effect.die(new Error(`Timed out waiting for ${waitDescription}`)),
152 }),
153 );
154});
155
156/**
157 * Resumes the thread subscription from a cursor on a measured client. The
158 * consumer runs in the client's scope, so closing the client stops it. Callers
159 * read items from the returned queue.
160 */
161export const subscribeThreadItems = Effect.fn("TransferBudget.subscribeThreadItems")(function* (
162 measured: MeasuredWsClient,
163 afterSequence: number,
164) {
165 const items = yield* Queue.unbounded<OrchestrationThreadStreamItem>();
166 yield* measured.client[ORCHESTRATION_WS_METHODS.subscribeThread]({
167 threadId: TRANSFER_THREAD_ID,
168 afterSequence,
169 requestCompletionMarker: true,
170 }).pipe(
171 Stream.runForEach((item) => Queue.offer(items, item).pipe(Effect.asVoid)),
172 Scope.provide(measured.scope),
173 Effect.forkIn(measured.scope),
174 );
175 return items;
176});
177
178/** Shell counterpart of subscribeThreadItems. */
179export const subscribeShellItems = Effect.fn("TransferBudget.subscribeShellItems")(function* (
180 measured: MeasuredWsClient,
181 afterSequence: number,
182) {
183 const items = yield* Queue.unbounded<OrchestrationShellStreamItem>();
184 yield* measured.client[ORCHESTRATION_WS_METHODS.subscribeShell]({
185 afterSequence,
186 requestCompletionMarker: true,
187 }).pipe(
188 Stream.runForEach((item) => Queue.offer(items, item).pipe(Effect.asVoid)),
189 Scope.provide(measured.scope),
190 Effect.forkIn(measured.scope),
191 );
192 return items;
193});
194
195/** Waits for the initial catch-up and reports whether it was a replay or a snapshot reset. */
196export const awaitSubscriptionSynchronized = Effect.fn(
197 "TransferBudget.awaitSubscriptionSynchronized",
198)(function* <Item extends OrchestrationThreadStreamItem | OrchestrationShellStreamItem>(
199 items: Queue.Queue<Item>,
200 waitDescription: string,
201) {
202 const initial = yield* collectQueueUntil(
203 items,
204 (item) => item.kind === "synchronized",
205 waitDescription,
206 );
207 return initial.some((item) => item.kind === "snapshot")
208 ? ("snapshot" as const)
209 : ("replay" as const);
210});
211
128212export { TRANSFER_HISTORY_TURN_COUNT, waitForTurnQuiesced };
apps/server/src/server.test.ts+181-64
......@@ -17,6 +17,8 @@ import {
1717 KeybindingRule,
1818 MessageId,
1919 ExternalLauncherCommandNotFoundError,
20 OrchestrationShellSnapshot,
21 type OrchestrationShellStreamItem,
2022 OrchestrationThreadDetailSnapshot,
2123 type OrchestrationThreadStreamItem,
2224 type OrchestrationThreadActivity,
......@@ -58,11 +60,11 @@ import * as ManagedRuntime from "effect/ManagedRuntime";
5860import * as Option from "effect/Option";
5961import * as Path from "effect/Path";
6062import * as PubSub from "effect/PubSub";
61import * as Queue from "effect/Queue";
6263import * as Ref from "effect/Ref";
6364import * as Schema from "effect/Schema";
6465import * as Stream from "effect/Stream";
6566import * as TestClock from "effect/testing/TestClock";
67import * as Tracer from "effect/Tracer";
6668import { ChildProcessSpawner } from "effect/unstable/process";
6769import {
6870 FetchHttpClient,
......@@ -82,26 +84,9 @@ const TEST_EPOCH = DateTime.makeUnsafe("1970-01-01T00:00:00.000Z");
8284const decodeTransferThreadSnapshot = Schema.decodeUnknownEffect(
8385 Schema.fromJsonString(OrchestrationThreadDetailSnapshot),
8486);
85
86const collectQueueUntil = Effect.fn("TransferBudget.collectQueueUntil")(function* <A>(
87 queue: Queue.Queue<A>,
88 predicate: (value: A) => boolean,
89 waitDescription: string,
90) {
91 return yield* Effect.gen(function* () {
92 const values: A[] = [];
93 while (true) {
94 const value = yield* Queue.take(queue);
95 values.push(value);
96 if (predicate(value)) return values;
97 }
98 }).pipe(
99 Effect.timeoutOrElse({
100 duration: "10 seconds",
101 orElse: () => Effect.die(new Error(`Timed out waiting for ${waitDescription}`)),
102 }),
103 );
104});
87const decodeTransferShellSnapshot = Schema.decodeUnknownEffect(
88 Schema.fromJsonString(OrchestrationShellSnapshot),
89);
10590
10691import * as BackgroundPolicy from "./background/BackgroundPolicy.ts";
10792import * as ServerConfig from "./config.ts";
......@@ -171,16 +156,19 @@ import * as Data from "effect/Data";
171156
172157import { makeOrchestrationIntegrationHarness } from "../integration/OrchestrationEngineHarness.integration.ts";
173158import {
174 countingWsRpcProtocolLayer,
175 makeCountingWsRpcClient,
176 makeWebSocketTransferRecorder,
177159 measureHttpGet,
160 openMeasuredWsClient,
178161 transferDelta,
179162} from "../integration/NetworkTransferMeasurement.integration.ts";
163import { makeSqlStatementCounter } from "../integration/SqlStatementCounter.integration.ts";
180164import {
165 awaitSubscriptionSynchronized,
166 collectQueueUntil,
181167 expectedMeasuredAssistantText,
182168 queueMeasuredTransferTurn,
183169 seedTransferBudgetHistory,
170 subscribeShellItems,
171 subscribeThreadItems,
184172 TRANSFER_HISTORY_TURN_COUNT,
185173 TRANSFER_MEASURED_TURN_CREATED_AT,
186174 TRANSFER_MEASURED_TURN_INDEX,
......@@ -9313,9 +9301,12 @@ it.live(
93139301
93149302 const runs = yield* Effect.forEach(
93159303 providers,
9316 (provider) =>
9317 Effect.acquireUseRelease(
9318 makeOrchestrationIntegrationHarness({ provider }),
9304 (provider) => {
9305 // One counter for the orchestration runtime and the HTTP/WS handlers,
9306 // so reactor writes and subscription reads land in the same total.
9307 const sqlCounter = makeSqlStatementCounter();
9308 return Effect.acquireUseRelease(
9309 makeOrchestrationIntegrationHarness({ provider, tracer: sqlCounter.tracer }),
93199310 (harness) =>
93209311 Effect.gen(function* () {
93219312 yield* seedTransferBudgetHistory(harness, provider);
......@@ -9328,19 +9319,10 @@ it.live(
93289319
93299320 const baseUrl = yield* getHttpServerUrl();
93309321 const cookie = yield* getAuthenticatedSessionCookieHeader();
9331
9332 const recorder = makeWebSocketTransferRecorder();
93339322 const wsUrl = baseUrl.replace(/^http:/, "ws:") + "/ws";
9334 const protocolLayer = countingWsRpcProtocolLayer({
9335 url: wsUrl,
9336 cookie,
9337 recorder,
9338 });
93399323
93409324 return yield* Effect.scoped(
93419325 Effect.gen(function* () {
9342 const client = yield* makeCountingWsRpcClient;
9343
93449326 const threadSnapshot = yield* measureHttpGet({
93459327 url: `${baseUrl}/api/orchestration/threads/${TRANSFER_THREAD_ID}`,
93469328 headers: { cookie },
......@@ -9354,29 +9336,78 @@ it.live(
93549336 decodedThread.thread.messages.length,
93559337 TRANSFER_HISTORY_TURN_COUNT * 2,
93569338 );
9339 const shellSnapshot = yield* measureHttpGet({
9340 url: `${baseUrl}/api/orchestration/shell`,
9341 headers: { cookie },
9342 });
9343 assert.equal(shellSnapshot.status, 200);
9344 const decodedShell = yield* decodeTransferShellSnapshot(
9345 Buffer.from(shellSnapshot.decodedBody).toString("utf8"),
9346 );
9347 assert.equal(decodedShell.threads.length, 1);
9348
9349 // Three sockets, the way real installs look: the capped
9350 // thread-only client, a shell-only socket that isolates the
9351 // sidebar cost, and a second device holding both.
9352 const threadClient = yield* openMeasuredWsClient({ url: wsUrl, cookie });
9353 const shellClient = yield* openMeasuredWsClient({ url: wsUrl, cookie });
9354 const secondClient = yield* openMeasuredWsClient({ url: wsUrl, cookie });
9355 assert.include(
9356 threadClient.recorder.negotiatedExtensions(),
9357 "permessage-deflate",
9358 );
93579359
9358 const threadItems = yield* Queue.unbounded<OrchestrationThreadStreamItem>();
9359 yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({
9360 threadId: TRANSFER_THREAD_ID,
9361 afterSequence: decodedThread.snapshotSequence,
9362 requestCompletionMarker: true,
9363 }).pipe(
9364 Stream.runForEach((item) =>
9365 Queue.offer(threadItems, item).pipe(Effect.asVoid),
9360 const threadItems = yield* subscribeThreadItems(
9361 threadClient,
9362 decodedThread.snapshotSequence,
9363 );
9364 const shellItems = yield* subscribeShellItems(
9365 shellClient,
9366 decodedShell.snapshotSequence,
9367 );
9368 const secondThreadItems = yield* subscribeThreadItems(
9369 secondClient,
9370 decodedThread.snapshotSequence,
9371 );
9372 const secondShellItems = yield* subscribeShellItems(
9373 secondClient,
9374 decodedShell.snapshotSequence,
9375 );
9376 assert.equal(
9377 yield* awaitSubscriptionSynchronized(
9378 threadItems,
9379 `${provider} thread subscription to synchronize`,
93669380 ),
9367 Effect.forkScoped,
9381 "replay",
93689382 );
9369 const initialThreadItems = yield* collectQueueUntil(
9370 threadItems,
9371 (item) => item.kind === "synchronized",
9372 `${provider} thread subscription to synchronize`,
9383 assert.equal(
9384 yield* awaitSubscriptionSynchronized(
9385 shellItems,
9386 `${provider} shell subscription to synchronize`,
9387 ),
9388 "replay",
9389 );
9390 assert.equal(
9391 yield* awaitSubscriptionSynchronized(
9392 secondThreadItems,
9393 `${provider} second client thread subscription to synchronize`,
9394 ),
9395 "replay",
9396 );
9397 assert.equal(
9398 yield* awaitSubscriptionSynchronized(
9399 secondShellItems,
9400 `${provider} second client shell subscription to synchronize`,
9401 ),
9402 "replay",
93739403 );
9374 assert.isFalse(initialThreadItems.some((item) => item.kind === "snapshot"));
9375 assert.include(recorder.negotiatedExtensions(), "permessage-deflate");
93769404
93779405 yield* queueMeasuredTransferTurn(harness, provider);
9378 const turnStartTotals = recorder.totals();
9379 yield* client[ORCHESTRATION_WS_METHODS.dispatchCommand]({
9406 const turnStartTotals = threadClient.recorder.totals();
9407 const shellTurnStartTotals = shellClient.recorder.totals();
9408 const secondTurnStartTotals = secondClient.recorder.totals();
9409 const turnStartSqlStatements = sqlCounter.count();
9410 yield* threadClient.client[ORCHESTRATION_WS_METHODS.dispatchCommand]({
93809411 type: "thread.turn.start",
93819412 commandId: CommandId.make(`transfer:${provider}:measured-turn`),
93829413 threadId: TRANSFER_THREAD_ID,
......@@ -9392,26 +9423,95 @@ it.live(
93929423 createdAt: TRANSFER_MEASURED_TURN_CREATED_AT,
93939424 });
93949425 yield* waitForTurnQuiesced(harness, TRANSFER_MEASURED_TURN_INDEX + 1);
9395 const finalThreadSequence = yield* harness.engine
9426 const finalSequences = yield* harness.engine
93969427 .readEvents(decodedThread.snapshotSequence, 10_000)
93979428 .pipe(
93989429 Stream.runFold(
9399 () => decodedThread.snapshotSequence,
9400 (sequence, event) =>
9401 event.aggregateId === TRANSFER_THREAD_ID && isThreadDetailEvent(event)
9402 ? Math.max(sequence, event.sequence)
9403 : sequence,
9430 () => ({
9431 detail: decodedThread.snapshotSequence,
9432 aggregate: decodedShell.snapshotSequence,
9433 }),
9434 (sequences, event) =>
9435 event.aggregateId !== TRANSFER_THREAD_ID
9436 ? sequences
9437 : {
9438 detail: isThreadDetailEvent(event)
9439 ? Math.max(sequences.detail, event.sequence)
9440 : sequences.detail,
9441 aggregate: Math.max(sequences.aggregate, event.sequence),
9442 },
94049443 ),
94059444 );
9445 const finalThreadSequence = finalSequences.detail;
94069446 assert.isAbove(finalThreadSequence, decodedThread.snapshotSequence);
94079447
9448 const reachedFinalThreadEvent = (item: OrchestrationThreadStreamItem) =>
9449 item.kind === "event" && item.event.sequence === finalThreadSequence;
9450 // Shell items carry the sequence of the latest coalesced
9451 // event for the thread, so the last one lands at or past
9452 // the final thread event.
9453 const reachedFinalShellEvent = (item: OrchestrationShellStreamItem) =>
9454 item.kind === "thread-upserted" && item.sequence >= finalSequences.aggregate;
94089455 yield* collectQueueUntil(
94099456 threadItems,
9410 (item) =>
9411 item.kind === "event" && item.event.sequence === finalThreadSequence,
9457 reachedFinalThreadEvent,
94129458 `${provider} thread stream to reach sequence ${finalThreadSequence}`,
94139459 );
9414 const measuredTurnWebSocket = transferDelta(turnStartTotals, recorder.totals());
9460 yield* collectQueueUntil(
9461 secondThreadItems,
9462 reachedFinalThreadEvent,
9463 `${provider} second client thread stream to reach sequence ${finalThreadSequence}`,
9464 );
9465 yield* collectQueueUntil(
9466 shellItems,
9467 reachedFinalShellEvent,
9468 `${provider} shell stream to reach sequence ${finalSequences.aggregate}`,
9469 );
9470 yield* collectQueueUntil(
9471 secondShellItems,
9472 reachedFinalShellEvent,
9473 `${provider} second client shell stream to reach sequence ${finalSequences.aggregate}`,
9474 );
9475 const measuredTurnWebSocket = transferDelta(
9476 turnStartTotals,
9477 threadClient.recorder.totals(),
9478 );
9479 const measuredTurnShellWebSocket = transferDelta(
9480 shellTurnStartTotals,
9481 shellClient.recorder.totals(),
9482 );
9483 const measuredTurnSecondClientWebSocket = transferDelta(
9484 secondTurnStartTotals,
9485 secondClient.recorder.totals(),
9486 );
9487 const measuredTurnSqlStatements = sqlCounter.count() - turnStartSqlStatements;
9488
9489 // The second device drops and comes back with the cursors it
9490 // held before the turn, one subscription at a time so the
9491 // catch-up bytes stay separable.
9492 yield* secondClient.close;
9493 const reconnectSqlStart = sqlCounter.count();
9494 const reconnected = yield* openMeasuredWsClient({ url: wsUrl, cookie });
9495 const reconnectStartTotals = reconnected.recorder.totals();
9496 const reconnectThreadItems = yield* subscribeThreadItems(
9497 reconnected,
9498 decodedThread.snapshotSequence,
9499 );
9500 const reconnectThreadMode = yield* awaitSubscriptionSynchronized(
9501 reconnectThreadItems,
9502 `${provider} reconnected thread subscription to synchronize`,
9503 );
9504 const reconnectThreadTotals = reconnected.recorder.totals();
9505 const reconnectShellItems = yield* subscribeShellItems(
9506 reconnected,
9507 decodedShell.snapshotSequence,
9508 );
9509 const reconnectShellMode = yield* awaitSubscriptionSynchronized(
9510 reconnectShellItems,
9511 `${provider} reconnected shell subscription to synchronize`,
9512 );
9513 const reconnectShellTotals = reconnected.recorder.totals();
9514 const reconnectSqlStatements = sqlCounter.count() - reconnectSqlStart;
94159515
94169516 const finalThreadSnapshot = yield* harness.snapshotQuery
94179517 .getThreadDetailSnapshot(TRANSFER_THREAD_ID)
......@@ -9436,12 +9536,29 @@ it.live(
94369536 provider,
94379537 threadSnapshot,
94389538 measuredTurnWebSocket,
9539 shellSnapshot,
9540 measuredTurnShellWebSocket,
9541 measuredTurnSecondClientWebSocket,
9542 reconnectThread: {
9543 mode: reconnectThreadMode,
9544 ...transferDelta(reconnectStartTotals, reconnectThreadTotals),
9545 },
9546 reconnectShell: {
9547 mode: reconnectShellMode,
9548 ...transferDelta(reconnectThreadTotals, reconnectShellTotals),
9549 },
9550 measuredTurnSqlStatements,
9551 reconnectSqlStatements,
94399552 } satisfies TransferBudgetRun;
9440 }).pipe(Effect.provide(protocolLayer)),
9553 }),
94419554 );
94429555 }),
94439556 (harness) => harness.dispose,
9444 ).pipe(Effect.provide(NodeHttpServerTestWithWsDeflate)),
9557 ).pipe(
9558 Effect.provideService(Tracer.Tracer, sqlCounter.tracer),
9559 Effect.provide(NodeHttpServerTestWithWsDeflate),
9560 );
9561 },
94459562 { concurrency: 1 },
94469563 );
94479564