authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-04-09 15:39:28-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-04-09 15:39:28-07:00
log9bedd7140ce22f7f05d3990cdb50cafcceae37de
treebf46c838f432519aa5bef0553e0a79cf976a1520
parent70f5dfcef78a8e153ab698c0b83661974a4fc033
signature Signed by PGP key B5690EEEBB952194

Debounce reconnect disconnect logging (#1862)

4 files changed, 123 insertions(+), 9 deletions(-)

apps/web/src/environments/runtime/connection.test.ts+74-2
...@@ -6,10 +6,12 @@ import type { WsRpcClient } from "~/rpc/wsRpcClient";...@@ -6,10 +6,12 @@ import type { WsRpcClient } from "~/rpc/wsRpcClient";
66
7function createTestClient(options?: {7function createTestClient(options?: {
8 readonly getSnapshot?: () => Promise<{ readonly snapshotSequence: number }>;8 readonly getSnapshot?: () => Promise<{ readonly snapshotSequence: number }>;
9 readonly replayEvents?: () => Promise<ReadonlyArray<any>>;
9}) {10}) {
10 const lifecycleListeners = new Set<(event: any) => void>();11 const lifecycleListeners = new Set<(event: any) => void>();
11 const configListeners = new Set<(event: any) => void>();12 const configListeners = new Set<(event: any) => void>();
12 const terminalListeners = new Set<(event: any) => void>();13 const terminalListeners = new Set<(event: any) => void>();
14 let domainResubscribe: (() => void) | undefined;
1315
14 const getSnapshot = vi.fn(16 const getSnapshot = vi.fn(
15 options?.getSnapshot ??17 options?.getSnapshot ??
...@@ -20,6 +22,7 @@ function createTestClient(options?: {...@@ -20,6 +22,7 @@ function createTestClient(options?: {
20 threads: [],22 threads: [],
21 }) as any),23 }) as any),
22 );24 );
25 const replayEvents = vi.fn(options?.replayEvents ?? (async () => []));
2326
24 const client = {27 const client = {
25 dispose: vi.fn(async () => undefined),28 dispose: vi.fn(async () => undefined),
...@@ -49,8 +52,15 @@ function createTestClient(options?: {...@@ -49,8 +52,15 @@ function createTestClient(options?: {
49 dispatchCommand: vi.fn(async () => undefined),52 dispatchCommand: vi.fn(async () => undefined),
50 getTurnDiff: vi.fn(async () => undefined),53 getTurnDiff: vi.fn(async () => undefined),
51 getFullThreadDiff: vi.fn(async () => undefined),54 getFullThreadDiff: vi.fn(async () => undefined),
52 replayEvents: vi.fn(async () => []),55 replayEvents,
53 onDomainEvent: () => () => undefined,56 onDomainEvent: vi.fn((_: (event: any) => void, options?: { onResubscribe?: () => void }) => {
57 domainResubscribe = options?.onResubscribe;
58 return () => {
59 if (domainResubscribe === options?.onResubscribe) {
60 domainResubscribe = undefined;
61 }
62 };
63 }),
54 },64 },
55 terminal: {65 terminal: {
56 open: vi.fn(async () => undefined),66 open: vi.fn(async () => undefined),
...@@ -114,6 +124,9 @@ function createTestClient(options?: {...@@ -114,6 +124,9 @@ function createTestClient(options?: {
114 });124 });
115 }125 }
116 },126 },
127 triggerDomainResubscribe: () => {
128 domainResubscribe?.();
129 },
117 };130 };
118}131}
119132
...@@ -213,4 +226,63 @@ describe("createEnvironmentConnection", () => {...@@ -213,4 +226,63 @@ describe("createEnvironmentConnection", () => {
213226
214 await connection.dispose();227 await connection.dispose();
215 });228 });
229
230 it("swallows replay recovery failures triggered by resubscribe", async () => {
231 const environmentId = EnvironmentId.makeUnsafe("env-1");
232 const snapshotError = new Error("snapshot failed");
233 let snapshotCalls = 0;
234 const { client, triggerDomainResubscribe } = createTestClient({
235 getSnapshot: async () => {
236 snapshotCalls += 1;
237 if (snapshotCalls === 1) {
238 return {
239 snapshotSequence: 1,
240 projects: [],
241 threads: [],
242 } as any;
243 }
244
245 throw snapshotError;
246 },
247 replayEvents: async () => {
248 throw new Error("SocketCloseError: 1006");
249 },
250 });
251
252 const connection = createEnvironmentConnection({
253 kind: "saved",
254 knownEnvironment: {
255 id: "env-1",
256 label: "Remote env",
257 source: "manual",
258 target: {
259 httpBaseUrl: "http://example.test",
260 wsBaseUrl: "ws://example.test",
261 },
262 environmentId,
263 },
264 client,
265 applyEventBatch: vi.fn(),
266 syncSnapshot: vi.fn(),
267 applyTerminalEvent: vi.fn(),
268 });
269
270 await Promise.resolve();
271 await Promise.resolve();
272
273 const onUnhandledRejection = vi.fn();
274 process.on("unhandledRejection", onUnhandledRejection);
275
276 try {
277 triggerDomainResubscribe();
278 await new Promise((resolve) => setTimeout(resolve, 0));
279 await new Promise((resolve) => setTimeout(resolve, 0));
280 } finally {
281 process.off("unhandledRejection", onUnhandledRejection);
282 }
283
284 expect(onUnhandledRejection).not.toHaveBeenCalled();
285
286 await connection.dispose();
287 });
216});288});
apps/web/src/environments/runtime/connection.ts+8-4
...@@ -128,6 +128,10 @@ export function createEnvironmentConnection(...@@ -128,6 +128,10 @@ export function createEnvironmentConnection(
128 queueMicrotask(flushPendingDomainEvents);128 queueMicrotask(flushPendingDomainEvents);
129 };129 };
130130
131 const scheduleReplayRecovery = (reason: "sequence-gap" | "resubscribe") => {
132 void runReplayRecovery(reason).catch(() => undefined);
133 };
134
131 const runReplayRecovery = async (reason: "sequence-gap" | "resubscribe"): Promise<void> => {135 const runReplayRecovery = async (reason: "sequence-gap" | "resubscribe"): Promise<void> => {
132 if (!recovery.beginReplayRecovery(reason)) {136 if (!recovery.beginReplayRecovery(reason)) {
133 return;137 return;
...@@ -172,7 +176,7 @@ export function createEnvironmentConnection(...@@ -172,7 +176,7 @@ export function createEnvironmentConnection(
172 return;176 return;
173 }177 }
174 }178 }
175 void runReplayRecovery(reason);179 scheduleReplayRecovery(reason);
176 } else if (replayCompletion.shouldReplay && import.meta.env.MODE !== "test") {180 } else if (replayCompletion.shouldReplay && import.meta.env.MODE !== "test") {
177 console.warn(181 console.warn(
178 "[orchestration-recovery]",182 "[orchestration-recovery]",
...@@ -198,7 +202,7 @@ export function createEnvironmentConnection(...@@ -198,7 +202,7 @@ export function createEnvironmentConnection(
198 if (!disposed) {202 if (!disposed) {
199 input.syncSnapshot(snapshot, environmentId);203 input.syncSnapshot(snapshot, environmentId);
200 if (recovery.completeSnapshotRecovery(snapshot.snapshotSequence)) {204 if (recovery.completeSnapshotRecovery(snapshot.snapshotSequence)) {
201 void runReplayRecovery("sequence-gap");205 scheduleReplayRecovery("sequence-gap");
202 }206 }
203 }207 }
204 } catch (error) {208 } catch (error) {
...@@ -245,7 +249,7 @@ export function createEnvironmentConnection(...@@ -245,7 +249,7 @@ export function createEnvironmentConnection(
245 }249 }
246 if (action === "recover") {250 if (action === "recover") {
247 flushPendingDomainEvents();251 flushPendingDomainEvents();
248 void runReplayRecovery("sequence-gap");252 scheduleReplayRecovery("sequence-gap");
249 }253 }
250 },254 },
251 {255 {
...@@ -254,7 +258,7 @@ export function createEnvironmentConnection(...@@ -254,7 +258,7 @@ export function createEnvironmentConnection(
254 return;258 return;
255 }259 }
256 flushPendingDomainEvents();260 flushPendingDomainEvents();
257 void runReplayRecovery("resubscribe");261 scheduleReplayRecovery("resubscribe");
258 },262 },
259 },263 },
260 );264 );
apps/web/src/rpc/wsTransport.test.ts+33
...@@ -707,6 +707,39 @@ describe("WsTransport", () => {...@@ -707,6 +707,39 @@ describe("WsTransport", () => {
707 await transport.dispose();707 await transport.dispose();
708 });708 });
709709
710 it("logs a transport disconnect once even when multiple subscriptions fail together", async () => {
711 const transport = new WsTransport("ws://localhost:3020");
712 const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
713
714 const unsubscribeA = transport.subscribe(
715 () => Stream.fail(new Error("SocketCloseError: 1006")),
716 vi.fn(),
717 { retryDelay: 10 },
718 );
719 const unsubscribeB = transport.subscribe(
720 () => Stream.fail(new Error("SocketCloseError: 1006")),
721 vi.fn(),
722 { retryDelay: 10 },
723 );
724
725 await waitFor(() => {
726 expect(sockets).toHaveLength(1);
727 });
728
729 getSocket().open();
730
731 await waitFor(() => {
732 expect(warnSpy).toHaveBeenCalledTimes(1);
733 });
734 expect(warnSpy).toHaveBeenCalledWith("WebSocket RPC subscription disconnected", {
735 error: "SocketCloseError: 1006",
736 });
737
738 unsubscribeA();
739 unsubscribeB();
740 await transport.dispose();
741 });
742
710 it("streams finite request events without re-subscribing", async () => {743 it("streams finite request events without re-subscribing", async () => {
711 const transport = new WsTransport("ws://localhost:3020");744 const transport = new WsTransport("ws://localhost:3020");
712 const listener = vi.fn();745 const listener = vi.fn();
apps/web/src/rpc/wsTransport.ts+8-3
...@@ -50,6 +50,7 @@ export class WsTransport {...@@ -50,6 +50,7 @@ export class WsTransport {
50 private readonly url: WsRpcProtocolSocketUrlProvider;50 private readonly url: WsRpcProtocolSocketUrlProvider;
51 private readonly lifecycleHandlers: WsProtocolLifecycleHandlers | undefined;51 private readonly lifecycleHandlers: WsProtocolLifecycleHandlers | undefined;
52 private disposed = false;52 private disposed = false;
53 private hasReportedTransportDisconnect = false;
53 private reconnectChain: Promise<void> = Promise.resolve();54 private reconnectChain: Promise<void> = Promise.resolve();
54 private session: TransportSession;55 private session: TransportSession;
5556
...@@ -136,6 +137,7 @@ export class WsTransport {...@@ -136,6 +137,7 @@ export class WsTransport {
136 listener,137 listener,
137 () => active,138 () => active,
138 () => {139 () => {
140 this.hasReportedTransportDisconnect = false;
139 hasReceivedValue = true;141 hasReceivedValue = true;
140 },142 },
141 );143 );
...@@ -156,9 +158,12 @@ export class WsTransport {...@@ -156,9 +158,12 @@ export class WsTransport {
156 return;158 return;
157 }159 }
158160
159 console.warn("WebSocket RPC subscription disconnected", {161 if (!this.hasReportedTransportDisconnect) {
160 error: formattedError,162 console.warn("WebSocket RPC subscription disconnected", {
161 });163 error: formattedError,
164 });
165 }
166 this.hasReportedTransportDisconnect = true;
162 await sleep(retryDelayMs);167 await sleep(retryDelayMs);
163 }168 }
164 }169 }