authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:38:13-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:38:13-07:00
logf36df1852423fef7f7855f86946c9f9a95c3429f
tree17ab412d7b615d0df9dc472c458199e858862785
parent8bd7745f0af918582807326d90e58d59324bb2a6
signature Signed by PGP key B5690EEEBB952194

fix(web): a rejected desktop-local credential is not retried every poll (#16273)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

4 files changed, 334 insertions(+), 13 deletions(-)

apps/web/src/connection/platform.test.ts+70
......@@ -7,10 +7,17 @@ import {
77} from "@t3tools/contracts";
88import { describe, expect, it } from "@effect/vitest";
99import * as Effect from "effect/Effect";
10import {
11 ConnectionBlockedError,
12 ConnectionTransientError,
13} from "@t3tools/client-runtime/connection";
1014
1115import {
1216 canRetainCachedPlatformRegistrationAfterRefreshFailure,
1317 canReuseCachedPlatformRegistration,
18 isRejectedBootstrapCredentialError,
19 isRejectedSecondaryBootstrap,
20 nextRejectedSecondaryBootstrap,
1421 primaryRegistrationToRetainAfterTopologyRead,
1522 provisionDesktopSshEnvironment,
1623 readPrimaryEnvironmentTargetResult,
......@@ -202,6 +209,69 @@ describe("desktop-local bearer cache", () => {
202209 });
203210});
204211
212describe("rejected desktop-local bootstrap tokens", () => {
213 it("backs off on a rejected signature and retries it on a growing, capped delay", () => {
214 const signature = "http://a|ws://a|old-token";
215 const first = nextRejectedSecondaryBootstrap(undefined, signature, 0);
216
217 expect(isRejectedSecondaryBootstrap(first, signature, 59_999)).toBe(true);
218 // A backend restarted on the same port accepts the same token again, so it is retried.
219 expect(isRejectedSecondaryBootstrap(first, signature, 60_000)).toBe(false);
220
221 const second = nextRejectedSecondaryBootstrap(first, signature, 60_000);
222 expect(second.retryAtEpochMs).toBe(180_000);
223
224 let backoff = second;
225 for (let attempt = 0; attempt < 10; attempt += 1) {
226 backoff = nextRejectedSecondaryBootstrap(backoff, signature, 0);
227 }
228 expect(backoff.delayMs).toBe(30 * 60_000);
229 });
230
231 it("retries at once when the token or endpoint changes", () => {
232 const rejected = nextRejectedSecondaryBootstrap(undefined, "http://a|ws://a|old-token", 0);
233
234 expect(isRejectedSecondaryBootstrap(rejected, "http://a|ws://a|new-token", 1)).toBe(false);
235 expect(isRejectedSecondaryBootstrap(rejected, "http://b|ws://b|old-token", 1)).toBe(false);
236 expect(isRejectedSecondaryBootstrap(undefined, "http://a|ws://a|old-token", 1)).toBe(false);
237 expect(nextRejectedSecondaryBootstrap(rejected, "http://a|ws://a|new-token", 1).delayMs).toBe(
238 60_000,
239 );
240 });
241
242 it("treats only authentication rejections as a dead credential", () => {
243 expect(
244 isRejectedBootstrapCredentialError(
245 new ConnectionBlockedError({ reason: "authentication", detail: "invalid" }),
246 ),
247 ).toBe(true);
248 expect(
249 isRejectedBootstrapCredentialError(
250 new ConnectionTransientError({ reason: "endpoint-unavailable", detail: "booting" }),
251 ),
252 ).toBe(false);
253 });
254});
255
256describe("desktop-local bearer cache across token changes", () => {
257 it("keeps a live bearer when only the bootstrap token rotates", () => {
258 // Cached by endpoint: a rotated token reuses the bearer instead of
259 // dropping the environment (and its drafts) when the new token is rejected.
260 const cached = {
261 signature: "http://a|ws://a",
262 registration: {} as never,
263 expiresAtEpochMs: 20_000,
264 refreshAtEpochMs: 15_000,
265 };
266
267 expect(canReuseCachedPlatformRegistration(cached, "http://a|ws://a", 10_000)).toBe(true);
268 expect(
269 canRetainCachedPlatformRegistrationAfterRefreshFailure(cached, "http://a|ws://a", 16_000),
270 ).toBe(true);
271 expect(canReuseCachedPlatformRegistration(cached, "http://b|ws://b", 10_000)).toBe(false);
272 });
273});
274
205275describe("primary topology cache", () => {
206276 const registration = {} as never;
207277 const cached = {
apps/web/src/connection/platform.ts+112-5
......@@ -9,6 +9,7 @@ import {
99 BearerConnectionRegistration,
1010 BearerConnectionTarget,
1111 ConnectionBlockedError,
12 type ConnectionAttemptError,
1213 ConnectionTransientError,
1314 Connectivity,
1415 mapRemoteEnvironmentError,
......@@ -420,7 +421,7 @@ const loadSecondaryConnectionRegistration = Effect.fn(
420421
421422// Poll cadence for the desktop bootstrap topology. There is no change event on
422423// the bridge, so the renderer polls; successful registrations are cached by a
423// signature of their endpoint + token until bearer credentials approach expiry.
424// signature of their endpoint until bearer credentials approach expiry.
424425const PLATFORM_POLL_INTERVAL = "3 seconds";
425426const SECONDARY_BEARER_REFRESH_SKEW_MS = 5_000;
426427
......@@ -447,6 +448,8 @@ interface CachedPlatformRegistration {
447448 readonly registration: PlatformConnectionRegistration;
448449 readonly expiresAtEpochMs?: number;
449450 readonly refreshAtEpochMs?: number;
451 /** The bootstrap token a desktop-local bearer was exchanged for. */
452 readonly bootstrapToken?: string;
450453}
451454
452455export type PrimaryEnvironmentTargetRead =
......@@ -499,6 +502,52 @@ export function canRetainCachedPlatformRegistrationAfterRefreshFailure(
499502 );
500503}
501504
505const REJECTED_BOOTSTRAP_RETRY_INITIAL_MS = 60_000;
506const REJECTED_BOOTSTRAP_RETRY_MAX_MS = 30 * 60_000;
507
508/** A bootstrap token a backend rejected, and when the poll may try it again. */
509export interface RejectedSecondaryBootstrap {
510 readonly signature: string;
511 readonly retryAtEpochMs: number;
512 readonly delayMs: number;
513}
514
515/**
516 * A backend that rejected a bootstrap token will usually keep rejecting it, so
517 * the poll backs off on that exact signature instead of re-presenting a dead
518 * credential every few seconds. It still retries on a capped backoff: a
519 * backend that restarts on the same port seeds a fresh grant for the same
520 * token, and nothing in the topology says it restarted. A new token or
521 * endpoint retries at once.
522 */
523export function isRejectedSecondaryBootstrap(
524 rejected: RejectedSecondaryBootstrap | undefined,
525 signature: string,
526 nowEpochMs: number,
527): rejected is RejectedSecondaryBootstrap {
528 return (
529 rejected !== undefined &&
530 rejected.signature === signature &&
531 nowEpochMs < rejected.retryAtEpochMs
532 );
533}
534
535export function nextRejectedSecondaryBootstrap(
536 previous: RejectedSecondaryBootstrap | undefined,
537 signature: string,
538 nowEpochMs: number,
539): RejectedSecondaryBootstrap {
540 const delayMs =
541 previous?.signature === signature
542 ? Math.min(previous.delayMs * 2, REJECTED_BOOTSTRAP_RETRY_MAX_MS)
543 : REJECTED_BOOTSTRAP_RETRY_INITIAL_MS;
544 return { signature, retryAtEpochMs: nowEpochMs + delayMs, delayMs };
545}
546
547export function isRejectedBootstrapCredentialError(error: ConnectionAttemptError): boolean {
548 return error._tag === "ConnectionBlockedError" && error.reason === "authentication";
549}
550
502551export function secondaryRegistrationsToRetainAfterTopologyRead(
503552 previous: ReadonlyMap<string, CachedPlatformRegistration>,
504553 topologyRead: DesktopSecondaryBootstrapsRead,
......@@ -523,6 +572,7 @@ const layerPlatformConnectionSource = Layer.effect(
523572 });
524573 }
525574 const cacheRef = yield* Ref.make(new Map<string, CachedPlatformRegistration>());
575 const rejectedRef = yield* Ref.make(new Map<string, RejectedSecondaryBootstrap>());
526576
527577 // Resolve the full set of platform-managed environments the host currently
528578 // reports: the primary (same-origin cookie auth) plus any desktop-local
......@@ -588,17 +638,45 @@ const layerPlatformConnectionSource = Layer.effect(
588638 cause: topologyRead.cause,
589639 });
590640 } else {
641 const rejected = yield* Ref.get(rejectedRef);
642 const nextRejected = new Map<string, RejectedSecondaryBootstrap>();
591643 for (const bootstrap of topologyRead.bootstraps) {
592 const signature = `${bootstrap.httpBaseUrl}|${bootstrap.wsBaseUrl}|${bootstrap.bootstrapToken ?? ""}`;
644 // The cached bearer belongs to the endpoint, not to the bootstrap
645 // token it was exchanged for: a new token must not drop a live
646 // session (its removal also clears the environment's drafts). The
647 // token only decides whether a rejected exchange is retried.
648 const endpointSignature = `${bootstrap.httpBaseUrl}|${bootstrap.wsBaseUrl}`;
649 const signature = `${endpointSignature}|${bootstrap.bootstrapToken ?? ""}`;
593650 const cached = previous.get(bootstrap.id);
651 // A new token from the desktop means something changed (rotation, a
652 // restarted backend): exchange it rather than reusing a bearer that
653 // may be dead. The old bearer stays registered if that exchange fails.
594654 if (
595655 cached !== undefined &&
596 canReuseCachedPlatformRegistration(cached, signature, nowEpochMs)
656 cached.bootstrapToken === bootstrap.bootstrapToken &&
657 canReuseCachedPlatformRegistration(cached, endpointSignature, nowEpochMs)
597658 ) {
598659 next.set(bootstrap.id, cached);
599660 registrations.push(cached.registration);
600661 continue;
601662 }
663 const previouslyRejected = rejected.get(bootstrap.id);
664 if (isRejectedSecondaryBootstrap(previouslyRejected, signature, nowEpochMs)) {
665 nextRejected.set(bootstrap.id, previouslyRejected);
666 // The bearer minted before the token died is still good until it expires.
667 if (
668 cached !== undefined &&
669 canRetainCachedPlatformRegistrationAfterRefreshFailure(
670 cached,
671 endpointSignature,
672 nowEpochMs,
673 )
674 ) {
675 next.set(bootstrap.id, cached);
676 registrations.push(cached.registration);
677 }
678 continue;
679 }
602680 const built = yield* loadSecondaryConnectionRegistration(bootstrap).pipe(
603681 Effect.tapError((error) =>
604682 Effect.logWarning("Could not connect a desktop-local backend.", {
......@@ -606,20 +684,49 @@ const layerPlatformConnectionSource = Layer.effect(
606684 error,
607685 }),
608686 ),
687 Effect.tapError((error) =>
688 isRejectedBootstrapCredentialError(error)
689 ? // Back off from when the rejection arrived; the exchanges in
690 // this poll can take longer than the first backoff step.
691 Clock.currentTimeMillis.pipe(
692 Effect.map((rejectedAtEpochMs) =>
693 nextRejected.set(
694 bootstrap.id,
695 nextRejectedSecondaryBootstrap(
696 previouslyRejected,
697 signature,
698 rejectedAtEpochMs,
699 ),
700 ),
701 ),
702 )
703 : Effect.void,
704 ),
609705 Effect.option,
610706 );
611707 if (Option.isSome(built)) {
612 const cacheEntry = { signature, ...built.value };
708 const cacheEntry = {
709 signature: endpointSignature,
710 ...(bootstrap.bootstrapToken === undefined
711 ? {}
712 : { bootstrapToken: bootstrap.bootstrapToken }),
713 ...built.value,
714 };
613715 next.set(bootstrap.id, cacheEntry);
614716 registrations.push(built.value.registration);
615717 } else if (
616718 cached !== undefined &&
617 canRetainCachedPlatformRegistrationAfterRefreshFailure(cached, signature, nowEpochMs)
719 canRetainCachedPlatformRegistrationAfterRefreshFailure(
720 cached,
721 endpointSignature,
722 nowEpochMs,
723 )
618724 ) {
619725 next.set(bootstrap.id, cached);
620726 registrations.push(cached.registration);
621727 }
622728 }
729 yield* Ref.set(rejectedRef, nextRejected);
623730 }
624731
625732 yield* Ref.set(cacheRef, next);
packages/client-runtime/src/connection/registry.test.ts+111
......@@ -158,6 +158,10 @@ const makeHarness = Effect.fn("TestEnvironmentRegistry.makeHarness")(function* (
158158 ) => Effect.Effect<void, Persistence.ConnectionPersistenceError>;
159159 readonly checkRoute?: (route: ConnectionRoute) => RouteCheck;
160160 readonly prepareRoute?: (target: ConnectionTarget) => ConnectionBlockedError | undefined;
161 /** Runs before preparing a route; may suspend to hold an attempt in flight. */
162 readonly beforePrepareRoute?: (
163 target: ConnectionTarget,
164 ) => Effect.Effect<ConnectionBlockedError | undefined>;
161165 readonly initialDisabled?: ReadonlyArray<EnvironmentId>;
162166 },
163167) {
......@@ -385,6 +389,10 @@ const makeHarness = Effect.fn("TestEnvironmentRegistry.makeHarness")(function* (
385389 target,
386390 };
387391 if (options?.prepareError) return yield* options.prepareError;
392 const heldError = options?.beforePrepareRoute
393 ? yield* options.beforePrepareRoute(target)
394 : undefined;
395 if (heldError !== undefined) return yield* heldError;
388396 const routeError = options?.prepareRoute?.(target);
389397 if (routeError !== undefined) return yield* routeError;
390398 yield* Ref.update(connectedRoutes, (current) => [...current, connectionRouteId(target)]);
......@@ -1270,6 +1278,109 @@ describe("EnvironmentRegistry", () => {
12701278 }),
12711279 );
12721280
1281 it.effect("a replaced platform bearer wakes a supervisor blocked on the old one", () =>
1282 Effect.gen(function* () {
1283 let storedCredentials: Ref.Ref<Map<string, ConnectionCredential>> | undefined;
1284 const harness = yield* makeHarness([], [], [], {
1285 // Authentication fails while the old bearer is stored, as the server
1286 // would answer once that session was revoked.
1287 beforePrepareRoute: (target) =>
1288 target._tag !== "BearerConnectionTarget" || storedCredentials === undefined
1289 ? Effect.succeed(undefined)
1290 : Ref.get(storedCredentials).pipe(
1291 Effect.map((current) =>
1292 current.get(target.connectionId)?.token === "old-bearer"
1293 ? new ConnectionBlockedError({ reason: "authentication", detail: "revoked" })
1294 : undefined,
1295 ),
1296 ),
1297 });
1298 storedCredentials = harness.storedCredentials;
1299
1300 yield* Effect.gen(function* () {
1301 const registry = yield* EnvironmentRegistry.EnvironmentRegistry;
1302 const registration = (token: string) =>
1303 new BearerConnectionRegistration({
1304 target: BEARER_TARGET,
1305 profile: BEARER_PROFILE,
1306 credential: new BearerConnectionCredential({ token }),
1307 });
1308 yield* registry.reconcilePlatform([registration("old-bearer")]);
1309 yield* awaitConnectionState(
1310 registry,
1311 BEARER_TARGET.environmentId,
1312 (state) => state.phase === "blocked",
1313 );
1314
1315 yield* registry.reconcilePlatform([registration("new-bearer")]);
1316
1317 yield* awaitConnectionState(
1318 registry,
1319 BEARER_TARGET.environmentId,
1320 (state) => state.phase === "connected",
1321 );
1322 }).pipe(Effect.provide(harness.layer), Effect.scoped);
1323 }),
1324 );
1325
1326 it.effect("a bearer replaced while an attempt is in flight retries with the new one", () =>
1327 Effect.gen(function* () {
1328 const attemptStarted = yield* Deferred.make<void>();
1329 const releaseFirstAttempt = yield* Deferred.make<void>();
1330 const attempts = yield* Ref.make(0);
1331 const harness = yield* makeHarness([], [], [], {
1332 // The first attempt read the old bearer; it stalls, then fails
1333 // authentication after the bearer has already been replaced.
1334 beforePrepareRoute: (target) =>
1335 target._tag !== "BearerConnectionTarget"
1336 ? Effect.succeed(undefined)
1337 : Ref.updateAndGet(attempts, (count) => count + 1).pipe(
1338 Effect.flatMap((count) =>
1339 count === 1
1340 ? Deferred.succeed(attemptStarted, undefined).pipe(
1341 Effect.andThen(Deferred.await(releaseFirstAttempt)),
1342 Effect.as(
1343 new ConnectionBlockedError({
1344 reason: "authentication",
1345 detail: "revoked",
1346 }),
1347 ),
1348 )
1349 : Effect.succeed(undefined),
1350 ),
1351 ),
1352 });
1353
1354 yield* Effect.gen(function* () {
1355 const registry = yield* EnvironmentRegistry.EnvironmentRegistry;
1356 const registration = (token: string) =>
1357 new BearerConnectionRegistration({
1358 target: BEARER_TARGET,
1359 profile: BEARER_PROFILE,
1360 credential: new BearerConnectionCredential({ token }),
1361 });
1362 yield* registry.reconcilePlatform([registration("old-bearer")]);
1363 yield* Deferred.await(attemptStarted).pipe(Effect.timeout("1 second"));
1364
1365 // A live state subscription, as the UI holds, keeps the runtime in place
1366 // so the stalled attempt's failure lands on the same supervisor.
1367 yield* registry
1368 .stateChanges(BEARER_TARGET.environmentId)
1369 .pipe(Stream.runDrain, Effect.forkScoped({ startImmediately: true }));
1370 yield* Effect.yieldNow;
1371 yield* registry.reconcilePlatform([registration("new-bearer")]);
1372 yield* Deferred.succeed(releaseFirstAttempt, undefined);
1373
1374 yield* awaitConnectionState(
1375 registry,
1376 BEARER_TARGET.environmentId,
1377 (state) => state.phase === "connected",
1378 );
1379 expect(yield* Ref.get(attempts)).toBeGreaterThanOrEqual(2);
1380 }).pipe(Effect.provide(harness.layer), Effect.scoped);
1381 }),
1382 );
1383
12731384 it.effect("starts a newly paired bearer environment without re-reading its profile", () =>
12741385 Effect.gen(function* () {
12751386 const harness = yield* makeHarness([]);
packages/client-runtime/src/connection/registry.ts+41-8
......@@ -15,6 +15,7 @@ import * as SubscriptionRef from "effect/SubscriptionRef";
1515import * as ClientCapabilities from "../platform/capabilities.ts";
1616import {
1717 type ConnectionCatalogEntry,
18 type ConnectionCredential,
1819 type ConnectionProfile,
1920 type ConnectionRegistration,
2021 type ConnectionRoute,
......@@ -654,15 +655,26 @@ export const make = Effect.gen(function* () {
654655 // on their own loopback origin, so they authenticate with a bearer
655656 // token instead of the primary's same-origin cookie. Stash it where
656657 // the resolver's bearer broker looks it up.
658 let bearerReplaced = false;
657659 if (registration._tag === "BearerConnectionRegistration") {
658 yield* credentials.put(registration.target.connectionId, registration.credential).pipe(
659 Effect.catch((error) =>
660 Effect.logWarning("Could not store the platform bearer credential.", {
661 environmentId: target.environmentId,
662 error,
663 }),
664 ),
665 );
660 const stored = yield* credentials
661 .get(registration.target.connectionId)
662 .pipe(Effect.orElseSucceed(() => Option.none<ConnectionCredential>()));
663 const changed =
664 Option.isSome(stored) && !Equal.equals(stored.value, registration.credential);
665 // Only a bearer that was actually stored can be retried with; a
666 // failed write leaves the rejected one in place.
667 bearerReplaced = yield* credentials
668 .put(registration.target.connectionId, registration.credential)
669 .pipe(
670 Effect.as(changed),
671 Effect.catch((error) =>
672 Effect.logWarning("Could not store the platform bearer credential.", {
673 environmentId: target.environmentId,
674 error,
675 }).pipe(Effect.as(false)),
676 ),
677 );
666678 }
667679
668680 if (persisted) {
......@@ -687,6 +699,27 @@ export const make = Effect.gen(function* () {
687699 }
688700
689701 yield* installEntryLocked(entry, { retainEquivalentRuntime: true });
702
703 // The catalog entry carries no credential, so a fresh bearer keeps
704 // the equivalent runtime. Anything not yet connected may be using the
705 // old bearer: an attempt in flight would fail with it and then wait
706 // blocked for a manual retry. Retry with the new one instead. A
707 // connected session keeps its socket.
708 if (bearerReplaced) {
709 const scope = (yield* SubscriptionRef.get(serviceScopes)).get(target.environmentId);
710 if (scope !== undefined) {
711 const state = yield* SubscriptionRef.get(scope.supervisor.state);
712 if (
713 state.phase === "connecting" ||
714 state.phase === "backoff" ||
715 (state.phase === "blocked" &&
716 state.lastFailure?._tag === "ConnectionBlockedError" &&
717 state.lastFailure.reason === "authentication")
718 ) {
719 yield* scope.supervisor.retryNow;
720 }
721 }
722 }
690723 }),
691724 );
692725 },