authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-25 21:58:26-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-25 21:58:26-07:00
loga21b42cec478b093cdc50cc2105b5368ea8b3546
treecfbb88da10131a7c76eb3bfb663d9692bd366642
parent5d707bb71de19eb0eb6fe4688602ee853aec2e3c
signature Signed by PGP key B5690EEEBB952194

fix(server): prune expired replay-protection files from the secrets directory (#13695)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

6 files changed, 293 insertions(+), 1 deletions(-)

apps/server/src/auth/dpop.ts+4-1
......@@ -18,6 +18,9 @@ import {
1818} from "./EnvironmentAuth.ts";
1919import * as ServerSecretStore from "./ServerSecretStore.ts";
2020
21/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */
22export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-";
23
2124export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => {
2225 switch (code) {
2326 case "time_window":
......@@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: {
9699 );
97100 yield* secretStore
98101 .create(
99 `dpop-proof-${replayKey}`,
102 `${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`,
100103 new TextEncoder().encode(
101104 [
102105 `thumbprint=${result.thumbprint}`,
apps/server/src/auth/replayMarkers.test.ts created+92
......@@ -0,0 +1,92 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, it } from "@effect/vitest";
3import * as DateTime from "effect/DateTime";
4import * as Duration from "effect/Duration";
5import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";
7import * as Layer from "effect/Layer";
8import * as Path from "effect/Path";
9import * as TestClock from "effect/testing/TestClock";
10
11import * as ServerConfig from "../config.ts";
12import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts";
13import * as ServerSecretStore from "./ServerSecretStore.ts";
14
15// Every secret name the server stores today. The last three stand for names
16// built from an id at runtime.
17const REAL_SECRET_NAMES = [
18 "server-signing-key",
19 "asset-access-signing-key",
20 "cloud-cli-oauth-token",
21 "cloud-cli-desired-link",
22 "cloud-link-ed25519-key-pair",
23 "cloud-link-ed25519-private-key",
24 "cloud-link-ed25519-public-key",
25 "cloud-mint-ed25519-public-key",
26 "cloud-endpoint-runtime-config",
27 "cloud-endpoint-confirmed-origin",
28 "cloud-linked-user-id",
29 "cloud-relay-url",
30 "cloud-relay-issuer",
31 "cloud-relay-environment-credential",
32 "cloud-publish-agent-activity",
33 "provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk",
34 "provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0",
35 "usage-limit-source-aHVi",
36];
37
38it.layer(NodeServices.layer)("replayMarkers", (it) => {
39 it.effect("prunes only replay markers older than the max age", () =>
40 Effect.gen(function* () {
41 const secretStore = yield* ServerSecretStore.ServerSecretStore;
42 const { secretsDir } = yield* ServerConfig.ServerConfig;
43 const fileSystem = yield* FileSystem.FileSystem;
44 const path = yield* Path.Path;
45 const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z");
46 const setAge = (fileName: string, age: Duration.Duration) => {
47 const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age));
48 return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime);
49 };
50 const writeAged = (name: string, age: Duration.Duration) =>
51 secretStore
52 .create(name, Uint8Array.from([1]))
53 .pipe(Effect.andThen(setAge(`${name}.bin`, age)));
54
55 const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1));
56 const expiredMarkers = [
57 "dpop-proof-old",
58 "cloud-mint-jti-old",
59 "cloud-mint-nonce-old",
60 "cloud-health-jti-old",
61 "cloud-health-nonce-old",
62 ];
63 for (const name of expiredMarkers) yield* writeAged(name, justExpired);
64 yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE);
65 for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30));
66 const pendingSetFile = "dpop-proof-pending.bin.0000.tmp";
67 yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1]));
68 yield* setAge(pendingSetFile, Duration.days(30));
69 yield* TestClock.setTime(DateTime.toEpochMillis(now));
70
71 yield* pruneExpiredReplayMarkers();
72
73 const remaining = yield* fileSystem.readDirectory(secretsDir);
74 assert.deepStrictEqual(
75 remaining.toSorted(),
76 [
77 ...REAL_SECRET_NAMES.map((name) => `${name}.bin`),
78 "dpop-proof-at-max-age.bin",
79 pendingSetFile,
80 ].toSorted(),
81 );
82 }).pipe(
83 Effect.provide(
84 ServerSecretStore.layer.pipe(
85 Layer.provideMerge(
86 ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }),
87 ),
88 ),
89 ),
90 ),
91 );
92});
apps/server/src/auth/replayMarkers.ts created+77
......@@ -0,0 +1,77 @@
1import * as Clock from "effect/Clock";
2import * as Duration from "effect/Duration";
3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as Path from "effect/Path";
8import * as Schedule from "effect/Schedule";
9
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts";
11import * as ServerConfig from "../config.ts";
12import { forkParked } from "../serverActivation.ts";
13import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts";
14
15const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES];
16
17/**
18 * How long a replay marker stays on disk. A marker only matters while its proof
19 * can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After
20 * that, the time check rejects a replay by itself. The sweep and the time check
21 * both use the wall clock, so a pruned marker can let a replay through only if
22 * the clock moves back by almost a day, or if the filesystem stamps mtimes almost
23 * a day behind. Markers are files, so a restart does not reset them.
24 */
25export const REPLAY_MARKER_MAX_AGE = Duration.days(1);
26
27/**
28 * Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`.
29 * `ServerSecretStore` saves each secret as `<name>.bin`, so only
30 * `<marker prefix>*.bin` names match. Other secrets and the `*.bin.<uuid>.tmp`
31 * files that `set` writes are never touched.
32 */
33export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () {
34 const fileSystem = yield* FileSystem.FileSystem;
35 const path = yield* Path.Path;
36 const { secretsDir } = yield* ServerConfig.ServerConfig;
37 const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE);
38 const markers = (yield* fileSystem.readDirectory(secretsDir)).filter(
39 (name) =>
40 name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)),
41 );
42 // `partition` visits every marker, so one locked file does not stop the sweep.
43 const [failures, removed] = yield* Effect.partition(markers, (name) => {
44 const markerPath = path.join(secretsDir, name);
45 return fileSystem.stat(markerPath).pipe(
46 Effect.flatMap((info) =>
47 Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff)
48 ? fileSystem.remove(markerPath).pipe(Effect.as(true))
49 : Effect.succeed(false),
50 ),
51 Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)),
52 );
53 });
54 yield* Effect.annotateCurrentSpan({
55 "replay_markers.matched": markers.length,
56 "replay_markers.removed": removed.filter(Boolean).length,
57 "replay_markers.failed": failures.length,
58 });
59 if (failures.length > 0) {
60 yield* Effect.logWarning("Failed to prune some replay markers", {
61 failed: failures.length,
62 cause: failures[0],
63 });
64 }
65});
66
67/** Prunes expired replay markers after server activation, then every hour. */
68export const layer = Layer.effectDiscard(
69 forkParked(
70 pruneExpiredReplayMarkers().pipe(
71 Effect.catch((cause) =>
72 Effect.logWarning("Failed to prune expired replay markers", { cause }),
73 ),
74 Effect.repeat(Schedule.spaced(Duration.hours(1))),
75 ),
76 ),
77);
apps/server/src/cloud/http.ts+7
......@@ -104,6 +104,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
104104const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
105105const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
106106const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
107/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
108export const CLOUD_REPLAY_MARKER_PREFIXES = [
109 CLOUD_MINT_NONCE_PREFIX,
110 CLOUD_MINT_JTI_PREFIX,
111 CLOUD_HEALTH_NONCE_PREFIX,
112 CLOUD_HEALTH_JTI_PREFIX,
113] as const;
107114const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
108115const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
109116// The desktop app stops its backends within seconds of writing the marker.
apps/server/src/server.test.ts+111
......@@ -176,6 +176,7 @@ import * as VcsProcess from "./vcs/VcsProcess.ts";
176176import * as GitWorkflowService from "./git/GitWorkflowService.ts";
177177import * as ReviewService from "./review/ReviewService.ts";
178178import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";
179import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts";
179180import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
180181import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
181182import * as PairingGrantStore from "./auth/PairingGrantStore.ts";
......@@ -2643,6 +2644,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
26432644 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
26442645 );
26452646
2647 it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () =>
2648 Effect.gen(function* () {
2649 yield* buildAppUnderTest();
2650
2651 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
2652 const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", {
2653 headers: { cookie: ownerCookie },
2654 body: yield* HttpBody.json({}),
2655 });
2656 const credential = (yield* credentialResponse.json) as { readonly credential: string };
2657 const tokenUrl = yield* getHttpServerUrl("/oauth/token");
2658 const acceptedAt = yield* DateTime.now;
2659 // The longest-lived proof: `iat` at the 5 s future skew the verifier allows.
2660 const dpop = makeDpopProof({
2661 method: "POST",
2662 url: tokenUrl,
2663 iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5,
2664 });
2665 const exchange = exchangeAccessToken(credential.credential, {
2666 headers: { dpop: dpop.proof },
2667 scope: "orchestration:read orchestration:operate terminal:operate review:write",
2668 });
2669
2670 assert.equal((yield* exchange).response.status, 200);
2671 // While the proof is fresh, only the replay marker rejects it.
2672 assert.equal((yield* exchange).body.dpopFailureReason, "replay");
2673 // Once the marker can be pruned, the time check rejects the proof by itself.
2674 yield* TestClock.setTime(
2675 acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
2676 );
2677 assert.equal((yield* exchange).body.dpopFailureReason, "time_window");
2678 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
2679 );
2680
26462681 it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () =>
26472682 Effect.gen(function* () {
26482683 yield* buildAppUnderTest();
......@@ -3710,6 +3745,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
37103745 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
37113746 );
37123747
3748 it.effect("rejects cloud replays by time alone once their markers can be pruned", () =>
3749 Effect.gen(function* () {
3750 yield* buildAppUnderTest();
3751
3752 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3753 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3754 publicKeyEncoding: { format: "pem", type: "spki" },
3755 });
3756 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3757 const relayConfigResponse = yield* fetchEffect(
3758 yield* getHttpServerUrl("/api/connect/relay-config"),
3759 {
3760 method: "POST",
3761 headers: { cookie: ownerCookie, "content-type": "application/json" },
3762 body: jsonRequestBody({
3763 relayUrl: "https://relay.example.test",
3764 cloudUserId: "user_123",
3765 environmentCredential: "t3env_test_credential",
3766 cloudMintPublicKey: cloudKeyPair.publicKey,
3767 endpointRuntime: null,
3768 }),
3769 },
3770 );
3771 assert.equal(relayConfigResponse.status, 200);
3772
3773 const acceptedAt = yield* DateTime.now;
3774 // The longest-lived proofs: `iat` at the 60 s future skew the handlers
3775 // allow, and the 5 minute maximum lifetime.
3776 const issuedAt = DateTime.add(acceptedAt, { minutes: 1 });
3777 const proofTimes = {
3778 issuedAt: DateTime.formatIso(issuedAt),
3779 expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })),
3780 };
3781 const requests = [
3782 [
3783 "/api/t3-connect/health",
3784 makeCloudEnvironmentHealthRequest({
3785 privateKey: cloudKeyPair.privateKey,
3786 environmentId: testEnvironmentDescriptor.environmentId,
3787 nonce: "cloud-health-nonce-pruned",
3788 ...proofTimes,
3789 }),
3790 ],
3791 [
3792 "/api/t3-connect/mint-credential",
3793 makeCloudMintCredentialRequest({
3794 privateKey: cloudKeyPair.privateKey,
3795 environmentId: testEnvironmentDescriptor.environmentId,
3796 clientProofKeyThumbprint: "client-proof-key-thumbprint",
3797 nonce: "cloud-mint-nonce-pruned",
3798 ...proofTimes,
3799 }),
3800 ],
3801 ] as const;
3802 const postAll = Effect.forEach(requests, ([pathname, request]) =>
3803 Effect.gen(function* () {
3804 const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), {
3805 method: "POST",
3806 headers: { "content-type": "application/json" },
3807 body: jsonRequestBody(request),
3808 });
3809 return response.status;
3810 }),
3811 );
3812
3813 assert.deepStrictEqual(yield* postAll, [200, 200]);
3814 // While the proofs are fresh, only the replay markers reject them (409).
3815 assert.deepStrictEqual(yield* postAll, [409, 409]);
3816 // Once the markers can be pruned, the time checks reject the proofs by themselves (401).
3817 yield* TestClock.setTime(
3818 acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
3819 );
3820 assert.deepStrictEqual(yield* postAll, [401, 401]);
3821 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3822 );
3823
37133824 it.effect(
37143825 "validates cloud proofs against the configured relay issuer, not the transport URL",
37153826 () =>
apps/server/src/server.ts+2
......@@ -124,6 +124,7 @@ import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts";
124124import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
125125import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
126126import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";
127import * as ReplayMarkers from "./auth/replayMarkers.ts";
127128import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
128129import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
129130import {
......@@ -502,6 +503,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard(
502503
503504const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(
504505 Layer.provideMerge(AntigravityInstallationRefreshLive),
506 Layer.provideMerge(ReplayMarkers.layer),
505507 Layer.provideMerge(ProviderAuthServiceLive),
506508 // Core Services
507509 Layer.provideMerge(ServerSettingsLayerLive),