authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-25 21:58:26-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-25 21:58:26-07:00
loga21b42cec478b093cdc50cc2105b5368ea8b3546
treecfbb88da10131a7c76eb3bfb663d9692bd366642
parent5d707bb71de19eb0eb6fe4688602ee853aec2e3c
signature Signed by PGP key B5690EEEBB952194

fix(server): prune expired replay-protection files from the secrets directory (#13695)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

6 files changed, 293 insertions(+), 1 deletions(-)

apps/server/src/auth/dpop.ts+4-1
...@@ -18,6 +18,9 @@ import {...@@ -18,6 +18,9 @@ import {
18} from "./EnvironmentAuth.ts";18} from "./EnvironmentAuth.ts";
19import * as ServerSecretStore from "./ServerSecretStore.ts";19import * as ServerSecretStore from "./ServerSecretStore.ts";
2020
21/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */
22export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-";
23
21export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => {24export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => {
22 switch (code) {25 switch (code) {
23 case "time_window":26 case "time_window":
...@@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: {...@@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: {
96 );99 );
97 yield* secretStore100 yield* secretStore
98 .create(101 .create(
99 `dpop-proof-${replayKey}`,102 `${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`,
100 new TextEncoder().encode(103 new TextEncoder().encode(
101 [104 [
102 `thumbprint=${result.thumbprint}`,105 `thumbprint=${result.thumbprint}`,
apps/server/src/auth/replayMarkers.test.ts created+92
...@@ -0,0 +1,92 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, it } from "@effect/vitest";
3import * as DateTime from "effect/DateTime";
4import * as Duration from "effect/Duration";
5import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";
7import * as Layer from "effect/Layer";
8import * as Path from "effect/Path";
9import * as TestClock from "effect/testing/TestClock";
10
11import * as ServerConfig from "../config.ts";
12import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts";
13import * as ServerSecretStore from "./ServerSecretStore.ts";
14
15// Every secret name the server stores today. The last three stand for names
16// built from an id at runtime.
17const REAL_SECRET_NAMES = [
18 "server-signing-key",
19 "asset-access-signing-key",
20 "cloud-cli-oauth-token",
21 "cloud-cli-desired-link",
22 "cloud-link-ed25519-key-pair",
23 "cloud-link-ed25519-private-key",
24 "cloud-link-ed25519-public-key",
25 "cloud-mint-ed25519-public-key",
26 "cloud-endpoint-runtime-config",
27 "cloud-endpoint-confirmed-origin",
28 "cloud-linked-user-id",
29 "cloud-relay-url",
30 "cloud-relay-issuer",
31 "cloud-relay-environment-credential",
32 "cloud-publish-agent-activity",
33 "provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk",
34 "provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0",
35 "usage-limit-source-aHVi",
36];
37
38it.layer(NodeServices.layer)("replayMarkers", (it) => {
39 it.effect("prunes only replay markers older than the max age", () =>
40 Effect.gen(function* () {
41 const secretStore = yield* ServerSecretStore.ServerSecretStore;
42 const { secretsDir } = yield* ServerConfig.ServerConfig;
43 const fileSystem = yield* FileSystem.FileSystem;
44 const path = yield* Path.Path;
45 const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z");
46 const setAge = (fileName: string, age: Duration.Duration) => {
47 const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age));
48 return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime);
49 };
50 const writeAged = (name: string, age: Duration.Duration) =>
51 secretStore
52 .create(name, Uint8Array.from([1]))
53 .pipe(Effect.andThen(setAge(`${name}.bin`, age)));
54
55 const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1));
56 const expiredMarkers = [
57 "dpop-proof-old",
58 "cloud-mint-jti-old",
59 "cloud-mint-nonce-old",
60 "cloud-health-jti-old",
61 "cloud-health-nonce-old",
62 ];
63 for (const name of expiredMarkers) yield* writeAged(name, justExpired);
64 yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE);
65 for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30));
66 const pendingSetFile = "dpop-proof-pending.bin.0000.tmp";
67 yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1]));
68 yield* setAge(pendingSetFile, Duration.days(30));
69 yield* TestClock.setTime(DateTime.toEpochMillis(now));
70
71 yield* pruneExpiredReplayMarkers();
72
73 const remaining = yield* fileSystem.readDirectory(secretsDir);
74 assert.deepStrictEqual(
75 remaining.toSorted(),
76 [
77 ...REAL_SECRET_NAMES.map((name) => `${name}.bin`),
78 "dpop-proof-at-max-age.bin",
79 pendingSetFile,
80 ].toSorted(),
81 );
82 }).pipe(
83 Effect.provide(
84 ServerSecretStore.layer.pipe(
85 Layer.provideMerge(
86 ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }),
87 ),
88 ),
89 ),
90 ),
91 );
92});
apps/server/src/auth/replayMarkers.ts created+77
...@@ -0,0 +1,77 @@
1import * as Clock from "effect/Clock";
2import * as Duration from "effect/Duration";
3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as Path from "effect/Path";
8import * as Schedule from "effect/Schedule";
9
10import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts";
11import * as ServerConfig from "../config.ts";
12import { forkParked } from "../serverActivation.ts";
13import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts";
14
15const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES];
16
17/**
18 * How long a replay marker stays on disk. A marker only matters while its proof
19 * can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After
20 * that, the time check rejects a replay by itself. The sweep and the time check
21 * both use the wall clock, so a pruned marker can let a replay through only if
22 * the clock moves back by almost a day, or if the filesystem stamps mtimes almost
23 * a day behind. Markers are files, so a restart does not reset them.
24 */
25export const REPLAY_MARKER_MAX_AGE = Duration.days(1);
26
27/**
28 * Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`.
29 * `ServerSecretStore` saves each secret as `<name>.bin`, so only
30 * `<marker prefix>*.bin` names match. Other secrets and the `*.bin.<uuid>.tmp`
31 * files that `set` writes are never touched.
32 */
33export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () {
34 const fileSystem = yield* FileSystem.FileSystem;
35 const path = yield* Path.Path;
36 const { secretsDir } = yield* ServerConfig.ServerConfig;
37 const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE);
38 const markers = (yield* fileSystem.readDirectory(secretsDir)).filter(
39 (name) =>
40 name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)),
41 );
42 // `partition` visits every marker, so one locked file does not stop the sweep.
43 const [failures, removed] = yield* Effect.partition(markers, (name) => {
44 const markerPath = path.join(secretsDir, name);
45 return fileSystem.stat(markerPath).pipe(
46 Effect.flatMap((info) =>
47 Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff)
48 ? fileSystem.remove(markerPath).pipe(Effect.as(true))
49 : Effect.succeed(false),
50 ),
51 Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)),
52 );
53 });
54 yield* Effect.annotateCurrentSpan({
55 "replay_markers.matched": markers.length,
56 "replay_markers.removed": removed.filter(Boolean).length,
57 "replay_markers.failed": failures.length,
58 });
59 if (failures.length > 0) {
60 yield* Effect.logWarning("Failed to prune some replay markers", {
61 failed: failures.length,
62 cause: failures[0],
63 });
64 }
65});
66
67/** Prunes expired replay markers after server activation, then every hour. */
68export const layer = Layer.effectDiscard(
69 forkParked(
70 pruneExpiredReplayMarkers().pipe(
71 Effect.catch((cause) =>
72 Effect.logWarning("Failed to prune expired replay markers", { cause }),
73 ),
74 Effect.repeat(Schedule.spaced(Duration.hours(1))),
75 ),
76 ),
77);
apps/server/src/cloud/http.ts+7
...@@ -104,6 +104,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";...@@ -104,6 +104,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
104const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";104const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
105const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";105const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
106const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";106const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
107/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
108export const CLOUD_REPLAY_MARKER_PREFIXES = [
109 CLOUD_MINT_NONCE_PREFIX,
110 CLOUD_MINT_JTI_PREFIX,
111 CLOUD_HEALTH_NONCE_PREFIX,
112 CLOUD_HEALTH_JTI_PREFIX,
113] as const;
107const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;114const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
108const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;115const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
109// The desktop app stops its backends within seconds of writing the marker.116// The desktop app stops its backends within seconds of writing the marker.
apps/server/src/server.test.ts+111
...@@ -176,6 +176,7 @@ import * as VcsProcess from "./vcs/VcsProcess.ts";...@@ -176,6 +176,7 @@ import * as VcsProcess from "./vcs/VcsProcess.ts";
176import * as GitWorkflowService from "./git/GitWorkflowService.ts";176import * as GitWorkflowService from "./git/GitWorkflowService.ts";
177import * as ReviewService from "./review/ReviewService.ts";177import * as ReviewService from "./review/ReviewService.ts";
178import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";178import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";
179import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts";
179import * as ServerSecretStore from "./auth/ServerSecretStore.ts";180import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
180import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";181import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
181import * as PairingGrantStore from "./auth/PairingGrantStore.ts";182import * as PairingGrantStore from "./auth/PairingGrantStore.ts";
...@@ -2643,6 +2644,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -2643,6 +2644,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
2643 }).pipe(Effect.provide(NodeHttpServer.layerTest)),2644 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
2644 );2645 );
26452646
2647 it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () =>
2648 Effect.gen(function* () {
2649 yield* buildAppUnderTest();
2650
2651 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
2652 const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", {
2653 headers: { cookie: ownerCookie },
2654 body: yield* HttpBody.json({}),
2655 });
2656 const credential = (yield* credentialResponse.json) as { readonly credential: string };
2657 const tokenUrl = yield* getHttpServerUrl("/oauth/token");
2658 const acceptedAt = yield* DateTime.now;
2659 // The longest-lived proof: `iat` at the 5 s future skew the verifier allows.
2660 const dpop = makeDpopProof({
2661 method: "POST",
2662 url: tokenUrl,
2663 iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5,
2664 });
2665 const exchange = exchangeAccessToken(credential.credential, {
2666 headers: { dpop: dpop.proof },
2667 scope: "orchestration:read orchestration:operate terminal:operate review:write",
2668 });
2669
2670 assert.equal((yield* exchange).response.status, 200);
2671 // While the proof is fresh, only the replay marker rejects it.
2672 assert.equal((yield* exchange).body.dpopFailureReason, "replay");
2673 // Once the marker can be pruned, the time check rejects the proof by itself.
2674 yield* TestClock.setTime(
2675 acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
2676 );
2677 assert.equal((yield* exchange).body.dpopFailureReason, "time_window");
2678 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
2679 );
2680
2646 it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () =>2681 it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () =>
2647 Effect.gen(function* () {2682 Effect.gen(function* () {
2648 yield* buildAppUnderTest();2683 yield* buildAppUnderTest();
...@@ -3710,6 +3745,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -3710,6 +3745,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
3710 }).pipe(Effect.provide(NodeHttpServer.layerTest)),3745 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3711 );3746 );
37123747
3748 it.effect("rejects cloud replays by time alone once their markers can be pruned", () =>
3749 Effect.gen(function* () {
3750 yield* buildAppUnderTest();
3751
3752 const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3753 privateKeyEncoding: { format: "pem", type: "pkcs8" },
3754 publicKeyEncoding: { format: "pem", type: "spki" },
3755 });
3756 const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3757 const relayConfigResponse = yield* fetchEffect(
3758 yield* getHttpServerUrl("/api/connect/relay-config"),
3759 {
3760 method: "POST",
3761 headers: { cookie: ownerCookie, "content-type": "application/json" },
3762 body: jsonRequestBody({
3763 relayUrl: "https://relay.example.test",
3764 cloudUserId: "user_123",
3765 environmentCredential: "t3env_test_credential",
3766 cloudMintPublicKey: cloudKeyPair.publicKey,
3767 endpointRuntime: null,
3768 }),
3769 },
3770 );
3771 assert.equal(relayConfigResponse.status, 200);
3772
3773 const acceptedAt = yield* DateTime.now;
3774 // The longest-lived proofs: `iat` at the 60 s future skew the handlers
3775 // allow, and the 5 minute maximum lifetime.
3776 const issuedAt = DateTime.add(acceptedAt, { minutes: 1 });
3777 const proofTimes = {
3778 issuedAt: DateTime.formatIso(issuedAt),
3779 expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })),
3780 };
3781 const requests = [
3782 [
3783 "/api/t3-connect/health",
3784 makeCloudEnvironmentHealthRequest({
3785 privateKey: cloudKeyPair.privateKey,
3786 environmentId: testEnvironmentDescriptor.environmentId,
3787 nonce: "cloud-health-nonce-pruned",
3788 ...proofTimes,
3789 }),
3790 ],
3791 [
3792 "/api/t3-connect/mint-credential",
3793 makeCloudMintCredentialRequest({
3794 privateKey: cloudKeyPair.privateKey,
3795 environmentId: testEnvironmentDescriptor.environmentId,
3796 clientProofKeyThumbprint: "client-proof-key-thumbprint",
3797 nonce: "cloud-mint-nonce-pruned",
3798 ...proofTimes,
3799 }),
3800 ],
3801 ] as const;
3802 const postAll = Effect.forEach(requests, ([pathname, request]) =>
3803 Effect.gen(function* () {
3804 const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), {
3805 method: "POST",
3806 headers: { "content-type": "application/json" },
3807 body: jsonRequestBody(request),
3808 });
3809 return response.status;
3810 }),
3811 );
3812
3813 assert.deepStrictEqual(yield* postAll, [200, 200]);
3814 // While the proofs are fresh, only the replay markers reject them (409).
3815 assert.deepStrictEqual(yield* postAll, [409, 409]);
3816 // Once the markers can be pruned, the time checks reject the proofs by themselves (401).
3817 yield* TestClock.setTime(
3818 acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
3819 );
3820 assert.deepStrictEqual(yield* postAll, [401, 401]);
3821 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
3822 );
3823
3713 it.effect(3824 it.effect(
3714 "validates cloud proofs against the configured relay issuer, not the transport URL",3825 "validates cloud proofs against the configured relay issuer, not the transport URL",
3715 () =>3826 () =>
apps/server/src/server.ts+2
...@@ -124,6 +124,7 @@ import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts";...@@ -124,6 +124,7 @@ import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts";
124import * as ServerEnvironment from "./environment/ServerEnvironment.ts";124import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
125import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";125import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
126import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";126import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";
127import * as ReplayMarkers from "./auth/replayMarkers.ts";
127import * as ServerSecretStore from "./auth/ServerSecretStore.ts";128import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
128import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";129import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
129import {130import {
...@@ -502,6 +503,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard(...@@ -502,6 +503,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard(
502503
503const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(504const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(
504 Layer.provideMerge(AntigravityInstallationRefreshLive),505 Layer.provideMerge(AntigravityInstallationRefreshLive),
506 Layer.provideMerge(ReplayMarkers.layer),
505 Layer.provideMerge(ProviderAuthServiceLive),507 Layer.provideMerge(ProviderAuthServiceLive),
506 // Core Services508 // Core Services
507 Layer.provideMerge(ServerSettingsLayerLive),509 Layer.provideMerge(ServerSettingsLayerLive),