authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 22:04:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 22:04:51-07:00
log25f0b2a1b7323563aa5d4e7f70e78a3889c5918e
tree63a33dec631f7cc7eebfcedbb4059fa45aa90386
parentff397a29523d3c63e5844b11608c0189640c7561
signature Signed by PGP key B5690EEEBB952194

fix(observability): T3 Connect tracing stops at the relay boundary (#16314)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

12 files changed, 373 insertions(+), 30 deletions(-)

apps/server/src/auth/http.test.ts+85-1
...@@ -1,12 +1,20 @@...@@ -1,12 +1,20 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { EnvironmentHttpApi } from "@t3tools/contracts";2import {
3 AuthSessionId,
4 EnvironmentAuthenticatedAuth,
5 EnvironmentHttpApi,
6} from "@t3tools/contracts";
7import { RelayClientTracer } from "@t3tools/shared/relayTracing";
3import { expect, it } from "@effect/vitest";8import { expect, it } from "@effect/vitest";
4import * as Context from "effect/Context";9import * as Context from "effect/Context";
5import * as Crypto from "effect/Crypto";10import * as Crypto from "effect/Crypto";
6import * as Effect from "effect/Effect";11import * as Effect from "effect/Effect";
7import * as Layer from "effect/Layer";12import * as Layer from "effect/Layer";
13import * as Option from "effect/Option";
8import * as Redacted from "effect/Redacted";14import * as Redacted from "effect/Redacted";
9import * as Schema from "effect/Schema";15import * as Schema from "effect/Schema";
16import * as Tracer from "effect/Tracer";
17import { HttpServerRequest } from "effect/http";
10import * as Etag from "effect/http/Etag";18import * as Etag from "effect/http/Etag";
11import * as HttpPlatform from "effect/http/HttpPlatform";19import * as HttpPlatform from "effect/http/HttpPlatform";
12import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";20import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
...@@ -134,3 +142,79 @@ it.effect("sets the selected browser session cookies through the HTTP route", ()...@@ -134,3 +142,79 @@ it.effect("sets the selected browser session cookies through the HTTP route", ()
134 );142 );
135 }).pipe(Effect.provide(NodeServices.layer)),143 }).pipe(Effect.provide(NodeServices.layer)),
136);144);
145
146it.effect("exports only verified T3 Connect requests", () =>
147 Effect.gen(function* () {
148 const productSpans: Array<string> = [];
149 const localSpans: Array<string> = [];
150 const collect = (into: Array<string>) =>
151 Tracer.make({
152 span: (options) => {
153 into.push(options.name);
154 return new Tracer.NativeSpan(options);
155 },
156 });
157 // "DPoP connect" is a T3 Connect session; any other DPoP token is rejected.
158 const environmentAuth = {
159 authenticateHttpRequest: (request: HttpServerRequest.HttpServerRequest) =>
160 (request.headers.authorization === "DPoP forged"
161 ? Effect.fail(
162 new EnvironmentAuth.ServerAuthInvalidCredentialError({ diagnostic: "forged" }),
163 )
164 : Effect.succeed({
165 sessionId: AuthSessionId.make("session-1"),
166 subject:
167 request.headers.authorization === "DPoP connect"
168 ? "cloud-connect"
169 : "cli-issued-session",
170 method: "bearer-access-token" as const,
171 scopes: ["orchestration:read" as const],
172 })
173 ).pipe(Effect.withSpan("EnvironmentAuth.authenticateHttpRequest")),
174 } as unknown as EnvironmentAuth.EnvironmentAuth["Service"];
175 const middleware = yield* Layer.build(AuthHttp.layerAuthenticatedAuth).pipe(
176 Effect.provideService(EnvironmentAuth.EnvironmentAuth, environmentAuth),
177 Effect.map(Context.get(EnvironmentAuthenticatedAuth)),
178 );
179 const handle = (authorization: string) =>
180 (
181 middleware as unknown as (
182 effect: Effect.Effect<void>,
183 ) => Effect.Effect<void, Error, HttpServerRequest.HttpServerRequest>
184 )(Effect.void.pipe(Effect.withSpan("environment.handler"))).pipe(
185 Effect.ignore,
186 Effect.provideService(
187 HttpServerRequest.HttpServerRequest,
188 HttpServerRequest.fromWeb(
189 new Request("https://environment.example.test/api/orchestration/shell", {
190 headers: {
191 authorization,
192 traceparent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
193 },
194 }),
195 ),
196 ),
197 Effect.provideService(RelayClientTracer, Option.some(collect(productSpans))),
198 Effect.withTracer(collect(localSpans)),
199 );
200
201 yield* handle("DPoP connect");
202 expect(productSpans).toEqual([
203 "environment.relay.request",
204 "EnvironmentAuth.authenticateHttpRequest",
205 "environment.handler",
206 ]);
207 expect(localSpans).toEqual(["EnvironmentAuth.authenticateHttpRequest"]);
208
209 productSpans.length = 0;
210 localSpans.length = 0;
211 yield* handle("DPoP forged");
212 expect(productSpans).toEqual([]);
213 expect(localSpans).toEqual(["EnvironmentAuth.authenticateHttpRequest"]);
214
215 localSpans.length = 0;
216 yield* handle("Bearer access-token");
217 expect(productSpans).toEqual([]);
218 expect(localSpans).toEqual(["EnvironmentAuth.authenticateHttpRequest", "environment.handler"]);
219 }).pipe(Effect.scoped),
220);
apps/server/src/auth/http.ts+7-3
...@@ -25,9 +25,9 @@ import {...@@ -25,9 +25,9 @@ import {
25import type { AuthEnvironmentScope, DpopFailureReason } from "@t3tools/contracts";25import type { AuthEnvironmentScope, DpopFailureReason } from "@t3tools/contracts";
26import { parseAllowedOAuthScope } from "@t3tools/shared/oauthScope";26import { parseAllowedOAuthScope } from "@t3tools/shared/oauthScope";
27import { causeErrorTag } from "@t3tools/shared/observability";27import { causeErrorTag } from "@t3tools/shared/observability";
28import * as Clock from "effect/Clock";
28import * as DateTime from "effect/DateTime";29import * as DateTime from "effect/DateTime";
29import * as Effect from "effect/Effect";30import * as Effect from "effect/Effect";
30import { identity } from "effect/Function";
31import * as Layer from "effect/Layer";31import * as Layer from "effect/Layer";
32import * as Cookies from "effect/http/Cookies";32import * as Cookies from "effect/http/Cookies";
33import * as HttpEffect from "effect/http/HttpEffect";33import * as HttpEffect from "effect/http/HttpEffect";
...@@ -205,6 +205,7 @@ export const layerAuthenticatedAuth = Layer.effect(...@@ -205,6 +205,7 @@ export const layerAuthenticatedAuth = Layer.effect(
205 return (httpEffect) =>205 return (httpEffect) =>
206 Effect.gen(function* () {206 Effect.gen(function* () {
207 const request = yield* HttpServerRequest.HttpServerRequest;207 const request = yield* HttpServerRequest.HttpServerRequest;
208 const startTime = yield* Clock.currentTimeNanos;
208 const session = yield* serverAuth.authenticateHttpRequest(request).pipe(209 const session = yield* serverAuth.authenticateHttpRequest(request).pipe(
209 Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) =>210 Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) =>
210 failEnvironmentAuthInvalid(211 failEnvironmentAuthInvalid(
...@@ -216,13 +217,16 @@ export const layerAuthenticatedAuth = Layer.effect(...@@ -216,13 +217,16 @@ export const layerAuthenticatedAuth = Layer.effect(
216 failEnvironmentInternal("internal_error", error),217 failEnvironmentInternal("internal_error", error),
217 ),218 ),
218 );219 );
219 return yield* httpEffect.pipe(220 const endTime = yield* Clock.currentTimeNanos;
221 const handler = httpEffect.pipe(
220 Effect.provideService(EnvironmentAuthenticatedPrincipal, {222 Effect.provideService(EnvironmentAuthenticatedPrincipal, {
221 ...session,223 ...session,
222 scopes: new Set(session.scopes),224 scopes: new Set(session.scopes),
223 }),225 }),
224 session.subject === "cloud-connect" ? traceAuthenticatedRelayRequest : identity,
225 );226 );
227 return yield* session.subject === "cloud-connect"
228 ? traceAuthenticatedRelayRequest(handler, { startTime, endTime })
229 : handler;
226 }).pipe(Effect.catchTags({ EnvironmentAuthInvalidError: appendDpopChallengeOnUnauthorized }));230 }).pipe(Effect.catchTags({ EnvironmentAuthInvalidError: appendDpopChallengeOnUnauthorized }));
227 }),231 }),
228);232);
apps/server/src/cloud/traceRelayRequest.test.ts+60-5
...@@ -5,7 +5,13 @@ import * as Tracer from "effect/Tracer";...@@ -5,7 +5,13 @@ import * as Tracer from "effect/Tracer";
5import { HttpServerRequest } from "effect/http";5import { HttpServerRequest } from "effect/http";
6import { RelayClientTracer } from "@t3tools/shared/relayTracing";6import { RelayClientTracer } from "@t3tools/shared/relayTracing";
77
8import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts";8import {
9 traceAuthenticatedRelayRequest,
10 traceLocalHandlerWork,
11 traceRelayRequest,
12} from "./traceRelayRequest.ts";
13
14const AUTHENTICATION = { startTime: 1_000n, endTime: 2_000n };
915
10describe("relay request tracing", () => {16describe("relay request tracing", () => {
11 it.effect("does not accept an unauthenticated request trace parent", () =>17 it.effect("does not accept an unauthenticated request trace parent", () =>
...@@ -58,15 +64,64 @@ describe("relay request tracing", () => {...@@ -58,15 +64,64 @@ describe("relay request tracing", () => {
5864
59 yield* traceAuthenticatedRelayRequest(65 yield* traceAuthenticatedRelayRequest(
60 Effect.void.pipe(Effect.withSpan("relay.mint.handler")),66 Effect.void.pipe(Effect.withSpan("relay.mint.handler")),
67 AUTHENTICATION,
61 ).pipe(68 ).pipe(
62 Effect.provideService(HttpServerRequest.HttpServerRequest, request),69 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
63 Effect.provideService(RelayClientTracer, Option.some(productTracer)),70 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
64 );71 );
6572
66 expect(spans).toHaveLength(1);73 expect(spans.map((span) => span.name)).toEqual([
67 const span = spans[0]!;74 "environment.relay.request",
68 expect(span.traceId).toBe("0123456789abcdef0123456789abcdef");75 "EnvironmentAuth.authenticateHttpRequest",
69 expect(Option.getOrUndefined(span.parent)?.spanId).toBe("0123456789abcdef");76 "relay.mint.handler",
77 ]);
78 const [relaySpan, authentication, handler] = spans;
79 expect(Option.getOrUndefined(authentication!.parent)?.spanId).toBe(relaySpan!.spanId);
80 expect(relaySpan!.traceId).toBe("0123456789abcdef0123456789abcdef");
81 expect(Option.getOrUndefined(relaySpan!.parent)?.spanId).toBe("0123456789abcdef");
82 expect(Option.getOrUndefined(handler!.parent)?.spanId).toBe(relaySpan!.spanId);
83 }),
84 );
85});
86
87describe("relay request tracing boundary", () => {
88 it.effect("exports a T3 Connect handler span but not its local work", () =>
89 Effect.gen(function* () {
90 const productSpans: Array<string> = [];
91 const localSpans: Array<string> = [];
92 const collect = (into: Array<string>) =>
93 Tracer.make({
94 span: (options) => {
95 into.push(options.name);
96 return new Tracer.NativeSpan(options);
97 },
98 });
99 const request = HttpServerRequest.fromWeb(
100 new Request("https://environment.example.test/api/orchestration/threads/thread-1"),
101 );
102
103 // Shaped like a real handler: an Effect.fn span whose service call runs
104 // on the local tracer.
105 const handler = Effect.fn("environment.orchestration.threadSnapshot")(function* () {
106 yield* Effect.void.pipe(
107 Effect.withSpan("sql.execute"),
108 Effect.withSpan("ServerSecretStore.get"),
109 traceLocalHandlerWork,
110 );
111 });
112
113 yield* traceAuthenticatedRelayRequest(handler(), AUTHENTICATION).pipe(
114 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
115 Effect.provideService(RelayClientTracer, Option.some(collect(productSpans))),
116 Effect.withTracer(collect(localSpans)),
117 );
118
119 expect(productSpans).toEqual([
120 "environment.relay.request",
121 "EnvironmentAuth.authenticateHttpRequest",
122 "environment.orchestration.threadSnapshot",
123 ]);
124 expect(localSpans).toEqual(["ServerSecretStore.get", "sql.execute"]);
70 }),125 }),
71 );126 );
72});127});
apps/server/src/cloud/traceRelayRequest.ts+65-9
...@@ -1,21 +1,77 @@...@@ -1,21 +1,77 @@
1import { withRelayClientTracing } from "@t3tools/shared/relayTracing";1import {
2 RelayClientTracer,
3 withLocalTracing,
4 withRelayClientTracing,
5} from "@t3tools/shared/relayTracing";
6import * as Clock from "effect/Clock";
7import * as Context from "effect/Context";
2import * as Effect from "effect/Effect";8import * as Effect from "effect/Effect";
9import * as Exit from "effect/Exit";
3import * as Option from "effect/Option";10import * as Option from "effect/Option";
11import * as Tracer from "effect/Tracer";
4import { HttpServerRequest, HttpTraceContext } from "effect/http";12import { HttpServerRequest, HttpTraceContext } from "effect/http";
513
14/**
15 * Exports every span of a handler that is itself T3 Connect work (the token
16 * exchange, the environment descriptor, credential minting), so the whole
17 * connection path can be measured.
18 */
6export const traceRelayRequest = <A, E, R>(19export const traceRelayRequest = <A, E, R>(
7 effect: Effect.Effect<A, E, R>,20 effect: Effect.Effect<A, E, R>,
8): Effect.Effect<A, E, R> => effect.pipe(withRelayClientTracing);21): Effect.Effect<A, E, R> => effect.pipe(withRelayClientTracing);
922
23/**
24 * Traces a request that arrived over T3 Connect, continuing the client's
25 * trace. The request's span and its authentication timing are exported, so
26 * their latency and errors are visible; what the handler then does on the
27 * user's machine (database reads, project indexing, processes) is not, unless
28 * the handler is connection work and opts back in with {@link traceRelayRequest}.
29 *
30 * Call it only after the session is verified as T3 Connect: an unverified
31 * request must not add spans to a trace it names. Authentication therefore
32 * runs on the local tracer, and its timing is recorded here afterwards.
33 */
10export const traceAuthenticatedRelayRequest = <A, E, R>(34export const traceAuthenticatedRelayRequest = <A, E, R>(
11 effect: Effect.Effect<A, E, R>,35 effect: Effect.Effect<A, E, R>,
36 authentication: { readonly startTime: bigint; readonly endTime: bigint },
12): Effect.Effect<A, E, R | HttpServerRequest.HttpServerRequest> =>37): Effect.Effect<A, E, R | HttpServerRequest.HttpServerRequest> =>
13 HttpServerRequest.HttpServerRequest.pipe(38 Effect.all([RelayClientTracer, HttpServerRequest.HttpServerRequest]).pipe(
14 Effect.flatMap((request) =>39 Effect.flatMap(([tracer, request]): Effect.Effect<A, E, R> => {
15 Option.match(HttpTraceContext.fromHeaders(request.headers), {40 if (Option.isNone(tracer)) return effect;
16 onNone: () => effect,41 const parent = HttpTraceContext.fromHeaders(request.headers);
17 onSome: (parent) => effect.pipe(Effect.withParentSpan(parent)),42 const sampled = Option.isNone(parent) || parent.value.sampled;
18 }),43 const startSpan = (
19 ),44 name: string,
20 withRelayClientTracing,45 kind: Tracer.SpanKind,
46 parent: Option.Option<Tracer.AnySpan>,
47 ) =>
48 tracer.value.span({
49 name,
50 parent,
51 annotations: Context.empty(),
52 links: [],
53 startTime: authentication.startTime,
54 kind,
55 root: Option.isNone(parent),
56 sampled,
57 });
58 const span = startSpan("environment.relay.request", "server", parent);
59 startSpan("EnvironmentAuth.authenticateHttpRequest", "internal", Option.some(span)).end(
60 authentication.endTime,
61 Exit.void,
62 );
63 return effect.pipe(
64 Effect.withParentSpan(span),
65 Effect.onExit((exit) =>
66 Clock.currentTimeNanos.pipe(Effect.map((endTime) => span.end(endTime, exit))),
67 ),
68 withRelayClientTracing,
69 );
70 }),
21 );71 );
72
73/**
74 * Keeps the work inside a relay-traced handler on the local tracer. The
75 * handler's span still records the timing and outcome.
76 */
77export const traceLocalHandlerWork = withLocalTracing;
apps/server/src/orchestration-v2/http.ts+7-3
...@@ -17,6 +17,7 @@ import {...@@ -17,6 +17,7 @@ import {
17 failEnvironmentNotFound,17 failEnvironmentNotFound,
18 requireEnvironmentScope,18 requireEnvironmentScope,
19} from "../auth/http.ts";19} from "../auth/http.ts";
20import { traceLocalHandlerWork } from "../cloud/traceRelayRequest.ts";
20import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";21import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";
21import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";22import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";
22import {23import {
...@@ -174,6 +175,7 @@ export const layer = HttpApiBuilder.group(...@@ -174,6 +175,7 @@ export const layer = HttpApiBuilder.group(
174 yield* annotateEnvironmentRequest(args.endpoint.name);175 yield* annotateEnvironmentRequest(args.endpoint.name);
175 yield* requireEnvironmentScope(AuthOrchestrationReadScope);176 yield* requireEnvironmentScope(AuthOrchestrationReadScope);
176 return yield* loadShellSnapshot().pipe(177 return yield* loadShellSnapshot().pipe(
178 traceLocalHandlerWork,
177 Effect.catch((cause) =>179 Effect.catch((cause) =>
178 failEnvironmentInternal("orchestration_snapshot_failed", cause),180 failEnvironmentInternal("orchestration_snapshot_failed", cause),
179 ),181 ),
...@@ -188,7 +190,7 @@ export const layer = HttpApiBuilder.group(...@@ -188,7 +190,7 @@ export const layer = HttpApiBuilder.group(
188 const snapshot = yield* loadThreadSnapshot(190 const snapshot = yield* loadThreadSnapshot(
189 args.params.threadId,191 args.params.threadId,
190 "orchestration_thread_snapshot_failed",192 "orchestration_thread_snapshot_failed",
191 );193 ).pipe(traceLocalHandlerWork);
192 return {194 return {
193 snapshotSequence: snapshot.snapshotSequence,195 snapshotSequence: snapshot.snapshotSequence,
194 projection: snapshot.projection,196 projection: snapshot.projection,
...@@ -200,7 +202,9 @@ export const layer = HttpApiBuilder.group(...@@ -200,7 +202,9 @@ export const layer = HttpApiBuilder.group(
200 Effect.fn("environment.orchestration.threadBoundedSnapshot")(function* (args) {202 Effect.fn("environment.orchestration.threadBoundedSnapshot")(function* (args) {
201 yield* annotateEnvironmentRequest(args.endpoint.name);203 yield* annotateEnvironmentRequest(args.endpoint.name);
202 yield* requireEnvironmentScope(AuthOrchestrationReadScope);204 yield* requireEnvironmentScope(AuthOrchestrationReadScope);
203 const snapshot = yield* loadThreadSnapshotWindow(args.params.threadId);205 const snapshot = yield* loadThreadSnapshotWindow(args.params.threadId).pipe(
206 traceLocalHandlerWork,
207 );
204 const bounded = buildBoundedThreadProjection({208 const bounded = buildBoundedThreadProjection({
205 projection: snapshot.projection,209 projection: snapshot.projection,
206 snapshotSequence: snapshot.snapshotSequence,210 snapshotSequence: snapshot.snapshotSequence,
...@@ -234,7 +238,7 @@ export const layer = HttpApiBuilder.group(...@@ -234,7 +238,7 @@ export const layer = HttpApiBuilder.group(
234 args.params.threadId,238 args.params.threadId,
235 anchorItemId,239 anchorItemId,
236 ThreadId.make(decodedCursor.st),240 ThreadId.make(decodedCursor.st),
237 );241 ).pipe(traceLocalHandlerWork);
238 const pageOrError = selectHistoryPageFromCursorOrError({242 const pageOrError = selectHistoryPageFromCursorOrError({
239 items: snapshot.projection.visibleTurnItems,243 items: snapshot.projection.visibleTurnItems,
240 cursor: args.query.cursor,244 cursor: args.query.cursor,
apps/server/src/project/http.ts+5-1
...@@ -12,6 +12,7 @@ import {...@@ -12,6 +12,7 @@ import {
12 failEnvironmentInvalidRequest,12 failEnvironmentInvalidRequest,
13 requireEnvironmentScope,13 requireEnvironmentScope,
14} from "../auth/http.ts";14} from "../auth/http.ts";
15import { traceLocalHandlerWork } from "../cloud/traceRelayRequest.ts";
15import * as ServerRuntimeStartup from "../serverRuntimeStartup.ts";16import * as ServerRuntimeStartup from "../serverRuntimeStartup.ts";
16import * as ProjectService from "./ProjectService.ts";17import * as ProjectService from "./ProjectService.ts";
17import { projectMutationOperation } from "./ProjectMutation.ts";18import { projectMutationOperation } from "./ProjectMutation.ts";
...@@ -43,6 +44,7 @@ export const layer = HttpApiBuilder.group(...@@ -43,6 +44,7 @@ export const layer = HttpApiBuilder.group(
43 yield* annotateEnvironmentRequest(args.endpoint.name);44 yield* annotateEnvironmentRequest(args.endpoint.name);
44 yield* requireEnvironmentScope(AuthOrchestrationReadScope);45 yield* requireEnvironmentScope(AuthOrchestrationReadScope);
45 return yield* projects.snapshot.pipe(46 return yield* projects.snapshot.pipe(
47 traceLocalHandlerWork,
46 Effect.catch((cause) => failEnvironmentInternal("project_snapshot_failed", cause)),48 Effect.catch((cause) => failEnvironmentInternal("project_snapshot_failed", cause)),
47 );49 );
48 }),50 }),
...@@ -53,7 +55,9 @@ export const layer = HttpApiBuilder.group(...@@ -53,7 +55,9 @@ export const layer = HttpApiBuilder.group(
53 yield* annotateEnvironmentRequest(args.endpoint.name);55 yield* annotateEnvironmentRequest(args.endpoint.name);
54 yield* requireEnvironmentScope(AuthOrchestrationOperateScope);56 yield* requireEnvironmentScope(AuthOrchestrationOperateScope);
55 const operation = projectMutationOperation(projects, args.payload);57 const operation = projectMutationOperation(projects, args.payload);
56 return yield* startup.enqueueCommand(operation).pipe(Effect.catch(failProjectMutation));58 return yield* startup
59 .enqueueCommand(operation)
60 .pipe(traceLocalHandlerWork, Effect.catch(failProjectMutation));
57 }),61 }),
58 );62 );
59 }),63 }),
apps/server/src/pullRequest/http.ts+2-1
...@@ -3,6 +3,7 @@ import * as Effect from "effect/Effect";...@@ -3,6 +3,7 @@ import * as Effect from "effect/Effect";
3import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";3import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
44
5import { annotateEnvironmentRequest, requireEnvironmentScope } from "../auth/http.ts";5import { annotateEnvironmentRequest, requireEnvironmentScope } from "../auth/http.ts";
6import { traceLocalHandlerWork } from "../cloud/traceRelayRequest.ts";
6import * as PullRequestService from "./PullRequestService.ts";7import * as PullRequestService from "./PullRequestService.ts";
78
8/** The patch is often the largest PR payload and benefits from HTTP compression and flow control. */9/** The patch is often the largest PR payload and benefits from HTTP compression and flow control. */
...@@ -16,7 +17,7 @@ export const layer = HttpApiBuilder.group(...@@ -16,7 +17,7 @@ export const layer = HttpApiBuilder.group(
16 Effect.fn("environment.pullRequests.diff")(function* (args) {17 Effect.fn("environment.pullRequests.diff")(function* (args) {
17 yield* annotateEnvironmentRequest(args.endpoint.name);18 yield* annotateEnvironmentRequest(args.endpoint.name);
18 yield* requireEnvironmentScope(AuthOrchestrationReadScope);19 yield* requireEnvironmentScope(AuthOrchestrationReadScope);
19 return yield* pullRequests.diff(args.payload);20 return yield* pullRequests.diff(args.payload).pipe(traceLocalHandlerWork);
20 }),21 }),
21 );22 );
22 }),23 }),
apps/server/src/relay/AgentAwarenessRelay.ts+3-2
...@@ -471,6 +471,8 @@ export const make = Effect.gen(function* () {...@@ -471,6 +471,8 @@ export const make = Effect.gen(function* () {
471 }471 }
472 const environmentId = yield* serverEnvironment.getEnvironmentId;472 const environmentId = yield* serverEnvironment.getEnvironmentId;
473473
474 // Only the relay interaction (signing the proof and the call) is exported;
475 // the reads that decide what to publish are the user's local work.
474 const publishState = (input: {476 const publishState = (input: {
475 readonly projectId: string | null;477 readonly projectId: string | null;
476 readonly state: RelayAgentActivityState | null;478 readonly state: RelayAgentActivityState | null;
...@@ -516,7 +518,7 @@ export const make = Effect.gen(function* () {...@@ -516,7 +518,7 @@ export const make = Effect.gen(function* () {
516 ok: response.ok,518 ok: response.ok,
517 deliveries: deliveryStats(response.deliveries),519 deliveries: deliveryStats(response.deliveries),
518 });520 });
519 });521 }).pipe(Effect.withSpan("relay.agent_activity.publish"), withRelayClientTracing);
520522
521 // Per-thread shell read: this publish runs for every activity-relevant523 // Per-thread shell read: this publish runs for every activity-relevant
522 // domain event, so materializing the full shell here would make the cost524 // domain event, so materializing the full shell here would make the cost
...@@ -659,7 +661,6 @@ export const make = Effect.gen(function* () {...@@ -659,7 +661,6 @@ export const make = Effect.gen(function* () {
659 }).pipe(Effect.andThen(schedulePublishRetry(threadId)));661 }).pipe(Effect.andThen(schedulePublishRetry(threadId)));
660 }),662 }),
661 Effect.withSpan("AgentAwarenessRelay.publishThread"),663 Effect.withSpan("AgentAwarenessRelay.publishThread"),
662 withRelayClientTracing,
663 );664 );
664 });665 });
665666
packages/client-runtime/src/state/environmentHttpAuth.test.ts+32
...@@ -10,12 +10,14 @@ import {...@@ -10,12 +10,14 @@ import {
10 OrchestrationV2ThreadBoundedSnapshot,10 OrchestrationV2ThreadBoundedSnapshot,
11 type OrchestrationV2ThreadHistoryPage,11 type OrchestrationV2ThreadHistoryPage,
12} from "@t3tools/contracts";12} from "@t3tools/contracts";
13import { RelayClientTracer } from "@t3tools/shared/relayTracing";
13import * as Deferred from "effect/Deferred";14import * as Deferred from "effect/Deferred";
14import * as Effect from "effect/Effect";15import * as Effect from "effect/Effect";
15import * as Fiber from "effect/Fiber";16import * as Fiber from "effect/Fiber";
16import * as Layer from "effect/Layer";17import * as Layer from "effect/Layer";
17import * as Option from "effect/Option";18import * as Option from "effect/Option";
18import * as Schema from "effect/Schema";19import * as Schema from "effect/Schema";
20import * as Tracer from "effect/Tracer";
19import { TestClock } from "effect/testing";21import { TestClock } from "effect/testing";
20import type { HttpClient } from "effect/http";22import type { HttpClient } from "effect/http";
2123
...@@ -653,6 +655,36 @@ describe("authenticated environment HTTP requests", () => {...@@ -653,6 +655,36 @@ describe("authenticated environment HTTP requests", () => {
653 );655 );
654});656});
655657
658describe("relay request tracing", () => {
659 it.effect("starts an exported trace for a T3 Connect request", () =>
660 Effect.gen(function* () {
661 const productSpans: Array<{ readonly name: string; readonly root: boolean }> = [];
662 const productTracer = Tracer.make({
663 span: (options) => {
664 productSpans.push({ name: options.name, root: Option.isNone(options.parent) });
665 return new Tracer.NativeSpan(options);
666 },
667 });
668 const harness = makeHarness(() => Response.json(DIFF_RESULT));
669
670 yield* PullRequestDiffLoader.fetchEnvironmentPullRequestDiff({
671 ...harness.input,
672 diff: DIFF,
673 }).pipe(
674 Effect.withSpan("mobile.screen.local"),
675 Effect.provide(harness.httpLayer),
676 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
677 );
678
679 expect(productSpans[0]).toEqual({
680 name: "clientRuntime.state.executeAuthenticatedEnvironmentHttpRequest",
681 root: true,
682 });
683 expect(productSpans.map((span) => span.name)).not.toContain("mobile.screen.local");
684 }),
685 );
686});
687
656describe("orchestration HTTP compatibility header", () => {688describe("orchestration HTTP compatibility header", () => {
657 it("announces the current protocol without replacing bearer or DPoP authentication", () => {689 it("announces the current protocol without replacing bearer or DPoP authentication", () => {
658 expect(690 expect(
packages/client-runtime/src/state/environmentHttpAuth.ts+23-3
...@@ -5,6 +5,7 @@ import {...@@ -5,6 +5,7 @@ import {
5 ORCHESTRATION_PROTOCOL_VERSION_TEXT,5 ORCHESTRATION_PROTOCOL_VERSION_TEXT,
6} from "@t3tools/contracts";6} from "@t3tools/contracts";
7import * as Result from "effect/Result";7import * as Result from "effect/Result";
8import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
8import { FetchHttpClient, type HttpMethod } from "effect/http";9import { FetchHttpClient, type HttpMethod } from "effect/http";
910
10import type { RemoteEnvironmentAuthorization } from "../authorization/service.ts";11import type { RemoteEnvironmentAuthorization } from "../authorization/service.ts";
...@@ -98,10 +99,29 @@ const buildEnvironmentAuthHeaders = (...@@ -98,10 +99,29 @@ const buildEnvironmentAuthHeaders = (
98 * Resolve relay credentials at request time without replacing the live socket.99 * Resolve relay credentials at request time without replacing the live socket.
99 * A rejected credential gets one refresh and retry, with a new request-bound100 * A rejected credential gets one refresh and retry, with a new request-bound
100 * proof. Cookie and bearer requests keep their existing authentication behavior.101 * proof. Cookie and bearer requests keep their existing authentication behavior.
102 *
103 * A DPoP request is T3 Connect work, so its span starts an exported trace that
104 * the environment continues; its local caller's span would leave that trace
105 * without a root.
101 */106 */
102export const executeAuthenticatedEnvironmentHttpRequest = Effect.fn(107export const executeAuthenticatedEnvironmentHttpRequest = <
103 "clientRuntime.state.executeAuthenticatedEnvironmentHttpRequest",108 Group extends Parameters<typeof makeEnvironmentHttpApiGroupClient>[1],
104)(function* <109 A,
110 E,
111 R,
112>(
113 input: Parameters<typeof executeEnvironmentRequest<Group, A, E, R>>[0],
114) =>
115 input.prepared.httpAuthorization?._tag === "Dpop"
116 ? executeEnvironmentRequest(input).pipe(
117 Effect.withSpan(ENVIRONMENT_REQUEST_SPAN, { root: true }),
118 withRelayClientTracing,
119 )
120 : executeEnvironmentRequest(input).pipe(Effect.withSpan(ENVIRONMENT_REQUEST_SPAN));
121
122const ENVIRONMENT_REQUEST_SPAN = "clientRuntime.state.executeAuthenticatedEnvironmentHttpRequest";
123
124const executeEnvironmentRequest = Effect.fnUntraced(function* <
105 Group extends Parameters<typeof makeEnvironmentHttpApiGroupClient>[1],125 Group extends Parameters<typeof makeEnvironmentHttpApiGroupClient>[1],
106 A,126 A,
107 E,127 E,
packages/shared/src/relayTracing.test.ts+47-1
...@@ -6,7 +6,7 @@ import * as Tracer from "effect/Tracer";...@@ -6,7 +6,7 @@ import * as Tracer from "effect/Tracer";
6import { FetchHttpClient } from "effect/http";6import { FetchHttpClient } from "effect/http";
7import { vi } from "vite-plus/test";7import { vi } from "vite-plus/test";
88
9import { RelayClientTracer, withRelayClientTracing } from "./relayTracing.ts";9import { RelayClientTracer, withLocalTracing, withRelayClientTracing } from "./relayTracing.ts";
10import * as RelayTracing from "./relayTracing.ts";10import * as RelayTracing from "./relayTracing.ts";
1111
12function collectingTracer(spans: Array<string>): Tracer.Tracer {12function collectingTracer(spans: Array<string>): Tracer.Tracer {
...@@ -101,3 +101,49 @@ describe("withRelayClientTracing", () => {...@@ -101,3 +101,49 @@ describe("withRelayClientTracing", () => {
101 );101 );
102 });102 });
103});103});
104
105describe("withLocalTracing", () => {
106 it.effect("keeps local work inside a relay span off the product tracer", () =>
107 Effect.gen(function* () {
108 const localSpans: Array<string> = [];
109 const productSpans: Array<string> = [];
110 const localTracer = collectingTracer(localSpans);
111 const productTracer = collectingTracer(productSpans);
112
113 yield* Effect.void.pipe(
114 Effect.withSpan("relay.connection.nested"),
115 Effect.andThen(
116 Effect.void.pipe(
117 Effect.withSpan("sql.execute"),
118 Effect.withSpan("ServerSecretStore.get"),
119 withLocalTracing,
120 ),
121 ),
122 Effect.withSpan("environment.orchestration.threadSnapshot"),
123 withRelayClientTracing,
124 Effect.provideService(RelayClientTracer, Option.some(productTracer)),
125 Effect.withTracer(localTracer),
126 );
127
128 expect(productSpans).toEqual([
129 "relay.connection.nested",
130 "environment.orchestration.threadSnapshot",
131 ]);
132 expect(localSpans).toEqual(["sql.execute", "ServerSecretStore.get"]);
133 }),
134 );
135
136 it.effect("leaves the current tracer alone outside relay tracing", () =>
137 Effect.gen(function* () {
138 const localSpans: Array<string> = [];
139
140 yield* Effect.void.pipe(
141 Effect.withSpan("sql.execute"),
142 withLocalTracing,
143 Effect.withTracer(collectingTracer(localSpans)),
144 );
145
146 expect(localSpans).toEqual(["sql.execute"]);
147 }),
148 );
149});
packages/shared/src/relayTracing.ts+37-1
...@@ -29,6 +29,19 @@ export class RelayClientTracer extends Context.Reference(...@@ -29,6 +29,19 @@ export class RelayClientTracer extends Context.Reference(
29 },29 },
30) {}30) {}
3131
32/**
33 * The tracer that was active before relay tracing took over, so work nested
34 * inside a relay span can return to it.
35 */
36class LocalTracer extends Context.Reference("@t3tools/shared/relayTracing/LocalTracer", {
37 defaultValue: () => Option.none<Tracer.Tracer>(),
38}) {}
39
40/**
41 * Exports every span `effect` creates through the product tracer. Use it only
42 * around T3 Connect work; wrap local work inside it with
43 * {@link withLocalTracing} so it stays off the product tracer.
44 */
32export const withRelayClientTracing = <A, E, R>(45export const withRelayClientTracing = <A, E, R>(
33 effect: Effect.Effect<A, E, R>,46 effect: Effect.Effect<A, E, R>,
34): Effect.Effect<A, E, R> =>47): Effect.Effect<A, E, R> =>
...@@ -36,7 +49,30 @@ export const withRelayClientTracing = <A, E, R>(...@@ -36,7 +49,30 @@ export const withRelayClientTracing = <A, E, R>(
36 Effect.flatMap(49 Effect.flatMap(
37 Option.match({50 Option.match({
38 onNone: () => effect,51 onNone: () => effect,
39 onSome: (tracer) => effect.pipe(Effect.provideService(Tracer.Tracer, tracer)),52 onSome: (tracer) =>
53 Tracer.Tracer.pipe(
54 Effect.flatMap((local) =>
55 effect.pipe(
56 Effect.provideService(Tracer.Tracer, tracer),
57 Effect.provideService(LocalTracer, Option.some(local)),
58 ),
59 ),
60 ),
61 }),
62 ),
63 );
64
65/**
66 * Runs `effect` on the tracer that was active outside relay tracing. Its spans
67 * stay in the local trace and are never exported, so a relay span can time the
68 * work without shipping what the user's machine did to answer it.
69 */
70export const withLocalTracing = <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
71 LocalTracer.pipe(
72 Effect.flatMap(
73 Option.match({
74 onNone: () => effect,
75 onSome: (local) => effect.pipe(Effect.provideService(Tracer.Tracer, local)),
40 }),76 }),
41 ),77 ),
42 );78 );