authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 23:13:47-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 23:13:47-07:00
log442735897f6f92af33798075baa12d4fd4d710dd
tree15770d3d9c33dc3100e97a1915705a74bec53686
parent83b982a85c87b4c0eda08beb9028e3a73f7c89b6
signature Signed by PGP key B5690EEEBB952194

fix(relay): export traces through one tracer, one request span each (#16382)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

6 files changed, 87 insertions(+), 133 deletions(-)

infra/relay/src/hooks/HookForwarder.test.ts+4-6
...@@ -615,15 +615,13 @@ describe("HookForwarder", () => {...@@ -615,15 +615,13 @@ describe("HookForwarder", () => {
615 });615 });
616 const harness = makeHarness();616 const harness = makeHarness();
617 const handler = yield* harness.httpEffect;617 const handler = yield* harness.httpEffect;
618 // As the worker runtime runs it: its own tracer around ours, off for hook618 // As the worker runtime runs it: its own tracer around ours, turned off
619 // paths. This checks the predicate; whether alchemy applies it per event619 // (see worker.ts). Whether alchemy applies the predicate per event is
620 // is only visible on a deployed worker (see worker.ts).620 // only visible on a deployed worker.
621 yield* HttpMiddleware.tracer(621 yield* HttpMiddleware.tracer(
622 traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),622 traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
623 ).pipe(623 ).pipe(
624 Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>624 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
625 HookForwarder.isRelayHookPath(request.url),
626 ),
627 Effect.withTracer(tracer),625 Effect.withTracer(tracer),
628 Effect.provideService(626 Effect.provideService(
629 HttpServerRequest.HttpServerRequest,627 HttpServerRequest.HttpServerRequest,
infra/relay/src/http/Api.test.ts+33
...@@ -30,6 +30,7 @@ import * as Tracer from "effect/Tracer";...@@ -30,6 +30,7 @@ import * as Tracer from "effect/Tracer";
30import * as Etag from "effect/http/Etag";30import * as Etag from "effect/http/Etag";
31import * as HttpEffect from "effect/http/HttpEffect";31import * as HttpEffect from "effect/http/HttpEffect";
32import * as HttpRouter from "effect/http/HttpRouter";32import * as HttpRouter from "effect/http/HttpRouter";
33import * as HttpMiddleware from "effect/http/HttpMiddleware";
33import * as HttpServerRequest from "effect/http/HttpServerRequest";34import * as HttpServerRequest from "effect/http/HttpServerRequest";
34import * as HttpServerResponse from "effect/http/HttpServerResponse";35import * as HttpServerResponse from "effect/http/HttpServerResponse";
35import * as HttpApi from "effect/http-api/HttpApi";36import * as HttpApi from "effect/http-api/HttpApi";
...@@ -1195,6 +1196,38 @@ describe("relay request tracing", () => {...@@ -1195,6 +1196,38 @@ describe("relay request tracing", () => {
1195 }),1196 }),
1196 );1197 );
11971198
1199 it.effect("records one server span inside the worker's disabled HTTP tracer", () =>
1200 Effect.gen(function* () {
1201 const spans: Array<Tracer.NativeSpan> = [];
1202 const tracer = Tracer.make({
1203 span: (options) => {
1204 const span = new Tracer.NativeSpan(options);
1205 spans.push(span);
1206 return span;
1207 },
1208 });
1209 const request = HttpServerRequest.fromWeb(
1210 new Request("https://relay.test/v1/mobile/devices", { method: "POST" }),
1211 );
1212
1213 // As the worker runtime runs it: its own tracer around ours, turned off.
1214 yield* HttpMiddleware.tracer(
1215 traceRelayHttpRequestWith(
1216 Effect.succeed(HttpServerResponse.empty({ status: 204 })),
1217 Layer.empty,
1218 ),
1219 ).pipe(
1220 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
1221 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
1222 Effect.withTracer(tracer),
1223 );
1224 yield* Effect.yieldNow;
1225
1226 expect(spans.filter((span) => span.kind === "server")).toHaveLength(1);
1227 expect(spans[0]?.attributes.get("url.path")).toBe("/v1/mobile/devices");
1228 }),
1229 );
1230
1198 it.effect("fails hung requests with a 504 before the client's 10s abort", () =>1231 it.effect("fails hung requests with a 504 before the client's 10s abort", () =>
1199 Effect.gen(function* () {1232 Effect.gen(function* () {
1200 const spans: Array<Tracer.NativeSpan> = [];1233 const spans: Array<Tracer.NativeSpan> = [];
infra/relay/src/http/Api.ts+7-3
...@@ -228,8 +228,12 @@ export const traceRelayHttpRequest = <E, R>(...@@ -228,8 +228,12 @@ export const traceRelayHttpRequest = <E, R>(
228 Effect.andThen(relayRequestDeadline(httpEffect)),228 Effect.andThen(relayRequestDeadline(httpEffect)),
229 );229 );
230 if (!isRelayHookPath(request.url)) {230 if (!isRelayHookPath(request.url)) {
231 // HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.231 return yield* HttpMiddleware.tracer(traced).pipe(
232 return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));232 // The worker turns its own request span off; this one is ours.
233 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
234 // HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
235 Effect.ensuring(Effect.yieldNow),
236 );
233 }237 }
234 // Hook URLs carry a secret token: the tracer and deadline log see a redacted238 // Hook URLs carry a secret token: the tracer and deadline log see a redacted
235 // request, while the route itself still receives the original. A webhook239 // request, while the route itself still receives the original. A webhook
...@@ -253,7 +257,7 @@ export const traceRelayHttpRequest = <E, R>(...@@ -253,7 +257,7 @@ export const traceRelayHttpRequest = <E, R>(
253 ),257 ),
254 ).pipe(258 ).pipe(
255 Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),259 Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
256 // The worker disables its own span for hook paths; this one is ours.260 // The worker turns its own request span off; this one is ours.
257 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),261 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
258 Effect.ensuring(Effect.yieldNow),262 Effect.ensuring(Effect.yieldNow),
259 );263 );
infra/relay/src/observability.test.ts+15-69
...@@ -1,46 +1,20 @@...@@ -1,46 +1,20 @@
1import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
2import { expect, it } from "@effect/vitest";1import { expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
4import * as Effect from "effect/Effect";2import * as Effect from "effect/Effect";
5import * as Redacted from "effect/Redacted";3import * as Tracer from "effect/Tracer";
6import * as Schema from "effect/Schema";
7import * as HttpServer from "effect/http/HttpServer";
8import * as HttpServerRequest from "effect/http/HttpServerRequest";
9import * as HttpServerResponse from "effect/http/HttpServerResponse";
10import type { OtlpTracer } from "effect/observability";
114
12import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";5import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
13import * as Observability from "./observability.ts";6import * as Observability from "./observability.ts";
147
15interface ExportedRequest {8it.effect("adds schema error fields to spans on the current tracer", () =>
16 readonly authorization: string | undefined;
17 readonly body: string;
18 readonly dataset: string | undefined;
19}
20
21const otlpAttributeValue = (value: {
22 readonly stringValue?: string | null;
23 readonly boolValue?: boolean | null;
24 readonly intValue?: string | number | null;
25 readonly doubleValue?: number | null;
26}) => value.stringValue ?? value.boolValue ?? value.intValue ?? value.doubleValue;
27
28const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
29
30it.effect("exports schema error fields as span attributes", () =>
31 Effect.gen(function* () {9 Effect.gen(function* () {
32 const exportedRequest = yield* Deferred.make<ExportedRequest>();10 const spans: Array<Tracer.NativeSpan> = [];
33 yield* HttpServer.serveEffect(11 const tracer = Tracer.make({
34 Effect.gen(function* () {12 span: (options) => {
35 const request = yield* HttpServerRequest.HttpServerRequest;13 const span = new Tracer.NativeSpan(options);
36 yield* Deferred.succeed(exportedRequest, {14 spans.push(span);
37 authorization: request.headers.authorization,15 return span;
38 body: yield* request.text,16 },
39 dataset: request.headers["x-axiom-dataset"],17 });
40 });
41 return HttpServerResponse.empty({ status: 204 });
42 }),
43 );
4418
45 yield* Effect.fail(19 yield* Effect.fail(
46 new EnvironmentConnector.EnvironmentConnectNotAuthorized({20 new EnvironmentConnector.EnvironmentConnectNotAuthorized({
...@@ -51,44 +25,16 @@ it.effect("exports schema error fields as span attributes", () =>...@@ -51,44 +25,16 @@ it.effect("exports schema error fields as span attributes", () =>
51 ).pipe(25 ).pipe(
52 Effect.withSpan("relay.test.schema_error"),26 Effect.withSpan("relay.test.schema_error"),
53 Effect.exit,27 Effect.exit,
54 Effect.provide(28 Observability.withSchemaErrorSpanAttributes,
55 Observability.layer({29 Effect.withTracer(tracer),
56 tracesEndpoint: "/v1/traces",
57 tracesDatasetName: "relay-test-traces",
58 ingestToken: Redacted.make("test-token"),
59 }),
60 ),
61 );30 );
6231
63 const request = yield* Deferred.await(exportedRequest).pipe(Effect.timeout("1 second"));32 expect(spans.map((span) => span.name)).toEqual(["relay.test.schema_error"]);
64 const payload = (yield* decodeJson(request.body)) as OtlpTracer.TraceData;33 expect(Object.fromEntries(spans[0]!.attributes)).toMatchObject({
65 const resourceAttributes = Object.fromEntries(
66 payload.resourceSpans
67 .flatMap((resourceSpan) => resourceSpan.resource.attributes)
68 .map((attribute) => [attribute.key, otlpAttributeValue(attribute.value)]),
69 );
70 const span = payload.resourceSpans
71 .flatMap((resourceSpan) => resourceSpan.scopeSpans)
72 .flatMap((scopeSpan) => scopeSpan.spans)
73 .find((candidate) => candidate.name === "relay.test.schema_error");
74 const attributes = Object.fromEntries(
75 (span?.attributes ?? []).map((attribute) => [
76 attribute.key,
77 otlpAttributeValue(attribute.value),
78 ]),
79 );
80
81 expect(request.authorization).toBe("Bearer test-token");
82 expect(request.dataset).toBe("relay-test-traces");
83 expect(resourceAttributes).toMatchObject({
84 "service.name": "t3code-relay",
85 "service.namespace": "t3code",
86 });
87 expect(attributes).toMatchObject({
88 "error.type": "EnvironmentConnectNotAuthorized",34 "error.type": "EnvironmentConnectNotAuthorized",
89 "error.environmentId": "environment-1",35 "error.environmentId": "environment-1",
90 "error.operation": "connect",36 "error.operation": "connect",
91 "error.reason": "managed_endpoint_allocation_not_ready",37 "error.reason": "managed_endpoint_allocation_not_ready",
92 });38 });
93 }).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),39 }),
94);40);
infra/relay/src/observability.ts+11-27
...@@ -4,12 +4,9 @@ import * as Output from "alchemy/Output";...@@ -4,12 +4,9 @@ import * as Output from "alchemy/Output";
4import * as Cause from "effect/Cause";4import * as Cause from "effect/Cause";
5import * as Effect from "effect/Effect";5import * as Effect from "effect/Effect";
6import * as Exit from "effect/Exit";6import * as Exit from "effect/Exit";
7import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";7import * as Option from "effect/Option";
9import * as Redacted from "effect/Redacted";
10import * as Schema from "effect/Schema";8import * as Schema from "effect/Schema";
11import * as Tracer from "effect/Tracer";9import * as Tracer from "effect/Tracer";
12import { OtlpExporter, OtlpSerialization, OtlpTracer } from "effect/observability";
1310
14import { relayResourceNameForStage } from "./deploymentConfig.ts";11import { relayResourceNameForStage } from "./deploymentConfig.ts";
1512
...@@ -212,27 +209,14 @@ const withSchemaErrorAttributes = (delegate: Tracer.Tracer): Tracer.Tracer =>...@@ -212,27 +209,14 @@ const withSchemaErrorAttributes = (delegate: Tracer.Tracer): Tracer.Tracer =>
212 ...(delegate.context ? { context: delegate.context } : {}),209 ...(delegate.context ? { context: delegate.context } : {}),
213 });210 });
214211
215export const layer = (input: {212/**
216 readonly tracesEndpoint: string;213 * Adds a failed span's schema error fields (`error.type`, `error.<field>`) to
217 readonly tracesDatasetName: string;214 * the span, on whichever tracer is current. Provide it around the work whose
218 readonly ingestToken: Redacted.Redacted<string>;215 * spans should carry them; the Worker's telemetry still owns export.
219}) =>216 */
220 Layer.effect(217export const withSchemaErrorSpanAttributes = <A, E, R>(
221 Tracer.Tracer,218 effect: Effect.Effect<A, E, R>,
222 OtlpTracer.make({219): Effect.Effect<A, E, R> =>
223 url: input.tracesEndpoint,220 Effect.flatMap(Effect.tracer, (tracer) =>
224 resource: {221 effect.pipe(Effect.withTracer(withSchemaErrorAttributes(tracer))),
225 serviceName: "t3code-relay",222 );
226 attributes: {
227 "service.namespace": "t3code",
228 "service.runtime": "cloudflare-worker",
229 "service.component": "relay",
230 },
231 },
232 headers: {
233 Authorization: `Bearer ${Redacted.value(input.ingestToken)}`,
234 "X-Axiom-Dataset": input.tracesDatasetName,
235 },
236 exportInterval: "1 second",
237 }).pipe(Effect.map(withSchemaErrorAttributes)),
238 ).pipe(Layer.provideMerge(OtlpExporter.layerFlusher), Layer.provide(OtlpSerialization.layerJson));
infra/relay/src/worker.ts+17-28
...@@ -134,7 +134,7 @@ export const layer = Api.make(...@@ -134,7 +134,7 @@ export const layer = Api.make(
134 const relayApiZone = yield* RelayApiZone;134 const relayApiZone = yield* RelayApiZone;
135 const managedEndpointZone = yield* ManagedEndpointZone;135 const managedEndpointZone = yield* ManagedEndpointZone;
136 const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;136 const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
137 const observability = yield* RelayObservability;137 yield* RelayObservability;
138138
139 //139 //
140 // 2. Create bindings140 // 2. Create bindings
...@@ -159,10 +159,6 @@ export const layer = Api.make(...@@ -159,10 +159,6 @@ export const layer = Api.make(
159 const apnsDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(apnsDeliveryQueue);159 const apnsDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(apnsDeliveryQueue);
160 const fcmDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(fcmDeliveryQueue);160 const fcmDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(fcmDeliveryQueue);
161161
162 const axiomDatasetName = yield* observability.traces.name;
163 const axiomIngestToken = yield* observability.workerIngestToken.token;
164 const axiomTracesEndpoint = yield* observability.traces.otelTracesEndpoint;
165
166 const clerkSecretKey = yield* Config.Redacted("CLERK_SECRET_KEY");162 const clerkSecretKey = yield* Config.Redacted("CLERK_SECRET_KEY");
167 const clerkPublishableKey = yield* Config.String("CLERK_PUBLISHABLE_KEY");163 const clerkPublishableKey = yield* Config.String("CLERK_PUBLISHABLE_KEY");
168 const clerkJwtAudience = yield* Config.String("CLERK_JWT_AUDIENCE");164 const clerkJwtAudience = yield* Config.String("CLERK_JWT_AUDIENCE");
...@@ -231,14 +227,6 @@ export const layer = Api.make(...@@ -231,14 +227,6 @@ export const layer = Api.make(
231 });227 });
232 });228 });
233229
234 const layerRelayTrace = Layer.unwrap(
235 Effect.all({
236 tracesDatasetName: axiomDatasetName,
237 tracesEndpoint: axiomTracesEndpoint,
238 ingestToken: axiomIngestToken,
239 }).pipe(Effect.map(Observability.layer)),
240 );
241
242 // Each managed endpoint's held webhook requests live in its own Durable Object.230 // Each managed endpoint's held webhook requests live in its own Durable Object.
243 const inboxCall =231 const inboxCall =
244 <A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>232 <A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
...@@ -442,8 +430,8 @@ export const layer = Api.make(...@@ -442,8 +430,8 @@ export const layer = Api.make(
442 { concurrency: 2, discard: true },430 { concurrency: 2, discard: true },
443 ).pipe(431 ).pipe(
444 Effect.withSpan("relay.cron.prune_expired_state"),432 Effect.withSpan("relay.cron.prune_expired_state"),
445 // Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends.433 Observability.withSchemaErrorSpanAttributes,
446 Effect.provide(Layer.merge(layerRuntime, layerRelayTrace)),434 Effect.provide(layerRuntime),
447 ),435 ),
448 );436 );
449437
...@@ -462,7 +450,11 @@ export const layer = Api.make(...@@ -462,7 +450,11 @@ export const layer = Api.make(
462 HttpRouter.toHttpEffect,450 HttpRouter.toHttpEffect,
463 Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG),451 Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG),
464 withoutCapturedParentSpan,452 withoutCapturedParentSpan,
465 Effect.flatMap((httpEffect) => traceRelayHttpRequestWith(httpEffect, layerRelayTrace)),453 Effect.map((httpEffect) =>
454 traceRelayHttpRequestWith(httpEffect, Layer.empty).pipe(
455 Observability.withSchemaErrorSpanAttributes,
456 ),
457 ),
466 );458 );
467459
468 return { fetch };460 return { fetch };
...@@ -477,20 +469,17 @@ export const layer = Api.make(...@@ -477,20 +469,17 @@ export const layer = Api.make(
477 Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),469 Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
478 Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),470 Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
479 Layer.provideMerge(HookInboxObject.layer),471 Layer.provideMerge(HookInboxObject.layer),
480 // The worker runtime opens its own HTTP span around ours. For webhook472 // The worker runtime opens its own HTTP span around ours. Ours carries
481 // paths it would record the raw URL, token included, and adopt the473 // the route, header redaction and webhook URL redaction, and drops a
482 // sender's traceparent, so only our redacted span covers those.474 // webhook sender's traceparent, so the runtime's span is off for every
483 // Registered as telemetry: request-time context is assembled per475 // request rather than duplicating it. Registered as telemetry:
484 // event, and only these layers are built into it.476 // request-time context is assembled per event, and only these layers
477 // are built into it.
485 Layer.provideMerge(478 Layer.provideMerge(
486 Alchemy.Telemetry.layer(479 Alchemy.Telemetry.layer(Layer.succeed(HttpMiddleware.TracerDisabledWhen)(() => true)),
487 Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
488 HookForwarder.isRelayHookPath(request.url),
489 ),
490 ),
491 ),480 ),
492 // Exports spans from events the HTTP tracer does not wrap, notably481 // The Worker's only trace exporter: every event (fetch, queue, cron,
493 // HookInboxObject calls and alarms, to the same Axiom dataset.482 // HookInboxObject calls and alarms) exports through it to Axiom.
494 Layer.provideMerge(483 Layer.provideMerge(
495 Layer.unwrap(484 Layer.unwrap(
496 Effect.map(RelayObservability, (observability) =>485 Effect.map(RelayObservability, (observability) =>