authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 23:13:47-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 23:13:47-07:00
log442735897f6f92af33798075baa12d4fd4d710dd
tree15770d3d9c33dc3100e97a1915705a74bec53686
parent83b982a85c87b4c0eda08beb9028e3a73f7c89b6
signature Signed by PGP key B5690EEEBB952194

fix(relay): export traces through one tracer, one request span each (#16382)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

6 files changed, 87 insertions(+), 133 deletions(-)

infra/relay/src/hooks/HookForwarder.test.ts+4-6
......@@ -615,15 +615,13 @@ describe("HookForwarder", () => {
615615 });
616616 const harness = makeHarness();
617617 const handler = yield* harness.httpEffect;
618 // As the worker runtime runs it: its own tracer around ours, off for hook
619 // paths. This checks the predicate; whether alchemy applies it per event
620 // is only visible on a deployed worker (see worker.ts).
618 // As the worker runtime runs it: its own tracer around ours, turned off
619 // (see worker.ts). Whether alchemy applies the predicate per event is
620 // only visible on a deployed worker.
621621 yield* HttpMiddleware.tracer(
622622 traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
623623 ).pipe(
624 Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>
625 HookForwarder.isRelayHookPath(request.url),
626 ),
624 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
627625 Effect.withTracer(tracer),
628626 Effect.provideService(
629627 HttpServerRequest.HttpServerRequest,
infra/relay/src/http/Api.test.ts+33
......@@ -30,6 +30,7 @@ import * as Tracer from "effect/Tracer";
3030import * as Etag from "effect/http/Etag";
3131import * as HttpEffect from "effect/http/HttpEffect";
3232import * as HttpRouter from "effect/http/HttpRouter";
33import * as HttpMiddleware from "effect/http/HttpMiddleware";
3334import * as HttpServerRequest from "effect/http/HttpServerRequest";
3435import * as HttpServerResponse from "effect/http/HttpServerResponse";
3536import * as HttpApi from "effect/http-api/HttpApi";
......@@ -1195,6 +1196,38 @@ describe("relay request tracing", () => {
11951196 }),
11961197 );
11971198
1199 it.effect("records one server span inside the worker's disabled HTTP tracer", () =>
1200 Effect.gen(function* () {
1201 const spans: Array<Tracer.NativeSpan> = [];
1202 const tracer = Tracer.make({
1203 span: (options) => {
1204 const span = new Tracer.NativeSpan(options);
1205 spans.push(span);
1206 return span;
1207 },
1208 });
1209 const request = HttpServerRequest.fromWeb(
1210 new Request("https://relay.test/v1/mobile/devices", { method: "POST" }),
1211 );
1212
1213 // As the worker runtime runs it: its own tracer around ours, turned off.
1214 yield* HttpMiddleware.tracer(
1215 traceRelayHttpRequestWith(
1216 Effect.succeed(HttpServerResponse.empty({ status: 204 })),
1217 Layer.empty,
1218 ),
1219 ).pipe(
1220 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
1221 Effect.provideService(HttpServerRequest.HttpServerRequest, request),
1222 Effect.withTracer(tracer),
1223 );
1224 yield* Effect.yieldNow;
1225
1226 expect(spans.filter((span) => span.kind === "server")).toHaveLength(1);
1227 expect(spans[0]?.attributes.get("url.path")).toBe("/v1/mobile/devices");
1228 }),
1229 );
1230
11981231 it.effect("fails hung requests with a 504 before the client's 10s abort", () =>
11991232 Effect.gen(function* () {
12001233 const spans: Array<Tracer.NativeSpan> = [];
infra/relay/src/http/Api.ts+7-3
......@@ -228,8 +228,12 @@ export const traceRelayHttpRequest = <E, R>(
228228 Effect.andThen(relayRequestDeadline(httpEffect)),
229229 );
230230 if (!isRelayHookPath(request.url)) {
231 // HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
232 return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));
231 return yield* HttpMiddleware.tracer(traced).pipe(
232 // The worker turns its own request span off; this one is ours.
233 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
234 // HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
235 Effect.ensuring(Effect.yieldNow),
236 );
233237 }
234238 // Hook URLs carry a secret token: the tracer and deadline log see a redacted
235239 // request, while the route itself still receives the original. A webhook
......@@ -253,7 +257,7 @@ export const traceRelayHttpRequest = <E, R>(
253257 ),
254258 ).pipe(
255259 Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
256 // The worker disables its own span for hook paths; this one is ours.
260 // The worker turns its own request span off; this one is ours.
257261 Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
258262 Effect.ensuring(Effect.yieldNow),
259263 );
infra/relay/src/observability.test.ts+15-69
......@@ -1,46 +1,20 @@
1import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
21import { expect, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
42import * as Effect from "effect/Effect";
5import * as Redacted from "effect/Redacted";
6import * as Schema from "effect/Schema";
7import * as HttpServer from "effect/http/HttpServer";
8import * as HttpServerRequest from "effect/http/HttpServerRequest";
9import * as HttpServerResponse from "effect/http/HttpServerResponse";
10import type { OtlpTracer } from "effect/observability";
3import * as Tracer from "effect/Tracer";
114
125import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
136import * as Observability from "./observability.ts";
147
15interface ExportedRequest {
16 readonly authorization: string | undefined;
17 readonly body: string;
18 readonly dataset: string | undefined;
19}
20
21const otlpAttributeValue = (value: {
22 readonly stringValue?: string | null;
23 readonly boolValue?: boolean | null;
24 readonly intValue?: string | number | null;
25 readonly doubleValue?: number | null;
26}) => value.stringValue ?? value.boolValue ?? value.intValue ?? value.doubleValue;
27
28const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
29
30it.effect("exports schema error fields as span attributes", () =>
8it.effect("adds schema error fields to spans on the current tracer", () =>
319 Effect.gen(function* () {
32 const exportedRequest = yield* Deferred.make<ExportedRequest>();
33 yield* HttpServer.serveEffect(
34 Effect.gen(function* () {
35 const request = yield* HttpServerRequest.HttpServerRequest;
36 yield* Deferred.succeed(exportedRequest, {
37 authorization: request.headers.authorization,
38 body: yield* request.text,
39 dataset: request.headers["x-axiom-dataset"],
40 });
41 return HttpServerResponse.empty({ status: 204 });
42 }),
43 );
10 const spans: Array<Tracer.NativeSpan> = [];
11 const tracer = Tracer.make({
12 span: (options) => {
13 const span = new Tracer.NativeSpan(options);
14 spans.push(span);
15 return span;
16 },
17 });
4418
4519 yield* Effect.fail(
4620 new EnvironmentConnector.EnvironmentConnectNotAuthorized({
......@@ -51,44 +25,16 @@ it.effect("exports schema error fields as span attributes", () =>
5125 ).pipe(
5226 Effect.withSpan("relay.test.schema_error"),
5327 Effect.exit,
54 Effect.provide(
55 Observability.layer({
56 tracesEndpoint: "/v1/traces",
57 tracesDatasetName: "relay-test-traces",
58 ingestToken: Redacted.make("test-token"),
59 }),
60 ),
28 Observability.withSchemaErrorSpanAttributes,
29 Effect.withTracer(tracer),
6130 );
6231
63 const request = yield* Deferred.await(exportedRequest).pipe(Effect.timeout("1 second"));
64 const payload = (yield* decodeJson(request.body)) as OtlpTracer.TraceData;
65 const resourceAttributes = Object.fromEntries(
66 payload.resourceSpans
67 .flatMap((resourceSpan) => resourceSpan.resource.attributes)
68 .map((attribute) => [attribute.key, otlpAttributeValue(attribute.value)]),
69 );
70 const span = payload.resourceSpans
71 .flatMap((resourceSpan) => resourceSpan.scopeSpans)
72 .flatMap((scopeSpan) => scopeSpan.spans)
73 .find((candidate) => candidate.name === "relay.test.schema_error");
74 const attributes = Object.fromEntries(
75 (span?.attributes ?? []).map((attribute) => [
76 attribute.key,
77 otlpAttributeValue(attribute.value),
78 ]),
79 );
80
81 expect(request.authorization).toBe("Bearer test-token");
82 expect(request.dataset).toBe("relay-test-traces");
83 expect(resourceAttributes).toMatchObject({
84 "service.name": "t3code-relay",
85 "service.namespace": "t3code",
86 });
87 expect(attributes).toMatchObject({
32 expect(spans.map((span) => span.name)).toEqual(["relay.test.schema_error"]);
33 expect(Object.fromEntries(spans[0]!.attributes)).toMatchObject({
8834 "error.type": "EnvironmentConnectNotAuthorized",
8935 "error.environmentId": "environment-1",
9036 "error.operation": "connect",
9137 "error.reason": "managed_endpoint_allocation_not_ready",
9238 });
93 }).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),
39 }),
9440);
infra/relay/src/observability.ts+11-27
......@@ -4,12 +4,9 @@ import * as Output from "alchemy/Output";
44import * as Cause from "effect/Cause";
55import * as Effect from "effect/Effect";
66import * as Exit from "effect/Exit";
7import * as Layer from "effect/Layer";
87import * as Option from "effect/Option";
9import * as Redacted from "effect/Redacted";
108import * as Schema from "effect/Schema";
119import * as Tracer from "effect/Tracer";
12import { OtlpExporter, OtlpSerialization, OtlpTracer } from "effect/observability";
1310
1411import { relayResourceNameForStage } from "./deploymentConfig.ts";
1512
......@@ -212,27 +209,14 @@ const withSchemaErrorAttributes = (delegate: Tracer.Tracer): Tracer.Tracer =>
212209 ...(delegate.context ? { context: delegate.context } : {}),
213210 });
214211
215export const layer = (input: {
216 readonly tracesEndpoint: string;
217 readonly tracesDatasetName: string;
218 readonly ingestToken: Redacted.Redacted<string>;
219}) =>
220 Layer.effect(
221 Tracer.Tracer,
222 OtlpTracer.make({
223 url: input.tracesEndpoint,
224 resource: {
225 serviceName: "t3code-relay",
226 attributes: {
227 "service.namespace": "t3code",
228 "service.runtime": "cloudflare-worker",
229 "service.component": "relay",
230 },
231 },
232 headers: {
233 Authorization: `Bearer ${Redacted.value(input.ingestToken)}`,
234 "X-Axiom-Dataset": input.tracesDatasetName,
235 },
236 exportInterval: "1 second",
237 }).pipe(Effect.map(withSchemaErrorAttributes)),
238 ).pipe(Layer.provideMerge(OtlpExporter.layerFlusher), Layer.provide(OtlpSerialization.layerJson));
212/**
213 * Adds a failed span's schema error fields (`error.type`, `error.<field>`) to
214 * the span, on whichever tracer is current. Provide it around the work whose
215 * spans should carry them; the Worker's telemetry still owns export.
216 */
217export const withSchemaErrorSpanAttributes = <A, E, R>(
218 effect: Effect.Effect<A, E, R>,
219): Effect.Effect<A, E, R> =>
220 Effect.flatMap(Effect.tracer, (tracer) =>
221 effect.pipe(Effect.withTracer(withSchemaErrorAttributes(tracer))),
222 );
infra/relay/src/worker.ts+17-28
......@@ -134,7 +134,7 @@ export const layer = Api.make(
134134 const relayApiZone = yield* RelayApiZone;
135135 const managedEndpointZone = yield* ManagedEndpointZone;
136136 const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
137 const observability = yield* RelayObservability;
137 yield* RelayObservability;
138138
139139 //
140140 // 2. Create bindings
......@@ -159,10 +159,6 @@ export const layer = Api.make(
159159 const apnsDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(apnsDeliveryQueue);
160160 const fcmDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(fcmDeliveryQueue);
161161
162 const axiomDatasetName = yield* observability.traces.name;
163 const axiomIngestToken = yield* observability.workerIngestToken.token;
164 const axiomTracesEndpoint = yield* observability.traces.otelTracesEndpoint;
165
166162 const clerkSecretKey = yield* Config.Redacted("CLERK_SECRET_KEY");
167163 const clerkPublishableKey = yield* Config.String("CLERK_PUBLISHABLE_KEY");
168164 const clerkJwtAudience = yield* Config.String("CLERK_JWT_AUDIENCE");
......@@ -231,14 +227,6 @@ export const layer = Api.make(
231227 });
232228 });
233229
234 const layerRelayTrace = Layer.unwrap(
235 Effect.all({
236 tracesDatasetName: axiomDatasetName,
237 tracesEndpoint: axiomTracesEndpoint,
238 ingestToken: axiomIngestToken,
239 }).pipe(Effect.map(Observability.layer)),
240 );
241
242230 // Each managed endpoint's held webhook requests live in its own Durable Object.
243231 const inboxCall =
244232 <A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
......@@ -442,8 +430,8 @@ export const layer = Api.make(
442430 { concurrency: 2, discard: true },
443431 ).pipe(
444432 Effect.withSpan("relay.cron.prune_expired_state"),
445 // Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends.
446 Effect.provide(Layer.merge(layerRuntime, layerRelayTrace)),
433 Observability.withSchemaErrorSpanAttributes,
434 Effect.provide(layerRuntime),
447435 ),
448436 );
449437
......@@ -462,7 +450,11 @@ export const layer = Api.make(
462450 HttpRouter.toHttpEffect,
463451 Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG),
464452 withoutCapturedParentSpan,
465 Effect.flatMap((httpEffect) => traceRelayHttpRequestWith(httpEffect, layerRelayTrace)),
453 Effect.map((httpEffect) =>
454 traceRelayHttpRequestWith(httpEffect, Layer.empty).pipe(
455 Observability.withSchemaErrorSpanAttributes,
456 ),
457 ),
466458 );
467459
468460 return { fetch };
......@@ -477,20 +469,17 @@ export const layer = Api.make(
477469 Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
478470 Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
479471 Layer.provideMerge(HookInboxObject.layer),
480 // The worker runtime opens its own HTTP span around ours. For webhook
481 // paths it would record the raw URL, token included, and adopt the
482 // sender's traceparent, so only our redacted span covers those.
483 // Registered as telemetry: request-time context is assembled per
484 // event, and only these layers are built into it.
472 // The worker runtime opens its own HTTP span around ours. Ours carries
473 // the route, header redaction and webhook URL redaction, and drops a
474 // webhook sender's traceparent, so the runtime's span is off for every
475 // request rather than duplicating it. Registered as telemetry:
476 // request-time context is assembled per event, and only these layers
477 // are built into it.
485478 Layer.provideMerge(
486 Alchemy.Telemetry.layer(
487 Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
488 HookForwarder.isRelayHookPath(request.url),
489 ),
490 ),
479 Alchemy.Telemetry.layer(Layer.succeed(HttpMiddleware.TracerDisabledWhen)(() => true)),
491480 ),
492 // Exports spans from events the HTTP tracer does not wrap, notably
493 // HookInboxObject calls and alarms, to the same Axiom dataset.
481 // The Worker's only trace exporter: every event (fetch, queue, cron,
482 // HookInboxObject calls and alarms) exports through it to Axiom.
494483 Layer.provideMerge(
495484 Layer.unwrap(
496485 Effect.map(RelayObservability, (observability) =>