authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-26 01:34:28-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-26 01:34:28-07:00
log8b873eab0d74b731bc0f387be84a8561eace789f
treef3ccb88561a7de0fc6a707cb741f46e1f475395a
parent99efaeab58297c7e346ee91a4f28044faed03c59
signature Signed by PGP key B5690EEEBB952194

perf(server): opening Diagnostics no longer loads the whole trace ring into memory (#13763)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 413 insertions(+), 369 deletions(-)

apps/server/src/cli/trace.ts+2-21
......@@ -13,11 +13,10 @@ import * as Effect from "effect/Effect";
1313import * as FileSystem from "effect/FileSystem";
1414import * as Option from "effect/Option";
1515import * as Schema from "effect/Schema";
16import * as Stream from "effect/Stream";
1716import { Command, Flag } from "effect/unstable/cli";
1817
1918import * as ServerConfig from "../config.ts";
20import { toRotatedTracePaths, TraceFileReadError } from "../diagnostics/TraceDiagnostics.ts";
19import { streamTraceFileLines, toRotatedTracePaths } from "../diagnostics/TraceDiagnostics.ts";
2120import { resolveBaseDir } from "../os-jank.ts";
2221import { baseDirFlag, DurationFromString, traceFileConfig, traceMaxFilesConfig } from "./config.ts";
2322
......@@ -187,27 +186,9 @@ const traceSummaryCommand = Command.make("summary", {
187186 ? (yield* Clock.currentTimeMillis) - Duration.toMillis(flags.since.value)
188187 : undefined;
189188 const summarizer = makeTraceSpanSummary(sinceMs);
190 // Stream each file so only one chunk of text is in memory at a time.
191189 yield* Effect.forEach(
192190 toRotatedTracePaths(traceFilePath, yield* traceMaxFilesConfig),
193 (path) =>
194 fs.stream(path).pipe(
195 Stream.decodeText,
196 Stream.splitLines,
197 Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(summarizer.addLine))),
198 Effect.catchTags({
199 PlatformError: (cause) =>
200 cause.reason._tag === "NotFound"
201 ? Effect.void
202 : Effect.fail(
203 new TraceFileReadError({
204 traceFilePath: path,
205 causeTag: cause.reason._tag,
206 cause,
207 }),
208 ),
209 }),
210 ),
191 (path) => streamTraceFileLines(fs, path, summarizer.addLine),
211192 { discard: true },
212193 );
213194 const summary = summarizer.finish();
apps/server/src/diagnostics/TraceDiagnostics.test.ts+207-137
......@@ -7,6 +7,7 @@ import * as Logger from "effect/Logger";
77import * as Option from "effect/Option";
88import * as PlatformError from "effect/PlatformError";
99import * as References from "effect/References";
10import * as Stream from "effect/Stream";
1011
1112import * as TraceDiagnostics from "./TraceDiagnostics.ts";
1213
......@@ -40,72 +41,78 @@ function record(input: {
4041 });
4142}
4243
44const traceFilePath = "/tmp/server.trace.ndjson";
45const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
46
47/** Aggregates whole lines in memory, the way diagnostics read traces before streaming. */
48function aggregateLines(lines: ReadonlyArray<string>) {
49 const aggregator = TraceDiagnostics.makeTraceDiagnosticsAggregator();
50 lines.forEach(aggregator.addLine);
51 return aggregator.finish({
52 traceFilePath,
53 scannedFilePaths: TraceDiagnostics.toRotatedTracePaths(traceFilePath, 1),
54 readAt,
55 });
56}
57
58/** Reads the trace file and one rotated backup through a fake file system. */
59function readTraces(fileSystem: Partial<FileSystem.FileSystem>) {
60 return TraceDiagnostics.readTraceDiagnostics({ traceFilePath, maxFiles: 1, readAt }).pipe(
61 Effect.provide(TraceDiagnostics.layer.pipe(Layer.provide(FileSystem.layerNoop(fileSystem)))),
62 );
63}
64
4365describe("TraceDiagnostics", () => {
4466 it.effect("aggregates failures, slow spans, log levels, and parse errors", () =>
4567 Effect.sync(() => {
46 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
47 traceFilePath: "/tmp/server.trace.ndjson",
48 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
49 slowSpanThresholdMs: 1_000,
50 files: [
51 {
52 path: "/tmp/server.trace.ndjson.1",
53 text: [
54 record({
55 name: "server.getConfig",
56 traceId: "trace-a",
57 spanId: "span-a",
58 startMs: 1_000,
59 durationMs: 50,
60 }),
61 "not-json",
62 ].join("\n"),
63 },
64 {
65 path: "/tmp/server.trace.ndjson",
66 text: [
67 record({
68 name: "orchestration.dispatch",
69 traceId: "trace-b",
70 spanId: "span-b",
71 startMs: 2_000,
72 durationMs: 1_500,
73 exit: { _tag: "Failure", cause: "Provider crashed" },
74 events: [
75 {
76 name: "provider failed",
77 timeUnixNano: ns(3_400),
78 attributes: { "effect.logLevel": "Error" },
79 },
80 ],
81 }),
82 record({
83 name: "orchestration.dispatch",
84 traceId: "trace-c",
85 spanId: "span-c",
86 startMs: 4_000,
87 durationMs: 250,
88 exit: { _tag: "Failure", cause: "Provider crashed" },
89 }),
90 record({
91 name: "git.status",
92 traceId: "trace-d",
93 spanId: "span-d",
94 startMs: 5_000,
95 durationMs: 25,
96 exit: { _tag: "Interrupted", cause: "Interrupted" },
97 events: [
98 {
99 name: "status delayed",
100 timeUnixNano: ns(5_010),
101 attributes: { "effect.logLevel": "Warning" },
102 },
103 ],
104 }),
105 ].join("\n"),
106 },
107 ],
108 });
68 const diagnostics = aggregateLines([
69 record({
70 name: "server.getConfig",
71 traceId: "trace-a",
72 spanId: "span-a",
73 startMs: 1_000,
74 durationMs: 50,
75 }),
76 "not-json",
77 record({
78 name: "orchestration.dispatch",
79 traceId: "trace-b",
80 spanId: "span-b",
81 startMs: 2_000,
82 durationMs: 1_500,
83 exit: { _tag: "Failure", cause: "Provider crashed" },
84 events: [
85 {
86 name: "provider failed",
87 timeUnixNano: ns(3_400),
88 attributes: { "effect.logLevel": "Error" },
89 },
90 ],
91 }),
92 record({
93 name: "orchestration.dispatch",
94 traceId: "trace-c",
95 spanId: "span-c",
96 startMs: 4_000,
97 durationMs: 250,
98 exit: { _tag: "Failure", cause: "Provider crashed" },
99 }),
100 record({
101 name: "git.status",
102 traceId: "trace-d",
103 spanId: "span-d",
104 startMs: 5_000,
105 durationMs: 25,
106 exit: { _tag: "Interrupted", cause: "Interrupted" },
107 events: [
108 {
109 name: "status delayed",
110 timeUnixNano: ns(5_010),
111 attributes: { "effect.logLevel": "Warning" },
112 },
113 ],
114 }),
115 ]);
109116
110117 assert.equal(diagnostics.recordCount, 4);
111118 assert.equal(DateTime.formatIso(diagnostics.readAt), "2026-05-05T10:00:00.000Z");
......@@ -139,47 +146,110 @@ describe("TraceDiagnostics", () => {
139146 );
140147
141148 it.effect("returns a not-found diagnostic when no files are available", () =>
142 Effect.sync(() => {
143 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
144 traceFilePath: "/tmp/missing.trace.ndjson",
145 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
146 files: [],
147 });
149 Effect.gen(function* () {
150 const diagnostics = yield* readTraces({});
148151
149152 assert.equal(diagnostics.recordCount, 0);
150153 assert.equal(Option.getOrUndefined(diagnostics.error)?.kind, "trace-file-not-found");
151154 }),
152155 );
153156
154 it.effect("preserves full failure causes and log messages", () =>
155 Effect.sync(() => {
156 const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim();
157 const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim();
158 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
159 traceFilePath: "/tmp/server.trace.ndjson",
160 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
161 files: [
162 {
163 path: "/tmp/server.trace.ndjson",
164 text: record({
165 name: "VcsProcess.run",
166 traceId: "trace-long",
167 spanId: "span-long",
157 it.effect("streams rotated files into the same result as reading them whole", () =>
158 Effect.gen(function* () {
159 // CRLF and LF endings plus multi-byte text, served one byte per chunk so
160 // chunks split lines, line endings, and characters.
161 const files = new Map([
162 [
163 `${traceFilePath}.1`,
164 [
165 record({
166 name: "server.getConfig",
167 traceId: "trace-a",
168 spanId: "span-a",
168169 startMs: 1_000,
170 durationMs: 50,
171 }),
172 "not-json",
173 record({
174 name: "orchestration.dispatch",
175 traceId: "trace-b",
176 spanId: "span-b",
177 startMs: 2_000,
178 durationMs: 1_500,
179 exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" },
180 }),
181 "",
182 ].join("\r\n"),
183 ],
184 [
185 traceFilePath,
186 [
187 record({
188 name: "git.status",
189 traceId: "trace-c",
190 spanId: "span-c",
191 startMs: 3_000,
169192 durationMs: 25,
170 exit: { _tag: "Failure", cause: longCause },
193 exit: { _tag: "Interrupted", cause: "Interrupted" },
171194 events: [
172195 {
173 name: longMessage,
174 timeUnixNano: ns(1_010),
196 name: "status delayed ⏳",
197 timeUnixNano: ns(3_010),
175198 attributes: { "effect.logLevel": "Warning" },
176199 },
177200 ],
178201 }),
179 },
202 "",
203 record({
204 name: "orchestration.dispatch",
205 traceId: "trace-d",
206 spanId: "span-d",
207 startMs: 4_000,
208 durationMs: 250,
209 exit: { _tag: "Failure", cause: "Provider crashed: café 🔥" },
210 }),
211 ].join("\n"),
180212 ],
213 ]);
214 const encoder = new TextEncoder();
215
216 const diagnostics = yield* readTraces({
217 stream: (path) =>
218 Stream.fromIterable(
219 Array.from(encoder.encode(files.get(path)), (byte) => Uint8Array.of(byte)),
220 ),
181221 });
182222
223 assert.equal(diagnostics.recordCount, 4);
224 assert.deepStrictEqual(
225 diagnostics,
226 aggregateLines([...files.values()].flatMap((text) => text.split(/\r?\n/))),
227 );
228 }),
229 );
230
231 it.effect("preserves full failure causes and log messages", () =>
232 Effect.sync(() => {
233 const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim();
234 const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim();
235 const diagnostics = aggregateLines([
236 record({
237 name: "VcsProcess.run",
238 traceId: "trace-long",
239 spanId: "span-long",
240 startMs: 1_000,
241 durationMs: 25,
242 exit: { _tag: "Failure", cause: longCause },
243 events: [
244 {
245 name: longMessage,
246 timeUnixNano: ns(1_010),
247 attributes: { "effect.logLevel": "Warning" },
248 },
249 ],
250 }),
251 ]);
252
183253 assert.equal(diagnostics.latestFailures[0]?.cause, longCause);
184254 assert.equal(diagnostics.commonFailures[0]?.cause, longCause);
185255 assert.equal(diagnostics.latestWarningAndErrorLogs[0]?.message, longMessage);
......@@ -188,45 +258,34 @@ describe("TraceDiagnostics", () => {
188258
189259 it.effect("keeps loaded trace data when one rotated trace file fails to read", () =>
190260 Effect.gen(function* () {
191 const traceFilePath = "/tmp/server.trace.ndjson";
192261 const readFailure = PlatformError.systemError({
193262 _tag: "PermissionDenied",
194263 module: "FileSystem",
195 method: "readFileString",
264 method: "open",
196265 description: "permission denied",
197266 pathOrDescriptor: `${traceFilePath}.1`,
198267 });
199 const fileSystemLayer = FileSystem.layerNoop({
200 readFileString: (path) =>
201 path === `${traceFilePath}.1`
202 ? Effect.fail(readFailure)
203 : Effect.succeed(
204 record({
205 name: "server.getConfig",
206 traceId: "trace-a",
207 spanId: "span-a",
208 startMs: 1_000,
209 durationMs: 50,
210 }),
211 ),
212 });
213268 const logAnnotations: Array<Record<string, unknown>> = [];
214269 const logger = Logger.make<unknown, void>((options) => {
215270 logAnnotations.push({ ...options.fiber.getRef(References.CurrentLogAnnotations) });
216271 });
217272
218 const diagnostics = yield* TraceDiagnostics.readTraceDiagnostics({
219 traceFilePath,
220 maxFiles: 1,
221 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
222 }).pipe(
223 Effect.provide(
224 Layer.mergeAll(
225 TraceDiagnostics.layer.pipe(Layer.provide(fileSystemLayer)),
226 Logger.layer([logger], { mergeWithExisting: false }),
227 ),
228 ),
229 );
273 const diagnostics = yield* readTraces({
274 stream: (path) =>
275 path === `${traceFilePath}.1`
276 ? Stream.fail(readFailure)
277 : Stream.make(
278 new TextEncoder().encode(
279 record({
280 name: "server.getConfig",
281 traceId: "trace-a",
282 spanId: "span-a",
283 startMs: 1_000,
284 durationMs: 50,
285 }),
286 ),
287 ),
288 }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
230289
231290 assert.equal(diagnostics.recordCount, 1);
232291 assert.equal(
......@@ -251,32 +310,43 @@ describe("TraceDiagnostics", () => {
251310 }),
252311 );
253312
254 it.effect("keeps only the slowest span occurrences while aggregating large inputs", () =>
313 it.effect("keeps only the top spans, failures, and warning logs from large inputs", () =>
255314 Effect.sync(() => {
256 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
257 traceFilePath: "/tmp/server.trace.ndjson",
258 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
259 files: [
260 {
261 path: "/tmp/server.trace.ndjson",
262 text: Array.from({ length: 25 }, (_, index) =>
263 record({
264 name: `span-${index}`,
265 traceId: `trace-${index}`,
266 spanId: `span-${index}`,
267 startMs: index * 1_000,
268 durationMs: index,
269 }),
270 ).join("\n"),
271 },
272 ],
273 });
315 // Shuffled, so some older records arrive after the lists are full.
316 const indexes = Array.from({ length: 30 }, (_, step) => (step * 7) % 30);
317 const diagnostics = aggregateLines(
318 indexes.map((index) =>
319 record({
320 name: `span-${index}`,
321 traceId: `trace-${index}`,
322 spanId: `span-${index}`,
323 startMs: index * 1_000,
324 durationMs: index,
325 exit: { _tag: "Failure", cause: "Provider crashed" },
326 events: [
327 {
328 name: `warning ${index}`,
329 timeUnixNano: ns(index * 1_000),
330 attributes: { "effect.logLevel": "Warning" },
331 },
332 ],
333 }),
334 ),
335 );
336 const newestTwenty = Array.from({ length: 20 }, (_, rank) => `trace-${29 - rank}`);
274337
275 assert.equal(diagnostics.recordCount, 25);
276 assert.equal(diagnostics.slowestSpans.length, 10);
338 assert.equal(diagnostics.recordCount, 30);
277339 assert.deepStrictEqual(
278340 diagnostics.slowestSpans.map((span) => span.durationMs),
279 [24, 23, 22, 21, 20, 19, 18, 17, 16, 15],
341 [29, 28, 27, 26, 25, 24, 23, 22, 21, 20],
342 );
343 assert.deepStrictEqual(
344 diagnostics.latestFailures.map((failure) => failure.traceId),
345 newestTwenty,
346 );
347 assert.deepStrictEqual(
348 diagnostics.latestWarningAndErrorLogs.map((log) => log.traceId),
349 newestTwenty,
280350 );
281351 }),
282352 );
apps/server/src/diagnostics/TraceDiagnostics.ts+204-211
......@@ -16,6 +16,7 @@ import * as Option from "effect/Option";
1616import * as PlatformError from "effect/PlatformError";
1717import * as Result from "effect/Result";
1818import * as Schema from "effect/Schema";
19import * as Stream from "effect/Stream";
1920
2021interface TraceRecordLike {
2122 readonly name?: unknown;
......@@ -63,16 +64,6 @@ export class TraceDiagnostics extends Context.Service<
6364 }
6465>()("t3/diagnostics/TraceDiagnostics") {}
6566
66interface TraceDiagnosticsInput {
67 readonly traceFilePath: string;
68 readonly files: ReadonlyArray<{ readonly path: string; readonly text: string }>;
69 readonly scannedFilePaths?: ReadonlyArray<string>;
70 readonly slowSpanThresholdMs?: number;
71 readonly readAt: DateTime.Utc;
72 readonly error?: TraceDiagnosticsErrorSummary;
73 readonly partialFailure?: boolean;
74}
75
7667interface TraceDiagnosticsErrorSummary {
7768 readonly kind: ServerTraceDiagnosticsErrorKind;
7869 readonly message: string;
......@@ -173,44 +164,50 @@ function isNotFoundError(error: PlatformError.PlatformError): boolean {
173164 return error.reason._tag === "NotFound";
174165}
175166
176function insertBoundedSlowestSpan(
177 slowestSpans: ServerTraceDiagnosticsSpanOccurrence[],
178 span: ServerTraceDiagnosticsSpanOccurrence,
167/**
168 * Adds `item` to `items`, which stays sorted by `order` and holds at most
169 * `limit` entries. Same result as a stable sort and slice over every item, but
170 * memory stays bounded however many items stream in.
171 */
172function insertBounded<A>(
173 items: A[],
174 item: A,
175 limit: number,
176 order: (left: A, right: A) => number,
179177): void {
180 if (
181 slowestSpans.length >= TOP_LIMIT &&
182 span.durationMs <= slowestSpans[slowestSpans.length - 1]!.durationMs
183 ) {
178 if (items.length >= limit && order(item, items[items.length - 1]!) >= 0) {
184179 return;
185180 }
186181
187 slowestSpans.push(span);
188 slowestSpans.sort((left, right) => right.durationMs - left.durationMs);
189 if (slowestSpans.length > TOP_LIMIT) {
190 slowestSpans.length = TOP_LIMIT;
182 items.push(item);
183 items.sort(order);
184 if (items.length > limit) {
185 items.length = limit;
191186 }
192187}
193188
194export function aggregateTraceDiagnostics(
195 input: TraceDiagnosticsInput,
196): ServerTraceDiagnosticsResult {
197 const readAt = input.readAt;
198 const slowSpanThresholdMs = input.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
199 const scannedFilePaths = input.scannedFilePaths ?? input.files.map((file) => file.path);
200 if (input.files.length === 0) {
201 return makeEmptyDiagnostics({
202 traceFilePath: input.traceFilePath,
203 scannedFilePaths,
204 readAt,
205 slowSpanThresholdMs,
206 error: input.error ?? {
207 kind: "trace-file-not-found",
208 message: "No local trace files were found.",
209 },
210 ...(input.partialFailure ? { partialFailure: true } : {}),
211 });
212 }
213
189const slowestFirst = (
190 left: ServerTraceDiagnosticsSpanOccurrence,
191 right: ServerTraceDiagnosticsSpanOccurrence,
192) => right.durationMs - left.durationMs;
193
194const latestEndedFirst = (
195 left: ServerTraceDiagnosticsRecentFailure,
196 right: ServerTraceDiagnosticsRecentFailure,
197) => DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt);
198
199const latestSeenFirst = (
200 left: ServerTraceDiagnosticsLogEvent,
201 right: ServerTraceDiagnosticsLogEvent,
202) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt);
203
204/**
205 * Folds trace NDJSON into diagnostics. Call `addLine` once per line as the
206 * rotated files stream in, then `finish` for the result.
207 */
208export function makeTraceDiagnosticsAggregator(
209 slowSpanThresholdMs = DEFAULT_SLOW_SPAN_THRESHOLD_MS,
210) {
214211 let parseErrorCount = 0;
215212 let recordCount = 0;
216213 let failureCount = 0;
......@@ -229,182 +226,184 @@ export function aggregateTraceDiagnostics(
229226 const latestWarningAndErrorLogs: ServerTraceDiagnosticsLogEvent[] = [];
230227 const logLevelCounts: Record<string, number> = {};
231228
232 for (const file of input.files) {
233 const lines = file.text.split(/\r?\n/);
234 for (const line of lines) {
235 if (line.trim().length === 0) continue;
236
237 let parsed: unknown;
238 try {
239 parsed = JSON.parse(line);
240 } catch {
241 parseErrorCount += 1;
242 continue;
243 }
229 const addLine = (line: string) => {
230 if (line.trim().length === 0) return;
244231
245 if (!isRecordObject(parsed)) {
246 parseErrorCount += 1;
247 continue;
248 }
232 let parsed: unknown;
233 try {
234 parsed = JSON.parse(line);
235 } catch {
236 parseErrorCount += 1;
237 return;
238 }
249239
250 const name = toStringValue(parsed.name);
251 const traceId = toStringValue(parsed.traceId);
252 const spanId = toStringValue(parsed.spanId);
253 const durationMs = toNumberValue(parsed.durationMs);
254 const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano);
255 const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano);
240 if (!isRecordObject(parsed)) {
241 parseErrorCount += 1;
242 return;
243 }
256244
257 if (!name || !traceId || !spanId || durationMs === null || !endedAt) {
258 parseErrorCount += 1;
259 continue;
260 }
245 const name = toStringValue(parsed.name);
246 const traceId = toStringValue(parsed.traceId);
247 const spanId = toStringValue(parsed.spanId);
248 const durationMs = toNumberValue(parsed.durationMs);
249 const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano);
250 const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano);
261251
262 recordCount += 1;
263 firstSpanAt =
264 startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt))
265 ? startedAt
266 : firstSpanAt;
267 lastSpanAt =
268 lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt;
269
270 const exitTag = readExitTag(parsed.exit);
271 const isFailure = exitTag === "Failure";
272 const isInterrupted = exitTag === "Interrupted";
273 if (isFailure) failureCount += 1;
274 if (isInterrupted) interruptionCount += 1;
275
276 const spanSummary = spansByName.get(name) ?? {
277 count: 0,
278 failureCount: 0,
279 totalDurationMs: 0,
280 maxDurationMs: 0,
281 };
282 spanSummary.count += 1;
283 spanSummary.totalDurationMs += durationMs;
284 spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs);
285 if (isFailure) spanSummary.failureCount += 1;
286 spansByName.set(name, spanSummary);
287
288 const spanItem = { name, durationMs, endedAt, traceId, spanId };
289 if (durationMs >= slowSpanThresholdMs) {
290 slowSpanCount += 1;
291 }
292 insertBoundedSlowestSpan(slowestSpans, spanItem);
293
294 if (isFailure) {
295 const cause = readExitCause(parsed.exit);
296 latestFailures.push({ ...spanItem, cause });
297
298 const failureKey = `${name}\0${cause}`;
299 const existing = failuresByKey.get(failureKey);
300 const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt);
301 failuresByKey.set(failureKey, {
302 name,
303 cause,
304 count: (existing?.count ?? 0) + 1,
305 lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt,
306 traceId: isLatestFailure ? traceId : existing!.traceId,
307 spanId: isLatestFailure ? spanId : existing!.spanId,
308 });
309 }
252 if (!name || !traceId || !spanId || durationMs === null || !endedAt) {
253 parseErrorCount += 1;
254 return;
255 }
310256
311 if (Array.isArray(parsed.events)) {
312 for (const rawEvent of parsed.events) {
313 if (!isTraceEvent(rawEvent)) continue;
314 const attributes = readEventAttributes(rawEvent);
315 const level = toStringValue(attributes["effect.logLevel"]);
316 if (!level) continue;
317
318 logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1;
319 const normalizedLevel = level.toLowerCase();
320 if (
321 normalizedLevel !== "warning" &&
322 normalizedLevel !== "warn" &&
323 normalizedLevel !== "error" &&
324 normalizedLevel !== "fatal"
325 ) {
326 continue;
327 }
328
329 const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt;
330 const message = toStringValue(rawEvent.name)?.trim() ?? "Log event";
331 latestWarningAndErrorLogs.push({
332 spanName: name,
333 level,
334 message,
335 seenAt,
336 traceId,
337 spanId,
338 });
257 recordCount += 1;
258 firstSpanAt =
259 startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt))
260 ? startedAt
261 : firstSpanAt;
262 lastSpanAt =
263 lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt;
264
265 const exitTag = readExitTag(parsed.exit);
266 const isFailure = exitTag === "Failure";
267 const isInterrupted = exitTag === "Interrupted";
268 if (isFailure) failureCount += 1;
269 if (isInterrupted) interruptionCount += 1;
270
271 const spanSummary = spansByName.get(name) ?? {
272 count: 0,
273 failureCount: 0,
274 totalDurationMs: 0,
275 maxDurationMs: 0,
276 };
277 spanSummary.count += 1;
278 spanSummary.totalDurationMs += durationMs;
279 spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs);
280 if (isFailure) spanSummary.failureCount += 1;
281 spansByName.set(name, spanSummary);
282
283 const spanItem = { name, durationMs, endedAt, traceId, spanId };
284 if (durationMs >= slowSpanThresholdMs) {
285 slowSpanCount += 1;
286 }
287 insertBounded(slowestSpans, spanItem, TOP_LIMIT, slowestFirst);
288
289 if (isFailure) {
290 const cause = readExitCause(parsed.exit);
291 insertBounded(latestFailures, { ...spanItem, cause }, RECENT_LIMIT, latestEndedFirst);
292
293 const failureKey = `${name}\0${cause}`;
294 const existing = failuresByKey.get(failureKey);
295 const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt);
296 failuresByKey.set(failureKey, {
297 name,
298 cause,
299 count: (existing?.count ?? 0) + 1,
300 lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt,
301 traceId: isLatestFailure ? traceId : existing!.traceId,
302 spanId: isLatestFailure ? spanId : existing!.spanId,
303 });
304 }
305
306 if (Array.isArray(parsed.events)) {
307 for (const rawEvent of parsed.events) {
308 if (!isTraceEvent(rawEvent)) continue;
309 const attributes = readEventAttributes(rawEvent);
310 const level = toStringValue(attributes["effect.logLevel"]);
311 if (!level) continue;
312
313 logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1;
314 const normalizedLevel = level.toLowerCase();
315 if (
316 normalizedLevel !== "warning" &&
317 normalizedLevel !== "warn" &&
318 normalizedLevel !== "error" &&
319 normalizedLevel !== "fatal"
320 ) {
321 continue;
339322 }
323
324 const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt;
325 const message = toStringValue(rawEvent.name)?.trim() ?? "Log event";
326 insertBounded(
327 latestWarningAndErrorLogs,
328 { spanName: name, level, message, seenAt, traceId, spanId },
329 RECENT_LIMIT,
330 latestSeenFirst,
331 );
340332 }
341333 }
342 }
343
344 const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()]
345 .map(([name, span]) => ({
346 name,
347 count: span.count,
348 failureCount: span.failureCount,
349 totalDurationMs: span.totalDurationMs,
350 averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0,
351 maxDurationMs: span.maxDurationMs,
352 }))
353 .toSorted((left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs)
354 .slice(0, TOP_LIMIT);
334 };
355335
356 return {
357 traceFilePath: input.traceFilePath,
358 scannedFilePaths,
359 readAt,
360 recordCount,
361 parseErrorCount,
362 firstSpanAt: Option.fromNullishOr(firstSpanAt),
363 lastSpanAt: Option.fromNullishOr(lastSpanAt),
364 failureCount,
365 interruptionCount,
366 slowSpanThresholdMs,
367 slowSpanCount,
368 logLevelCounts,
369 topSpansByCount,
370 slowestSpans,
371 commonFailures: [...failuresByKey.values()]
372 .toSorted(
373 (left, right) =>
374 right.count - left.count ||
375 DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt),
376 )
377 .slice(0, TOP_LIMIT),
378 latestFailures: latestFailures
336 const finish = (input: {
337 readonly traceFilePath: string;
338 readonly scannedFilePaths: ReadonlyArray<string>;
339 readonly readAt: DateTime.Utc;
340 readonly error?: TraceDiagnosticsErrorSummary;
341 readonly partialFailure?: boolean;
342 }): ServerTraceDiagnosticsResult => {
343 const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()]
344 .map(([name, span]) => ({
345 name,
346 count: span.count,
347 failureCount: span.failureCount,
348 totalDurationMs: span.totalDurationMs,
349 averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0,
350 maxDurationMs: span.maxDurationMs,
351 }))
379352 .toSorted(
380 (left, right) =>
381 DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt),
353 (left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs,
382354 )
383 .slice(0, RECENT_LIMIT),
384 latestWarningAndErrorLogs: latestWarningAndErrorLogs
385 .toSorted(
386 (left, right) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt),
387 )
388 .slice(0, RECENT_LIMIT),
389 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
390 error: Option.fromNullishOr(input.error),
355 .slice(0, TOP_LIMIT);
356
357 return {
358 traceFilePath: input.traceFilePath,
359 scannedFilePaths: input.scannedFilePaths,
360 readAt: input.readAt,
361 recordCount,
362 parseErrorCount,
363 firstSpanAt: Option.fromNullishOr(firstSpanAt),
364 lastSpanAt: Option.fromNullishOr(lastSpanAt),
365 failureCount,
366 interruptionCount,
367 slowSpanThresholdMs,
368 slowSpanCount,
369 logLevelCounts,
370 topSpansByCount,
371 slowestSpans,
372 commonFailures: [...failuresByKey.values()]
373 .toSorted(
374 (left, right) =>
375 right.count - left.count ||
376 DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt),
377 )
378 .slice(0, TOP_LIMIT),
379 latestFailures,
380 latestWarningAndErrorLogs,
381 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
382 error: Option.fromNullishOr(input.error),
383 };
391384 };
392}
393385
394type TraceFileReadResult =
395 | { readonly _tag: "Loaded"; readonly path: string; readonly text: string }
396 | { readonly _tag: "Missing"; readonly path: string };
386 return { addLine, finish };
387}
397388
398function readTraceFile(
389/**
390 * Feeds each line of one trace file to `onLine`, streaming so only one chunk of
391 * text is in memory at a time. Succeeds with false when the file does not exist.
392 */
393export function streamTraceFileLines(
399394 fileSystem: FileSystem.FileSystem,
400395 path: string,
401): Effect.Effect<TraceFileReadResult, TraceFileReadError> {
402 return fileSystem.readFileString(path).pipe(
403 Effect.map((text): TraceFileReadResult => ({ _tag: "Loaded", path, text })),
396 onLine: (line: string) => void,
397): Effect.Effect<boolean, TraceFileReadError> {
398 return fileSystem.stream(path).pipe(
399 Stream.decodeText,
400 Stream.splitLines,
401 Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(onLine))),
402 Effect.as(true),
404403 Effect.catchTags({
405404 PlatformError: (cause) =>
406405 isNotFoundError(cause)
407 ? Effect.succeed<TraceFileReadResult>({ _tag: "Missing", path })
406 ? Effect.succeed(false)
408407 : Effect.fail(
409408 new TraceFileReadError({
410409 traceFilePath: path,
......@@ -425,10 +424,11 @@ export const make = Effect.gen(function* () {
425424 const readAt = options.readAt ?? (yield* DateTime.now);
426425 const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
427426 const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles);
427 const aggregator = makeTraceDiagnosticsAggregator(slowSpanThresholdMs);
428428 const results = yield* Effect.forEach(
429429 paths,
430430 (path) =>
431 readTraceFile(fileSystem, path).pipe(
431 streamTraceFileLines(fileSystem, path, aggregator.addLine).pipe(
432432 Effect.tapError((cause) =>
433433 Effect.logWarning("Failed to read local trace file.").pipe(
434434 Effect.annotateLogs({
......@@ -440,15 +440,10 @@ export const make = Effect.gen(function* () {
440440 ),
441441 Effect.result,
442442 ),
443 {
444 concurrency: 1,
445 },
446 );
447 const files = results.flatMap((result) =>
448 Result.isSuccess(result) && result.success._tag === "Loaded"
449 ? [{ path: result.success.path, text: result.success.text }]
450 : [],
443 // Every file feeds one aggregator, so read them one at a time, oldest first.
444 { concurrency: 1 },
451445 );
446 const foundFile = results.some((result) => Result.isSuccess(result) && result.success);
452447 const readFailure = results.find(Result.isFailure);
453448 const readFailureError = readFailure
454449 ? ({
......@@ -457,7 +452,7 @@ export const make = Effect.gen(function* () {
457452 } satisfies TraceDiagnosticsErrorSummary)
458453 : undefined;
459454
460 if (files.length === 0) {
455 if (!foundFile) {
461456 return makeEmptyDiagnostics({
462457 traceFilePath: options.traceFilePath,
463458 scannedFilePaths: paths,
......@@ -472,12 +467,10 @@ export const make = Effect.gen(function* () {
472467 });
473468 }
474469
475 return aggregateTraceDiagnostics({
470 return aggregator.finish({
476471 traceFilePath: options.traceFilePath,
477 files,
478472 scannedFilePaths: paths,
479473 readAt,
480 slowSpanThresholdMs,
481474 ...(readFailureError ? { partialFailure: true, error: readFailureError } : {}),
482475 });
483476 },