authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-05-05 21:22:50-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-05-06 04:22:50+00:00
loga2ff50dbba0725a53dfdb76184a54a621645ddd5
treecc9b4b69eb9cdc2bf0357db569f1cb30513fadcb
parent166bce0389e2068ae48c5936692eb315b17a269d
signature Signed by PGP key B5690EEEBB952194

Add process and trace diagnostics views (#2532)

Co-authored-by: codex <codex@users.noreply.github.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com>

26 files changed, 3300 insertions(+), 145 deletions(-)

apps/server/src/diagnostics/ProcessDiagnostics.test.ts created+252
......@@ -0,0 +1,252 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Option from "effect/Option";
6import * as Sink from "effect/Sink";
7import * as Stream from "effect/Stream";
8import { ChildProcessSpawner } from "effect/unstable/process";
9
10import * as ProcessDiagnostics from "./ProcessDiagnostics.ts";
11
12const encoder = new TextEncoder();
13
14function mockHandle(result: {
15 readonly stdout?: string;
16 readonly stderr?: string;
17 readonly code?: number;
18}) {
19 return ChildProcessSpawner.makeHandle({
20 pid: ChildProcessSpawner.ProcessId(1),
21 exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.code ?? 0)),
22 isRunning: Effect.succeed(false),
23 kill: () => Effect.void,
24 unref: Effect.succeed(Effect.void),
25 stdin: Sink.drain,
26 stdout: Stream.make(encoder.encode(result.stdout ?? "")),
27 stderr: Stream.make(encoder.encode(result.stderr ?? "")),
28 all: Stream.empty,
29 getInputFd: () => Sink.drain,
30 getOutputFd: () => Stream.empty,
31 });
32}
33
34describe("ProcessDiagnostics", () => {
35 it.effect("parses POSIX ps rows with full commands", () =>
36 Effect.sync(() => {
37 const rows = ProcessDiagnostics.parsePosixProcessRows(
38 [
39 " 10 1 10 Ss 0.0 1024 01:02.03 /usr/bin/node server.js",
40 " 11 10 10 S+ 12.5 20480 00:04 codex app-server --config /tmp/one two",
41 ].join("\n"),
42 );
43
44 expect(rows).toEqual([
45 {
46 pid: 10,
47 ppid: 1,
48 pgid: 10,
49 status: "Ss",
50 cpuPercent: 0,
51 rssBytes: 1024 * 1024,
52 elapsed: "01:02.03",
53 command: "/usr/bin/node server.js",
54 },
55 {
56 pid: 11,
57 ppid: 10,
58 pgid: 10,
59 status: "S+",
60 cpuPercent: 12.5,
61 rssBytes: 20480 * 1024,
62 elapsed: "00:04",
63 command: "codex app-server --config /tmp/one two",
64 },
65 ]);
66 }),
67 );
68
69 it.effect("aggregates only descendants of the server process", () =>
70 Effect.sync(() => {
71 const diagnostics = ProcessDiagnostics.aggregateProcessDiagnostics({
72 serverPid: 100,
73 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
74 rows: [
75 {
76 pid: 100,
77 ppid: 1,
78 pgid: 100,
79 status: "S",
80 cpuPercent: 0,
81 rssBytes: 1_000,
82 elapsed: "01:00",
83 command: "t3 server",
84 },
85 {
86 pid: 101,
87 ppid: 100,
88 pgid: 100,
89 status: "S",
90 cpuPercent: 1.5,
91 rssBytes: 2_000,
92 elapsed: "00:20",
93 command: "codex app-server",
94 },
95 {
96 pid: 102,
97 ppid: 101,
98 pgid: 100,
99 status: "R",
100 cpuPercent: 3.25,
101 rssBytes: 4_000,
102 elapsed: "00:05",
103 command: "git status",
104 },
105 {
106 pid: 200,
107 ppid: 1,
108 pgid: 200,
109 status: "S",
110 cpuPercent: 99,
111 rssBytes: 8_000,
112 elapsed: "00:01",
113 command: "unrelated",
114 },
115 {
116 pid: 201,
117 ppid: 100,
118 pgid: 100,
119 status: "R",
120 cpuPercent: 9,
121 rssBytes: 9_000,
122 elapsed: "00:00",
123 command: "ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=",
124 },
125 ],
126 });
127
128 expect(diagnostics.serverPid).toBe(100);
129 expect(DateTime.formatIso(diagnostics.readAt)).toBe("2026-05-05T10:00:00.000Z");
130 expect(diagnostics.processCount).toBe(2);
131 expect(diagnostics.totalRssBytes).toBe(6_000);
132 expect(diagnostics.totalCpuPercent).toBe(4.75);
133 expect(diagnostics.processes.map((process) => process.pid)).toEqual([101, 102]);
134 expect(diagnostics.processes.map((process) => process.depth)).toEqual([0, 1]);
135 expect(Option.getOrNull(diagnostics.processes[0]!.pgid)).toBe(100);
136 expect(diagnostics.processes[0]?.childPids).toEqual([102]);
137 }),
138 );
139
140 it.effect("preserves ascending sibling order for nested descendants", () =>
141 Effect.sync(() => {
142 const diagnostics = ProcessDiagnostics.aggregateProcessDiagnostics({
143 serverPid: 100,
144 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
145 rows: [
146 {
147 pid: 101,
148 ppid: 100,
149 pgid: 100,
150 status: "S",
151 cpuPercent: 0,
152 rssBytes: 100,
153 elapsed: "00:10",
154 command: "agent",
155 },
156 {
157 pid: 103,
158 ppid: 101,
159 pgid: 100,
160 status: "S",
161 cpuPercent: 0,
162 rssBytes: 100,
163 elapsed: "00:10",
164 command: "child-b",
165 },
166 {
167 pid: 102,
168 ppid: 101,
169 pgid: 100,
170 status: "S",
171 cpuPercent: 0,
172 rssBytes: 100,
173 elapsed: "00:10",
174 command: "child-a",
175 },
176 ],
177 });
178
179 expect(diagnostics.processes.map((process) => process.pid)).toEqual([101, 102, 103]);
180 }),
181 );
182
183 it.effect("queries processes through the ChildProcessSpawner service", () =>
184 Effect.gen(function* () {
185 const commands: Array<{ readonly command: string; readonly args: ReadonlyArray<string> }> =
186 [];
187 const spawnerLayer = Layer.succeed(
188 ChildProcessSpawner.ChildProcessSpawner,
189 ChildProcessSpawner.make((command) => {
190 const childProcess = command as unknown as {
191 readonly command: string;
192 readonly args: ReadonlyArray<string>;
193 };
194 commands.push({ command: childProcess.command, args: childProcess.args });
195 return Effect.succeed(
196 mockHandle({
197 stdout: [
198 ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`,
199 ` 4242 ${process.pid} ${process.pid} S 1.5 2048 00:04 agent`,
200 ].join("\n"),
201 }),
202 );
203 }),
204 );
205 const layer = ProcessDiagnostics.layer.pipe(Layer.provide(spawnerLayer));
206
207 const diagnostics = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
208 Effect.flatMap((pd) => pd.read),
209 Effect.provide(layer),
210 );
211
212 expect(diagnostics.processes.map((process) => process.pid)).toEqual([4242]);
213 expect(commands).toEqual([
214 {
215 command: "ps",
216 args: ["-axo", "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command="],
217 },
218 ]);
219 }),
220 );
221
222 it.effect("does not allow signaling the diagnostics query process", () =>
223 Effect.gen(function* () {
224 const spawnerLayer = Layer.succeed(
225 ChildProcessSpawner.ChildProcessSpawner,
226 ChildProcessSpawner.make(() =>
227 Effect.succeed(
228 mockHandle({
229 stdout: [
230 ` ${process.pid} 1 ${process.pid} Ss 0.0 1024 01:02.03 t3 server`,
231 ` 4242 ${process.pid} ${process.pid} R 1.5 2048 00:00 ps -axo pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=`,
232 ].join("\n"),
233 }),
234 ),
235 ),
236 );
237 const layer = ProcessDiagnostics.layer.pipe(Layer.provide(spawnerLayer));
238
239 const result = yield* Effect.service(ProcessDiagnostics.ProcessDiagnostics).pipe(
240 Effect.flatMap((pd) => pd.signal({ pid: 4242, signal: "SIGINT" })),
241 Effect.provide(layer),
242 );
243
244 expect(result).toEqual({
245 pid: 4242,
246 signal: "SIGINT",
247 signaled: false,
248 message: Option.some("Process 4242 is not a live descendant of the T3 server."),
249 });
250 }),
251 );
252});
apps/server/src/diagnostics/ProcessDiagnostics.ts created+461
......@@ -0,0 +1,461 @@
1import type {
2 ServerProcessDiagnosticsEntry,
3 ServerProcessDiagnosticsResult,
4 ServerProcessSignal,
5 ServerSignalProcessResult,
6} from "@t3tools/contracts";
7import * as Context from "effect/Context";
8import * as DateTime from "effect/DateTime";
9import * as Duration from "effect/Duration";
10import * as Effect from "effect/Effect";
11import * as Layer from "effect/Layer";
12import * as Option from "effect/Option";
13import * as Schema from "effect/Schema";
14import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
15
16import { collectUint8StreamText } from "../stream/collectUint8StreamText.ts";
17
18interface ProcessRow {
19 readonly pid: number;
20 readonly ppid: number;
21 readonly pgid: number | null;
22 readonly status: string;
23 readonly cpuPercent: number;
24 readonly rssBytes: number;
25 readonly elapsed: string;
26 readonly command: string;
27}
28
29const PROCESS_QUERY_TIMEOUT_MS = 1_000;
30const POSIX_PROCESS_QUERY_COMMAND = "pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=";
31const PROCESS_QUERY_MAX_OUTPUT_BYTES = 2 * 1024 * 1024;
32
33export interface ProcessDiagnosticsShape {
34 readonly read: Effect.Effect<ServerProcessDiagnosticsResult>;
35 readonly signal: (input: {
36 readonly pid: number;
37 readonly signal: ServerProcessSignal;
38 }) => Effect.Effect<ServerSignalProcessResult>;
39}
40
41export class ProcessDiagnostics extends Context.Service<
42 ProcessDiagnostics,
43 ProcessDiagnosticsShape
44>()("t3/diagnostics/ProcessDiagnostics") {}
45
46class ProcessDiagnosticsError extends Schema.TaggedErrorClass<ProcessDiagnosticsError>()(
47 "ProcessDiagnosticsError",
48 {
49 message: Schema.String,
50 cause: Schema.optional(Schema.Defect),
51 },
52) {}
53
54function toProcessDiagnosticsError(message: string, cause?: unknown): ProcessDiagnosticsError {
55 return new ProcessDiagnosticsError({
56 message,
57 ...(cause === undefined ? {} : { cause }),
58 });
59}
60
61function parsePositiveInt(value: string): number | null {
62 const parsed = Number.parseInt(value, 10);
63 return Number.isInteger(parsed) && parsed > 0 ? parsed : null;
64}
65
66function parseNonNegativeInt(value: string): number | null {
67 const parsed = Number.parseInt(value, 10);
68 return Number.isInteger(parsed) && parsed >= 0 ? parsed : null;
69}
70
71function parseNumber(value: string): number | null {
72 const parsed = Number.parseFloat(value);
73 return Number.isFinite(parsed) ? parsed : null;
74}
75
76export function parsePosixProcessRows(output: string): ReadonlyArray<ProcessRow> {
77 const rows: ProcessRow[] = [];
78 const rowPattern =
79 /^\s*(\d+)\s+(\d+)\s+(-?\d+)\s+(\S+)\s+([+-]?(?:\d+\.?\d*|\.\d+))\s+(\d+)\s+(\S+)\s+(.+)$/;
80
81 for (const line of output.split(/\r?\n/)) {
82 if (line.trim().length === 0) continue;
83
84 const match = rowPattern.exec(line);
85 if (!match) continue;
86
87 const pidText = match[1];
88 const ppidText = match[2];
89 const pgidText = match[3];
90 const status = match[4];
91 const cpuText = match[5];
92 const rssText = match[6];
93 const elapsed = match[7];
94 const command = match[8];
95 if (
96 pidText === undefined ||
97 ppidText === undefined ||
98 pgidText === undefined ||
99 status === undefined ||
100 cpuText === undefined ||
101 rssText === undefined ||
102 elapsed === undefined ||
103 command === undefined
104 ) {
105 continue;
106 }
107
108 const pid = parsePositiveInt(pidText);
109 const ppid = parseNonNegativeInt(ppidText);
110 const pgid = Number.parseInt(pgidText, 10);
111 const cpuPercent = parseNumber(cpuText);
112 const rssKiB = parseNonNegativeInt(rssText);
113 if (
114 pid === null ||
115 ppid === null ||
116 !Number.isInteger(pgid) ||
117 cpuPercent === null ||
118 rssKiB === null ||
119 !status ||
120 !elapsed ||
121 !command
122 ) {
123 continue;
124 }
125
126 rows.push({
127 pid,
128 ppid,
129 pgid,
130 status,
131 cpuPercent,
132 rssBytes: rssKiB * 1024,
133 elapsed,
134 command,
135 });
136 }
137
138 return rows;
139}
140
141function normalizeWindowsProcessRow(value: unknown): ProcessRow | null {
142 if (typeof value !== "object" || value === null) return null;
143 const record = value as Record<string, unknown>;
144 const pid = typeof record.ProcessId === "number" ? record.ProcessId : null;
145 const ppid = typeof record.ParentProcessId === "number" ? record.ParentProcessId : null;
146 const commandLine =
147 typeof record.CommandLine === "string" && record.CommandLine.trim().length > 0
148 ? record.CommandLine
149 : typeof record.Name === "string"
150 ? record.Name
151 : null;
152 const workingSet =
153 typeof record.WorkingSetSize === "number" && Number.isFinite(record.WorkingSetSize)
154 ? Math.max(0, Math.round(record.WorkingSetSize))
155 : 0;
156 const cpuPercent =
157 typeof record.PercentProcessorTime === "number" && Number.isFinite(record.PercentProcessorTime)
158 ? Math.max(0, record.PercentProcessorTime)
159 : 0;
160
161 if (!pid || pid <= 0 || ppid === null || ppid < 0 || !commandLine) return null;
162 return {
163 pid,
164 ppid,
165 pgid: null,
166 status: typeof record.Status === "string" && record.Status.length > 0 ? record.Status : "Live",
167 cpuPercent,
168 rssBytes: workingSet,
169 elapsed: "",
170 command: commandLine,
171 };
172}
173
174function parseWindowsProcessRows(output: string): ReadonlyArray<ProcessRow> {
175 if (output.trim().length === 0) return [];
176 try {
177 const parsed = JSON.parse(output) as unknown;
178 const records = Array.isArray(parsed) ? parsed : [parsed];
179 return records.flatMap((record) => {
180 const row = normalizeWindowsProcessRow(record);
181 return row ? [row] : [];
182 });
183 } catch {
184 return [];
185 }
186}
187
188function buildDescendantEntries(
189 rows: ReadonlyArray<ProcessRow>,
190 serverPid: number,
191): ReadonlyArray<ServerProcessDiagnosticsEntry> {
192 const childrenByParent = new Map<number, ProcessRow[]>();
193 for (const row of rows) {
194 const children = childrenByParent.get(row.ppid) ?? [];
195 children.push(row);
196 childrenByParent.set(row.ppid, children);
197 }
198
199 const entries: ServerProcessDiagnosticsEntry[] = [];
200 const visited = new Set<number>();
201 const stack = [...(childrenByParent.get(serverPid) ?? [])]
202 .toSorted((left, right) => left.pid - right.pid)
203 .map((row) => ({ row, depth: 0 }));
204
205 while (stack.length > 0) {
206 const item = stack.shift();
207 if (!item || visited.has(item.row.pid)) continue;
208 visited.add(item.row.pid);
209
210 const children = [...(childrenByParent.get(item.row.pid) ?? [])].toSorted(
211 (left, right) => left.pid - right.pid,
212 );
213 entries.push({
214 pid: item.row.pid,
215 ppid: item.row.ppid,
216 pgid: Option.fromNullishOr(item.row.pgid),
217 status: item.row.status,
218 cpuPercent: item.row.cpuPercent,
219 rssBytes: item.row.rssBytes,
220 elapsed: item.row.elapsed || "n/a",
221 command: item.row.command,
222 depth: item.depth,
223 childPids: children.map((child) => child.pid),
224 });
225
226 stack.unshift(...children.map((row) => ({ row, depth: item.depth + 1 })));
227 }
228
229 return entries;
230}
231
232function isDiagnosticsQueryProcess(row: ProcessRow, serverPid: number): boolean {
233 if (row.ppid !== serverPid) return false;
234
235 const command = row.command.trim();
236 return (
237 /(?:^|[/\\])ps\s+-axo\s+pid=,ppid=,pgid=,stat=,pcpu=,rss=,etime=,command=/.test(command) ||
238 (/\bpowershell(?:\.exe)?\b/i.test(command) &&
239 /\bGet-CimInstance\s+Win32_Process\b/i.test(command))
240 );
241}
242
243function makeResult(input: {
244 readonly serverPid: number;
245 readonly rows: ReadonlyArray<ProcessRow>;
246 readonly readAt: DateTime.Utc;
247 readonly error?: string;
248}): ServerProcessDiagnosticsResult {
249 const readAt = input.readAt;
250 const rows = input.rows.filter((row) => !isDiagnosticsQueryProcess(row, input.serverPid));
251 const processes = buildDescendantEntries(rows, input.serverPid);
252 const totalRssBytes = processes.reduce((total, process) => total + process.rssBytes, 0);
253 const totalCpuPercent = processes.reduce((total, process) => total + process.cpuPercent, 0);
254
255 return {
256 serverPid: input.serverPid,
257 readAt,
258 processCount: processes.length,
259 totalRssBytes,
260 totalCpuPercent,
261 processes,
262 error: input.error ? Option.some({ message: input.error }) : Option.none(),
263 };
264}
265
266interface ProcessOutput {
267 readonly exitCode: number;
268 readonly stdout: string;
269 readonly stderr: string;
270}
271
272const runProcess = Effect.fn("runProcess")(
273 function* (input: {
274 readonly command: string;
275 readonly args: ReadonlyArray<string>;
276 readonly errorMessage: string;
277 }) {
278 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
279 const child = yield* spawner.spawn(
280 ChildProcess.make(input.command, input.args, {
281 cwd: process.cwd(),
282 shell: process.platform === "win32",
283 }),
284 );
285 const [stdout, stderr, exitCode] = yield* Effect.all(
286 [
287 collectUint8StreamText({
288 stream: child.stdout,
289 maxBytes: PROCESS_QUERY_MAX_OUTPUT_BYTES,
290 truncatedMarker: "\n\n[truncated]",
291 }),
292 collectUint8StreamText({
293 stream: child.stderr,
294 maxBytes: PROCESS_QUERY_MAX_OUTPUT_BYTES,
295 truncatedMarker: "\n\n[truncated]",
296 }),
297 child.exitCode,
298 ],
299 { concurrency: "unbounded" },
300 );
301
302 return {
303 exitCode,
304 stdout: stdout.text,
305 stderr: stderr.text,
306 } satisfies ProcessOutput;
307 },
308 (effect, input) =>
309 effect.pipe(
310 Effect.scoped,
311 Effect.timeoutOption(Duration.millis(PROCESS_QUERY_TIMEOUT_MS)),
312 Effect.flatMap((result) =>
313 Option.match(result, {
314 onNone: () => Effect.fail(toProcessDiagnosticsError(`${input.errorMessage} timed out.`)),
315 onSome: Effect.succeed,
316 }),
317 ),
318 Effect.mapError((cause) =>
319 Schema.is(ProcessDiagnosticsError)(cause)
320 ? cause
321 : toProcessDiagnosticsError(input.errorMessage, cause),
322 ),
323 ),
324);
325
326function readPosixProcessRows(): Effect.Effect<
327 ReadonlyArray<ProcessRow>,
328 ProcessDiagnosticsError,
329 ChildProcessSpawner.ChildProcessSpawner
330> {
331 return runProcess({
332 command: "ps",
333 args: ["-axo", POSIX_PROCESS_QUERY_COMMAND],
334 errorMessage: "Failed to query process diagnostics.",
335 }).pipe(
336 Effect.flatMap((result) =>
337 result.exitCode !== 0
338 ? Effect.fail(toProcessDiagnosticsError(result.stderr.trim() || "ps failed."))
339 : Effect.succeed(parsePosixProcessRows(result.stdout)),
340 ),
341 );
342}
343
344function readWindowsProcessRows(): Effect.Effect<
345 ReadonlyArray<ProcessRow>,
346 ProcessDiagnosticsError,
347 ChildProcessSpawner.ChildProcessSpawner
348> {
349 const command = [
350 "$processes = Get-CimInstance Win32_Process | ForEach-Object {",
351 '$perf = Get-CimInstance Win32_PerfFormattedData_PerfProc_Process -Filter "IDProcess = $($_.ProcessId)" -ErrorAction SilentlyContinue;',
352 "[pscustomobject]@{ ProcessId = $_.ProcessId; ParentProcessId = $_.ParentProcessId; Name = $_.Name; CommandLine = $_.CommandLine; Status = $_.Status; WorkingSetSize = $_.WorkingSetSize; PercentProcessorTime = if ($perf) { $perf.PercentProcessorTime } else { 0 } }",
353 "};",
354 "$processes | ConvertTo-Json -Compress -Depth 3",
355 ].join(" ");
356
357 return runProcess({
358 command: "powershell.exe",
359 args: ["-NoProfile", "-NonInteractive", "-Command", command],
360 errorMessage: "Failed to query process diagnostics.",
361 }).pipe(
362 Effect.flatMap((result) =>
363 result.exitCode !== 0
364 ? Effect.fail(
365 toProcessDiagnosticsError(result.stderr.trim() || "PowerShell process query failed."),
366 )
367 : Effect.succeed(parseWindowsProcessRows(result.stdout)),
368 ),
369 );
370}
371
372const readProcessRows = (platform = process.platform) =>
373 platform === "win32" ? readWindowsProcessRows() : readPosixProcessRows();
374
375export function aggregateProcessDiagnostics(input: {
376 readonly serverPid: number;
377 readonly rows: ReadonlyArray<ProcessRow>;
378 readonly readAt: DateTime.Utc;
379}): ServerProcessDiagnosticsResult {
380 return makeResult(input);
381}
382
383function assertDescendantPid(
384 pid: number,
385): Effect.Effect<void, ProcessDiagnosticsError, ChildProcessSpawner.ChildProcessSpawner> {
386 if (pid === process.pid) {
387 return Effect.fail(toProcessDiagnosticsError("Refusing to signal the T3 server process."));
388 }
389
390 return readProcessRows().pipe(
391 Effect.flatMap((rows) => {
392 const filteredRows = rows.filter((row) => !isDiagnosticsQueryProcess(row, process.pid));
393 const descendant = buildDescendantEntries(filteredRows, process.pid).some(
394 (entry) => entry.pid === pid,
395 );
396 return descendant
397 ? Effect.void
398 : Effect.fail(
399 toProcessDiagnosticsError(`Process ${pid} is not a live descendant of the T3 server.`),
400 );
401 }),
402 );
403}
404
405export const make = Effect.fn("makeProcessDiagnostics")(function* () {
406 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
407
408 const read: ProcessDiagnosticsShape["read"] = Effect.gen(function* () {
409 const readAt = yield* DateTime.now;
410 const rows = yield* readProcessRows().pipe(
411 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
412 );
413 return makeResult({ serverPid: process.pid, rows, readAt });
414 }).pipe(
415 Effect.catch((error: ProcessDiagnosticsError) =>
416 DateTime.now.pipe(
417 Effect.map((readAt) =>
418 makeResult({ serverPid: process.pid, rows: [], readAt, error: error.message }),
419 ),
420 ),
421 ),
422 );
423
424 const signal: ProcessDiagnosticsShape["signal"] = Effect.fn("ProcessDiagnostics.signal")(
425 function* (input) {
426 return yield* assertDescendantPid(input.pid).pipe(
427 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
428 Effect.flatMap(() =>
429 Effect.try({
430 try: () => {
431 process.kill(input.pid, input.signal);
432 return {
433 pid: input.pid,
434 signal: input.signal,
435 signaled: true,
436 message: Option.none(),
437 };
438 },
439 catch: (cause) =>
440 toProcessDiagnosticsError(
441 `Failed to signal process ${input.pid} with ${input.signal}.`,
442 cause,
443 ),
444 }),
445 ),
446 Effect.catch((error: ProcessDiagnosticsError) =>
447 Effect.succeed({
448 pid: input.pid,
449 signal: input.signal,
450 signaled: false,
451 message: Option.some(error.message),
452 }),
453 ),
454 );
455 },
456 );
457
458 return ProcessDiagnostics.of({ read, signal });
459});
460
461export const layer = Layer.effect(ProcessDiagnostics, make());
apps/server/src/diagnostics/TraceDiagnostics.test.ts created+258
......@@ -0,0 +1,258 @@
1import { assert, describe, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
5import * as Layer from "effect/Layer";
6import * as Option from "effect/Option";
7import * as PlatformError from "effect/PlatformError";
8
9import * as TraceDiagnostics from "./TraceDiagnostics.ts";
10
11function ns(ms: number): string {
12 return String(BigInt(ms) * 1_000_000n);
13}
14
15function record(input: {
16 readonly name: string;
17 readonly traceId: string;
18 readonly spanId: string;
19 readonly startMs: number;
20 readonly durationMs: number;
21 readonly exit?: { readonly _tag: "Success" | "Failure" | "Interrupted"; readonly cause?: string };
22 readonly events?: ReadonlyArray<unknown>;
23}) {
24 return JSON.stringify({
25 type: "effect-span",
26 name: input.name,
27 traceId: input.traceId,
28 spanId: input.spanId,
29 sampled: true,
30 kind: "internal",
31 startTimeUnixNano: ns(input.startMs),
32 endTimeUnixNano: ns(input.startMs + input.durationMs),
33 durationMs: input.durationMs,
34 attributes: {},
35 events: input.events ?? [],
36 links: [],
37 exit: input.exit ?? { _tag: "Success" },
38 });
39}
40
41describe("TraceDiagnostics", () => {
42 it.effect("aggregates failures, slow spans, log levels, and parse errors", () =>
43 Effect.sync(() => {
44 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
45 traceFilePath: "/tmp/server.trace.ndjson",
46 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
47 slowSpanThresholdMs: 1_000,
48 files: [
49 {
50 path: "/tmp/server.trace.ndjson.1",
51 text: [
52 record({
53 name: "server.getConfig",
54 traceId: "trace-a",
55 spanId: "span-a",
56 startMs: 1_000,
57 durationMs: 50,
58 }),
59 "not-json",
60 ].join("\n"),
61 },
62 {
63 path: "/tmp/server.trace.ndjson",
64 text: [
65 record({
66 name: "orchestration.dispatch",
67 traceId: "trace-b",
68 spanId: "span-b",
69 startMs: 2_000,
70 durationMs: 1_500,
71 exit: { _tag: "Failure", cause: "Provider crashed" },
72 events: [
73 {
74 name: "provider failed",
75 timeUnixNano: ns(3_400),
76 attributes: { "effect.logLevel": "Error" },
77 },
78 ],
79 }),
80 record({
81 name: "orchestration.dispatch",
82 traceId: "trace-c",
83 spanId: "span-c",
84 startMs: 4_000,
85 durationMs: 250,
86 exit: { _tag: "Failure", cause: "Provider crashed" },
87 }),
88 record({
89 name: "git.status",
90 traceId: "trace-d",
91 spanId: "span-d",
92 startMs: 5_000,
93 durationMs: 25,
94 exit: { _tag: "Interrupted", cause: "Interrupted" },
95 events: [
96 {
97 name: "status delayed",
98 timeUnixNano: ns(5_010),
99 attributes: { "effect.logLevel": "Warning" },
100 },
101 ],
102 }),
103 ].join("\n"),
104 },
105 ],
106 });
107
108 assert.equal(diagnostics.recordCount, 4);
109 assert.equal(DateTime.formatIso(diagnostics.readAt), "2026-05-05T10:00:00.000Z");
110 assert.equal(
111 Option.match(diagnostics.firstSpanAt, {
112 onNone: () => null,
113 onSome: DateTime.formatIso,
114 }),
115 "1970-01-01T00:00:01.000Z",
116 );
117 assert.equal(
118 Option.match(diagnostics.lastSpanAt, {
119 onNone: () => null,
120 onSome: DateTime.formatIso,
121 }),
122 "1970-01-01T00:00:05.025Z",
123 );
124 assert.equal(diagnostics.parseErrorCount, 1);
125 assert.equal(diagnostics.failureCount, 2);
126 assert.equal(diagnostics.interruptionCount, 1);
127 assert.equal(diagnostics.slowSpanCount, 1);
128 assert.equal(diagnostics.logLevelCounts.Error, 1);
129 assert.equal(diagnostics.logLevelCounts.Warning, 1);
130 assert.equal(diagnostics.commonFailures[0]?.name, "orchestration.dispatch");
131 assert.equal(diagnostics.commonFailures[0]?.count, 2);
132 assert.equal(diagnostics.latestFailures[0]?.traceId, "trace-c");
133 assert.equal(diagnostics.slowestSpans[0]?.traceId, "trace-b");
134 assert.equal(diagnostics.latestWarningAndErrorLogs[0]?.message, "status delayed");
135 assert.equal(diagnostics.topSpansByCount[0]?.name, "orchestration.dispatch");
136 }),
137 );
138
139 it.effect("returns a not-found diagnostic when no files are available", () =>
140 Effect.sync(() => {
141 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
142 traceFilePath: "/tmp/missing.trace.ndjson",
143 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
144 files: [],
145 });
146
147 assert.equal(diagnostics.recordCount, 0);
148 assert.equal(Option.getOrUndefined(diagnostics.error)?.kind, "trace-file-not-found");
149 }),
150 );
151
152 it.effect("preserves full failure causes and log messages", () =>
153 Effect.sync(() => {
154 const longCause = `VcsProcessSpawnError: ${"missing executable ".repeat(80)}`.trim();
155 const longMessage = `provider warning: ${"retrying command ".repeat(80)}`.trim();
156 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
157 traceFilePath: "/tmp/server.trace.ndjson",
158 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
159 files: [
160 {
161 path: "/tmp/server.trace.ndjson",
162 text: record({
163 name: "VcsProcess.run",
164 traceId: "trace-long",
165 spanId: "span-long",
166 startMs: 1_000,
167 durationMs: 25,
168 exit: { _tag: "Failure", cause: longCause },
169 events: [
170 {
171 name: longMessage,
172 timeUnixNano: ns(1_010),
173 attributes: { "effect.logLevel": "Warning" },
174 },
175 ],
176 }),
177 },
178 ],
179 });
180
181 assert.equal(diagnostics.latestFailures[0]?.cause, longCause);
182 assert.equal(diagnostics.commonFailures[0]?.cause, longCause);
183 assert.equal(diagnostics.latestWarningAndErrorLogs[0]?.message, longMessage);
184 }),
185 );
186
187 it.effect("keeps loaded trace data when one rotated trace file fails to read", () =>
188 Effect.gen(function* () {
189 const traceFilePath = "/tmp/server.trace.ndjson";
190 const fileSystemLayer = FileSystem.layerNoop({
191 readFileString: (path) =>
192 path === `${traceFilePath}.1`
193 ? Effect.fail(
194 PlatformError.systemError({
195 _tag: "PermissionDenied",
196 module: "FileSystem",
197 method: "readFileString",
198 description: "permission denied",
199 pathOrDescriptor: path,
200 }),
201 )
202 : Effect.succeed(
203 record({
204 name: "server.getConfig",
205 traceId: "trace-a",
206 spanId: "span-a",
207 startMs: 1_000,
208 durationMs: 50,
209 }),
210 ),
211 });
212
213 const diagnostics = yield* TraceDiagnostics.readTraceDiagnostics({
214 traceFilePath,
215 maxFiles: 1,
216 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
217 }).pipe(Effect.provide(TraceDiagnostics.layer.pipe(Layer.provide(fileSystemLayer))));
218
219 assert.equal(diagnostics.recordCount, 1);
220 assert.equal(
221 Option.getOrElse(diagnostics.partialFailure, () => false),
222 true,
223 );
224 assert.equal(Option.getOrUndefined(diagnostics.error)?.kind, "trace-file-read-failed");
225 assert.deepStrictEqual(diagnostics.scannedFilePaths, [`${traceFilePath}.1`, traceFilePath]);
226 }),
227 );
228
229 it.effect("keeps only the slowest span occurrences while aggregating large inputs", () =>
230 Effect.sync(() => {
231 const diagnostics = TraceDiagnostics.aggregateTraceDiagnostics({
232 traceFilePath: "/tmp/server.trace.ndjson",
233 readAt: DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"),
234 files: [
235 {
236 path: "/tmp/server.trace.ndjson",
237 text: Array.from({ length: 25 }, (_, index) =>
238 record({
239 name: `span-${index}`,
240 traceId: `trace-${index}`,
241 spanId: `span-${index}`,
242 startMs: index * 1_000,
243 durationMs: index,
244 }),
245 ).join("\n"),
246 },
247 ],
248 });
249
250 assert.equal(diagnostics.recordCount, 25);
251 assert.equal(diagnostics.slowestSpans.length, 10);
252 assert.deepStrictEqual(
253 diagnostics.slowestSpans.map((span) => span.durationMs),
254 [24, 23, 22, 21, 20, 19, 18, 17, 16, 15],
255 );
256 }),
257 );
258});
apps/server/src/diagnostics/TraceDiagnostics.ts created+470
......@@ -0,0 +1,470 @@
1import type {
2 ServerTraceDiagnosticsErrorKind,
3 ServerTraceDiagnosticsFailureSummary,
4 ServerTraceDiagnosticsLogEvent,
5 ServerTraceDiagnosticsRecentFailure,
6 ServerTraceDiagnosticsResult,
7 ServerTraceDiagnosticsSpanOccurrence,
8 ServerTraceDiagnosticsSpanSummary,
9} from "@t3tools/contracts";
10import * as Context from "effect/Context";
11import * as DateTime from "effect/DateTime";
12import * as Effect from "effect/Effect";
13import * as FileSystem from "effect/FileSystem";
14import * as Layer from "effect/Layer";
15import * as Option from "effect/Option";
16import * as PlatformError from "effect/PlatformError";
17
18interface TraceRecordLike {
19 readonly name?: unknown;
20 readonly traceId?: unknown;
21 readonly spanId?: unknown;
22 readonly startTimeUnixNano?: unknown;
23 readonly endTimeUnixNano?: unknown;
24 readonly durationMs?: unknown;
25 readonly exit?: unknown;
26 readonly events?: unknown;
27}
28
29interface TraceEventLike {
30 readonly name?: unknown;
31 readonly timeUnixNano?: unknown;
32 readonly attributes?: unknown;
33}
34
35export interface TraceDiagnosticsOptions {
36 readonly traceFilePath: string;
37 readonly maxFiles: number;
38 readonly slowSpanThresholdMs?: number;
39 readonly readAt?: DateTime.Utc;
40}
41
42export interface TraceDiagnosticsShape {
43 readonly read: (options: TraceDiagnosticsOptions) => Effect.Effect<ServerTraceDiagnosticsResult>;
44}
45
46export class TraceDiagnostics extends Context.Service<TraceDiagnostics, TraceDiagnosticsShape>()(
47 "t3/diagnostics/TraceDiagnostics",
48) {}
49
50interface TraceDiagnosticsInput {
51 readonly traceFilePath: string;
52 readonly files: ReadonlyArray<{ readonly path: string; readonly text: string }>;
53 readonly scannedFilePaths?: ReadonlyArray<string>;
54 readonly slowSpanThresholdMs?: number;
55 readonly readAt: DateTime.Utc;
56 readonly error?: TraceDiagnosticsErrorSummary;
57 readonly partialFailure?: boolean;
58}
59
60interface TraceDiagnosticsErrorSummary {
61 readonly kind: ServerTraceDiagnosticsErrorKind;
62 readonly message: string;
63}
64
65const DEFAULT_SLOW_SPAN_THRESHOLD_MS = 1_000;
66const TOP_LIMIT = 10;
67const RECENT_LIMIT = 20;
68function toRotatedTracePaths(traceFilePath: string, maxFiles: number): ReadonlyArray<string> {
69 const backupCount = Math.max(0, Math.floor(maxFiles));
70 const backups = Array.from(
71 { length: backupCount },
72 (_, index) => `${traceFilePath}.${backupCount - index}`,
73 );
74 return [...backups, traceFilePath];
75}
76
77function isRecordObject(value: unknown): value is TraceRecordLike {
78 return typeof value === "object" && value !== null;
79}
80
81function toStringValue(value: unknown): string | null {
82 return typeof value === "string" && value.trim().length > 0 ? value : null;
83}
84
85function toNumberValue(value: unknown): number | null {
86 return typeof value === "number" && Number.isFinite(value) ? value : null;
87}
88
89function unixNanoToDateTime(value: unknown): DateTime.Utc | null {
90 const text = toStringValue(value);
91 if (!text) return null;
92
93 try {
94 const millis = Number(BigInt(text) / 1_000_000n);
95 if (!Number.isFinite(millis)) return null;
96 return DateTime.makeUnsafe(millis);
97 } catch {
98 return null;
99 }
100}
101
102function isAfter(left: DateTime.Utc, right: DateTime.Utc): boolean {
103 return DateTime.toEpochMillis(left) > DateTime.toEpochMillis(right);
104}
105
106function isBefore(left: DateTime.Utc, right: DateTime.Utc): boolean {
107 return DateTime.toEpochMillis(left) < DateTime.toEpochMillis(right);
108}
109
110function readExitTag(exit: unknown): string | null {
111 if (!isRecordObject(exit) || !("_tag" in exit)) return null;
112 return toStringValue(exit._tag);
113}
114
115function readExitCause(exit: unknown): string {
116 if (!isRecordObject(exit) || !("cause" in exit)) return "Failure";
117 return toStringValue(exit.cause)?.trim() ?? "Failure";
118}
119
120function isTraceEvent(value: unknown): value is TraceEventLike {
121 return typeof value === "object" && value !== null;
122}
123
124function readEventAttributes(event: TraceEventLike): Readonly<Record<string, unknown>> {
125 return typeof event.attributes === "object" && event.attributes !== null
126 ? (event.attributes as Readonly<Record<string, unknown>>)
127 : {};
128}
129
130function makeEmptyDiagnostics(input: {
131 readonly traceFilePath: string;
132 readonly scannedFilePaths: ReadonlyArray<string>;
133 readonly readAt: DateTime.Utc;
134 readonly slowSpanThresholdMs: number;
135 readonly error?: TraceDiagnosticsErrorSummary;
136 readonly partialFailure?: boolean;
137}): ServerTraceDiagnosticsResult {
138 return {
139 traceFilePath: input.traceFilePath,
140 scannedFilePaths: [...input.scannedFilePaths],
141 readAt: input.readAt,
142 recordCount: 0,
143 parseErrorCount: 0,
144 firstSpanAt: Option.none(),
145 lastSpanAt: Option.none(),
146 failureCount: 0,
147 interruptionCount: 0,
148 slowSpanThresholdMs: input.slowSpanThresholdMs,
149 slowSpanCount: 0,
150 logLevelCounts: {},
151 topSpansByCount: [],
152 slowestSpans: [],
153 commonFailures: [],
154 latestFailures: [],
155 latestWarningAndErrorLogs: [],
156 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
157 error: Option.fromNullishOr(input.error),
158 };
159}
160
161function isNotFoundError(error: PlatformError.PlatformError): boolean {
162 return error.reason._tag === "NotFound";
163}
164
165function platformErrorMessage(error: PlatformError.PlatformError): string {
166 return error.message || String(error);
167}
168
169function insertBoundedSlowestSpan(
170 slowestSpans: ServerTraceDiagnosticsSpanOccurrence[],
171 span: ServerTraceDiagnosticsSpanOccurrence,
172): void {
173 if (
174 slowestSpans.length >= TOP_LIMIT &&
175 span.durationMs <= slowestSpans[slowestSpans.length - 1]!.durationMs
176 ) {
177 return;
178 }
179
180 slowestSpans.push(span);
181 slowestSpans.sort((left, right) => right.durationMs - left.durationMs);
182 if (slowestSpans.length > TOP_LIMIT) {
183 slowestSpans.length = TOP_LIMIT;
184 }
185}
186
187export function aggregateTraceDiagnostics(
188 input: TraceDiagnosticsInput,
189): ServerTraceDiagnosticsResult {
190 const readAt = input.readAt;
191 const slowSpanThresholdMs = input.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
192 const scannedFilePaths = input.scannedFilePaths ?? input.files.map((file) => file.path);
193 if (input.files.length === 0) {
194 return makeEmptyDiagnostics({
195 traceFilePath: input.traceFilePath,
196 scannedFilePaths,
197 readAt,
198 slowSpanThresholdMs,
199 error: input.error ?? {
200 kind: "trace-file-not-found",
201 message: "No local trace files were found.",
202 },
203 ...(input.partialFailure ? { partialFailure: true } : {}),
204 });
205 }
206
207 let parseErrorCount = 0;
208 let recordCount = 0;
209 let failureCount = 0;
210 let interruptionCount = 0;
211 let slowSpanCount = 0;
212 let firstSpanAt: DateTime.Utc | null = null;
213 let lastSpanAt: DateTime.Utc | null = null;
214
215 const spansByName = new Map<
216 string,
217 { count: number; failureCount: number; totalDurationMs: number; maxDurationMs: number }
218 >();
219 const failuresByKey = new Map<string, ServerTraceDiagnosticsFailureSummary>();
220 const latestFailures: ServerTraceDiagnosticsRecentFailure[] = [];
221 const slowestSpans: ServerTraceDiagnosticsSpanOccurrence[] = [];
222 const latestWarningAndErrorLogs: ServerTraceDiagnosticsLogEvent[] = [];
223 const logLevelCounts: Record<string, number> = {};
224
225 for (const file of input.files) {
226 const lines = file.text.split(/\r?\n/);
227 for (const line of lines) {
228 if (line.trim().length === 0) continue;
229
230 let parsed: unknown;
231 try {
232 parsed = JSON.parse(line);
233 } catch {
234 parseErrorCount += 1;
235 continue;
236 }
237
238 if (!isRecordObject(parsed)) {
239 parseErrorCount += 1;
240 continue;
241 }
242
243 const name = toStringValue(parsed.name);
244 const traceId = toStringValue(parsed.traceId);
245 const spanId = toStringValue(parsed.spanId);
246 const durationMs = toNumberValue(parsed.durationMs);
247 const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano);
248 const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano);
249
250 if (!name || !traceId || !spanId || durationMs === null || !endedAt) {
251 parseErrorCount += 1;
252 continue;
253 }
254
255 recordCount += 1;
256 firstSpanAt =
257 startedAt && (firstSpanAt === null || isBefore(startedAt, firstSpanAt))
258 ? startedAt
259 : firstSpanAt;
260 lastSpanAt = lastSpanAt === null || isAfter(endedAt, lastSpanAt) ? endedAt : lastSpanAt;
261
262 const exitTag = readExitTag(parsed.exit);
263 const isFailure = exitTag === "Failure";
264 const isInterrupted = exitTag === "Interrupted";
265 if (isFailure) failureCount += 1;
266 if (isInterrupted) interruptionCount += 1;
267
268 const spanSummary = spansByName.get(name) ?? {
269 count: 0,
270 failureCount: 0,
271 totalDurationMs: 0,
272 maxDurationMs: 0,
273 };
274 spanSummary.count += 1;
275 spanSummary.totalDurationMs += durationMs;
276 spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs);
277 if (isFailure) spanSummary.failureCount += 1;
278 spansByName.set(name, spanSummary);
279
280 const spanItem = { name, durationMs, endedAt, traceId, spanId };
281 if (durationMs >= slowSpanThresholdMs) {
282 slowSpanCount += 1;
283 }
284 insertBoundedSlowestSpan(slowestSpans, spanItem);
285
286 if (isFailure) {
287 const cause = readExitCause(parsed.exit);
288 latestFailures.push({ ...spanItem, cause });
289
290 const failureKey = `${name}\0${cause}`;
291 const existing = failuresByKey.get(failureKey);
292 const isLatestFailure = !existing || isAfter(endedAt, existing.lastSeenAt);
293 failuresByKey.set(failureKey, {
294 name,
295 cause,
296 count: (existing?.count ?? 0) + 1,
297 lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt,
298 traceId: isLatestFailure ? traceId : existing!.traceId,
299 spanId: isLatestFailure ? spanId : existing!.spanId,
300 });
301 }
302
303 if (Array.isArray(parsed.events)) {
304 for (const rawEvent of parsed.events) {
305 if (!isTraceEvent(rawEvent)) continue;
306 const attributes = readEventAttributes(rawEvent);
307 const level = toStringValue(attributes["effect.logLevel"]);
308 if (!level) continue;
309
310 logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1;
311 const normalizedLevel = level.toLowerCase();
312 if (
313 normalizedLevel !== "warning" &&
314 normalizedLevel !== "warn" &&
315 normalizedLevel !== "error" &&
316 normalizedLevel !== "fatal"
317 ) {
318 continue;
319 }
320
321 const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt;
322 const message = toStringValue(rawEvent.name)?.trim() ?? "Log event";
323 latestWarningAndErrorLogs.push({
324 spanName: name,
325 level,
326 message,
327 seenAt,
328 traceId,
329 spanId,
330 });
331 }
332 }
333 }
334 }
335
336 const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()]
337 .map(([name, span]) => ({
338 name,
339 count: span.count,
340 failureCount: span.failureCount,
341 totalDurationMs: span.totalDurationMs,
342 averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0,
343 maxDurationMs: span.maxDurationMs,
344 }))
345 .toSorted((left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs)
346 .slice(0, TOP_LIMIT);
347
348 return {
349 traceFilePath: input.traceFilePath,
350 scannedFilePaths,
351 readAt,
352 recordCount,
353 parseErrorCount,
354 firstSpanAt: Option.fromNullishOr(firstSpanAt),
355 lastSpanAt: Option.fromNullishOr(lastSpanAt),
356 failureCount,
357 interruptionCount,
358 slowSpanThresholdMs,
359 slowSpanCount,
360 logLevelCounts,
361 topSpansByCount,
362 slowestSpans,
363 commonFailures: [...failuresByKey.values()]
364 .toSorted(
365 (left, right) =>
366 right.count - left.count ||
367 DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt),
368 )
369 .slice(0, TOP_LIMIT),
370 latestFailures: latestFailures
371 .toSorted(
372 (left, right) =>
373 DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt),
374 )
375 .slice(0, RECENT_LIMIT),
376 latestWarningAndErrorLogs: latestWarningAndErrorLogs
377 .toSorted(
378 (left, right) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt),
379 )
380 .slice(0, RECENT_LIMIT),
381 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
382 error: Option.fromNullishOr(input.error),
383 };
384}
385
386type TraceFileReadResult =
387 | { readonly _tag: "Loaded"; readonly path: string; readonly text: string }
388 | { readonly _tag: "Missing"; readonly path: string }
389 | { readonly _tag: "Failed"; readonly path: string; readonly message: string };
390
391function readTraceFile(
392 fileSystem: FileSystem.FileSystem,
393 path: string,
394): Effect.Effect<TraceFileReadResult> {
395 return fileSystem.readFileString(path).pipe(
396 Effect.map((text) => ({ _tag: "Loaded" as const, path, text })),
397 Effect.catch((error: PlatformError.PlatformError) =>
398 Effect.succeed(
399 isNotFoundError(error)
400 ? { _tag: "Missing" as const, path }
401 : { _tag: "Failed" as const, path, message: platformErrorMessage(error) },
402 ),
403 ),
404 );
405}
406
407export const make = Effect.fn("makeTraceDiagnostics")(function* () {
408 const fileSystem = yield* FileSystem.FileSystem;
409
410 const read: TraceDiagnosticsShape["read"] = Effect.fn("TraceDiagnostics.read")(
411 function* (options) {
412 const readAt = options.readAt ?? (yield* DateTime.now);
413 const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
414 const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles);
415 const results = yield* Effect.all(
416 paths.map((path) => readTraceFile(fileSystem, path)),
417 {
418 concurrency: 1,
419 },
420 );
421 const files = results.flatMap((result) =>
422 result._tag === "Loaded" ? [{ path: result.path, text: result.text }] : [],
423 );
424 const readFailure = results.find((result) => result._tag === "Failed");
425 const readFailureError = readFailure
426 ? ({
427 kind: "trace-file-read-failed",
428 message: readFailure.message.trim() || `Failed to read ${readFailure.path}.`,
429 } satisfies TraceDiagnosticsErrorSummary)
430 : undefined;
431
432 if (files.length === 0) {
433 return makeEmptyDiagnostics({
434 traceFilePath: options.traceFilePath,
435 scannedFilePaths: paths,
436 readAt,
437 slowSpanThresholdMs,
438 error:
439 readFailureError ??
440 ({
441 kind: "trace-file-not-found",
442 message: "No local trace files were found.",
443 } satisfies TraceDiagnosticsErrorSummary),
444 });
445 }
446
447 return aggregateTraceDiagnostics({
448 traceFilePath: options.traceFilePath,
449 files,
450 scannedFilePaths: paths,
451 readAt,
452 slowSpanThresholdMs,
453 ...(readFailureError ? { partialFailure: true, error: readFailureError } : {}),
454 });
455 },
456 );
457
458 return TraceDiagnostics.of({ read });
459});
460
461export const layer = Layer.effect(TraceDiagnostics, make());
462
463export function readTraceDiagnostics(
464 options: TraceDiagnosticsOptions,
465): Effect.Effect<ServerTraceDiagnosticsResult, never, TraceDiagnostics> {
466 return Effect.gen(function* () {
467 const diagnostics = yield* TraceDiagnostics;
468 return yield* diagnostics.read(options);
469 });
470}
apps/server/src/observability/Metrics.test.ts+45-1
......@@ -1,6 +1,7 @@
11import { assert, describe, it } from "@effect/vitest";
22import { ProviderDriverKind } from "@t3tools/contracts";
3import { Effect, Metric } from "effect";
3import { Duration, Effect, Fiber, Metric } from "effect";
4import { TestClock } from "effect/testing";
45
56import { withMetrics } from "./Metrics.ts";
67
......@@ -15,6 +16,18 @@ const hasMetricSnapshot = (
1516 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
1617 );
1718
19const findHistogramSnapshot = (
20 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
21 id: string,
22 attributes: Readonly<Record<string, string>>,
23) =>
24 snapshots.find(
25 (snapshot): snapshot is Extract<Metric.Metric.Snapshot, { readonly type: "Histogram" }> =>
26 snapshot.type === "Histogram" &&
27 snapshot.id === id &&
28 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
29 );
30
1831describe("withMetrics", () => {
1932 it.effect("supports pipe-style usage", () =>
2033 Effect.gen(function* () {
......@@ -110,4 +123,35 @@ describe("withMetrics", () => {
110123 );
111124 }),
112125 );
126
127 it.effect("records timer durations from nanosecond clock readings", () =>
128 Effect.gen(function* () {
129 const duration = Duration.nanos(1_500_000n);
130 const timer = Metric.timer("with_metrics_nanos_duration");
131
132 yield* Effect.gen(function* () {
133 const fiber = yield* Effect.sleep(duration).pipe(
134 withMetrics({
135 timer,
136 attributes: {
137 operation: "nanos",
138 },
139 }),
140 Effect.forkChild,
141 );
142
143 yield* Effect.yieldNow;
144 yield* TestClock.adjust(duration);
145 yield* Fiber.join(fiber);
146 }).pipe(Effect.provide(TestClock.layer()));
147
148 const snapshots = yield* Metric.snapshot;
149 const snapshot = findHistogramSnapshot(snapshots, "with_metrics_nanos_duration", {
150 operation: "nanos",
151 });
152
153 assert.equal(snapshot?.state.count, 1);
154 assert.equal(snapshot?.state.sum, 1.5);
155 }),
156 );
113157});
apps/server/src/observability/Metrics.ts+5-3
......@@ -1,4 +1,4 @@
1import { Duration, Effect, Exit, Metric } from "effect";
1import { Clock, Duration, Effect, Exit, Metric } from "effect";
22import { dual } from "effect/Function";
33
44import {
......@@ -96,9 +96,11 @@ const withMetricsImpl = <A, E, R>(
9696 options: WithMetricsOptions,
9797): Effect.Effect<A, E, R> =>
9898 Effect.gen(function* () {
99 const startedAt = Date.now();
99 const startedAt = yield* Clock.currentTimeNanos;
100100 const exit = yield* Effect.exit(effect);
101 const duration = Duration.millis(Math.max(0, Date.now() - startedAt));
101 const endedAt = yield* Clock.currentTimeNanos;
102 const elapsedNanos = endedAt > startedAt ? endedAt - startedAt : 0n;
103 const duration = Duration.nanos(elapsedNanos);
102104 const baseAttributes =
103105 typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
104106
apps/server/src/observability/RpcInstrumentation.test.ts+147-1
......@@ -1,5 +1,7 @@
11import { assert, describe, it } from "@effect/vitest";
2import { Effect, Exit, Metric, Stream } from "effect";
2import { WS_METHODS } from "@t3tools/contracts";
3import { Duration, Effect, Exit, Fiber, Metric, Stream, Tracer } from "effect";
4import { TestClock } from "effect/testing";
35
46import {
57 observeRpcEffect,
......@@ -18,6 +20,44 @@ const hasMetricSnapshot = (
1820 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
1921 );
2022
23const findHistogramSnapshot = (
24 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
25 id: string,
26 attributes: Readonly<Record<string, string>>,
27) =>
28 snapshots.find(
29 (snapshot): snapshot is Extract<Metric.Metric.Snapshot, { readonly type: "Histogram" }> =>
30 snapshot.type === "Histogram" &&
31 snapshot.id === id &&
32 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
33 );
34
35const collectSpanNames = <A, E, R>(
36 effect: Effect.Effect<A, E, R>,
37): Effect.Effect<ReadonlyArray<string>, E, R> =>
38 Effect.gen(function* () {
39 const spanNames: Array<string> = [];
40 const tracer = Tracer.make({
41 span: (options) => {
42 const span = new Tracer.NativeSpan(options);
43 const end = span.end.bind(span);
44
45 span.end = (endTime, exit) => {
46 end(endTime, exit);
47 if (span.sampled) {
48 spanNames.push(span.name);
49 }
50 };
51
52 return span;
53 },
54 });
55
56 yield* effect.pipe(Effect.withTracer(tracer));
57
58 return spanNames;
59 });
60
2161describe("RpcInstrumentation", () => {
2262 it.effect("records success metrics for unary RPC handlers", () =>
2363 Effect.gen(function* () {
......@@ -129,6 +169,37 @@ describe("RpcInstrumentation", () => {
129169 }),
130170 );
131171
172 it.effect("records direct stream durations from nanosecond clock readings", () =>
173 Effect.gen(function* () {
174 const duration = Duration.nanos(1_500_000n);
175 const events = yield* Effect.gen(function* () {
176 const fiber = yield* Stream.runCollect(
177 observeRpcStream(
178 WS_METHODS.serverGetProcessDiagnostics,
179 Stream.fromEffect(Effect.sleep(duration).pipe(Effect.as("ok"))),
180 {
181 "rpc.aggregate": "test",
182 },
183 ),
184 ).pipe(Effect.forkChild);
185
186 yield* Effect.yieldNow;
187 yield* TestClock.adjust(duration);
188 return yield* Fiber.join(fiber);
189 }).pipe(Effect.provide(TestClock.layer()));
190
191 assert.deepStrictEqual(Array.from(events), ["ok"]);
192
193 const snapshots = yield* Metric.snapshot;
194 const snapshot = findHistogramSnapshot(snapshots, "t3_rpc_request_duration", {
195 method: WS_METHODS.serverGetProcessDiagnostics,
196 });
197
198 assert.equal(snapshot?.state.count, 1);
199 assert.equal(snapshot?.state.sum, 1.5);
200 }),
201 );
202
132203 it.effect("records failure outcomes when a stream RPC effect produces a failing stream", () =>
133204 Effect.gen(function* () {
134205 const exit = yield* Stream.runCollect(
......@@ -158,4 +229,79 @@ describe("RpcInstrumentation", () => {
158229 );
159230 }),
160231 );
232
233 it.effect("records spans for traced stream RPC handlers", () =>
234 Effect.gen(function* () {
235 const spanNames = yield* collectSpanNames(
236 Stream.runCollect(
237 observeRpcStream(
238 "rpc.instrumentation.traced.stream",
239 Stream.fromEffect(
240 Effect.succeed("ok").pipe(Effect.withSpan("rpc.instrumentation.traced.stream.child")),
241 ),
242 { "rpc.aggregate": "test" },
243 ),
244 ),
245 );
246
247 assert.equal(spanNames.includes("ws.rpc.rpc.instrumentation.traced.stream"), true);
248 assert.equal(spanNames.includes("rpc.instrumentation.traced.stream.child"), true);
249 }),
250 );
251
252 it.effect("does not create spans for disabled unary RPC handlers", () =>
253 Effect.gen(function* () {
254 const spanNames = yield* collectSpanNames(
255 observeRpcEffect(
256 WS_METHODS.serverGetTraceDiagnostics,
257 Effect.succeed("ok").pipe(Effect.withSpan("rpc.instrumentation.disabled.unary.child")),
258 { "rpc.aggregate": "test" },
259 ),
260 );
261
262 assert.deepStrictEqual(spanNames, []);
263 }),
264 );
265
266 it.effect("does not create spans for disabled direct stream RPC handlers", () =>
267 Effect.gen(function* () {
268 const spanNames = yield* collectSpanNames(
269 Stream.runCollect(
270 observeRpcStream(
271 WS_METHODS.serverGetTraceDiagnostics,
272 Stream.fromEffect(
273 Effect.succeed("ok").pipe(
274 Effect.withSpan("rpc.instrumentation.disabled.stream.child"),
275 ),
276 ),
277 { "rpc.aggregate": "test" },
278 ),
279 ),
280 );
281
282 assert.deepStrictEqual(spanNames, []);
283 }),
284 );
285
286 it.effect("does not create spans for disabled stream effect RPC handlers", () =>
287 Effect.gen(function* () {
288 const spanNames = yield* collectSpanNames(
289 Stream.runCollect(
290 observeRpcStreamEffect(
291 WS_METHODS.serverGetTraceDiagnostics,
292 Effect.succeed(
293 Stream.fromEffect(
294 Effect.succeed("ok").pipe(
295 Effect.withSpan("rpc.instrumentation.disabled.stream.effect.consume"),
296 ),
297 ),
298 ).pipe(Effect.withSpan("rpc.instrumentation.disabled.stream.effect.create")),
299 { "rpc.aggregate": "test" },
300 ),
301 ),
302 );
303
304 assert.deepStrictEqual(spanNames, []);
305 }),
306 );
161307});
apps/server/src/observability/RpcInstrumentation.ts+78-30
......@@ -1,26 +1,71 @@
1import { Duration, Effect, Exit, Metric, Stream } from "effect";
1import { WS_METHODS } from "@t3tools/contracts";
2import { Clock, Duration, Effect, Exit, Metric, References, Stream } from "effect";
23
34import { outcomeFromExit } from "./Attributes.ts";
45import { metricAttributes, rpcRequestDuration, rpcRequestsTotal, withMetrics } from "./Metrics.ts";
56
6const annotateRpcSpan = (
7const RPC_SPAN_PREFIX = "ws.rpc";
8const DEFAULT_RPC_SPAN_ATTRIBUTES = {
9 "rpc.transport": "websocket",
10 "rpc.system": "effect-rpc",
11} as const;
12const RPC_METHODS_WITH_TRACING_DISABLED: ReadonlySet<string> = new Set([
13 WS_METHODS.serverGetTraceDiagnostics,
14 WS_METHODS.serverGetProcessDiagnostics,
15 WS_METHODS.serverSignalProcess,
16]);
17
18function shouldTraceRpc(method: string): boolean {
19 return !RPC_METHODS_WITH_TRACING_DISABLED.has(method);
20}
21
22const rpcSpanAttributes = (
723 method: string,
824 traceAttributes?: Readonly<Record<string, unknown>>,
9): Effect.Effect<void, never, never> =>
10 Effect.annotateCurrentSpan({
11 "rpc.method": method,
12 ...traceAttributes,
13 });
25): Record<string, unknown> => ({
26 ...DEFAULT_RPC_SPAN_ATTRIBUTES,
27 "rpc.method": method,
28 ...traceAttributes,
29});
30
31const withRpcEffectTracing = <A, E, R>(
32 method: string,
33 effect: Effect.Effect<A, E, R>,
34 traceAttributes?: Readonly<Record<string, unknown>>,
35): Effect.Effect<A, E, R> =>
36 shouldTraceRpc(method)
37 ? effect.pipe(
38 Effect.withSpan(`${RPC_SPAN_PREFIX}.${method}`, {
39 attributes: rpcSpanAttributes(method, traceAttributes),
40 }),
41 )
42 : effect.pipe(Effect.provideService(References.TracerEnabled, false));
43
44const withRpcStreamTracing = <A, E, R>(
45 method: string,
46 stream: Stream.Stream<A, E, R>,
47 traceAttributes?: Readonly<Record<string, unknown>>,
48): Stream.Stream<A, E, R> =>
49 shouldTraceRpc(method)
50 ? stream.pipe(
51 Stream.withSpan(`${RPC_SPAN_PREFIX}.${method}`, {
52 attributes: rpcSpanAttributes(method, traceAttributes),
53 }),
54 )
55 : stream.pipe(Stream.provideService(References.TracerEnabled, false));
1456
1557const recordRpcStreamMetrics = <E>(
1658 method: string,
17 startedAt: number,
59 startedAt: bigint,
1860 exit: Exit.Exit<unknown, E>,
1961): Effect.Effect<void, never, never> =>
2062 Effect.gen(function* () {
63 const endedAt = yield* Clock.currentTimeNanos;
64 const elapsedNanos = endedAt > startedAt ? endedAt - startedAt : 0n;
65
2166 yield* Metric.update(
2267 Metric.withAttributes(rpcRequestDuration, metricAttributes({ method })),
23 Duration.millis(Math.max(0, Date.now() - startedAt)),
68 Duration.nanos(elapsedNanos),
2469 );
2570 yield* Metric.update(
2671 Metric.withAttributes(
......@@ -38,43 +83,43 @@ export const observeRpcEffect = <A, E, R>(
3883 method: string,
3984 effect: Effect.Effect<A, E, R>,
4085 traceAttributes?: Readonly<Record<string, unknown>>,
41): Effect.Effect<A, E, R> =>
42 Effect.gen(function* () {
43 yield* annotateRpcSpan(method, traceAttributes);
86): Effect.Effect<A, E, R> => {
87 const instrumented = effect.pipe(
88 withMetrics({
89 counter: rpcRequestsTotal,
90 timer: rpcRequestDuration,
91 attributes: {
92 method,
93 },
94 }),
95 );
4496
45 return yield* effect.pipe(
46 withMetrics({
47 counter: rpcRequestsTotal,
48 timer: rpcRequestDuration,
49 attributes: {
50 method,
51 },
52 }),
53 );
54 });
97 return withRpcEffectTracing(method, instrumented, traceAttributes);
98};
5599
56100export const observeRpcStream = <A, E, R>(
57101 method: string,
58102 stream: Stream.Stream<A, E, R>,
59103 traceAttributes?: Readonly<Record<string, unknown>>,
60): Stream.Stream<A, E, R> =>
61 Stream.unwrap(
104): Stream.Stream<A, E, R> => {
105 const instrumented = Stream.unwrap(
62106 Effect.gen(function* () {
63 yield* annotateRpcSpan(method, traceAttributes);
64 const startedAt = Date.now();
107 const startedAt = yield* Clock.currentTimeNanos;
65108 return stream.pipe(Stream.onExit((exit) => recordRpcStreamMetrics(method, startedAt, exit)));
66109 }),
67110 );
68111
112 return withRpcStreamTracing(method, instrumented, traceAttributes);
113};
114
69115export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectError, EffectContext>(
70116 method: string,
71117 effect: Effect.Effect<Stream.Stream<A, StreamError, StreamContext>, EffectError, EffectContext>,
72118 traceAttributes?: Readonly<Record<string, unknown>>,
73): Stream.Stream<A, StreamError | EffectError, StreamContext | EffectContext> =>
74 Stream.unwrap(
119): Stream.Stream<A, StreamError | EffectError, StreamContext | EffectContext> => {
120 const instrumented = Stream.unwrap(
75121 Effect.gen(function* () {
76 yield* annotateRpcSpan(method, traceAttributes);
77 const startedAt = Date.now();
122 const startedAt = yield* Clock.currentTimeNanos;
78123 const exit = yield* Effect.exit(effect);
79124
80125 if (Exit.isFailure(exit)) {
......@@ -87,3 +132,6 @@ export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectErro
87132 );
88133 }),
89134 );
135
136 return withRpcStreamTracing(method, instrumented, traceAttributes);
137};
apps/server/src/server.test.ts+48
......@@ -114,6 +114,8 @@ import * as GitWorkflowService from "./git/GitWorkflowService.ts";
114114import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";
115115import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";
116116import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";
117import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
118import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
117119
118120const defaultProjectId = ProjectId.make("project-default");
119121const defaultThreadId = ThreadId.make("thread-default");
......@@ -543,6 +545,52 @@ const buildAppUnderTest = (options?: {
543545 ...options?.layers?.open,
544546 }),
545547 ),
548 Layer.provide(
549 Layer.mock(ProcessDiagnostics.ProcessDiagnostics)({
550 read: Effect.succeed({
551 serverPid: process.pid,
552 readAt: TEST_EPOCH,
553 processCount: 0,
554 totalRssBytes: 0,
555 totalCpuPercent: 0,
556 processes: [],
557 error: Option.none(),
558 }),
559 signal: (input) =>
560 Effect.succeed({
561 pid: input.pid,
562 signal: input.signal,
563 signaled: true,
564 message: Option.none(),
565 }),
566 }),
567 ),
568 Layer.provide(
569 Layer.mock(TraceDiagnostics.TraceDiagnostics)({
570 read: () =>
571 Effect.succeed({
572 traceFilePath: "",
573 scannedFilePaths: [],
574 readAt: TEST_EPOCH,
575 recordCount: 0,
576 parseErrorCount: 0,
577 firstSpanAt: Option.none(),
578 lastSpanAt: Option.none(),
579 failureCount: 0,
580 interruptionCount: 0,
581 slowSpanThresholdMs: 1_000,
582 slowSpanCount: 0,
583 logLevelCounts: {},
584 topSpansByCount: [],
585 slowestSpans: [],
586 commonFailures: [],
587 latestFailures: [],
588 latestWarningAndErrorLogs: [],
589 partialFailure: Option.none(),
590 error: Option.none(),
591 }),
592 }),
593 ),
546594 Layer.provide(gitManagerLayer),
547595 Layer.provide(gitVcsDriverLayer),
548596 Layer.provide(gitWorkflowLayer),
apps/server/src/server.ts+4
......@@ -74,6 +74,8 @@ import {
7474} from "./auth/http.ts";
7575import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";
7676import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";
77import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
78import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
7779import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts";
7880import {
7981 clearPersistedServerRuntimeState,
......@@ -276,6 +278,8 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(
276278
277279const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe(
278280 // Misc.
281 Layer.provideMerge(ProcessDiagnostics.layer),
282 Layer.provideMerge(TraceDiagnostics.layer),
279283 Layer.provideMerge(AnalyticsServiceLayerLive),
280284 Layer.provideMerge(OpenLive),
281285 Layer.provideMerge(ServerLifecycleEventsLive),
apps/server/src/vcs/GitVcsDriverCore.ts+9-15
......@@ -528,7 +528,7 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
528528 };
529529});
530530
531const collectOutput = Effect.fn("collectOutput")(function* <E>(
531const collectOutput = Effect.fnUntraced(function* <E>(
532532 input: Pick<GitVcsDriver.ExecuteGitInput, "operation" | "cwd" | "args">,
533533 stream: Stream.Stream<Uint8Array, E>,
534534 maxOutputBytes: number,
......@@ -541,7 +541,7 @@ const collectOutput = Effect.fn("collectOutput")(function* <E>(
541541 let lineBuffer = "";
542542 let truncated = false;
543543
544 const emitCompleteLines = Effect.fn("emitCompleteLines")(function* (flush: boolean) {
544 const emitCompleteLines = Effect.fnUntraced(function* (flush: boolean) {
545545 let newlineIndex = lineBuffer.indexOf("\n");
546546 while (newlineIndex >= 0) {
547547 const line = lineBuffer.slice(0, newlineIndex).replace(/\r$/, "");
......@@ -561,7 +561,7 @@ const collectOutput = Effect.fn("collectOutput")(function* <E>(
561561 }
562562 });
563563
564 const processChunk = Effect.fn("processChunk")(function* (chunk: Uint8Array) {
564 const processChunk = Effect.fnUntraced(function* (chunk: Uint8Array) {
565565 if (truncateOutputAtMaxBytes && truncated) {
566566 return;
567567 }
......@@ -602,20 +602,14 @@ const collectOutput = Effect.fn("collectOutput")(function* <E>(
602602 };
603603});
604604
605export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* (options?: {
606 executeOverride?: GitVcsDriver.GitVcsDriverShape["execute"];
607}) {
605export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* () {
608606 const fileSystem = yield* FileSystem.FileSystem;
609607 const path = yield* Path.Path;
608 const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
610609 const { worktreesDir } = yield* ServerConfig;
611610
612 let executeRaw: GitVcsDriver.GitVcsDriverShape["execute"];
613
614 if (options?.executeOverride) {
615 executeRaw = options.executeOverride;
616 } else {
617 const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
618 executeRaw = Effect.fnUntraced(function* (input) {
611 const executeRaw: GitVcsDriver.GitVcsDriverShape["execute"] = Effect.fnUntraced(
612 function* (input) {
619613 const commandInput = {
620614 ...input,
621615 args: [...input.args],
......@@ -712,8 +706,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
712706 }),
713707 ),
714708 );
715 });
716 }
709 },
710 );
717711
718712 const execute: GitVcsDriver.GitVcsDriverShape["execute"] = (input) =>
719713 executeRaw(input).pipe(
apps/server/src/ws.ts+23-5
......@@ -55,6 +55,8 @@ import { ProjectSetupScriptRunner } from "./project/Services/ProjectSetupScriptR
5555import { RepositoryIdentityResolver } from "./project/Services/RepositoryIdentityResolver.ts";
5656import { ServerEnvironment } from "./environment/Services/ServerEnvironment.ts";
5757import { ServerAuth } from "./auth/Services/ServerAuth.ts";
58import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
59import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
5860import * as SourceControlDiscoveryLayer from "./sourceControl/SourceControlDiscovery.ts";
5961import { SourceControlRepositoryService } from "./sourceControl/SourceControlRepositoryService.ts";
6062import * as AzureDevOpsCli from "./sourceControl/AzureDevOpsCli.ts";
......@@ -168,6 +170,7 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>
168170 const sourceControlRepositories = yield* SourceControlRepositoryService;
169171 const bootstrapCredentials = yield* BootstrapCredentialService;
170172 const sessions = yield* SessionCredentialService;
173 const processDiagnostics = yield* ProcessDiagnostics.ProcessDiagnostics;
171174 const serverCommandId = (tag: string) =>
172175 CommandId.make(`server:${tag}:${crypto.randomUUID()}`);
173176
......@@ -837,6 +840,25 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>
837840 "rpc.aggregate": "server",
838841 },
839842 ),
843 [WS_METHODS.serverGetTraceDiagnostics]: (_input) =>
844 observeRpcEffect(
845 WS_METHODS.serverGetTraceDiagnostics,
846 TraceDiagnostics.readTraceDiagnostics({
847 traceFilePath: config.serverTracePath,
848 maxFiles: config.traceMaxFiles,
849 }),
850 {
851 "rpc.aggregate": "server",
852 },
853 ),
854 [WS_METHODS.serverGetProcessDiagnostics]: (_input) =>
855 observeRpcEffect(WS_METHODS.serverGetProcessDiagnostics, processDiagnostics.read, {
856 "rpc.aggregate": "server",
857 }),
858 [WS_METHODS.serverSignalProcess]: (input) =>
859 observeRpcEffect(WS_METHODS.serverSignalProcess, processDiagnostics.signal(input), {
860 "rpc.aggregate": "server",
861 }),
840862 [WS_METHODS.sourceControlLookupRepository]: (input) =>
841863 observeRpcEffect(
842864 WS_METHODS.sourceControlLookupRepository,
......@@ -1162,11 +1184,7 @@ export const websocketRpcRouteLayer = Layer.unwrap(
11621184 const sessions = yield* SessionCredentialService;
11631185 const session = yield* serverAuth.authenticateWebSocketUpgrade(request);
11641186 const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, {
1165 spanPrefix: "ws.rpc",
1166 spanAttributes: {
1167 "rpc.transport": "websocket",
1168 "rpc.system": "effect-rpc",
1169 },
1187 disableTracing: true,
11701188 }).pipe(
11711189 Effect.provide(
11721190 makeWsRpcLayer(session.sessionId).pipe(
apps/web/src/components/settings/DiagnosticsSettings.tsx created+982
......@@ -0,0 +1,982 @@
1import {
2 AlertTriangleIcon,
3 ChevronDownIcon,
4 ChevronRightIcon,
5 CopyIcon,
6 FolderOpenIcon,
7 InfoIcon,
8 RefreshCwIcon,
9} from "lucide-react";
10import { useCallback, useMemo, useState, type ReactNode } from "react";
11import type { ServerProcessDiagnosticsEntry, ServerProcessSignal } from "@t3tools/contracts";
12import { DateTime, Option } from "effect";
13
14import { ensureLocalApi } from "../../localApi";
15import { cn } from "../../lib/utils";
16import { resolveAndPersistPreferredEditor } from "../../editorPreferences";
17import { formatRelativeTime } from "../../timestampFormat";
18import { useServerAvailableEditors, useServerObservability } from "../../rpc/serverState";
19import { useProcessDiagnostics } from "../../lib/processDiagnosticsState";
20import { useTraceDiagnostics } from "../../lib/traceDiagnosticsState";
21import { Button } from "../ui/button";
22import { ScrollArea } from "../ui/scroll-area";
23import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
24import { toastManager } from "../ui/toast";
25import { SettingsPageContainer, SettingsSection, useRelativeTimeTick } from "./settingsLayout";
26
27const NUMBER_FORMAT = new Intl.NumberFormat();
28
29function formatCount(value: number): string {
30 return NUMBER_FORMAT.format(value);
31}
32
33function formatDuration(value: number): string {
34 if (value < 1_000) return `${Math.round(value)} ms`;
35 return `${(value / 1_000).toFixed(value >= 10_000 ? 1 : 2)} s`;
36}
37
38function formatBytes(value: number): string {
39 if (value < 1024) return `${value} B`;
40 const units = ["KB", "MB", "GB"] as const;
41 let unitIndex = -1;
42 let next = value;
43 do {
44 next /= 1024;
45 unitIndex += 1;
46 } while (next >= 1024 && unitIndex < units.length - 1);
47 return `${next.toFixed(next >= 10 ? 1 : 2)} ${units[unitIndex]}`;
48}
49
50function formatRelative(value: DateTime.Utc | null): string {
51 if (!value) return "No trace records";
52 const relative = formatRelativeTime(DateTime.formatIso(value));
53 return relative.suffix ? `${relative.value} ${relative.suffix}` : relative.value;
54}
55
56function formatRelativeNoWrap(value: DateTime.Utc | null): string {
57 return formatRelative(value).replaceAll(" ", "\u00a0");
58}
59
60function shortenTraceId(traceId: string): string {
61 if (traceId.length <= 32) return traceId;
62 return `${traceId.slice(0, 18)}...${traceId.slice(-10)}`;
63}
64
65function isStaleProcessSignalMessage(message: string | undefined): boolean {
66 return message?.includes("not a live descendant") ?? false;
67}
68
69function StatBlock({
70 label,
71 value,
72 tooltip,
73 tone = "default",
74}: {
75 label: string;
76 value: string;
77 tooltip?: ReactNode;
78 tone?: "default" | "warning" | "danger";
79}) {
80 return (
81 <div className="min-w-0 border-border/60 px-4 py-3 sm:px-5">
82 <div className="flex min-w-0 items-center gap-1.5 text-[11px] font-medium uppercase tracking-[0.08em] text-muted-foreground/70">
83 <span className="min-w-0 truncate">{label}</span>
84 {tooltip ? (
85 <Tooltip>
86 <TooltipTrigger
87 render={
88 <button
89 type="button"
90 className="inline-flex size-3.5 shrink-0 items-center justify-center rounded-sm text-muted-foreground/60 hover:text-foreground"
91 aria-label={`${label} details`}
92 >
93 <InfoIcon className="size-3" />
94 </button>
95 }
96 />
97 <TooltipPopup
98 side="top"
99 className="max-w-[min(300px,calc(100vw-2rem))] whitespace-normal text-left text-[11px] leading-relaxed text-wrap"
100 >
101 {tooltip}
102 </TooltipPopup>
103 </Tooltip>
104 ) : null}
105 </div>
106 <div
107 className={cn(
108 "mt-1 truncate font-mono text-lg font-semibold tabular-nums text-foreground",
109 tone === "warning" && "text-amber-600 dark:text-amber-400",
110 tone === "danger" && "text-destructive",
111 )}
112 >
113 {value}
114 </div>
115 </div>
116 );
117}
118
119function StatsGrid({ children }: { children: ReactNode }) {
120 return (
121 <div className="relative grid grid-cols-2 sm:grid-cols-4">
122 <span
123 className="pointer-events-none absolute inset-y-0 left-1/2 w-px bg-border/60"
124 aria-hidden
125 />
126 <span
127 className="pointer-events-none absolute inset-x-0 top-1/2 h-px bg-border/60 sm:hidden"
128 aria-hidden
129 />
130 <span
131 className="pointer-events-none absolute inset-y-0 left-1/4 hidden w-px bg-border/60 sm:block"
132 aria-hidden
133 />
134 <span
135 className="pointer-events-none absolute inset-y-0 left-3/4 hidden w-px bg-border/60 sm:block"
136 aria-hidden
137 />
138 {children}
139 </div>
140 );
141}
142
143function EmptyRows({ label }: { label: string }) {
144 return <div className="px-4 py-4 text-xs text-muted-foreground sm:px-5">{label}</div>;
145}
146
147function ExpandableText({
148 text,
149 className,
150 collapsedClassName = "line-clamp-3",
151 expandLabel = "Show full error",
152}: {
153 text: string;
154 className?: string;
155 collapsedClassName?: string;
156 expandLabel?: string;
157}) {
158 const [expanded, setExpanded] = useState(false);
159 const canExpand = text.length > 180 || text.includes("\n");
160
161 return (
162 <div className={cn("min-w-0", className)}>
163 <div
164 className={cn(
165 "whitespace-pre-wrap break-words",
166 !expanded && canExpand ? collapsedClassName : null,
167 )}
168 >
169 {text}
170 </div>
171 {canExpand ? (
172 <button
173 type="button"
174 className="mt-1 text-[11px] font-medium text-foreground/70 underline-offset-2 hover:text-foreground hover:underline"
175 onClick={() => setExpanded((value) => !value)}
176 >
177 {expanded ? "Show less" : expandLabel}
178 </button>
179 ) : null}
180 </div>
181 );
182}
183
184function DiagnosticsTable({
185 headers,
186 children,
187 minTableWidth = "min-w-[640px]",
188 columnWidths,
189}: {
190 headers: ReadonlyArray<string>;
191 children: ReactNode;
192 minTableWidth?: string;
193 columnWidths?: ReadonlyArray<string>;
194}) {
195 return (
196 <ScrollArea
197 chainVerticalScroll
198 scrollFade
199 hideScrollbars
200 className="w-full max-w-full rounded-none"
201 >
202 <table
203 className={cn("w-full text-left text-xs", minTableWidth, columnWidths && "table-fixed")}
204 >
205 {columnWidths ? (
206 <colgroup>
207 {headers.map((header, index) => (
208 <col key={header} className={columnWidths[index]} />
209 ))}
210 </colgroup>
211 ) : null}
212 <thead className="border-b border-border/60 text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">
213 <tr>
214 {headers.map((header, index) => (
215 <th
216 key={header}
217 className={cn(
218 "whitespace-nowrap px-4 py-2.5 font-semibold first:sm:pl-5 last:sm:pr-5",
219 !columnWidths && index === headers.length - 1 && "w-px",
220 )}
221 >
222 {header.replaceAll(" ", "\u00a0")}
223 </th>
224 ))}
225 </tr>
226 </thead>
227 <tbody className="divide-y divide-border/60">{children}</tbody>
228 </table>
229 </ScrollArea>
230 );
231}
232
233function TraceIdCell({ traceId }: { traceId: string }) {
234 const [copied, setCopied] = useState(false);
235 const copyTraceId = useCallback(() => {
236 void navigator.clipboard
237 ?.writeText(traceId)
238 .then(() => {
239 setCopied(true);
240 window.setTimeout(() => setCopied(false), 1_200);
241 })
242 .catch(() => undefined);
243 }, [traceId]);
244
245 return (
246 <div className="flex w-full min-w-0 max-w-full items-center gap-2">
247 <Tooltip>
248 <TooltipTrigger
249 render={
250 <span className="min-w-0 flex-1 truncate font-mono text-[11px]">
251 {shortenTraceId(traceId)}
252 </span>
253 }
254 />
255 <TooltipPopup
256 side="top"
257 className="max-w-[min(520px,calc(100vw-2rem))] break-all font-mono text-[11px]"
258 >
259 {traceId}
260 </TooltipPopup>
261 </Tooltip>
262 <Tooltip>
263 <TooltipTrigger
264 render={
265 <button
266 type="button"
267 className="inline-flex size-5 shrink-0 items-center justify-center rounded-sm text-muted-foreground hover:bg-accent hover:text-foreground"
268 aria-label={copied ? "Copied trace ID" : "Copy trace ID"}
269 onClick={copyTraceId}
270 >
271 <CopyIcon className="size-3" />
272 </button>
273 }
274 />
275 <TooltipPopup side="top">{copied ? "Copied" : "Copy full trace ID"}</TooltipPopup>
276 </Tooltip>
277 </div>
278 );
279}
280
281function formatProcessName(command: string): string {
282 const firstToken = command.trim().split(/\s+/)[0];
283 if (!firstToken) return command;
284 const normalized = firstToken.replace(/^['"]|['"]$/g, "");
285 const segments = normalized.split(/[\\/]/).filter(Boolean);
286 return segments.at(-1) ?? normalized;
287}
288
289function formatProcessType(process: ServerProcessDiagnosticsEntry): string {
290 if (process.depth > 0) return "Subprocess";
291 if (/\b(codex|claude|opencode|cursor)\b/i.test(process.command)) return "Agent";
292 return "Process";
293}
294
295function ProcessNameCell({
296 process,
297 isExpanded,
298 onToggle,
299}: {
300 process: ServerProcessDiagnosticsEntry;
301 isExpanded: boolean;
302 onToggle: (pid: number) => void;
303}) {
304 const name = formatProcessName(process.command);
305 const hasChildren = process.childPids.length > 0;
306 const ChevronIcon = isExpanded ? ChevronDownIcon : ChevronRightIcon;
307
308 return (
309 <div
310 className="grid min-w-0 grid-cols-[1.25rem_0.375rem_minmax(0,1fr)] items-center gap-2"
311 style={{ paddingLeft: `${Math.min(process.depth, 8) * 16}px` }}
312 >
313 {hasChildren ? (
314 <button
315 type="button"
316 className="inline-flex size-5 items-center justify-center rounded-sm text-muted-foreground hover:bg-accent hover:text-foreground"
317 aria-label={isExpanded ? `Collapse ${name}` : `Expand ${name}`}
318 onClick={() => onToggle(process.pid)}
319 >
320 <ChevronIcon className="size-3.5" />
321 </button>
322 ) : (
323 <span className="size-5 shrink-0" aria-hidden="true" />
324 )}
325 <span className="size-1.5 shrink-0 rounded-full bg-emerald-500/80" />
326 <span className="min-w-0 truncate font-medium text-foreground">{name}</span>
327 </div>
328 );
329}
330
331function ProcessSignalActions({
332 process,
333 isSignaling,
334 onSignal,
335}: {
336 process: ServerProcessDiagnosticsEntry;
337 isSignaling: boolean;
338 onSignal: (pid: number, signal: ServerProcessSignal) => void;
339}) {
340 return (
341 <div className="flex items-center justify-end gap-1.5">
342 <Tooltip>
343 <TooltipTrigger
344 render={
345 <button
346 type="button"
347 disabled={isSignaling}
348 className="text-[11px] font-medium text-muted-foreground underline-offset-2 hover:text-foreground hover:underline disabled:pointer-events-none disabled:opacity-50"
349 onClick={() => onSignal(process.pid, "SIGINT")}
350 >
351 INT
352 </button>
353 }
354 />
355 <TooltipPopup side="top">Send SIGINT</TooltipPopup>
356 </Tooltip>
357 <Tooltip>
358 <TooltipTrigger
359 render={
360 <button
361 type="button"
362 disabled={isSignaling}
363 className="text-[11px] font-medium text-destructive underline-offset-2 hover:underline disabled:pointer-events-none disabled:opacity-50"
364 onClick={() => onSignal(process.pid, "SIGKILL")}
365 >
366 KILL
367 </button>
368 }
369 />
370 <TooltipPopup side="top">Send SIGKILL</TooltipPopup>
371 </Tooltip>
372 </div>
373 );
374}
375
376function ProcessDiagnosticsTable({
377 processes,
378 signalingPid,
379 onSignal,
380 emptyLabel,
381}: {
382 processes: ReadonlyArray<ServerProcessDiagnosticsEntry>;
383 signalingPid: number | null;
384 onSignal: (pid: number, signal: ServerProcessSignal) => void;
385 emptyLabel?: string;
386}) {
387 const [collapsedPids, setCollapsedPids] = useState<ReadonlySet<number>>(() => new Set());
388 const visibleProcesses = useMemo(() => {
389 const visible: ServerProcessDiagnosticsEntry[] = [];
390 let hiddenChildDepth: number | null = null;
391
392 for (const process of processes) {
393 if (hiddenChildDepth !== null) {
394 if (process.depth > hiddenChildDepth) continue;
395 hiddenChildDepth = null;
396 }
397
398 visible.push(process);
399 if (collapsedPids.has(process.pid)) {
400 hiddenChildDepth = process.depth;
401 }
402 }
403
404 return visible;
405 }, [collapsedPids, processes]);
406
407 const toggleProcess = useCallback((pid: number) => {
408 setCollapsedPids((previous) => {
409 const next = new Set(previous);
410 if (next.has(pid)) {
411 next.delete(pid);
412 } else {
413 next.add(pid);
414 }
415 return next;
416 });
417 }, []);
418
419 return (
420 <ScrollArea
421 chainVerticalScroll
422 scrollFade
423 hideScrollbars
424 className="w-full max-w-full rounded-none border-t border-border/60"
425 >
426 <table className="w-full min-w-[1040px] table-fixed text-left text-xs">
427 <colgroup>
428 <col className="w-[24%]" />
429 <col className="w-[8%]" />
430 <col className="w-[10%]" />
431 <col className="w-[33%]" />
432 <col className="w-[8%]" />
433 <col className="w-[11%]" />
434 <col className="w-[6%]" />
435 </colgroup>
436 <thead className="border-b border-border/60 bg-muted/15 text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">
437 <tr>
438 <th className="px-4 py-2 font-semibold sm:pl-5">Name</th>
439 <th className="px-3 py-2 text-right font-semibold">CPU</th>
440 <th className="px-3 py-2 text-right font-semibold">Memory</th>
441 <th className="px-3 py-2 font-semibold">Command</th>
442 <th className="px-3 py-2 text-right font-semibold">PID</th>
443 <th className="px-3 py-2 font-semibold">Type</th>
444 <th className="px-2 py-2 text-right font-semibold sm:pr-4">Kill</th>
445 </tr>
446 </thead>
447 <tbody className="divide-y divide-border/50">
448 {visibleProcesses.length === 0 ? (
449 <tr>
450 <td colSpan={7} className="px-4 py-4 text-xs text-muted-foreground sm:px-5">
451 {emptyLabel ?? "No live descendant processes found."}
452 </td>
453 </tr>
454 ) : null}
455 {visibleProcesses.map((process) => (
456 <tr key={process.pid} className="hover:bg-muted/20">
457 <td className="px-4 py-2 align-middle sm:pl-5">
458 <ProcessNameCell
459 process={process}
460 isExpanded={!collapsedPids.has(process.pid)}
461 onToggle={toggleProcess}
462 />
463 </td>
464 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
465 {process.cpuPercent.toFixed(1)}%
466 </td>
467 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
468 {formatBytes(process.rssBytes)}
469 </td>
470 <td className="px-3 py-2 align-middle text-muted-foreground">
471 <Tooltip>
472 <TooltipTrigger
473 render={<span className="block truncate">{process.command}</span>}
474 />
475 <TooltipPopup
476 side="top"
477 className="max-w-[min(440px,calc(100vw-2rem))] whitespace-normal break-words text-left font-mono text-[11px] leading-relaxed text-wrap"
478 >
479 {process.command}
480 </TooltipPopup>
481 </Tooltip>
482 </td>
483 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums text-muted-foreground">
484 {process.pid}
485 </td>
486 <td className="truncate px-3 py-2 align-middle text-muted-foreground">
487 {formatProcessType(process)}
488 </td>
489 <td className="px-2 py-2 align-middle sm:pr-4">
490 <ProcessSignalActions
491 process={process}
492 isSignaling={signalingPid === process.pid}
493 onSignal={onSignal}
494 />
495 </td>
496 </tr>
497 ))}
498 </tbody>
499 </table>
500 </ScrollArea>
501 );
502}
503
504function DiagnosticsLastChecked({ checkedAt }: { checkedAt: DateTime.Utc | null }) {
505 useRelativeTimeTick();
506 const relative = checkedAt ? formatRelativeTime(DateTime.formatIso(checkedAt)) : null;
507
508 if (!relative) {
509 return <span className="text-[11px] text-muted-foreground/50">Checking</span>;
510 }
511
512 return (
513 <span className="text-[11px] text-muted-foreground/60">
514 {relative.suffix ? (
515 <>
516 Checked <span className="font-mono tabular-nums">{relative.value}</span> {relative.suffix}
517 </>
518 ) : (
519 <>Checked {relative.value}</>
520 )}
521 </span>
522 );
523}
524
525function DiagnosticsRefreshButton({
526 isPending,
527 label,
528 onClick,
529}: {
530 isPending: boolean;
531 label: string;
532 onClick: () => void;
533}) {
534 return (
535 <Tooltip>
536 <TooltipTrigger
537 render={
538 <Button
539 size="icon-xs"
540 variant="ghost"
541 className="size-5 rounded-sm p-0 text-muted-foreground hover:text-foreground"
542 disabled={isPending}
543 onClick={onClick}
544 aria-label={label}
545 >
546 <RefreshCwIcon className={cn("size-3", isPending && "animate-spin")} />
547 </Button>
548 }
549 />
550 <TooltipPopup side="top">{label}</TooltipPopup>
551 </Tooltip>
552 );
553}
554
555export function DiagnosticsSettingsPanel() {
556 const observability = useServerObservability();
557 const availableEditors = useServerAvailableEditors();
558 const { data, error, isPending, refresh } = useTraceDiagnostics();
559 const {
560 data: processData,
561 error: processError,
562 isPending: isProcessPending,
563 refresh: refreshProcesses,
564 } = useProcessDiagnostics();
565 const [isOpeningLogsDirectory, setIsOpeningLogsDirectory] = useState(false);
566 const [openLogsDirectoryError, setOpenLogsDirectoryError] = useState<string | null>(null);
567 const [signalingPid, setSignalingPid] = useState<number | null>(null);
568
569 const openLogsDirectory = useCallback(() => {
570 const logsDirectoryPath = observability?.logsDirectoryPath ?? null;
571 if (!logsDirectoryPath) return;
572
573 const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
574 if (!editor) {
575 setOpenLogsDirectoryError("No available editors found.");
576 return;
577 }
578
579 setIsOpeningLogsDirectory(true);
580 setOpenLogsDirectoryError(null);
581 void ensureLocalApi()
582 .shell.openInEditor(logsDirectoryPath, editor)
583 .catch((error: unknown) => {
584 setOpenLogsDirectoryError(
585 error instanceof Error ? error.message : "Unable to open logs folder.",
586 );
587 })
588 .finally(() => {
589 setIsOpeningLogsDirectory(false);
590 });
591 }, [availableEditors, observability?.logsDirectoryPath]);
592
593 const isInitialLoading = isPending && data === null;
594 const isProcessInitialLoading = isProcessPending && processData === null;
595 const signalProcess = useCallback(
596 (pid: number, signal: ServerProcessSignal) => {
597 if (
598 signal === "SIGKILL" &&
599 !window.confirm(`Send SIGKILL to process ${pid}? This cannot be handled by the process.`)
600 ) {
601 return;
602 }
603
604 setSignalingPid(pid);
605 void ensureLocalApi()
606 .server.signalProcess({ pid, signal })
607 .then((result) => {
608 if (!result.signaled) {
609 const message = Option.getOrUndefined(result.message);
610 refreshProcesses();
611 if (isStaleProcessSignalMessage(message)) {
612 toastManager.add({
613 type: "info",
614 title: "Process already exited",
615 description:
616 "The process is not a child of the T3 Server. It might already have exited.",
617 });
618 return;
619 }
620
621 toastManager.add({
622 type: "error",
623 title: `Could not send ${signal}`,
624 description: message ?? `Failed to send ${signal}.`,
625 });
626 return;
627 }
628 refreshProcesses();
629 })
630 .catch((error: unknown) => {
631 toastManager.add({
632 type: "error",
633 title: `Could not send ${signal}`,
634 description: error instanceof Error ? error.message : `Failed to send ${signal}.`,
635 });
636 })
637 .finally(() => {
638 setSignalingPid(null);
639 });
640 },
641 [refreshProcesses],
642 );
643
644 const processDiagnosticsError = processData ? Option.getOrNull(processData.error) : null;
645 const traceDiagnosticsError = data ? Option.getOrNull(data.error) : null;
646 const traceDiagnosticsPartialFailure = data
647 ? Option.getOrElse(data.partialFailure, () => false)
648 : false;
649
650 return (
651 <SettingsPageContainer>
652 <SettingsSection
653 title="Live Processes"
654 headerAction={
655 <div className="flex items-center gap-1.5">
656 <DiagnosticsLastChecked checkedAt={processData?.readAt ?? null} />
657 <DiagnosticsRefreshButton
658 isPending={isProcessPending}
659 label="Refresh process diagnostics"
660 onClick={refreshProcesses}
661 />
662 </div>
663 }
664 >
665 <StatsGrid>
666 <StatBlock
667 label="Child Processes"
668 value={processData ? formatCount(processData.processCount) : "..."}
669 />
670 <StatBlock
671 label="CPU"
672 value={processData ? `${processData.totalCpuPercent.toFixed(1)}%` : "..."}
673 tooltip="Total CPU across live child processes of the current server process. The desktop shell and other parent processes are not included."
674 />
675 <StatBlock
676 label="Memory"
677 value={processData ? formatBytes(processData.totalRssBytes) : "..."}
678 tooltip="Total resident memory across live child processes of the current server process. The desktop shell and other parent processes are not included."
679 />
680 <StatBlock
681 label="Server PID"
682 value={processData ? String(processData.serverPid) : "..."}
683 />
684 </StatsGrid>
685 {processDiagnosticsError || processError ? (
686 <div className="space-y-2 border-t border-border/60 px-4 py-3 text-xs text-muted-foreground sm:px-5">
687 {processDiagnosticsError ? (
688 <div className="flex items-start gap-2 text-destructive">
689 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
690 <span>{processDiagnosticsError.message}</span>
691 </div>
692 ) : null}
693 {processError ? (
694 <div className="flex items-start gap-2 text-destructive">
695 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
696 <span>{processError}</span>
697 </div>
698 ) : null}
699 </div>
700 ) : null}
701 <ProcessDiagnosticsTable
702 processes={processData?.processes ?? []}
703 signalingPid={signalingPid}
704 onSignal={signalProcess}
705 emptyLabel={
706 isProcessInitialLoading
707 ? "Loading live processes..."
708 : "No live descendant processes found."
709 }
710 />
711 </SettingsSection>
712
713 <SettingsSection
714 title="Trace Diagnostics"
715 headerAction={
716 <div className="flex items-center gap-1.5">
717 <DiagnosticsLastChecked checkedAt={data?.readAt ?? null} />
718 <Tooltip>
719 <TooltipTrigger
720 render={
721 <Button
722 size="icon-xs"
723 variant="ghost"
724 className="size-5 rounded-sm p-0 text-muted-foreground hover:text-foreground"
725 disabled={!observability?.logsDirectoryPath || isOpeningLogsDirectory}
726 onClick={openLogsDirectory}
727 aria-label="Open logs folder"
728 >
729 <FolderOpenIcon className="size-3" />
730 </Button>
731 }
732 />
733 <TooltipPopup side="top">Open logs folder</TooltipPopup>
734 </Tooltip>
735 <DiagnosticsRefreshButton
736 isPending={isPending}
737 label="Refresh trace diagnostics"
738 onClick={refresh}
739 />
740 </div>
741 }
742 >
743 <StatsGrid>
744 <StatBlock label="Spans" value={data ? formatCount(data.recordCount) : "..."} />
745 <StatBlock
746 label="Failures"
747 value={data ? formatCount(data.failureCount) : "..."}
748 tone={data && data.failureCount > 0 ? "danger" : "default"}
749 />
750 <StatBlock
751 label="Slow Spans"
752 value={data ? formatCount(data.slowSpanCount) : "..."}
753 tooltip={
754 data
755 ? `Spans with a duration of ${formatDuration(data.slowSpanThresholdMs)} or longer.`
756 : "Spans at or above the configured slow-span threshold."
757 }
758 tone={data && data.slowSpanCount > 0 ? "warning" : "default"}
759 />
760 <StatBlock
761 label="Parse Errors"
762 value={data ? formatCount(data.parseErrorCount) : "..."}
763 tone={data && data.parseErrorCount > 0 ? "warning" : "default"}
764 />
765 </StatsGrid>
766 {openLogsDirectoryError || traceDiagnosticsError || error ? (
767 <div className="space-y-2 border-t border-border/60 px-4 py-3 text-xs text-muted-foreground sm:px-5">
768 {openLogsDirectoryError ? (
769 <div className="flex items-start gap-2 text-destructive">
770 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
771 <span>{openLogsDirectoryError}</span>
772 </div>
773 ) : null}
774 {traceDiagnosticsError ? (
775 <div
776 className={cn(
777 "flex items-start gap-2",
778 traceDiagnosticsPartialFailure
779 ? "text-amber-600 dark:text-amber-400"
780 : "text-destructive",
781 )}
782 >
783 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
784 <span>
785 {traceDiagnosticsPartialFailure
786 ? `Some trace files could not be read, so diagnostics may be incomplete. ${traceDiagnosticsError.message}`
787 : traceDiagnosticsError.message}
788 </span>
789 </div>
790 ) : null}
791 {error ? (
792 <div className="flex items-start gap-2 text-destructive">
793 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
794 <span>{error}</span>
795 </div>
796 ) : null}
797 </div>
798 ) : null}
799 </SettingsSection>
800
801 <SettingsSection title="Latest Failures">
802 {data && data.latestFailures.length > 0 ? (
803 <DiagnosticsTable headers={["Span", "Cause", "Duration", "Ended"]}>
804 {data.latestFailures.map((failure) => (
805 <tr key={`${failure.traceId}:${failure.spanId}`}>
806 <td className="px-4 py-3 align-top text-xs font-medium text-foreground first:sm:pl-5">
807 {failure.name}
808 </td>
809 <td className="max-w-[360px] px-4 py-3 align-top text-muted-foreground">
810 <ExpandableText text={failure.cause} />
811 </td>
812 <td className="px-4 py-3 align-top font-mono tabular-nums">
813 {formatDuration(failure.durationMs)}
814 </td>
815 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums text-muted-foreground last:sm:pr-5">
816 {formatRelativeNoWrap(failure.endedAt)}
817 </td>
818 </tr>
819 ))}
820 </DiagnosticsTable>
821 ) : (
822 <EmptyRows label={isInitialLoading ? "Loading failures..." : "No failed spans found."} />
823 )}
824 </SettingsSection>
825
826 <SettingsSection title="Most Common Failures">
827 {data && data.commonFailures.length > 0 ? (
828 <DiagnosticsTable
829 headers={["Span", "Count", "Cause", "Last Seen"]}
830 minTableWidth="min-w-[760px]"
831 >
832 {data.commonFailures.map((failure) => (
833 <tr key={`${failure.name}:${failure.cause}`}>
834 <td className="px-4 py-3 align-top text-xs font-medium text-foreground first:sm:pl-5">
835 {failure.name}
836 </td>
837 <td className="px-4 py-3 align-top font-mono tabular-nums">
838 {formatCount(failure.count)}
839 </td>
840 <td className="max-w-[360px] px-4 py-3 align-top text-muted-foreground">
841 <ExpandableText text={failure.cause} />
842 </td>
843 <td className="w-px whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums text-muted-foreground last:sm:pr-5">
844 {formatRelativeNoWrap(failure.lastSeenAt)}
845 </td>
846 </tr>
847 ))}
848 </DiagnosticsTable>
849 ) : (
850 <EmptyRows
851 label={isInitialLoading ? "Loading failure groups..." : "No repeated failures found."}
852 />
853 )}
854 </SettingsSection>
855
856 <SettingsSection title="Slowest Spans">
857 {data && data.slowestSpans.length > 0 ? (
858 <DiagnosticsTable
859 headers={["Span", "Duration", "Ended", "Trace"]}
860 minTableWidth="min-w-[900px]"
861 columnWidths={["w-[44%]", "w-[14%]", "w-[12%]", "w-[30%]"]}
862 >
863 {data.slowestSpans.map((span) => (
864 <tr key={`${span.traceId}:${span.spanId}`}>
865 <td className="px-4 py-3 align-top text-xs font-medium text-foreground first:sm:pl-5">
866 {span.name}
867 </td>
868 <td className="px-4 py-3 align-top font-mono tabular-nums">
869 {formatDuration(span.durationMs)}
870 </td>
871 <td className="w-px whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums text-muted-foreground">
872 {formatRelativeNoWrap(span.endedAt)}
873 </td>
874 <td className="min-w-0 whitespace-nowrap px-4 py-3 align-top text-muted-foreground last:sm:pr-5">
875 <TraceIdCell traceId={span.traceId} />
876 </td>
877 </tr>
878 ))}
879 </DiagnosticsTable>
880 ) : (
881 <EmptyRows label={isInitialLoading ? "Loading slow spans..." : "No spans found."} />
882 )}
883 </SettingsSection>
884
885 <SettingsSection title="Span Logs">
886 {data && data.latestWarningAndErrorLogs.length > 0 ? (
887 <ScrollArea
888 chainVerticalScroll
889 scrollFade
890 hideScrollbars
891 className="w-full max-w-full rounded-none"
892 >
893 <table className="w-full min-w-[920px] table-fixed text-left text-xs">
894 <colgroup>
895 <col className="w-[11%]" />
896 <col className="w-[9%]" />
897 <col className="w-[24%]" />
898 <col className="w-[26%]" />
899 <col className="w-[30%]" />
900 </colgroup>
901 <thead className="border-b border-border/60 text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">
902 <tr>
903 <th className="whitespace-nowrap px-4 py-2.5 font-semibold sm:pl-5">Time</th>
904 <th className="whitespace-nowrap px-4 py-2.5 font-semibold">Level</th>
905 <th className="whitespace-nowrap px-4 py-2.5 font-semibold">Span</th>
906 <th className="whitespace-nowrap px-4 py-2.5 font-semibold">Message</th>
907 <th className="whitespace-nowrap px-4 py-2.5 font-semibold sm:pr-5">Trace</th>
908 </tr>
909 </thead>
910 <tbody className="divide-y divide-border/60">
911 {data.latestWarningAndErrorLogs.map((event) => (
912 <tr
913 key={`${event.traceId}:${event.spanId}:${DateTime.formatIso(event.seenAt)}:${event.message}`}
914 className="hover:bg-muted/15"
915 >
916 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums text-muted-foreground sm:pl-5">
917 {formatRelativeNoWrap(event.seenAt)}
918 </td>
919 <td className="px-4 py-3 align-top">
920 <span className="inline-flex rounded bg-muted px-1.5 py-0.5 font-mono text-[11px] font-medium uppercase text-foreground/80">
921 {event.level}
922 </span>
923 </td>
924 <td className="px-4 py-3 align-top">
925 <div className="truncate font-medium text-foreground">{event.spanName}</div>
926 </td>
927 <td className="px-4 py-3 align-top text-muted-foreground">
928 <ExpandableText
929 collapsedClassName="line-clamp-2"
930 expandLabel="Show full message"
931 text={event.message}
932 />
933 </td>
934 <td className="min-w-0 whitespace-nowrap px-4 py-3 align-top text-muted-foreground sm:pr-5">
935 <TraceIdCell traceId={event.traceId} />
936 </td>
937 </tr>
938 ))}
939 </tbody>
940 </table>
941 </ScrollArea>
942 ) : (
943 <EmptyRows
944 label={isInitialLoading ? "Loading recent logs..." : "No warnings or errors found."}
945 />
946 )}
947 </SettingsSection>
948
949 <SettingsSection title="Top Span Names">
950 {data && data.topSpansByCount.length > 0 ? (
951 <DiagnosticsTable
952 headers={["Span", "Count", "Failures", "Average", "Max"]}
953 minTableWidth="min-w-[760px]"
954 columnWidths={["w-[48%]", "w-[13%]", "w-[13%]", "w-[13%]", "w-[13%]"]}
955 >
956 {data.topSpansByCount.map((span) => (
957 <tr key={span.name}>
958 <td className="px-4 py-3 align-top text-xs font-medium text-foreground first:sm:pl-5">
959 {span.name}
960 </td>
961 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums">
962 {formatCount(span.count)}
963 </td>
964 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums">
965 {formatCount(span.failureCount)}
966 </td>
967 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums">
968 {formatDuration(span.averageDurationMs)}
969 </td>
970 <td className="whitespace-nowrap px-4 py-3 align-top font-mono tabular-nums last:sm:pr-5">
971 {formatDuration(span.maxDurationMs)}
972 </td>
973 </tr>
974 ))}
975 </DiagnosticsTable>
976 ) : (
977 <EmptyRows label={isInitialLoading ? "Loading span names..." : "No spans found."} />
978 )}
979 </SettingsSection>
980 </SettingsPageContainer>
981 );
982}
apps/web/src/components/settings/SettingsPanels.browser.tsx+68-8
......@@ -20,15 +20,40 @@ import { DateTime, Option } from "effect";
2020import { page } from "vitest/browser";
2121import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2222import { render } from "vitest-browser-react";
23import type { ReactNode } from "react";
24import {
25 RouterProvider,
26 createMemoryHistory,
27 createRootRoute,
28 createRoute,
29 createRouter,
30} from "@tanstack/react-router";
2331
2432import { __resetLocalApiForTests } from "../../localApi";
2533import { AppAtomRegistryProvider, resetAppAtomRegistryForTests } from "../../rpc/atomRegistry";
2634import { resetServerStateForTests, setServerConfigSnapshot } from "../../rpc/serverState";
2735import { useUiStateStore } from "../../uiStateStore";
2836import { ConnectionsSettings } from "./ConnectionsSettings";
37import { DiagnosticsSettingsPanel } from "./DiagnosticsSettings";
2938import { GeneralSettingsPanel, ProviderSettingsPanel } from "./SettingsPanels";
3039import { SourceControlSettingsPanel } from "./SourceControlSettings";
3140
41function renderWithTestRouter(children: ReactNode) {
42 const rootRoute = createRootRoute({
43 component: () => children,
44 });
45 const indexRoute = createRoute({
46 getParentRoute: () => rootRoute,
47 path: "/",
48 });
49 const router = createRouter({
50 routeTree: rootRoute.addChildren([indexRoute]),
51 history: createMemoryHistory({ initialEntries: ["/"] }),
52 });
53
54 return render(<RouterProvider router={router} />);
55}
56
3257const authAccessHarness = vi.hoisted(() => {
3358 type Snapshot = AuthAccessSnapshot;
3459 let snapshot: Snapshot = {
......@@ -694,25 +719,24 @@ describe("GeneralSettingsPanel observability", () => {
694719 await expect.element(page.getByText("http://127.0.0.1:3773/").first()).toBeInTheDocument();
695720 });
696721
697 it("shows diagnostics inside About with a single logs-folder action", async () => {
722 it("shows diagnostics inside About with a diagnostics link", async () => {
698723 setServerConfigSnapshot(createBaseServerConfig());
699724
700 mounted = await render(
725 mounted = await renderWithTestRouter(
701726 <AppAtomRegistryProvider>
702727 <GeneralSettingsPanel />
703728 </AppAtomRegistryProvider>,
704729 );
705730
706731 await expect.element(page.getByText("About")).toBeInTheDocument();
707 await expect.element(page.getByText("Diagnostics")).toBeInTheDocument();
708 await expect.element(page.getByText("Open logs folder")).toBeInTheDocument();
709732 await expect
710 .element(page.getByText("/repo/project/.t3/logs", { exact: true }))
733 .element(page.getByRole("heading", { name: "Diagnostics", exact: true }))
711734 .toBeInTheDocument();
735 await expect.element(page.getByRole("link", { name: "View diagnostics" })).toBeInTheDocument();
712736 await expect
713737 .element(
714738 page.getByText(
715 "Local trace file. OTLP exporting traces to http://localhost:4318/v1/traces.",
739 "Local trace file. Exporting OTEL traces to http://localhost:4318/v1/traces.",
716740 ),
717741 )
718742 .toBeInTheDocument();
......@@ -1020,20 +1044,56 @@ describe("GeneralSettingsPanel observability", () => {
10201044 it("opens the logs folder in the preferred editor", async () => {
10211045 const openInEditor = vi.fn<LocalApi["shell"]["openInEditor"]>().mockResolvedValue(undefined);
10221046 window.nativeApi = {
1047 persistence: {
1048 getClientSettings: vi.fn().mockResolvedValue(null),
1049 setClientSettings: vi.fn().mockResolvedValue(undefined),
1050 },
10231051 shell: {
10241052 openInEditor,
10251053 },
1054 server: {
1055 getProcessDiagnostics: vi.fn().mockResolvedValue({
1056 serverPid: 1234,
1057 readAt: makeUtc("2036-04-07T00:00:00.000Z"),
1058 processCount: 0,
1059 totalRssBytes: 0,
1060 totalCpuPercent: 0,
1061 processes: [],
1062 error: Option.none(),
1063 }),
1064 getTraceDiagnostics: vi.fn().mockResolvedValue({
1065 traceFilePath: "/repo/project/.t3/traces.jsonl",
1066 scannedFilePaths: ["/repo/project/.t3/traces.jsonl"],
1067 readAt: makeUtc("2036-04-07T00:00:00.000Z"),
1068 recordCount: 0,
1069 parseErrorCount: 0,
1070 firstSpanAt: Option.none(),
1071 lastSpanAt: Option.none(),
1072 failureCount: 0,
1073 interruptionCount: 0,
1074 slowSpanThresholdMs: 5_000,
1075 slowSpanCount: 0,
1076 logLevelCounts: {},
1077 topSpansByCount: [],
1078 slowestSpans: [],
1079 commonFailures: [],
1080 latestFailures: [],
1081 latestWarningAndErrorLogs: [],
1082 partialFailure: Option.none(),
1083 error: Option.none(),
1084 }),
1085 },
10261086 } as unknown as LocalApi;
10271087
10281088 setServerConfigSnapshot(createBaseServerConfig());
10291089
10301090 mounted = await render(
10311091 <AppAtomRegistryProvider>
1032 <GeneralSettingsPanel />
1092 <DiagnosticsSettingsPanel />
10331093 </AppAtomRegistryProvider>,
10341094 );
10351095
1036 const openLogsButton = page.getByText("Open logs folder");
1096 const openLogsButton = page.getByLabelText("Open logs folder");
10371097 await openLogsButton.click();
10381098
10391099 expect(openInEditor).toHaveBeenCalledWith("/repo/project/.t3/logs", "cursor");
apps/web/src/components/settings/SettingsPanels.logic.test.ts+42-1
......@@ -5,7 +5,48 @@ import {
55 type ProviderInstanceConfig,
66} from "@t3tools/contracts";
77import { describe, expect, it } from "vitest";
8import { buildProviderInstanceUpdatePatch } from "./SettingsPanels.logic";
8import {
9 buildProviderInstanceUpdatePatch,
10 formatDiagnosticsDescription,
11} from "./SettingsPanels.logic";
12
13describe("formatDiagnosticsDescription", () => {
14 it("collapses trace and metric URLs that share the same OTEL base path", () => {
15 expect(
16 formatDiagnosticsDescription({
17 localTracingEnabled: true,
18 otlpTracesEnabled: true,
19 otlpTracesUrl: "http://localhost:4318/v1/traces",
20 otlpMetricsEnabled: true,
21 otlpMetricsUrl: "http://localhost:4318/v1/metrics",
22 }),
23 ).toBe("Local trace file. Exporting OTEL to http://localhost:4318/v1/{traces,metrics}.");
24 });
25
26 it("keeps separate trace and metric URLs when their base paths differ", () => {
27 expect(
28 formatDiagnosticsDescription({
29 localTracingEnabled: true,
30 otlpTracesEnabled: true,
31 otlpTracesUrl: "http://localhost:4318/v1/traces",
32 otlpMetricsEnabled: true,
33 otlpMetricsUrl: "http://localhost:9000/v1/metrics",
34 }),
35 ).toBe(
36 "Local trace file. Exporting OTEL traces to http://localhost:4318/v1/traces and metrics to http://localhost:9000/v1/metrics.",
37 );
38 });
39
40 it("omits OTEL text when no exporter is enabled", () => {
41 expect(
42 formatDiagnosticsDescription({
43 localTracingEnabled: true,
44 otlpTracesEnabled: false,
45 otlpMetricsEnabled: false,
46 }),
47 ).toBe("Local trace file.");
48 });
49});
950
1051describe("buildProviderInstanceUpdatePatch", () => {
1152 it("promotes an edited default provider into providerInstances and resets the legacy provider", () => {
apps/web/src/components/settings/SettingsPanels.logic.ts+48
......@@ -7,6 +7,54 @@ import type {
77} from "@t3tools/contracts";
88import { DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts/settings";
99
10function collapseOtelSignalsUrl(input: {
11 readonly tracesUrl: string;
12 readonly metricsUrl: string;
13}): string | null {
14 const tracesSuffix = "/traces";
15 const metricsSuffix = "/metrics";
16 if (!input.tracesUrl.endsWith(tracesSuffix) || !input.metricsUrl.endsWith(metricsSuffix)) {
17 return null;
18 }
19
20 const tracesBase = input.tracesUrl.slice(0, -tracesSuffix.length);
21 const metricsBase = input.metricsUrl.slice(0, -metricsSuffix.length);
22 if (tracesBase !== metricsBase) {
23 return null;
24 }
25
26 return `${tracesBase}/{traces,metrics}`;
27}
28
29export function formatDiagnosticsDescription(input: {
30 readonly localTracingEnabled: boolean;
31 readonly otlpTracesEnabled: boolean;
32 readonly otlpTracesUrl?: string | undefined;
33 readonly otlpMetricsEnabled: boolean;
34 readonly otlpMetricsUrl?: string | undefined;
35}): string {
36 const mode = input.localTracingEnabled ? "Local trace file" : "Terminal logs only";
37 const tracesUrl = input.otlpTracesEnabled ? input.otlpTracesUrl : undefined;
38 const metricsUrl = input.otlpMetricsEnabled ? input.otlpMetricsUrl : undefined;
39
40 if (tracesUrl && metricsUrl) {
41 const collapsedUrl = collapseOtelSignalsUrl({ tracesUrl, metricsUrl });
42 return collapsedUrl
43 ? `${mode}. Exporting OTEL to ${collapsedUrl}.`
44 : `${mode}. Exporting OTEL traces to ${tracesUrl} and metrics to ${metricsUrl}.`;
45 }
46
47 if (tracesUrl) {
48 return `${mode}. Exporting OTEL traces to ${tracesUrl}.`;
49 }
50
51 if (metricsUrl) {
52 return `${mode}. Exporting OTEL metrics to ${metricsUrl}.`;
53 }
54
55 return `${mode}.`;
56}
57
1058export function buildProviderInstanceUpdatePatch(input: {
1159 readonly settings: Pick<ServerSettings, "providers" | "providerInstances">;
1260 readonly instanceId: ProviderInstanceId;
apps/web/src/components/settings/SettingsPanels.tsx+15-81
......@@ -1,5 +1,6 @@
11import { ArchiveIcon, ArchiveX, LoaderIcon, PlusIcon, RefreshCwIcon } from "lucide-react";
22import { useQueryClient } from "@tanstack/react-query";
3import { Link } from "@tanstack/react-router";
34import { useCallback, useMemo, useRef, useState } from "react";
45import {
56 defaultInstanceIdForDriver,
......@@ -24,7 +25,6 @@ import {
2425} from "../../components/desktopUpdate.logic";
2526import { ProviderModelPicker } from "../chat/ProviderModelPicker";
2627import { TraitsPicker } from "../chat/TraitsPicker";
27import { resolveAndPersistPreferredEditor } from "../../editorPreferences";
2828import { isElectron } from "../../env";
2929import { useTheme } from "../../hooks/useTheme";
3030import { useSettings, useUpdateSettings } from "../../hooks/useSettings";
......@@ -66,7 +66,10 @@ import {
6666} from "../ProviderUpdateLaunchNotification.logic";
6767import { ProviderInstanceCard } from "./ProviderInstanceCard";
6868import { DRIVER_OPTIONS, getDriverOption } from "./providerDriverMeta";
69import { buildProviderInstanceUpdatePatch } from "./SettingsPanels.logic";
69import {
70 buildProviderInstanceUpdatePatch,
71 formatDiagnosticsDescription,
72} from "./SettingsPanels.logic";
7073import {
7174 SettingResetButton,
7275 SettingsPageContainer,
......@@ -75,11 +78,7 @@ import {
7578 useRelativeTimeTick,
7679} from "./settingsLayout";
7780import { ProjectFavicon } from "../ProjectFavicon";
78import {
79 useServerAvailableEditors,
80 useServerObservability,
81 useServerProviders,
82} from "../../rpc/serverState";
81import { useServerObservability, useServerProviders } from "../../rpc/serverState";
8382
8483const THEME_OPTIONS = [
8584 {
......@@ -442,27 +441,15 @@ export function GeneralSettingsPanel() {
442441 const { theme, setTheme } = useTheme();
443442 const settings = useSettings();
444443 const { updateSettings } = useUpdateSettings();
445 const [openingPathByTarget, setOpeningPathByTarget] = useState({
446 logsDirectory: false,
447 });
448 const [openPathErrorByTarget, setOpenPathErrorByTarget] = useState<
449 Partial<Record<"logsDirectory", string | null>>
450 >({});
451 const availableEditors = useServerAvailableEditors();
452444 const observability = useServerObservability();
453445 const serverProviders = useServerProviders();
454 const logsDirectoryPath = observability?.logsDirectoryPath ?? null;
455 const diagnosticsDescription = (() => {
456 const exports: string[] = [];
457 if (observability?.otlpTracesEnabled && observability.otlpTracesUrl) {
458 exports.push(`traces to ${observability.otlpTracesUrl}`);
459 }
460 if (observability?.otlpMetricsEnabled && observability.otlpMetricsUrl) {
461 exports.push(`metrics to ${observability.otlpMetricsUrl}`);
462 }
463 const mode = observability?.localTracingEnabled ? "Local trace file" : "Terminal logs only";
464 return exports.length > 0 ? `${mode}. OTLP exporting ${exports.join(" and ")}.` : `${mode}.`;
465 })();
446 const diagnosticsDescription = formatDiagnosticsDescription({
447 localTracingEnabled: observability?.localTracingEnabled ?? false,
448 otlpTracesEnabled: observability?.otlpTracesEnabled ?? false,
449 otlpTracesUrl: observability?.otlpTracesUrl,
450 otlpMetricsEnabled: observability?.otlpMetricsEnabled ?? false,
451 otlpMetricsUrl: observability?.otlpMetricsUrl,
452 });
466453
467454 const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);
468455 const textGenInstanceId = textGenerationModelSelection.instanceId;
......@@ -487,44 +474,6 @@ export function GeneralSettingsPanel() {
487474 DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,
488475 );
489476
490 const openInPreferredEditor = useCallback(
491 (target: "logsDirectory", path: string | null, failureMessage: string) => {
492 if (!path) return;
493 setOpenPathErrorByTarget((existing) => ({ ...existing, [target]: null }));
494 setOpeningPathByTarget((existing) => ({ ...existing, [target]: true }));
495
496 const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
497 if (!editor) {
498 setOpenPathErrorByTarget((existing) => ({
499 ...existing,
500 [target]: "No available editors found.",
501 }));
502 setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
503 return;
504 }
505
506 void ensureLocalApi()
507 .shell.openInEditor(path, editor)
508 .catch((error) => {
509 setOpenPathErrorByTarget((existing) => ({
510 ...existing,
511 [target]: error instanceof Error ? error.message : failureMessage,
512 }));
513 })
514 .finally(() => {
515 setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
516 });
517 },
518 [availableEditors],
519 );
520
521 const openLogsDirectory = useCallback(() => {
522 openInPreferredEditor("logsDirectory", logsDirectoryPath, "Unable to open logs folder.");
523 }, [logsDirectoryPath, openInPreferredEditor]);
524
525 const openDiagnosticsError = openPathErrorByTarget.logsDirectory ?? null;
526 const isOpeningLogsDirectory = openingPathByTarget.logsDirectory;
527
528477 return (
529478 <SettingsPageContainer>
530479 <SettingsSection title="General">
......@@ -914,24 +863,9 @@ export function GeneralSettingsPanel() {
914863 <SettingsRow
915864 title="Diagnostics"
916865 description={diagnosticsDescription}
917 status={
918 <>
919 <span className="block break-all font-mono text-[11px] text-foreground">
920 {logsDirectoryPath ?? "Resolving logs directory..."}
921 </span>
922 {openDiagnosticsError ? (
923 <span className="mt-1 block text-destructive">{openDiagnosticsError}</span>
924 ) : null}
925 </>
926 }
927866 control={
928 <Button
929 size="xs"
930 variant="outline"
931 disabled={!logsDirectoryPath || isOpeningLogsDirectory}
932 onClick={openLogsDirectory}
933 >
934 {isOpeningLogsDirectory ? "Opening..." : "Open logs folder"}
867 <Button render={<Link to="/settings/diagnostics" />} size="xs" variant="outline">
868 View diagnostics
935869 </Button>
936870 }
937871 />
apps/web/src/lib/processDiagnosticsState.ts created+63
......@@ -0,0 +1,63 @@
1import { useAtomValue } from "@effect/atom-react";
2import type { ServerProcessDiagnosticsResult } from "@t3tools/contracts";
3import { Cause, Effect, Option } from "effect";
4import { AsyncResult, Atom } from "effect/unstable/reactivity";
5import { useCallback } from "react";
6
7import { ensureLocalApi } from "../localApi";
8import { appAtomRegistry } from "../rpc/atomRegistry";
9
10const PROCESS_DIAGNOSTICS_STALE_TIME_MS = 2_000;
11const PROCESS_DIAGNOSTICS_IDLE_TTL_MS = 5 * 60_000;
12
13const processDiagnosticsAtom = Atom.make(
14 Effect.promise(() => ensureLocalApi().server.getProcessDiagnostics()),
15).pipe(
16 Atom.swr({
17 staleTime: PROCESS_DIAGNOSTICS_STALE_TIME_MS,
18 revalidateOnMount: true,
19 }),
20 Atom.setIdleTTL(PROCESS_DIAGNOSTICS_IDLE_TTL_MS),
21 Atom.withLabel("process-diagnostics"),
22);
23
24export interface ProcessDiagnosticsState {
25 readonly data: ServerProcessDiagnosticsResult | null;
26 readonly error: string | null;
27 readonly isPending: boolean;
28 readonly refresh: () => void;
29}
30
31function formatProcessDiagnosticsError(error: unknown): string {
32 return error instanceof Error ? error.message : "Failed to load process diagnostics.";
33}
34
35function readProcessDiagnosticsError(
36 result: AsyncResult.AsyncResult<ServerProcessDiagnosticsResult, unknown>,
37): string | null {
38 if (result._tag !== "Failure") {
39 return null;
40 }
41
42 const squashed = Cause.squash(result.cause);
43 return formatProcessDiagnosticsError(squashed);
44}
45
46export function refreshProcessDiagnostics(): void {
47 appAtomRegistry.refresh(processDiagnosticsAtom);
48}
49
50export function useProcessDiagnostics(): ProcessDiagnosticsState {
51 const result = useAtomValue(processDiagnosticsAtom);
52 const data = Option.getOrNull(AsyncResult.value(result));
53 const refresh = useCallback(() => {
54 refreshProcessDiagnostics();
55 }, []);
56
57 return {
58 data,
59 error: readProcessDiagnosticsError(result),
60 isPending: result.waiting,
61 refresh,
62 };
63}
apps/web/src/lib/traceDiagnosticsState.ts created+63
......@@ -0,0 +1,63 @@
1import { useAtomValue } from "@effect/atom-react";
2import type { ServerTraceDiagnosticsResult } from "@t3tools/contracts";
3import { Cause, Effect, Option } from "effect";
4import { AsyncResult, Atom } from "effect/unstable/reactivity";
5import { useCallback } from "react";
6
7import { ensureLocalApi } from "../localApi";
8import { appAtomRegistry } from "../rpc/atomRegistry";
9
10const TRACE_DIAGNOSTICS_STALE_TIME_MS = 5_000;
11const TRACE_DIAGNOSTICS_IDLE_TTL_MS = 5 * 60_000;
12
13const traceDiagnosticsAtom = Atom.make(
14 Effect.promise(() => ensureLocalApi().server.getTraceDiagnostics()),
15).pipe(
16 Atom.swr({
17 staleTime: TRACE_DIAGNOSTICS_STALE_TIME_MS,
18 revalidateOnMount: true,
19 }),
20 Atom.setIdleTTL(TRACE_DIAGNOSTICS_IDLE_TTL_MS),
21 Atom.withLabel("trace-diagnostics"),
22);
23
24export interface TraceDiagnosticsState {
25 readonly data: ServerTraceDiagnosticsResult | null;
26 readonly error: string | null;
27 readonly isPending: boolean;
28 readonly refresh: () => void;
29}
30
31function formatTraceDiagnosticsError(error: unknown): string {
32 return error instanceof Error ? error.message : "Failed to load trace diagnostics.";
33}
34
35function readTraceDiagnosticsError(
36 result: AsyncResult.AsyncResult<ServerTraceDiagnosticsResult, unknown>,
37): string | null {
38 if (result._tag !== "Failure") {
39 return null;
40 }
41
42 const squashed = Cause.squash(result.cause);
43 return formatTraceDiagnosticsError(squashed);
44}
45
46export function refreshTraceDiagnostics(): void {
47 appAtomRegistry.refresh(traceDiagnosticsAtom);
48}
49
50export function useTraceDiagnostics(): TraceDiagnosticsState {
51 const result = useAtomValue(traceDiagnosticsAtom);
52 const data = Option.getOrNull(AsyncResult.value(result));
53 const refresh = useCallback(() => {
54 refreshTraceDiagnostics();
55 }, []);
56
57 return {
58 data,
59 error: readTraceDiagnosticsError(result),
60 isPending: result.waiting,
61 refresh,
62 };
63}
apps/web/src/localApi.ts+12
......@@ -147,6 +147,18 @@ function createBrowserLocalApi(rpcClient?: WsRpcClient): LocalApi {
147147 rpcClient
148148 ? rpcClient.server.discoverSourceControl()
149149 : Promise.reject(unavailableLocalBackendError()),
150 getTraceDiagnostics: () =>
151 rpcClient
152 ? rpcClient.server.getTraceDiagnostics()
153 : Promise.reject(unavailableLocalBackendError()),
154 getProcessDiagnostics: () =>
155 rpcClient
156 ? rpcClient.server.getProcessDiagnostics()
157 : Promise.reject(unavailableLocalBackendError()),
158 signalProcess: (input) =>
159 rpcClient
160 ? rpcClient.server.signalProcess(input)
161 : Promise.reject(unavailableLocalBackendError()),
150162 },
151163 };
152164}
apps/web/src/routeTree.gen.ts+21
......@@ -17,6 +17,7 @@ import { Route as SettingsSourceControlRouteImport } from './routes/settings.sou
1717import { Route as SettingsProvidersRouteImport } from './routes/settings.providers'
1818import { Route as SettingsKeybindingsRouteImport } from './routes/settings.keybindings'
1919import { Route as SettingsGeneralRouteImport } from './routes/settings.general'
20import { Route as SettingsDiagnosticsRouteImport } from './routes/settings.diagnostics'
2021import { Route as SettingsConnectionsRouteImport } from './routes/settings.connections'
2122import { Route as SettingsArchivedRouteImport } from './routes/settings.archived'
2223import { Route as ChatDraftDraftIdRouteImport } from './routes/_chat.draft.$draftId'
......@@ -61,6 +62,11 @@ const SettingsGeneralRoute = SettingsGeneralRouteImport.update({
6162 path: '/general',
6263 getParentRoute: () => SettingsRoute,
6364} as any)
65const SettingsDiagnosticsRoute = SettingsDiagnosticsRouteImport.update({
66 id: '/diagnostics',
67 path: '/diagnostics',
68 getParentRoute: () => SettingsRoute,
69} as any)
6470const SettingsConnectionsRoute = SettingsConnectionsRouteImport.update({
6571 id: '/connections',
6672 path: '/connections',
......@@ -89,6 +95,7 @@ export interface FileRoutesByFullPath {
8995 '/settings': typeof SettingsRouteWithChildren
9096 '/settings/archived': typeof SettingsArchivedRoute
9197 '/settings/connections': typeof SettingsConnectionsRoute
98 '/settings/diagnostics': typeof SettingsDiagnosticsRoute
9299 '/settings/general': typeof SettingsGeneralRoute
93100 '/settings/keybindings': typeof SettingsKeybindingsRoute
94101 '/settings/providers': typeof SettingsProvidersRoute
......@@ -101,6 +108,7 @@ export interface FileRoutesByTo {
101108 '/settings': typeof SettingsRouteWithChildren
102109 '/settings/archived': typeof SettingsArchivedRoute
103110 '/settings/connections': typeof SettingsConnectionsRoute
111 '/settings/diagnostics': typeof SettingsDiagnosticsRoute
104112 '/settings/general': typeof SettingsGeneralRoute
105113 '/settings/keybindings': typeof SettingsKeybindingsRoute
106114 '/settings/providers': typeof SettingsProvidersRoute
......@@ -116,6 +124,7 @@ export interface FileRoutesById {
116124 '/settings': typeof SettingsRouteWithChildren
117125 '/settings/archived': typeof SettingsArchivedRoute
118126 '/settings/connections': typeof SettingsConnectionsRoute
127 '/settings/diagnostics': typeof SettingsDiagnosticsRoute
119128 '/settings/general': typeof SettingsGeneralRoute
120129 '/settings/keybindings': typeof SettingsKeybindingsRoute
121130 '/settings/providers': typeof SettingsProvidersRoute
......@@ -132,6 +141,7 @@ export interface FileRouteTypes {
132141 | '/settings'
133142 | '/settings/archived'
134143 | '/settings/connections'
144 | '/settings/diagnostics'
135145 | '/settings/general'
136146 | '/settings/keybindings'
137147 | '/settings/providers'
......@@ -144,6 +154,7 @@ export interface FileRouteTypes {
144154 | '/settings'
145155 | '/settings/archived'
146156 | '/settings/connections'
157 | '/settings/diagnostics'
147158 | '/settings/general'
148159 | '/settings/keybindings'
149160 | '/settings/providers'
......@@ -158,6 +169,7 @@ export interface FileRouteTypes {
158169 | '/settings'
159170 | '/settings/archived'
160171 | '/settings/connections'
172 | '/settings/diagnostics'
161173 | '/settings/general'
162174 | '/settings/keybindings'
163175 | '/settings/providers'
......@@ -231,6 +243,13 @@ declare module '@tanstack/react-router' {
231243 preLoaderRoute: typeof SettingsGeneralRouteImport
232244 parentRoute: typeof SettingsRoute
233245 }
246 '/settings/diagnostics': {
247 id: '/settings/diagnostics'
248 path: '/diagnostics'
249 fullPath: '/settings/diagnostics'
250 preLoaderRoute: typeof SettingsDiagnosticsRouteImport
251 parentRoute: typeof SettingsRoute
252 }
234253 '/settings/connections': {
235254 id: '/settings/connections'
236255 path: '/connections'
......@@ -279,6 +298,7 @@ const ChatRouteWithChildren = ChatRoute._addFileChildren(ChatRouteChildren)
279298interface SettingsRouteChildren {
280299 SettingsArchivedRoute: typeof SettingsArchivedRoute
281300 SettingsConnectionsRoute: typeof SettingsConnectionsRoute
301 SettingsDiagnosticsRoute: typeof SettingsDiagnosticsRoute
282302 SettingsGeneralRoute: typeof SettingsGeneralRoute
283303 SettingsKeybindingsRoute: typeof SettingsKeybindingsRoute
284304 SettingsProvidersRoute: typeof SettingsProvidersRoute
......@@ -288,6 +308,7 @@ interface SettingsRouteChildren {
288308const SettingsRouteChildren: SettingsRouteChildren = {
289309 SettingsArchivedRoute: SettingsArchivedRoute,
290310 SettingsConnectionsRoute: SettingsConnectionsRoute,
311 SettingsDiagnosticsRoute: SettingsDiagnosticsRoute,
291312 SettingsGeneralRoute: SettingsGeneralRoute,
292313 SettingsKeybindingsRoute: SettingsKeybindingsRoute,
293314 SettingsProvidersRoute: SettingsProvidersRoute,
apps/web/src/routes/settings.diagnostics.tsx created+7
......@@ -0,0 +1,7 @@
1import { createFileRoute } from "@tanstack/react-router";
2
3import { DiagnosticsSettingsPanel } from "../components/settings/DiagnosticsSettings";
4
5export const Route = createFileRoute("/settings/diagnostics")({
6 component: DiagnosticsSettingsPanel,
7});
apps/web/src/rpc/wsRpcClient.ts+17
......@@ -129,6 +129,11 @@ export interface WsRpcClient {
129129 readonly discoverSourceControl: RpcUnaryNoArgMethod<
130130 typeof WS_METHODS.serverDiscoverSourceControl
131131 >;
132 readonly getTraceDiagnostics: RpcUnaryNoArgMethod<typeof WS_METHODS.serverGetTraceDiagnostics>;
133 readonly getProcessDiagnostics: RpcUnaryNoArgMethod<
134 typeof WS_METHODS.serverGetProcessDiagnostics
135 >;
136 readonly signalProcess: RpcUnaryMethod<typeof WS_METHODS.serverSignalProcess>;
132137 readonly subscribeConfig: RpcStreamMethod<typeof WS_METHODS.subscribeServerConfig>;
133138 readonly subscribeLifecycle: RpcStreamMethod<typeof WS_METHODS.subscribeServerLifecycle>;
134139 readonly subscribeAuthAccess: RpcStreamMethod<typeof WS_METHODS.subscribeAuthAccess>;
......@@ -247,6 +252,18 @@ export function createWsRpcClient(transport: WsTransport): WsRpcClient {
247252 transport.request((client) => client[WS_METHODS.serverUpdateSettings]({ patch })),
248253 discoverSourceControl: () =>
249254 transport.request((client) => client[WS_METHODS.serverDiscoverSourceControl]({})),
255 getTraceDiagnostics: () =>
256 transport.request((client) =>
257 client[WS_METHODS.serverGetTraceDiagnostics]({}).pipe(Effect.withTracerEnabled(false)),
258 ),
259 getProcessDiagnostics: () =>
260 transport.request((client) =>
261 client[WS_METHODS.serverGetProcessDiagnostics]({}).pipe(Effect.withTracerEnabled(false)),
262 ),
263 signalProcess: (input) =>
264 transport.request((client) =>
265 client[WS_METHODS.serverSignalProcess](input).pipe(Effect.withTracerEnabled(false)),
266 ),
250267 subscribeConfig: (listener, options) =>
251268 transport.subscribe((client) => client[WS_METHODS.subscribeServerConfig]({}), listener, {
252269 ...options,
packages/contracts/src/ipc.ts+7
......@@ -28,9 +28,13 @@ import type {
2828import type { ProviderInstanceId } from "./providerInstance.ts";
2929import type {
3030 ServerConfig,
31 ServerProcessDiagnosticsResult,
3132 ServerProviderUpdateInput,
3233 ServerProviderUpdatedPayload,
3334 ServerRemoveKeybindingResult,
35 ServerSignalProcessInput,
36 ServerSignalProcessResult,
37 ServerTraceDiagnosticsResult,
3438 ServerUpsertKeybindingResult,
3539} from "./server.ts";
3640import type {
......@@ -304,6 +308,9 @@ export interface LocalApi {
304308 getSettings: () => Promise<ServerSettings>;
305309 updateSettings: (patch: ServerSettingsPatch) => Promise<ServerSettings>;
306310 discoverSourceControl: () => Promise<SourceControlDiscoveryResult>;
311 getTraceDiagnostics: () => Promise<ServerTraceDiagnosticsResult>;
312 getProcessDiagnostics: () => Promise<ServerProcessDiagnosticsResult>;
313 signalProcess: (input: ServerSignalProcessInput) => Promise<ServerSignalProcessResult>;
307314 };
308315}
309316
packages/contracts/src/rpc.ts+25
......@@ -77,6 +77,10 @@ import {
7777 ServerRemoveKeybindingInput,
7878 ServerRemoveKeybindingResult,
7979 ServerProviderUpdatedPayload,
80 ServerTraceDiagnosticsResult,
81 ServerProcessDiagnosticsResult,
82 ServerSignalProcessInput,
83 ServerSignalProcessResult,
8084 ServerUpsertKeybindingInput,
8185 ServerUpsertKeybindingResult,
8286} from "./server.ts";
......@@ -139,6 +143,9 @@ export const WS_METHODS = {
139143 serverGetSettings: "server.getSettings",
140144 serverUpdateSettings: "server.updateSettings",
141145 serverDiscoverSourceControl: "server.discoverSourceControl",
146 serverGetTraceDiagnostics: "server.getTraceDiagnostics",
147 serverGetProcessDiagnostics: "server.getProcessDiagnostics",
148 serverSignalProcess: "server.signalProcess",
142149
143150 // Source control methods
144151 sourceControlLookupRepository: "sourceControl.lookupRepository",
......@@ -207,6 +214,21 @@ export const WsServerDiscoverSourceControlRpc = Rpc.make(WS_METHODS.serverDiscov
207214 success: SourceControlDiscoveryResult,
208215});
209216
217export const WsServerGetTraceDiagnosticsRpc = Rpc.make(WS_METHODS.serverGetTraceDiagnostics, {
218 payload: Schema.Struct({}),
219 success: ServerTraceDiagnosticsResult,
220});
221
222export const WsServerGetProcessDiagnosticsRpc = Rpc.make(WS_METHODS.serverGetProcessDiagnostics, {
223 payload: Schema.Struct({}),
224 success: ServerProcessDiagnosticsResult,
225});
226
227export const WsServerSignalProcessRpc = Rpc.make(WS_METHODS.serverSignalProcess, {
228 payload: ServerSignalProcessInput,
229 success: ServerSignalProcessResult,
230});
231
210232export const WsSourceControlLookupRepositoryRpc = Rpc.make(
211233 WS_METHODS.sourceControlLookupRepository,
212234 {
......@@ -439,6 +461,9 @@ export const WsRpcGroup = RpcGroup.make(
439461 WsServerGetSettingsRpc,
440462 WsServerUpdateSettingsRpc,
441463 WsServerDiscoverSourceControlRpc,
464 WsServerGetTraceDiagnosticsRpc,
465 WsServerGetProcessDiagnosticsRpc,
466 WsServerSignalProcessRpc,
442467 WsSourceControlLookupRepositoryRpc,
443468 WsSourceControlCloneRepositoryRpc,
444469 WsSourceControlPublishRepositoryRpc,
packages/contracts/src/server.ts+130
......@@ -4,6 +4,7 @@ import { ServerAuthDescriptor } from "./auth.ts";
44import {
55 IsoDateTime,
66 NonNegativeInt,
7 PositiveInt,
78 ProjectId,
89 ThreadId,
910 TrimmedNonEmptyString,
......@@ -211,6 +212,135 @@ export const ServerObservability = Schema.Struct({
211212});
212213export type ServerObservability = typeof ServerObservability.Type;
213214
215export const ServerTraceDiagnosticsErrorKind = Schema.Literals([
216 "trace-file-not-found",
217 "trace-file-read-failed",
218]);
219export type ServerTraceDiagnosticsErrorKind = typeof ServerTraceDiagnosticsErrorKind.Type;
220
221export const ServerTraceDiagnosticsSpanSummary = Schema.Struct({
222 name: TrimmedNonEmptyString,
223 count: NonNegativeInt,
224 failureCount: NonNegativeInt,
225 totalDurationMs: Schema.Number,
226 averageDurationMs: Schema.Number,
227 maxDurationMs: Schema.Number,
228});
229export type ServerTraceDiagnosticsSpanSummary = typeof ServerTraceDiagnosticsSpanSummary.Type;
230
231export const ServerTraceDiagnosticsFailureSummary = Schema.Struct({
232 name: TrimmedNonEmptyString,
233 cause: TrimmedNonEmptyString,
234 count: NonNegativeInt,
235 lastSeenAt: Schema.DateTimeUtc,
236 traceId: TrimmedNonEmptyString,
237 spanId: TrimmedNonEmptyString,
238});
239export type ServerTraceDiagnosticsFailureSummary = typeof ServerTraceDiagnosticsFailureSummary.Type;
240
241export const ServerTraceDiagnosticsRecentFailure = Schema.Struct({
242 name: TrimmedNonEmptyString,
243 cause: TrimmedNonEmptyString,
244 durationMs: Schema.Number,
245 endedAt: Schema.DateTimeUtc,
246 traceId: TrimmedNonEmptyString,
247 spanId: TrimmedNonEmptyString,
248});
249export type ServerTraceDiagnosticsRecentFailure = typeof ServerTraceDiagnosticsRecentFailure.Type;
250
251export const ServerTraceDiagnosticsSpanOccurrence = Schema.Struct({
252 name: TrimmedNonEmptyString,
253 durationMs: Schema.Number,
254 endedAt: Schema.DateTimeUtc,
255 traceId: TrimmedNonEmptyString,
256 spanId: TrimmedNonEmptyString,
257});
258export type ServerTraceDiagnosticsSpanOccurrence = typeof ServerTraceDiagnosticsSpanOccurrence.Type;
259
260export const ServerTraceDiagnosticsLogEvent = Schema.Struct({
261 spanName: TrimmedNonEmptyString,
262 level: TrimmedNonEmptyString,
263 message: TrimmedNonEmptyString,
264 seenAt: Schema.DateTimeUtc,
265 traceId: TrimmedNonEmptyString,
266 spanId: TrimmedNonEmptyString,
267});
268export type ServerTraceDiagnosticsLogEvent = typeof ServerTraceDiagnosticsLogEvent.Type;
269
270export const ServerTraceDiagnosticsResult = Schema.Struct({
271 traceFilePath: TrimmedNonEmptyString,
272 scannedFilePaths: Schema.Array(TrimmedNonEmptyString),
273 readAt: Schema.DateTimeUtc,
274 recordCount: NonNegativeInt,
275 parseErrorCount: NonNegativeInt,
276 firstSpanAt: Schema.Option(Schema.DateTimeUtc),
277 lastSpanAt: Schema.Option(Schema.DateTimeUtc),
278 failureCount: NonNegativeInt,
279 interruptionCount: NonNegativeInt,
280 slowSpanThresholdMs: NonNegativeInt,
281 slowSpanCount: NonNegativeInt,
282 logLevelCounts: Schema.Record(TrimmedNonEmptyString, NonNegativeInt),
283 topSpansByCount: Schema.Array(ServerTraceDiagnosticsSpanSummary),
284 slowestSpans: Schema.Array(ServerTraceDiagnosticsSpanOccurrence),
285 commonFailures: Schema.Array(ServerTraceDiagnosticsFailureSummary),
286 latestFailures: Schema.Array(ServerTraceDiagnosticsRecentFailure),
287 latestWarningAndErrorLogs: Schema.Array(ServerTraceDiagnosticsLogEvent),
288 partialFailure: Schema.Option(Schema.Boolean),
289 error: Schema.Option(
290 Schema.Struct({
291 kind: ServerTraceDiagnosticsErrorKind,
292 message: TrimmedNonEmptyString,
293 }),
294 ),
295});
296export type ServerTraceDiagnosticsResult = typeof ServerTraceDiagnosticsResult.Type;
297
298export const ServerProcessSignal = Schema.Literals(["SIGINT", "SIGKILL"]);
299export type ServerProcessSignal = typeof ServerProcessSignal.Type;
300
301export const ServerProcessDiagnosticsEntry = Schema.Struct({
302 pid: PositiveInt,
303 ppid: NonNegativeInt,
304 pgid: Schema.Option(Schema.Int),
305 status: TrimmedNonEmptyString,
306 cpuPercent: Schema.Number,
307 rssBytes: NonNegativeInt,
308 elapsed: TrimmedNonEmptyString,
309 command: TrimmedNonEmptyString,
310 depth: NonNegativeInt,
311 childPids: Schema.Array(PositiveInt),
312});
313export type ServerProcessDiagnosticsEntry = typeof ServerProcessDiagnosticsEntry.Type;
314
315export const ServerProcessDiagnosticsResult = Schema.Struct({
316 serverPid: PositiveInt,
317 readAt: Schema.DateTimeUtc,
318 processCount: NonNegativeInt,
319 totalRssBytes: NonNegativeInt,
320 totalCpuPercent: Schema.Number,
321 processes: Schema.Array(ServerProcessDiagnosticsEntry),
322 error: Schema.Option(
323 Schema.Struct({
324 message: TrimmedNonEmptyString,
325 }),
326 ),
327});
328export type ServerProcessDiagnosticsResult = typeof ServerProcessDiagnosticsResult.Type;
329
330export const ServerSignalProcessInput = Schema.Struct({
331 pid: PositiveInt,
332 signal: ServerProcessSignal,
333});
334export type ServerSignalProcessInput = typeof ServerSignalProcessInput.Type;
335
336export const ServerSignalProcessResult = Schema.Struct({
337 pid: PositiveInt,
338 signal: ServerProcessSignal,
339 signaled: Schema.Boolean,
340 message: Schema.Option(TrimmedNonEmptyString),
341});
342export type ServerSignalProcessResult = typeof ServerSignalProcessResult.Type;
343
214344export const ServerConfig = Schema.Struct({
215345 environment: ExecutionEnvironmentDescriptor,
216346 auth: ServerAuthDescriptor,