authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-05-14 09:14:32+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-05-14 09:14:32+02:00
log9e632f5cec7315d65c2df4d3bc7be49bfe1385f3
tree6c8cebf5f4621e9fc4dfd66ed16be3c8d5723821
parent4120e94598c7e4237ce9ecfd53d0449eeb0e0d40
signature Signed by PGP key B5690EEEBB952194

[codex] add diagnostics resource history (#2685)

Co-authored-by: Julius Marminge <julius@macmini.local>

15 files changed, 1075 insertions(+), 13 deletions(-)

apps/server/src/diagnostics/ProcessDiagnostics.ts+4-4
...@@ -15,7 +15,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";...@@ -15,7 +15,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
1515
16import { collectUint8StreamText } from "../stream/collectUint8StreamText.ts";16import { collectUint8StreamText } from "../stream/collectUint8StreamText.ts";
1717
18interface ProcessRow {18export interface ProcessRow {
19 readonly pid: number;19 readonly pid: number;
20 readonly ppid: number;20 readonly ppid: number;
21 readonly pgid: number | null;21 readonly pgid: number | null;
...@@ -186,7 +186,7 @@ function parseWindowsProcessRows(output: string): ReadonlyArray<ProcessRow> {...@@ -186,7 +186,7 @@ function parseWindowsProcessRows(output: string): ReadonlyArray<ProcessRow> {
186 }186 }
187}187}
188188
189function buildDescendantEntries(189export function buildDescendantEntries(
190 rows: ReadonlyArray<ProcessRow>,190 rows: ReadonlyArray<ProcessRow>,
191 serverPid: number,191 serverPid: number,
192): ReadonlyArray<ServerProcessDiagnosticsEntry> {192): ReadonlyArray<ServerProcessDiagnosticsEntry> {
...@@ -230,7 +230,7 @@ function buildDescendantEntries(...@@ -230,7 +230,7 @@ function buildDescendantEntries(
230 return entries;230 return entries;
231}231}
232232
233function isDiagnosticsQueryProcess(row: ProcessRow, serverPid: number): boolean {233export function isDiagnosticsQueryProcess(row: ProcessRow, serverPid: number): boolean {
234 if (row.ppid !== serverPid) return false;234 if (row.ppid !== serverPid) return false;
235235
236 const command = row.command.trim();236 const command = row.command.trim();
...@@ -370,7 +370,7 @@ function readWindowsProcessRows(): Effect.Effect<...@@ -370,7 +370,7 @@ function readWindowsProcessRows(): Effect.Effect<
370 );370 );
371}371}
372372
373const readProcessRows = (platform = process.platform) =>373export const readProcessRows = (platform = process.platform) =>
374 platform === "win32" ? readWindowsProcessRows() : readPosixProcessRows();374 platform === "win32" ? readWindowsProcessRows() : readPosixProcessRows();
375375
376export function aggregateProcessDiagnostics(input: {376export function aggregateProcessDiagnostics(input: {
apps/server/src/diagnostics/ProcessResourceMonitor.test.ts created+231
...@@ -0,0 +1,231 @@
1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Option from "effect/Option";
5
6import {
7 aggregateProcessResourceHistory,
8 collectMonitoredSamples,
9} from "./ProcessResourceMonitor.ts";
10
11describe("ProcessResourceMonitor", () => {
12 it.effect("samples the server root process and descendants", () =>
13 Effect.sync(() => {
14 const sampledAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
15 const samples = collectMonitoredSamples({
16 serverPid: 100,
17 sampledAt,
18 sampledAtMs: DateTime.toEpochMillis(sampledAt),
19 rows: [
20 {
21 pid: 100,
22 ppid: 1,
23 pgid: 100,
24 status: "S",
25 cpuPercent: 2,
26 rssBytes: 1_000,
27 elapsed: "01:00",
28 command: "t3 server",
29 },
30 {
31 pid: 101,
32 ppid: 100,
33 pgid: 100,
34 status: "S",
35 cpuPercent: 10,
36 rssBytes: 2_000,
37 elapsed: "00:20",
38 command: "codex app-server",
39 },
40 {
41 pid: 102,
42 ppid: 101,
43 pgid: 100,
44 status: "R",
45 cpuPercent: 50,
46 rssBytes: 3_000,
47 elapsed: "00:05",
48 command: "rg needle",
49 },
50 {
51 pid: 200,
52 ppid: 1,
53 pgid: 200,
54 status: "R",
55 cpuPercent: 99,
56 rssBytes: 9_000,
57 elapsed: "00:05",
58 command: "unrelated",
59 },
60 ],
61 });
62
63 expect(samples.map((sample) => sample.pid)).toEqual([100, 101, 102]);
64 expect(samples.map((sample) => sample.depth)).toEqual([0, 1, 2]);
65 expect(samples[0]?.isServerRoot).toBe(true);
66 expect(samples[1]?.isServerRoot).toBe(false);
67 }),
68 );
69
70 it.effect("rolls samples up by process and CPU time", () =>
71 Effect.sync(() => {
72 const firstAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
73 const secondAt = DateTime.makeUnsafe("2026-05-05T10:00:05.000Z");
74 const samples = [
75 ...collectMonitoredSamples({
76 serverPid: 100,
77 sampledAt: firstAt,
78 sampledAtMs: DateTime.toEpochMillis(firstAt),
79 rows: [
80 {
81 pid: 100,
82 ppid: 1,
83 pgid: 100,
84 status: "S",
85 cpuPercent: 10,
86 rssBytes: 1_000,
87 elapsed: "01:00",
88 command: "t3 server",
89 },
90 ],
91 }),
92 ...collectMonitoredSamples({
93 serverPid: 100,
94 sampledAt: secondAt,
95 sampledAtMs: DateTime.toEpochMillis(secondAt),
96 rows: [
97 {
98 pid: 100,
99 ppid: 1,
100 pgid: 100,
101 status: "S",
102 cpuPercent: 30,
103 rssBytes: 2_000,
104 elapsed: "01:05",
105 command: "t3 server",
106 },
107 ],
108 }),
109 ];
110
111 const result = aggregateProcessResourceHistory({
112 samples,
113 readAt: secondAt,
114 readAtMs: DateTime.toEpochMillis(secondAt),
115 windowMs: 60_000,
116 bucketMs: 10_000,
117 lastError: null,
118 });
119
120 expect(Option.isNone(result.error)).toBe(true);
121 expect(result.topProcesses).toHaveLength(1);
122 expect(result.topProcesses[0]?.avgCpuPercent).toBe(20);
123 expect(result.topProcesses[0]?.maxCpuPercent).toBe(30);
124 expect(result.topProcesses[0]?.cpuSecondsApprox).toBe(2);
125 expect(result.totalCpuSecondsApprox).toBe(2);
126 expect(result.buckets.some((bucket) => bucket.maxCpuPercent === 30)).toBe(true);
127 }),
128 );
129
130 it.effect("keeps a process grouped when elapsed time drifts between samples", () =>
131 Effect.sync(() => {
132 const firstAt = DateTime.makeUnsafe("2026-05-05T10:00:00.400Z");
133 const secondAt = DateTime.makeUnsafe("2026-05-05T10:00:05.900Z");
134 const samples = [
135 ...collectMonitoredSamples({
136 serverPid: 100,
137 sampledAt: firstAt,
138 sampledAtMs: DateTime.toEpochMillis(firstAt),
139 rows: [
140 {
141 pid: 100,
142 ppid: 1,
143 pgid: 100,
144 status: "S",
145 cpuPercent: 1,
146 rssBytes: 1_000,
147 elapsed: "01:00",
148 command: "t3 server",
149 },
150 ],
151 }),
152 ...collectMonitoredSamples({
153 serverPid: 100,
154 sampledAt: secondAt,
155 sampledAtMs: DateTime.toEpochMillis(secondAt),
156 rows: [
157 {
158 pid: 100,
159 ppid: 1,
160 pgid: 100,
161 status: "S",
162 cpuPercent: 2,
163 rssBytes: 2_000,
164 elapsed: "01:06",
165 command: "t3 server",
166 },
167 ],
168 }),
169 ];
170
171 const result = aggregateProcessResourceHistory({
172 samples,
173 readAt: secondAt,
174 readAtMs: DateTime.toEpochMillis(secondAt),
175 windowMs: 60_000,
176 bucketMs: 10_000,
177 lastError: null,
178 });
179
180 expect(result.topProcesses).toHaveLength(1);
181 expect(result.topProcesses[0]?.isServerRoot).toBe(true);
182 expect(result.topProcesses[0]?.sampleCount).toBe(2);
183 expect(result.topProcesses[0]?.maxRssBytes).toBe(2_000);
184 }),
185 );
186
187 it.effect("returns all process summaries in the selected window", () =>
188 Effect.sync(() => {
189 const sampledAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
190 const samples = collectMonitoredSamples({
191 serverPid: 100,
192 sampledAt,
193 sampledAtMs: DateTime.toEpochMillis(sampledAt),
194 rows: [
195 {
196 pid: 100,
197 ppid: 1,
198 pgid: 100,
199 status: "S",
200 cpuPercent: 1,
201 rssBytes: 1_000,
202 elapsed: "01:00",
203 command: "t3 server",
204 },
205 ...Array.from({ length: 35 }, (_, index) => ({
206 pid: 200 + index,
207 ppid: index === 0 ? 100 : 199 + index,
208 pgid: 100,
209 status: "S",
210 cpuPercent: 35 - index,
211 rssBytes: 2_000 + index,
212 elapsed: "00:10",
213 command: `worker ${index}`,
214 })),
215 ],
216 });
217
218 const result = aggregateProcessResourceHistory({
219 samples,
220 readAt: sampledAt,
221 readAtMs: DateTime.toEpochMillis(sampledAt),
222 windowMs: 60_000,
223 bucketMs: 10_000,
224 lastError: null,
225 });
226
227 expect(result.topProcesses).toHaveLength(36);
228 expect(result.topProcesses.some((process) => process.command === "worker 34")).toBe(true);
229 }),
230 );
231});
apps/server/src/diagnostics/ProcessResourceMonitor.ts created+299
...@@ -0,0 +1,299 @@
1import type {
2 ServerProcessResourceHistoryBucket,
3 ServerProcessResourceHistoryInput,
4 ServerProcessResourceHistoryResult,
5 ServerProcessResourceHistorySummary,
6} from "@t3tools/contracts";
7import * as Context from "effect/Context";
8import * as DateTime from "effect/DateTime";
9import * as Effect from "effect/Effect";
10import * as Layer from "effect/Layer";
11import * as Option from "effect/Option";
12import * as Ref from "effect/Ref";
13import { ChildProcessSpawner } from "effect/unstable/process";
14
15import {
16 buildDescendantEntries,
17 isDiagnosticsQueryProcess,
18 type ProcessRow,
19 readProcessRows,
20} from "./ProcessDiagnostics.ts";
21
22const SAMPLE_INTERVAL_MS = 5_000;
23const RETENTION_MS = 60 * 60_000;
24const MAX_RETAINED_SAMPLES = 20_000;
25
26export interface ProcessResourceSample {
27 readonly sampledAt: DateTime.Utc;
28 readonly sampledAtMs: number;
29 readonly processKey: string;
30 readonly pid: number;
31 readonly ppid: number;
32 readonly command: string;
33 readonly cpuPercent: number;
34 readonly rssBytes: number;
35 readonly depth: number;
36 readonly isServerRoot: boolean;
37}
38
39interface MonitorState {
40 readonly samples: ReadonlyArray<ProcessResourceSample>;
41 readonly lastError: string | null;
42}
43
44export interface ProcessResourceMonitorShape {
45 readonly readHistory: (
46 input: ServerProcessResourceHistoryInput,
47 ) => Effect.Effect<ServerProcessResourceHistoryResult>;
48}
49
50export class ProcessResourceMonitor extends Context.Service<
51 ProcessResourceMonitor,
52 ProcessResourceMonitorShape
53>()("t3/diagnostics/ProcessResourceMonitor") {}
54
55function dateTimeFromMillis(ms: number): DateTime.Utc {
56 return DateTime.makeUnsafe(ms);
57}
58
59function sampleKey(row: Pick<ProcessRow, "pid" | "command">): string {
60 return `${row.pid}:${row.command}`;
61}
62
63function findServerRootRow(rows: ReadonlyArray<ProcessRow>, serverPid: number): ProcessRow | null {
64 return rows.find((row) => row.pid === serverPid) ?? null;
65}
66
67export function collectMonitoredSamples(input: {
68 readonly rows: ReadonlyArray<ProcessRow>;
69 readonly serverPid: number;
70 readonly sampledAt: DateTime.Utc;
71 readonly sampledAtMs: number;
72}): ReadonlyArray<ProcessResourceSample> {
73 const rows = input.rows.filter((row) => !isDiagnosticsQueryProcess(row, input.serverPid));
74 const root = findServerRootRow(rows, input.serverPid);
75 const descendants = buildDescendantEntries(rows, input.serverPid);
76 const samples: ProcessResourceSample[] = [];
77
78 if (root) {
79 samples.push({
80 sampledAt: input.sampledAt,
81 sampledAtMs: input.sampledAtMs,
82 processKey: sampleKey(root),
83 pid: root.pid,
84 ppid: root.ppid,
85 command: root.command,
86 cpuPercent: root.cpuPercent,
87 rssBytes: root.rssBytes,
88 depth: 0,
89 isServerRoot: true,
90 });
91 }
92
93 for (const process of descendants) {
94 samples.push({
95 sampledAt: input.sampledAt,
96 sampledAtMs: input.sampledAtMs,
97 processKey: sampleKey(process),
98 pid: process.pid,
99 ppid: process.ppid,
100 command: process.command,
101 cpuPercent: process.cpuPercent,
102 rssBytes: process.rssBytes,
103 depth: process.depth + 1,
104 isServerRoot: false,
105 });
106 }
107
108 return samples;
109}
110
111function trimSamples(
112 samples: ReadonlyArray<ProcessResourceSample>,
113 nowMs: number,
114): ReadonlyArray<ProcessResourceSample> {
115 const minSampledAtMs = nowMs - RETENTION_MS;
116 const retained = samples.filter((sample) => sample.sampledAtMs >= minSampledAtMs);
117 return retained.length <= MAX_RETAINED_SAMPLES
118 ? retained
119 : retained.slice(retained.length - MAX_RETAINED_SAMPLES);
120}
121
122function summarizeProcesses(
123 samples: ReadonlyArray<ProcessResourceSample>,
124): ReadonlyArray<ServerProcessResourceHistorySummary> {
125 const groups = new Map<string, ProcessResourceSample[]>();
126 for (const sample of samples) {
127 const processSamples = groups.get(sample.processKey) ?? [];
128 processSamples.push(sample);
129 groups.set(sample.processKey, processSamples);
130 }
131
132 return [...groups.entries()]
133 .map(([processKey, processSamples]) => {
134 const sorted = processSamples.toSorted((left, right) => left.sampledAtMs - right.sampledAtMs);
135 const first = sorted[0]!;
136 const latest = sorted[sorted.length - 1]!;
137 const cpuPercentTotal = sorted.reduce((total, sample) => total + sample.cpuPercent, 0);
138 const maxCpuPercent = Math.max(...sorted.map((sample) => sample.cpuPercent));
139 const maxRssBytes = Math.max(...sorted.map((sample) => sample.rssBytes));
140 const cpuSecondsApprox = sorted.reduce(
141 (total, sample) => total + (sample.cpuPercent / 100) * (SAMPLE_INTERVAL_MS / 1_000),
142 0,
143 );
144
145 return {
146 processKey,
147 pid: latest.pid,
148 ppid: latest.ppid,
149 command: latest.command,
150 depth: latest.depth,
151 isServerRoot: latest.isServerRoot,
152 firstSeenAt: first.sampledAt,
153 lastSeenAt: latest.sampledAt,
154 currentCpuPercent: latest.cpuPercent,
155 avgCpuPercent: cpuPercentTotal / sorted.length,
156 maxCpuPercent,
157 cpuSecondsApprox,
158 currentRssBytes: latest.rssBytes,
159 maxRssBytes,
160 sampleCount: sorted.length,
161 } satisfies ServerProcessResourceHistorySummary;
162 })
163 .toSorted((left, right) => right.cpuSecondsApprox - left.cpuSecondsApprox);
164}
165
166function buildBuckets(input: {
167 readonly samples: ReadonlyArray<ProcessResourceSample>;
168 readonly nowMs: number;
169 readonly windowMs: number;
170 readonly bucketMs: number;
171}): ReadonlyArray<ServerProcessResourceHistoryBucket> {
172 const bucketMs = Math.max(1_000, input.bucketMs);
173 const windowStartMs = input.nowMs - input.windowMs;
174 const buckets: ServerProcessResourceHistoryBucket[] = [];
175
176 for (let startedAtMs = windowStartMs; startedAtMs < input.nowMs; startedAtMs += bucketMs) {
177 const endedAtMs = Math.min(input.nowMs, startedAtMs + bucketMs);
178 const bucketSamples = input.samples.filter(
179 (sample) =>
180 sample.sampledAtMs >= startedAtMs &&
181 (endedAtMs === input.nowMs
182 ? sample.sampledAtMs <= endedAtMs
183 : sample.sampledAtMs < endedAtMs),
184 );
185 const samplesByRead = new Map<number, ProcessResourceSample[]>();
186 for (const sample of bucketSamples) {
187 const samplesAtTime = samplesByRead.get(sample.sampledAtMs) ?? [];
188 samplesAtTime.push(sample);
189 samplesByRead.set(sample.sampledAtMs, samplesAtTime);
190 }
191
192 const readTotals = [...samplesByRead.values()].map((samplesAtTime) => ({
193 cpuPercent: samplesAtTime.reduce((total, sample) => total + sample.cpuPercent, 0),
194 rssBytes: samplesAtTime.reduce((total, sample) => total + sample.rssBytes, 0),
195 processCount: samplesAtTime.length,
196 }));
197 const avgCpuPercent =
198 readTotals.length === 0
199 ? 0
200 : readTotals.reduce((total, read) => total + read.cpuPercent, 0) / readTotals.length;
201
202 buckets.push({
203 startedAt: dateTimeFromMillis(startedAtMs),
204 endedAt: dateTimeFromMillis(endedAtMs),
205 avgCpuPercent,
206 maxCpuPercent: readTotals.length ? Math.max(...readTotals.map((read) => read.cpuPercent)) : 0,
207 maxRssBytes: readTotals.length ? Math.max(...readTotals.map((read) => read.rssBytes)) : 0,
208 maxProcessCount: readTotals.length
209 ? Math.max(...readTotals.map((read) => read.processCount))
210 : 0,
211 });
212 }
213
214 return buckets;
215}
216
217export function aggregateProcessResourceHistory(input: {
218 readonly samples: ReadonlyArray<ProcessResourceSample>;
219 readonly readAt: DateTime.Utc;
220 readonly readAtMs: number;
221 readonly windowMs: number;
222 readonly bucketMs: number;
223 readonly lastError: string | null;
224}): ServerProcessResourceHistoryResult {
225 const windowMs = Math.max(1_000, input.windowMs);
226 const bucketMs = Math.max(1_000, input.bucketMs);
227 const minSampledAtMs = input.readAtMs - windowMs;
228 const samples = input.samples.filter((sample) => sample.sampledAtMs >= minSampledAtMs);
229 const topProcesses = summarizeProcesses(samples);
230 const totalCpuSecondsApprox = samples.reduce(
231 (total, sample) => total + (sample.cpuPercent / 100) * (SAMPLE_INTERVAL_MS / 1_000),
232 0,
233 );
234
235 return {
236 readAt: input.readAt,
237 windowMs,
238 bucketMs,
239 sampleIntervalMs: SAMPLE_INTERVAL_MS,
240 retainedSampleCount: input.samples.length,
241 totalCpuSecondsApprox,
242 buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }),
243 topProcesses,
244 error: input.lastError ? Option.some({ message: input.lastError }) : Option.none(),
245 };
246}
247
248export const make = Effect.fn("makeProcessResourceMonitor")(function* () {
249 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
250 const state = yield* Ref.make<MonitorState>({ samples: [], lastError: null });
251
252 const sampleOnce = Effect.gen(function* () {
253 const sampledAt = yield* DateTime.now;
254 const sampledAtMs = DateTime.toEpochMillis(sampledAt);
255 const rows = yield* readProcessRows().pipe(
256 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
257 );
258 const samples = collectMonitoredSamples({
259 rows,
260 serverPid: process.pid,
261 sampledAt,
262 sampledAtMs,
263 });
264 yield* Ref.update(state, (current) => ({
265 samples: trimSamples([...current.samples, ...samples], sampledAtMs),
266 lastError: null,
267 }));
268 }).pipe(
269 Effect.catch((error: unknown) =>
270 Ref.update(state, (current) => ({
271 ...current,
272 lastError: error instanceof Error ? error.message : "Failed to sample process resources.",
273 })),
274 ),
275 );
276
277 yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe(
278 Effect.forkScoped,
279 );
280
281 const readHistory: ProcessResourceMonitorShape["readHistory"] = (input) =>
282 Effect.gen(function* () {
283 const readAt = yield* DateTime.now;
284 const readAtMs = DateTime.toEpochMillis(readAt);
285 const current = yield* Ref.get(state);
286 return aggregateProcessResourceHistory({
287 samples: current.samples,
288 readAt,
289 readAtMs,
290 windowMs: input.windowMs,
291 bucketMs: input.bucketMs,
292 lastError: current.lastError,
293 });
294 });
295
296 return ProcessResourceMonitor.of({ readHistory });
297});
298
299export const layer = Layer.effect(ProcessResourceMonitor, make());
apps/server/src/observability/RpcInstrumentation.ts+1
...@@ -18,6 +18,7 @@ const DEFAULT_RPC_SPAN_ATTRIBUTES = {...@@ -18,6 +18,7 @@ const DEFAULT_RPC_SPAN_ATTRIBUTES = {
18const RPC_METHODS_WITH_TRACING_DISABLED: ReadonlySet<string> = new Set([18const RPC_METHODS_WITH_TRACING_DISABLED: ReadonlySet<string> = new Set([
19 WS_METHODS.serverGetTraceDiagnostics,19 WS_METHODS.serverGetTraceDiagnostics,
20 WS_METHODS.serverGetProcessDiagnostics,20 WS_METHODS.serverGetProcessDiagnostics,
21 WS_METHODS.serverGetProcessResourceHistory,
21 WS_METHODS.serverSignalProcess,22 WS_METHODS.serverSignalProcess,
22]);23]);
2324
apps/server/src/server.test.ts+17
...@@ -115,6 +115,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe...@@ -115,6 +115,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe
115import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";115import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";
116import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";116import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";
117import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";117import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
118import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts";
118import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";119import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
119import * as Data from "effect/Data";120import * as Data from "effect/Data";
120121
...@@ -564,6 +565,22 @@ const buildAppUnderTest = (options?: {...@@ -564,6 +565,22 @@ const buildAppUnderTest = (options?: {
564 }),565 }),
565 }),566 }),
566 ),567 ),
568 Layer.provide(
569 Layer.mock(ProcessResourceMonitor.ProcessResourceMonitor)({
570 readHistory: (input) =>
571 Effect.succeed({
572 readAt: TEST_EPOCH,
573 windowMs: input.windowMs,
574 bucketMs: input.bucketMs,
575 sampleIntervalMs: 5_000,
576 retainedSampleCount: 0,
577 totalCpuSecondsApprox: 0,
578 buckets: [],
579 topProcesses: [],
580 error: Option.none(),
581 }),
582 }),
583 ),
567 Layer.provide(584 Layer.provide(
568 Layer.mock(TraceDiagnostics.TraceDiagnostics)({585 Layer.mock(TraceDiagnostics.TraceDiagnostics)({
569 read: () =>586 read: () =>
apps/server/src/server.ts+2
...@@ -76,6 +76,7 @@ import {...@@ -76,6 +76,7 @@ import {
76import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";76import { ServerSecretStoreLive } from "./auth/Layers/ServerSecretStore.ts";
77import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";77import { ServerAuthLive } from "./auth/Layers/ServerAuth.ts";
78import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";78import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
79import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts";
79import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";80import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
80import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts";81import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts";
81import {82import {
...@@ -280,6 +281,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(...@@ -280,6 +281,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(
280const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe(281const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe(
281 // Misc.282 // Misc.
282 Layer.provideMerge(ProcessDiagnostics.layer),283 Layer.provideMerge(ProcessDiagnostics.layer),
284 Layer.provideMerge(ProcessResourceMonitor.layer),
283 Layer.provideMerge(TraceDiagnostics.layer),285 Layer.provideMerge(TraceDiagnostics.layer),
284 Layer.provideMerge(AnalyticsServiceLayerLive),286 Layer.provideMerge(AnalyticsServiceLayerLive),
285 Layer.provideMerge(ExternalLauncher.layer),287 Layer.provideMerge(ExternalLauncher.layer),
apps/server/src/ws.ts+10
...@@ -66,6 +66,7 @@ import { RepositoryIdentityResolver } from "./project/Services/RepositoryIdentit...@@ -66,6 +66,7 @@ import { RepositoryIdentityResolver } from "./project/Services/RepositoryIdentit
66import { ServerEnvironment } from "./environment/Services/ServerEnvironment.ts";66import { ServerEnvironment } from "./environment/Services/ServerEnvironment.ts";
67import { ServerAuth } from "./auth/Services/ServerAuth.ts";67import { ServerAuth } from "./auth/Services/ServerAuth.ts";
68import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";68import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts";
69import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts";
69import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";70import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
70import * as SourceControlDiscoveryLayer from "./sourceControl/SourceControlDiscovery.ts";71import * as SourceControlDiscoveryLayer from "./sourceControl/SourceControlDiscovery.ts";
71import { SourceControlRepositoryService } from "./sourceControl/SourceControlRepositoryService.ts";72import { SourceControlRepositoryService } from "./sourceControl/SourceControlRepositoryService.ts";
...@@ -193,6 +194,7 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>...@@ -193,6 +194,7 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>
193 const bootstrapCredentials = yield* BootstrapCredentialService;194 const bootstrapCredentials = yield* BootstrapCredentialService;
194 const sessions = yield* SessionCredentialService;195 const sessions = yield* SessionCredentialService;
195 const processDiagnostics = yield* ProcessDiagnostics.ProcessDiagnostics;196 const processDiagnostics = yield* ProcessDiagnostics.ProcessDiagnostics;
197 const processResourceMonitor = yield* ProcessResourceMonitor.ProcessResourceMonitor;
196 const serverCommandId = (tag: string) =>198 const serverCommandId = (tag: string) =>
197 CommandId.make(`server:${tag}:${crypto.randomUUID()}`);199 CommandId.make(`server:${tag}:${crypto.randomUUID()}`);
198200
...@@ -909,6 +911,14 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>...@@ -909,6 +911,14 @@ const makeWsRpcLayer = (currentSessionId: AuthSessionId) =>
909 observeRpcEffect(WS_METHODS.serverGetProcessDiagnostics, processDiagnostics.read, {911 observeRpcEffect(WS_METHODS.serverGetProcessDiagnostics, processDiagnostics.read, {
910 "rpc.aggregate": "server",912 "rpc.aggregate": "server",
911 }),913 }),
914 [WS_METHODS.serverGetProcessResourceHistory]: (input) =>
915 observeRpcEffect(
916 WS_METHODS.serverGetProcessResourceHistory,
917 processResourceMonitor.readHistory(input),
918 {
919 "rpc.aggregate": "server",
920 },
921 ),
912 [WS_METHODS.serverSignalProcess]: (input) =>922 [WS_METHODS.serverSignalProcess]: (input) =>
913 observeRpcEffect(WS_METHODS.serverSignalProcess, processDiagnostics.signal(input), {923 observeRpcEffect(WS_METHODS.serverSignalProcess, processDiagnostics.signal(input), {
914 "rpc.aggregate": "server",924 "rpc.aggregate": "server",
apps/web/src/components/settings/DiagnosticsSettings.tsx+335-8
...@@ -8,7 +8,11 @@ import {...@@ -8,7 +8,11 @@ import {
8 RefreshCwIcon,8 RefreshCwIcon,
9} from "lucide-react";9} from "lucide-react";
10import { useCallback, useMemo, useState, type ReactNode } from "react";10import { useCallback, useMemo, useState, type ReactNode } from "react";
11import type { ServerProcessDiagnosticsEntry, ServerProcessSignal } from "@t3tools/contracts";11import type {
12 ServerProcessDiagnosticsEntry,
13 ServerProcessResourceHistorySummary,
14 ServerProcessSignal,
15} from "@t3tools/contracts";
12import * as DateTime from "effect/DateTime";16import * as DateTime from "effect/DateTime";
13import * as Option from "effect/Option";17import * as Option from "effect/Option";
1418
...@@ -17,7 +21,10 @@ import { cn } from "../../lib/utils";...@@ -17,7 +21,10 @@ import { cn } from "../../lib/utils";
17import { resolveAndPersistPreferredEditor } from "../../editorPreferences";21import { resolveAndPersistPreferredEditor } from "../../editorPreferences";
18import { formatRelativeTime } from "../../timestampFormat";22import { formatRelativeTime } from "../../timestampFormat";
19import { useServerAvailableEditors, useServerObservability } from "../../rpc/serverState";23import { useServerAvailableEditors, useServerObservability } from "../../rpc/serverState";
20import { useProcessDiagnostics } from "../../lib/processDiagnosticsState";24import {
25 useProcessDiagnostics,
26 useProcessResourceHistory,
27} from "../../lib/processDiagnosticsState";
21import { useTraceDiagnostics } from "../../lib/traceDiagnosticsState";28import { useTraceDiagnostics } from "../../lib/traceDiagnosticsState";
22import { Button } from "../ui/button";29import { Button } from "../ui/button";
23import { ScrollArea } from "../ui/scroll-area";30import { ScrollArea } from "../ui/scroll-area";
...@@ -309,7 +316,7 @@ function ProcessNameCell({...@@ -309,7 +316,7 @@ function ProcessNameCell({
309 return (316 return (
310 <div317 <div
311 className="grid min-w-0 grid-cols-[1.25rem_0.375rem_minmax(0,1fr)] items-center gap-2"318 className="grid min-w-0 grid-cols-[1.25rem_0.375rem_minmax(0,1fr)] items-center gap-2"
312 style={{ paddingLeft: `${Math.min(process.depth, 8) * 16}px` }}319 style={{ paddingLeft: `${Math.min(process.depth, 6) * 10}px` }}
313 >320 >
314 {hasChildren ? (321 {hasChildren ? (
315 <button322 <button
...@@ -324,7 +331,17 @@ function ProcessNameCell({...@@ -324,7 +331,17 @@ function ProcessNameCell({
324 <span className="size-5 shrink-0" aria-hidden="true" />331 <span className="size-5 shrink-0" aria-hidden="true" />
325 )}332 )}
326 <span className="size-1.5 shrink-0 rounded-full bg-emerald-500/80" />333 <span className="size-1.5 shrink-0 rounded-full bg-emerald-500/80" />
327 <span className="min-w-0 truncate font-medium text-foreground">{name}</span>334 <Tooltip>
335 <TooltipTrigger
336 render={<span className="min-w-0 truncate font-medium text-foreground">{name}</span>}
337 />
338 <TooltipPopup
339 side="top"
340 className="max-w-[min(440px,calc(100vw-2rem))] whitespace-normal break-words text-left font-mono text-[11px] leading-relaxed text-wrap"
341 >
342 {process.command}
343 </TooltipPopup>
344 </Tooltip>
328 </div>345 </div>
329 );346 );
330}347}
...@@ -422,7 +439,7 @@ function ProcessDiagnosticsTable({...@@ -422,7 +439,7 @@ function ProcessDiagnosticsTable({
422 chainVerticalScroll439 chainVerticalScroll
423 scrollFade440 scrollFade
424 hideScrollbars441 hideScrollbars
425 className="w-full max-w-full rounded-none border-t border-border/60"442 className="max-h-[min(64vh,44rem)] w-full max-w-full rounded-none border-t border-border/60"
426 >443 >
427 <table className="w-full min-w-[1040px] table-fixed text-left text-xs">444 <table className="w-full min-w-[1040px] table-fixed text-left text-xs">
428 <colgroup>445 <colgroup>
...@@ -434,7 +451,7 @@ function ProcessDiagnosticsTable({...@@ -434,7 +451,7 @@ function ProcessDiagnosticsTable({
434 <col className="w-[11%]" />451 <col className="w-[11%]" />
435 <col className="w-[6%]" />452 <col className="w-[6%]" />
436 </colgroup>453 </colgroup>
437 <thead className="border-b border-border/60 bg-muted/15 text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">454 <thead className="sticky top-0 z-10 border-b border-border/60 bg-card text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">
438 <tr>455 <tr>
439 <th className="px-4 py-2 font-semibold sm:pl-5">Name</th>456 <th className="px-4 py-2 font-semibold sm:pl-5">Name</th>
440 <th className="px-3 py-2 text-right font-semibold">CPU</th>457 <th className="px-3 py-2 text-right font-semibold">CPU</th>
...@@ -442,7 +459,7 @@ function ProcessDiagnosticsTable({...@@ -442,7 +459,7 @@ function ProcessDiagnosticsTable({
442 <th className="px-3 py-2 font-semibold">Command</th>459 <th className="px-3 py-2 font-semibold">Command</th>
443 <th className="px-3 py-2 text-right font-semibold">PID</th>460 <th className="px-3 py-2 text-right font-semibold">PID</th>
444 <th className="px-3 py-2 font-semibold">Type</th>461 <th className="px-3 py-2 font-semibold">Type</th>
445 <th className="px-2 py-2 text-right font-semibold sm:pr-4">Kill</th>462 <th className="p-2 text-right font-semibold sm:pr-4">Kill</th>
446 </tr>463 </tr>
447 </thead>464 </thead>
448 <tbody className="divide-y divide-border/50">465 <tbody className="divide-y divide-border/50">
...@@ -487,7 +504,7 @@ function ProcessDiagnosticsTable({...@@ -487,7 +504,7 @@ function ProcessDiagnosticsTable({
487 <td className="truncate px-3 py-2 align-middle text-muted-foreground">504 <td className="truncate px-3 py-2 align-middle text-muted-foreground">
488 {formatProcessType(process)}505 {formatProcessType(process)}
489 </td>506 </td>
490 <td className="px-2 py-2 align-middle sm:pr-4">507 <td className="p-2 align-middle sm:pr-4">
491 <ProcessSignalActions508 <ProcessSignalActions
492 process={process}509 process={process}
493 isSignaling={signalingPid === process.pid}510 isSignaling={signalingPid === process.pid}
...@@ -502,6 +519,238 @@ function ProcessDiagnosticsTable({...@@ -502,6 +519,238 @@ function ProcessDiagnosticsTable({
502 );519 );
503}520}
504521
522const RESOURCE_HISTORY_WINDOWS = [
523 { label: "5m", windowMs: 5 * 60_000, bucketMs: 30_000 },
524 { label: "15m", windowMs: 15 * 60_000, bucketMs: 60_000 },
525 { label: "30m", windowMs: 30 * 60_000, bucketMs: 2 * 60_000 },
526 { label: "1h", windowMs: 60 * 60_000, bucketMs: 5 * 60_000 },
527] as const;
528
529function formatCpuTime(seconds: number): string {
530 if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 1 : 2)}s`;
531 const minutes = seconds / 60;
532 if (minutes < 60) return `${minutes.toFixed(minutes >= 10 ? 1 : 2)}m`;
533 return `${(minutes / 60).toFixed(2)}h`;
534}
535
536function formatShortProcessName(command: string): string {
537 const name = formatProcessName(command);
538 return name.length > 42 ? `${name.slice(0, 39)}...` : name;
539}
540
541function ResourceHistoryProcessNameCell({
542 process,
543 visualDepth,
544}: {
545 process: ServerProcessResourceHistorySummary;
546 visualDepth: number;
547}) {
548 const name = formatShortProcessName(process.command);
549
550 return (
551 <div
552 className="grid min-w-0 grid-cols-[1.25rem_0.375rem_minmax(0,1fr)] items-center gap-2"
553 style={{ paddingLeft: `${Math.min(visualDepth, 6) * 10}px` }}
554 aria-label={`${process.isServerRoot ? "Root" : "Child"} process ${name}`}
555 >
556 <span className="size-5 shrink-0" aria-hidden="true" />
557 <span
558 className={cn(
559 "size-1.5 shrink-0 rounded-full",
560 process.isServerRoot ? "bg-amber-500/90" : "bg-emerald-500/80",
561 )}
562 />
563 <Tooltip>
564 <TooltipTrigger
565 render={<span className="min-w-0 truncate font-medium text-foreground">{name}</span>}
566 />
567 <TooltipPopup
568 side="top"
569 className="max-w-[min(440px,calc(100vw-2rem))] whitespace-normal break-words text-left font-mono text-[11px] leading-relaxed text-wrap"
570 >
571 {process.command}
572 </TooltipPopup>
573 </Tooltip>
574 </div>
575 );
576}
577
578function ProcessResourceHistoryChart({
579 buckets,
580}: {
581 buckets: ReadonlyArray<{
582 readonly startedAt: DateTime.Utc;
583 readonly avgCpuPercent: number;
584 readonly maxCpuPercent: number;
585 }>;
586}) {
587 const maxCpuPercent = Math.max(1, ...buckets.map((bucket) => bucket.maxCpuPercent));
588
589 return (
590 <div className="border-t border-border/60 px-4 py-3 sm:px-5">
591 <div className="flex h-28 items-end gap-1 overflow-hidden rounded-sm bg-muted/10 p-2">
592 {buckets.map((bucket) => {
593 const peakHeight = Math.max(2, (bucket.maxCpuPercent / maxCpuPercent) * 100);
594 const averageHeight = Math.max(2, (bucket.avgCpuPercent / maxCpuPercent) * 100);
595 return (
596 <Tooltip key={DateTime.formatIso(bucket.startedAt)}>
597 <TooltipTrigger
598 render={
599 <div className="flex h-full min-w-1 flex-1 items-end">
600 <div
601 className="relative h-full w-full"
602 aria-label={`Average CPU ${bucket.avgCpuPercent.toFixed(1)}%, peak CPU ${bucket.maxCpuPercent.toFixed(1)}%`}
603 >
604 <div
605 className="absolute inset-x-0 bottom-0 rounded-t-sm bg-foreground/15 transition-colors"
606 style={{ height: `${peakHeight}%` }}
607 />
608 <div
609 className="absolute inset-x-0 bottom-0 rounded-t-sm bg-foreground/60 transition-colors"
610 style={{ height: `${averageHeight}%` }}
611 />
612 </div>
613 </div>
614 }
615 />
616 <TooltipPopup side="top">
617 Avg {bucket.avgCpuPercent.toFixed(1)}%, peak {bucket.maxCpuPercent.toFixed(1)}%
618 </TooltipPopup>
619 </Tooltip>
620 );
621 })}
622 </div>
623 </div>
624 );
625}
626
627function ResourceHistoryWindowSelector({
628 selectedWindowMs,
629 onSelect,
630}: {
631 selectedWindowMs: number;
632 onSelect: (windowMs: number) => void;
633}) {
634 return (
635 <div className="flex items-center rounded-md border border-border/60 p-0.5">
636 {RESOURCE_HISTORY_WINDOWS.map((option) => (
637 <button
638 key={option.windowMs}
639 type="button"
640 className={cn(
641 "h-6 rounded-sm px-2 text-[11px] font-medium text-muted-foreground hover:text-foreground",
642 selectedWindowMs === option.windowMs && "bg-muted text-foreground",
643 )}
644 onClick={() => onSelect(option.windowMs)}
645 >
646 {option.label}
647 </button>
648 ))}
649 </div>
650 );
651}
652
653function ProcessResourceHistoryTable({
654 processes,
655 emptyLabel,
656}: {
657 processes: ReadonlyArray<ServerProcessResourceHistorySummary>;
658 emptyLabel: string;
659}) {
660 const shallowestChildDepth = processes.reduce<number | null>((minDepth, process) => {
661 if (process.isServerRoot) return minDepth;
662 return minDepth === null ? process.depth : Math.min(minDepth, process.depth);
663 }, null);
664
665 return (
666 <ScrollArea
667 chainVerticalScroll
668 scrollFade
669 hideScrollbars
670 className="max-h-[min(64vh,44rem)] w-full max-w-full border-t border-border/60"
671 >
672 <table className="w-full min-w-[980px] table-fixed text-left text-xs">
673 <colgroup>
674 <col className="w-[24%]" />
675 <col className="w-[10%]" />
676 <col className="w-[10%]" />
677 <col className="w-[10%]" />
678 <col className="w-[10%]" />
679 <col className="w-[10%]" />
680 <col className="w-[16%]" />
681 <col className="w-[10%]" />
682 </colgroup>
683 <thead className="sticky top-0 z-10 border-b border-border/60 bg-card text-[11px] uppercase tracking-[0.08em] text-muted-foreground/70">
684 <tr>
685 <th className="px-4 py-2 font-semibold sm:pl-5">Process</th>
686 <th className="px-3 py-2 text-right font-semibold">CPU Time</th>
687 <th className="px-3 py-2 text-right font-semibold">Current</th>
688 <th className="px-3 py-2 text-right font-semibold">Average</th>
689 <th className="px-3 py-2 text-right font-semibold">Peak</th>
690 <th className="px-3 py-2 text-right font-semibold">Max Mem</th>
691 <th className="px-3 py-2 font-semibold">Command</th>
692 <th className="px-3 py-2 text-right font-semibold sm:pr-5">PID</th>
693 </tr>
694 </thead>
695 <tbody className="divide-y divide-border/50">
696 {processes.length === 0 ? (
697 <tr>
698 <td colSpan={8} className="px-4 py-4 text-xs text-muted-foreground sm:px-5">
699 {emptyLabel}
700 </td>
701 </tr>
702 ) : null}
703 {processes.map((process) => (
704 <tr key={process.processKey} className="hover:bg-muted/20">
705 <td className="px-4 py-2 align-middle sm:pl-5">
706 <ResourceHistoryProcessNameCell
707 process={process}
708 visualDepth={
709 process.isServerRoot || shallowestChildDepth === null
710 ? 0
711 : Math.max(1, process.depth - shallowestChildDepth + 1)
712 }
713 />
714 </td>
715 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
716 {formatCpuTime(process.cpuSecondsApprox)}
717 </td>
718 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
719 {process.currentCpuPercent.toFixed(1)}%
720 </td>
721 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
722 {process.avgCpuPercent.toFixed(1)}%
723 </td>
724 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
725 {process.maxCpuPercent.toFixed(1)}%
726 </td>
727 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums">
728 {formatBytes(process.maxRssBytes)}
729 </td>
730 <td className="px-3 py-2 align-middle text-muted-foreground">
731 <Tooltip>
732 <TooltipTrigger
733 render={<span className="block truncate">{process.command}</span>}
734 />
735 <TooltipPopup
736 side="top"
737 className="max-w-[min(440px,calc(100vw-2rem))] whitespace-normal break-words text-left font-mono text-[11px] leading-relaxed text-wrap"
738 >
739 {process.command}
740 </TooltipPopup>
741 </Tooltip>
742 </td>
743 <td className="px-3 py-2 text-right align-middle font-mono tabular-nums text-muted-foreground sm:pr-5">
744 {process.pid}
745 </td>
746 </tr>
747 ))}
748 </tbody>
749 </table>
750 </ScrollArea>
751 );
752}
753
505function DiagnosticsLastChecked({ checkedAt }: { checkedAt: DateTime.Utc | null }) {754function DiagnosticsLastChecked({ checkedAt }: { checkedAt: DateTime.Utc | null }) {
506 useRelativeTimeTick();755 useRelativeTimeTick();
507 const relative = checkedAt ? formatRelativeTime(DateTime.formatIso(checkedAt)) : null;756 const relative = checkedAt ? formatRelativeTime(DateTime.formatIso(checkedAt)) : null;
...@@ -556,6 +805,10 @@ function DiagnosticsRefreshButton({...@@ -556,6 +805,10 @@ function DiagnosticsRefreshButton({
556export function DiagnosticsSettingsPanel() {805export function DiagnosticsSettingsPanel() {
557 const observability = useServerObservability();806 const observability = useServerObservability();
558 const availableEditors = useServerAvailableEditors();807 const availableEditors = useServerAvailableEditors();
808 const [resourceWindowMs, setResourceWindowMs] = useState(15 * 60_000);
809 const selectedResourceWindow =
810 RESOURCE_HISTORY_WINDOWS.find((option) => option.windowMs === resourceWindowMs) ??
811 RESOURCE_HISTORY_WINDOWS[1];
559 const { data, error, isPending, refresh } = useTraceDiagnostics();812 const { data, error, isPending, refresh } = useTraceDiagnostics();
560 const {813 const {
561 data: processData,814 data: processData,
...@@ -563,6 +816,15 @@ export function DiagnosticsSettingsPanel() {...@@ -563,6 +816,15 @@ export function DiagnosticsSettingsPanel() {
563 isPending: isProcessPending,816 isPending: isProcessPending,
564 refresh: refreshProcesses,817 refresh: refreshProcesses,
565 } = useProcessDiagnostics();818 } = useProcessDiagnostics();
819 const {
820 data: resourceData,
821 error: resourceError,
822 isPending: isResourcePending,
823 refresh: refreshResources,
824 } = useProcessResourceHistory({
825 windowMs: selectedResourceWindow.windowMs,
826 bucketMs: selectedResourceWindow.bucketMs,
827 });
566 const [isOpeningLogsDirectory, setIsOpeningLogsDirectory] = useState(false);828 const [isOpeningLogsDirectory, setIsOpeningLogsDirectory] = useState(false);
567 const [openLogsDirectoryError, setOpenLogsDirectoryError] = useState<string | null>(null);829 const [openLogsDirectoryError, setOpenLogsDirectoryError] = useState<string | null>(null);
568 const [signalingPid, setSignalingPid] = useState<number | null>(null);830 const [signalingPid, setSignalingPid] = useState<number | null>(null);
...@@ -643,6 +905,7 @@ export function DiagnosticsSettingsPanel() {...@@ -643,6 +905,7 @@ export function DiagnosticsSettingsPanel() {
643 );905 );
644906
645 const processDiagnosticsError = processData ? Option.getOrNull(processData.error) : null;907 const processDiagnosticsError = processData ? Option.getOrNull(processData.error) : null;
908 const processResourceError = resourceData ? Option.getOrNull(resourceData.error) : null;
646 const traceDiagnosticsError = data ? Option.getOrNull(data.error) : null;909 const traceDiagnosticsError = data ? Option.getOrNull(data.error) : null;
647 const traceDiagnosticsPartialFailure = data910 const traceDiagnosticsPartialFailure = data
648 ? Option.getOrElse(data.partialFailure, () => false)911 ? Option.getOrElse(data.partialFailure, () => false)
...@@ -711,6 +974,70 @@ export function DiagnosticsSettingsPanel() {...@@ -711,6 +974,70 @@ export function DiagnosticsSettingsPanel() {
711 />974 />
712 </SettingsSection>975 </SettingsSection>
713976
977 <SettingsSection
978 title="Resource History"
979 headerAction={
980 <div className="flex items-center gap-1.5">
981 <ResourceHistoryWindowSelector
982 selectedWindowMs={resourceWindowMs}
983 onSelect={setResourceWindowMs}
984 />
985 <DiagnosticsLastChecked checkedAt={resourceData?.readAt ?? null} />
986 <DiagnosticsRefreshButton
987 isPending={isResourcePending}
988 label="Refresh resource history"
989 onClick={refreshResources}
990 />
991 </div>
992 }
993 >
994 <StatsGrid>
995 <StatBlock
996 label="CPU Time"
997 value={resourceData ? formatCpuTime(resourceData.totalCpuSecondsApprox) : "..."}
998 tooltip="Approximate active CPU time for the T3 server root process and its descendants during the selected window. It grows only while sampled processes use CPU and older samples leave as the window moves."
999 />
1000 <StatBlock
1001 label="Samples"
1002 value={resourceData ? formatCount(resourceData.retainedSampleCount) : "..."}
1003 tooltip="In-memory process samples retained by the server. This resets when the server restarts."
1004 />
1005 <StatBlock
1006 label="Interval"
1007 value={resourceData ? formatDuration(resourceData.sampleIntervalMs) : "..."}
1008 />
1009 <StatBlock
1010 label="Processes"
1011 value={resourceData ? formatCount(resourceData.topProcesses.length) : "..."}
1012 />
1013 </StatsGrid>
1014 {processResourceError || resourceError ? (
1015 <div className="space-y-2 border-t border-border/60 px-4 py-3 text-xs text-muted-foreground sm:px-5">
1016 {processResourceError ? (
1017 <div className="flex items-start gap-2 text-destructive">
1018 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
1019 <span>{processResourceError.message}</span>
1020 </div>
1021 ) : null}
1022 {resourceError ? (
1023 <div className="flex items-start gap-2 text-destructive">
1024 <AlertTriangleIcon className="mt-0.5 size-3.5 shrink-0" />
1025 <span>{resourceError}</span>
1026 </div>
1027 ) : null}
1028 </div>
1029 ) : null}
1030 <ProcessResourceHistoryChart buckets={resourceData?.buckets ?? []} />
1031 <ProcessResourceHistoryTable
1032 processes={resourceData?.topProcesses ?? []}
1033 emptyLabel={
1034 isResourcePending && resourceData === null
1035 ? "Collecting process resource samples..."
1036 : "No process resource samples found for this window."
1037 }
1038 />
1039 </SettingsSection>
1040
714 <SettingsSection1041 <SettingsSection
715 title="Trace Diagnostics"1042 title="Trace Diagnostics"
716 headerAction={1043 headerAction={
apps/web/src/components/settings/SettingsPanels.browser.tsx+18
...@@ -13,6 +13,7 @@ import {...@@ -13,6 +13,7 @@ import {
13 ProviderDriverKind,13 ProviderDriverKind,
14 ProviderInstanceId,14 ProviderInstanceId,
15 type ServerConfig,15 type ServerConfig,
16 type ServerProcessResourceHistoryResult,
16 type ServerProvider,17 type ServerProvider,
17 type SourceControlDiscoveryResult,18 type SourceControlDiscoveryResult,
18} from "@t3tools/contracts";19} from "@t3tools/contracts";
...@@ -268,6 +269,20 @@ function makeUtc(value: string) {...@@ -268,6 +269,20 @@ function makeUtc(value: string) {
268 return DateTime.makeUnsafe(value);269 return DateTime.makeUnsafe(value);
269}270}
270271
272function createEmptyProcessResourceHistoryResult(): ServerProcessResourceHistoryResult {
273 return {
274 readAt: makeUtc("2036-04-07T00:00:00.000Z"),
275 windowMs: 15 * 60_000,
276 bucketMs: 60_000,
277 sampleIntervalMs: 5_000,
278 retainedSampleCount: 0,
279 totalCpuSecondsApprox: 0,
280 buckets: [],
281 topProcesses: [],
282 error: Option.none(),
283 };
284}
285
271function makePairingLink(input: {286function makePairingLink(input: {
272 readonly id: string;287 readonly id: string;
273 readonly credential: string;288 readonly credential: string;
...@@ -1065,6 +1080,9 @@ describe("GeneralSettingsPanel observability", () => {...@@ -1065,6 +1080,9 @@ describe("GeneralSettingsPanel observability", () => {
1065 processes: [],1080 processes: [],
1066 error: Option.none(),1081 error: Option.none(),
1067 }),1082 }),
1083 getProcessResourceHistory: vi
1084 .fn()
1085 .mockResolvedValue(createEmptyProcessResourceHistoryResult()),
1068 getTraceDiagnostics: vi.fn().mockResolvedValue({1086 getTraceDiagnostics: vi.fn().mockResolvedValue({
1069 traceFilePath: "/repo/project/.t3/traces.jsonl",1087 traceFilePath: "/repo/project/.t3/traces.jsonl",
1070 scannedFilePaths: ["/repo/project/.t3/traces.jsonl"],1088 scannedFilePaths: ["/repo/project/.t3/traces.jsonl"],
apps/web/src/lib/processDiagnosticsState.ts+76-1
...@@ -1,5 +1,8 @@...@@ -1,5 +1,8 @@
1import { useAtomValue } from "@effect/atom-react";1import { useAtomValue } from "@effect/atom-react";
2import type { ServerProcessDiagnosticsResult } from "@t3tools/contracts";2import type {
3 ServerProcessDiagnosticsResult,
4 ServerProcessResourceHistoryResult,
5} from "@t3tools/contracts";
3import * as Cause from "effect/Cause";6import * as Cause from "effect/Cause";
4import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
5import * as Option from "effect/Option";8import * as Option from "effect/Option";
...@@ -11,6 +14,8 @@ import { appAtomRegistry } from "../rpc/atomRegistry";...@@ -11,6 +14,8 @@ import { appAtomRegistry } from "../rpc/atomRegistry";
1114
12const PROCESS_DIAGNOSTICS_STALE_TIME_MS = 2_000;15const PROCESS_DIAGNOSTICS_STALE_TIME_MS = 2_000;
13const PROCESS_DIAGNOSTICS_IDLE_TTL_MS = 5 * 60_000;16const PROCESS_DIAGNOSTICS_IDLE_TTL_MS = 5 * 60_000;
17const PROCESS_RESOURCE_HISTORY_STALE_TIME_MS = 5_000;
18const PROCESS_RESOURCE_HISTORY_INPUT_SEPARATOR = ":";
1419
15const processDiagnosticsAtom = Atom.make(20const processDiagnosticsAtom = Atom.make(
16 Effect.promise(() => ensureLocalApi().server.getProcessDiagnostics()),21 Effect.promise(() => ensureLocalApi().server.getProcessDiagnostics()),
...@@ -23,6 +28,38 @@ const processDiagnosticsAtom = Atom.make(...@@ -23,6 +28,38 @@ const processDiagnosticsAtom = Atom.make(
23 Atom.withLabel("process-diagnostics"),28 Atom.withLabel("process-diagnostics"),
24);29);
2530
31function formatProcessResourceHistoryKey(input: {
32 readonly windowMs: number;
33 readonly bucketMs: number;
34}): string {
35 return `${input.windowMs}${PROCESS_RESOURCE_HISTORY_INPUT_SEPARATOR}${input.bucketMs}`;
36}
37
38function parseProcessResourceHistoryKey(key: string): {
39 readonly windowMs: number;
40 readonly bucketMs: number;
41} {
42 const [windowMs = "0", bucketMs = "0"] = key.split(PROCESS_RESOURCE_HISTORY_INPUT_SEPARATOR);
43 return {
44 windowMs: Number(windowMs),
45 bucketMs: Number(bucketMs),
46 };
47}
48
49const processResourceHistoryAtom = Atom.family((key: string) => {
50 const input = parseProcessResourceHistoryKey(key);
51 return Atom.make(
52 Effect.promise(() => ensureLocalApi().server.getProcessResourceHistory(input)),
53 ).pipe(
54 Atom.swr({
55 staleTime: PROCESS_RESOURCE_HISTORY_STALE_TIME_MS,
56 revalidateOnMount: true,
57 }),
58 Atom.setIdleTTL(PROCESS_DIAGNOSTICS_IDLE_TTL_MS),
59 Atom.withLabel(`process-resource-history:${key}`),
60 );
61});
62
26export interface ProcessDiagnosticsState {63export interface ProcessDiagnosticsState {
27 readonly data: ServerProcessDiagnosticsResult | null;64 readonly data: ServerProcessDiagnosticsResult | null;
28 readonly error: string | null;65 readonly error: string | null;
...@@ -30,6 +67,13 @@ export interface ProcessDiagnosticsState {...@@ -30,6 +67,13 @@ export interface ProcessDiagnosticsState {
30 readonly refresh: () => void;67 readonly refresh: () => void;
31}68}
3269
70export interface ProcessResourceHistoryState {
71 readonly data: ServerProcessResourceHistoryResult | null;
72 readonly error: string | null;
73 readonly isPending: boolean;
74 readonly refresh: () => void;
75}
76
33function formatProcessDiagnosticsError(error: unknown): string {77function formatProcessDiagnosticsError(error: unknown): string {
34 return error instanceof Error ? error.message : "Failed to load process diagnostics.";78 return error instanceof Error ? error.message : "Failed to load process diagnostics.";
35}79}
...@@ -45,6 +89,17 @@ function readProcessDiagnosticsError(...@@ -45,6 +89,17 @@ function readProcessDiagnosticsError(
45 return formatProcessDiagnosticsError(squashed);89 return formatProcessDiagnosticsError(squashed);
46}90}
4791
92function readProcessResourceHistoryError(
93 result: AsyncResult.AsyncResult<ServerProcessResourceHistoryResult, unknown>,
94): string | null {
95 if (result._tag !== "Failure") {
96 return null;
97 }
98
99 const squashed = Cause.squash(result.cause);
100 return formatProcessDiagnosticsError(squashed);
101}
102
48export function refreshProcessDiagnostics(): void {103export function refreshProcessDiagnostics(): void {
49 appAtomRegistry.refresh(processDiagnosticsAtom);104 appAtomRegistry.refresh(processDiagnosticsAtom);
50}105}
...@@ -63,3 +118,23 @@ export function useProcessDiagnostics(): ProcessDiagnosticsState {...@@ -63,3 +118,23 @@ export function useProcessDiagnostics(): ProcessDiagnosticsState {
63 refresh,118 refresh,
64 };119 };
65}120}
121
122export function useProcessResourceHistory(input: {
123 readonly windowMs: number;
124 readonly bucketMs: number;
125}): ProcessResourceHistoryState {
126 const atom = processResourceHistoryAtom(formatProcessResourceHistoryKey(input));
127 const result = useAtomValue(atom);
128 const data = Option.getOrNull(AsyncResult.value(result));
129
130 const refresh = useCallback(() => {
131 appAtomRegistry.refresh(atom);
132 }, [atom]);
133
134 return {
135 data,
136 error: readProcessResourceHistoryError(result),
137 isPending: result.waiting,
138 refresh,
139 };
140}
apps/web/src/localApi.ts+4
...@@ -155,6 +155,10 @@ function createBrowserLocalApi(rpcClient?: WsRpcClient): LocalApi {...@@ -155,6 +155,10 @@ function createBrowserLocalApi(rpcClient?: WsRpcClient): LocalApi {
155 rpcClient155 rpcClient
156 ? rpcClient.server.getProcessDiagnostics()156 ? rpcClient.server.getProcessDiagnostics()
157 : Promise.reject(unavailableLocalBackendError()),157 : Promise.reject(unavailableLocalBackendError()),
158 getProcessResourceHistory: (input) =>
159 rpcClient
160 ? rpcClient.server.getProcessResourceHistory(input)
161 : Promise.reject(unavailableLocalBackendError()),
158 signalProcess: (input) =>162 signalProcess: (input) =>
159 rpcClient163 rpcClient
160 ? rpcClient.server.signalProcess(input)164 ? rpcClient.server.signalProcess(input)
apps/web/src/rpc/wsRpcClient.ts+9
...@@ -135,6 +135,9 @@ export interface WsRpcClient {...@@ -135,6 +135,9 @@ export interface WsRpcClient {
135 readonly getProcessDiagnostics: RpcUnaryNoArgMethod<135 readonly getProcessDiagnostics: RpcUnaryNoArgMethod<
136 typeof WS_METHODS.serverGetProcessDiagnostics136 typeof WS_METHODS.serverGetProcessDiagnostics
137 >;137 >;
138 readonly getProcessResourceHistory: RpcUnaryMethod<
139 typeof WS_METHODS.serverGetProcessResourceHistory
140 >;
138 readonly signalProcess: RpcUnaryMethod<typeof WS_METHODS.serverSignalProcess>;141 readonly signalProcess: RpcUnaryMethod<typeof WS_METHODS.serverSignalProcess>;
139 readonly subscribeConfig: RpcStreamMethod<typeof WS_METHODS.subscribeServerConfig>;142 readonly subscribeConfig: RpcStreamMethod<typeof WS_METHODS.subscribeServerConfig>;
140 readonly subscribeLifecycle: RpcStreamMethod<typeof WS_METHODS.subscribeServerLifecycle>;143 readonly subscribeLifecycle: RpcStreamMethod<typeof WS_METHODS.subscribeServerLifecycle>;
...@@ -266,6 +269,12 @@ export function createWsRpcClient(transport: WsTransport): WsRpcClient {...@@ -266,6 +269,12 @@ export function createWsRpcClient(transport: WsTransport): WsRpcClient {
266 transport.request((client) =>269 transport.request((client) =>
267 client[WS_METHODS.serverGetProcessDiagnostics]({}).pipe(Effect.withTracerEnabled(false)),270 client[WS_METHODS.serverGetProcessDiagnostics]({}).pipe(Effect.withTracerEnabled(false)),
268 ),271 ),
272 getProcessResourceHistory: (input) =>
273 transport.request((client) =>
274 client[WS_METHODS.serverGetProcessResourceHistory](input).pipe(
275 Effect.withTracerEnabled(false),
276 ),
277 ),
269 signalProcess: (input) =>278 signalProcess: (input) =>
270 transport.request((client) =>279 transport.request((client) =>
271 client[WS_METHODS.serverSignalProcess](input).pipe(Effect.withTracerEnabled(false)),280 client[WS_METHODS.serverSignalProcess](input).pipe(Effect.withTracerEnabled(false)),
packages/contracts/src/ipc.ts+5
...@@ -29,6 +29,8 @@ import type { ProviderInstanceId } from "./providerInstance.ts";...@@ -29,6 +29,8 @@ import type { ProviderInstanceId } from "./providerInstance.ts";
29import type {29import type {
30 ServerConfig,30 ServerConfig,
31 ServerProcessDiagnosticsResult,31 ServerProcessDiagnosticsResult,
32 ServerProcessResourceHistoryInput,
33 ServerProcessResourceHistoryResult,
32 ServerProviderUpdateInput,34 ServerProviderUpdateInput,
33 ServerProviderUpdatedPayload,35 ServerProviderUpdatedPayload,
34 ServerRemoveKeybindingResult,36 ServerRemoveKeybindingResult,
...@@ -475,6 +477,9 @@ export interface LocalApi {...@@ -475,6 +477,9 @@ export interface LocalApi {
475 discoverSourceControl: () => Promise<SourceControlDiscoveryResult>;477 discoverSourceControl: () => Promise<SourceControlDiscoveryResult>;
476 getTraceDiagnostics: () => Promise<ServerTraceDiagnosticsResult>;478 getTraceDiagnostics: () => Promise<ServerTraceDiagnosticsResult>;
477 getProcessDiagnostics: () => Promise<ServerProcessDiagnosticsResult>;479 getProcessDiagnostics: () => Promise<ServerProcessDiagnosticsResult>;
480 getProcessResourceHistory: (
481 input: ServerProcessResourceHistoryInput,
482 ) => Promise<ServerProcessResourceHistoryResult>;
478 signalProcess: (input: ServerSignalProcessInput) => Promise<ServerSignalProcessResult>;483 signalProcess: (input: ServerSignalProcessInput) => Promise<ServerSignalProcessResult>;
479 };484 };
480}485}
packages/contracts/src/rpc.ts+12
...@@ -79,6 +79,8 @@ import {...@@ -79,6 +79,8 @@ import {
79 ServerProviderUpdatedPayload,79 ServerProviderUpdatedPayload,
80 ServerTraceDiagnosticsResult,80 ServerTraceDiagnosticsResult,
81 ServerProcessDiagnosticsResult,81 ServerProcessDiagnosticsResult,
82 ServerProcessResourceHistoryInput,
83 ServerProcessResourceHistoryResult,
82 ServerSignalProcessInput,84 ServerSignalProcessInput,
83 ServerSignalProcessResult,85 ServerSignalProcessResult,
84 ServerUpsertKeybindingInput,86 ServerUpsertKeybindingInput,
...@@ -145,6 +147,7 @@ export const WS_METHODS = {...@@ -145,6 +147,7 @@ export const WS_METHODS = {
145 serverDiscoverSourceControl: "server.discoverSourceControl",147 serverDiscoverSourceControl: "server.discoverSourceControl",
146 serverGetTraceDiagnostics: "server.getTraceDiagnostics",148 serverGetTraceDiagnostics: "server.getTraceDiagnostics",
147 serverGetProcessDiagnostics: "server.getProcessDiagnostics",149 serverGetProcessDiagnostics: "server.getProcessDiagnostics",
150 serverGetProcessResourceHistory: "server.getProcessResourceHistory",
148 serverSignalProcess: "server.signalProcess",151 serverSignalProcess: "server.signalProcess",
149152
150 // Source control methods153 // Source control methods
...@@ -224,6 +227,14 @@ export const WsServerGetProcessDiagnosticsRpc = Rpc.make(WS_METHODS.serverGetPro...@@ -224,6 +227,14 @@ export const WsServerGetProcessDiagnosticsRpc = Rpc.make(WS_METHODS.serverGetPro
224 success: ServerProcessDiagnosticsResult,227 success: ServerProcessDiagnosticsResult,
225});228});
226229
230export const WsServerGetProcessResourceHistoryRpc = Rpc.make(
231 WS_METHODS.serverGetProcessResourceHistory,
232 {
233 payload: ServerProcessResourceHistoryInput,
234 success: ServerProcessResourceHistoryResult,
235 },
236);
237
227export const WsServerSignalProcessRpc = Rpc.make(WS_METHODS.serverSignalProcess, {238export const WsServerSignalProcessRpc = Rpc.make(WS_METHODS.serverSignalProcess, {
228 payload: ServerSignalProcessInput,239 payload: ServerSignalProcessInput,
229 success: ServerSignalProcessResult,240 success: ServerSignalProcessResult,
...@@ -472,6 +483,7 @@ export const WsRpcGroup = RpcGroup.make(...@@ -472,6 +483,7 @@ export const WsRpcGroup = RpcGroup.make(
472 WsServerDiscoverSourceControlRpc,483 WsServerDiscoverSourceControlRpc,
473 WsServerGetTraceDiagnosticsRpc,484 WsServerGetTraceDiagnosticsRpc,
474 WsServerGetProcessDiagnosticsRpc,485 WsServerGetProcessDiagnosticsRpc,
486 WsServerGetProcessResourceHistoryRpc,
475 WsServerSignalProcessRpc,487 WsServerSignalProcessRpc,
476 WsSourceControlLookupRepositoryRpc,488 WsSourceControlLookupRepositoryRpc,
477 WsSourceControlCloneRepositoryRpc,489 WsSourceControlCloneRepositoryRpc,
packages/contracts/src/server.ts+52
...@@ -328,6 +328,58 @@ export const ServerProcessDiagnosticsResult = Schema.Struct({...@@ -328,6 +328,58 @@ export const ServerProcessDiagnosticsResult = Schema.Struct({
328});328});
329export type ServerProcessDiagnosticsResult = typeof ServerProcessDiagnosticsResult.Type;329export type ServerProcessDiagnosticsResult = typeof ServerProcessDiagnosticsResult.Type;
330330
331export const ServerProcessResourceHistoryInput = Schema.Struct({
332 windowMs: NonNegativeInt,
333 bucketMs: NonNegativeInt,
334});
335export type ServerProcessResourceHistoryInput = typeof ServerProcessResourceHistoryInput.Type;
336
337export const ServerProcessResourceHistoryBucket = Schema.Struct({
338 startedAt: Schema.DateTimeUtc,
339 endedAt: Schema.DateTimeUtc,
340 avgCpuPercent: Schema.Number,
341 maxCpuPercent: Schema.Number,
342 maxRssBytes: NonNegativeInt,
343 maxProcessCount: NonNegativeInt,
344});
345export type ServerProcessResourceHistoryBucket = typeof ServerProcessResourceHistoryBucket.Type;
346
347export const ServerProcessResourceHistorySummary = Schema.Struct({
348 processKey: TrimmedNonEmptyString,
349 pid: PositiveInt,
350 ppid: NonNegativeInt,
351 command: TrimmedNonEmptyString,
352 depth: NonNegativeInt,
353 isServerRoot: Schema.Boolean,
354 firstSeenAt: Schema.DateTimeUtc,
355 lastSeenAt: Schema.DateTimeUtc,
356 currentCpuPercent: Schema.Number,
357 avgCpuPercent: Schema.Number,
358 maxCpuPercent: Schema.Number,
359 cpuSecondsApprox: Schema.Number,
360 currentRssBytes: NonNegativeInt,
361 maxRssBytes: NonNegativeInt,
362 sampleCount: NonNegativeInt,
363});
364export type ServerProcessResourceHistorySummary = typeof ServerProcessResourceHistorySummary.Type;
365
366export const ServerProcessResourceHistoryResult = Schema.Struct({
367 readAt: Schema.DateTimeUtc,
368 windowMs: NonNegativeInt,
369 bucketMs: NonNegativeInt,
370 sampleIntervalMs: NonNegativeInt,
371 retainedSampleCount: NonNegativeInt,
372 totalCpuSecondsApprox: Schema.Number,
373 buckets: Schema.Array(ServerProcessResourceHistoryBucket),
374 topProcesses: Schema.Array(ServerProcessResourceHistorySummary),
375 error: Schema.Option(
376 Schema.Struct({
377 message: TrimmedNonEmptyString,
378 }),
379 ),
380});
381export type ServerProcessResourceHistoryResult = typeof ServerProcessResourceHistoryResult.Type;
382
331export const ServerSignalProcessInput = Schema.Struct({383export const ServerSignalProcessInput = Schema.Struct({
332 pid: PositiveInt,384 pid: PositiveInt,
333 signal: ServerProcessSignal,385 signal: ServerProcessSignal,