From d84ebe831922167d556e16cb40a238a904e0f14a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 20 Jun 2026 11:11:15 -0700 Subject: [PATCH] [codex] Structure process resource sampling failures (#3415) Co-authored-by: codex --- .../ProcessResourceMonitor.test.ts | 33 +++++++++- .../src/diagnostics/ProcessResourceMonitor.ts | 66 ++++++++++++++----- packages/contracts/src/server.ts | 11 ++++ 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/apps/server/src/diagnostics/ProcessResourceMonitor.test.ts b/apps/server/src/diagnostics/ProcessResourceMonitor.test.ts index 49a9676ab11df72d3e2756f2a80a4fce97017041..d9c4eb06ef184edd7f41dedb0ac818c00880f243 100644 --- a/apps/server/src/diagnostics/ProcessResourceMonitor.test.ts +++ b/apps/server/src/diagnostics/ProcessResourceMonitor.test.ts @@ -111,7 +111,7 @@ describe("ProcessResourceMonitor", () => { readAtMs: DateTime.toEpochMillis(secondAt), windowMs: 60_000, bucketMs: 10_000, - lastError: null, + lastFailure: null, }); expect(Option.isNone(result.error)).toBe(true); @@ -171,7 +171,7 @@ describe("ProcessResourceMonitor", () => { readAtMs: DateTime.toEpochMillis(secondAt), windowMs: 60_000, bucketMs: 10_000, - lastError: null, + lastFailure: null, }); expect(result.topProcesses).toHaveLength(1); @@ -218,11 +218,38 @@ describe("ProcessResourceMonitor", () => { readAtMs: DateTime.toEpochMillis(sampledAt), windowMs: 60_000, bucketMs: 10_000, - lastError: null, + lastFailure: null, }); expect(result.topProcesses).toHaveLength(36); expect(result.topProcesses.some((process) => process.command === "worker 34")).toBe(true); }), ); + + it.effect("exposes bounded failure diagnostics while retaining the exact cause", () => + Effect.sync(() => { + const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z"); + const cause = new Error("stderr included credential=secret-value"); + const failure = new ProcessResourceMonitor.ProcessResourceSamplingError({ + failureTag: "ProcessDiagnosticsQueryFailedError", + cause, + }); + + const result = ProcessResourceMonitor.aggregateProcessResourceHistory({ + samples: [], + readAt, + readAtMs: DateTime.toEpochMillis(readAt), + windowMs: 60_000, + bucketMs: 10_000, + lastFailure: failure, + }); + + expect(failure.cause).toBe(cause); + expect(Option.getOrThrow(result.error)).toEqual({ + failureTag: "ProcessDiagnosticsQueryFailedError", + message: "Failed to sample process resources (ProcessDiagnosticsQueryFailedError).", + }); + expect(Option.getOrThrow(result.error).message).not.toContain("secret-value"); + }), + ); }); diff --git a/apps/server/src/diagnostics/ProcessResourceMonitor.ts b/apps/server/src/diagnostics/ProcessResourceMonitor.ts index b6e71dd24237ac5048dbe3da7b7840e6089d029b..6030e4172e1dfff51b81fe97a158f51e9cf36612 100644 --- a/apps/server/src/diagnostics/ProcessResourceMonitor.ts +++ b/apps/server/src/diagnostics/ProcessResourceMonitor.ts @@ -1,8 +1,10 @@ -import type { - ServerProcessResourceHistoryBucket, - ServerProcessResourceHistoryInput, - ServerProcessResourceHistoryResult, - ServerProcessResourceHistorySummary, +import { + ServerProcessResourceHistoryFailureTag, + type ServerProcessResourceHistoryBucket, + type ServerProcessResourceHistoryFailureTag as ServerProcessResourceHistoryFailureTagType, + type ServerProcessResourceHistoryInput, + type ServerProcessResourceHistoryResult, + type ServerProcessResourceHistorySummary, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -10,6 +12,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import * as ProcessDiagnostics from "./ProcessDiagnostics.ts"; @@ -31,9 +34,21 @@ export interface ProcessResourceSample { readonly isServerRoot: boolean; } +export class ProcessResourceSamplingError extends Schema.TaggedErrorClass()( + "ProcessResourceSamplingError", + { + failureTag: ServerProcessResourceHistoryFailureTag, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Failed to sample process resources (${this.failureTag}).`; + } +} + interface MonitorState { readonly samples: ReadonlyArray; - readonly lastError: string | null; + readonly lastFailure: ProcessResourceSamplingError | null; } export class ProcessResourceMonitor extends Context.Service< @@ -218,7 +233,7 @@ export function aggregateProcessResourceHistory(input: { readonly readAtMs: number; readonly windowMs: number; readonly bucketMs: number; - readonly lastError: string | null; + readonly lastFailure: ProcessResourceSamplingError | null; }): ServerProcessResourceHistoryResult { const windowMs = Math.max(1_000, input.windowMs); const bucketMs = Math.max(1_000, input.bucketMs); @@ -239,13 +254,29 @@ export function aggregateProcessResourceHistory(input: { totalCpuSecondsApprox, buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }), topProcesses, - error: input.lastError ? Option.some({ message: input.lastError }) : Option.none(), + error: input.lastFailure + ? Option.some({ + failureTag: input.lastFailure.failureTag, + message: input.lastFailure.message, + }) + : Option.none(), }; } export const make = Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const state = yield* Ref.make({ samples: [], lastError: null }); + const state = yield* Ref.make({ samples: [], lastFailure: null }); + + const recordSamplingFailure = (cause: { + readonly _tag: ServerProcessResourceHistoryFailureTagType; + }) => + Ref.update(state, (current) => ({ + ...current, + lastFailure: new ProcessResourceSamplingError({ + failureTag: cause._tag, + cause, + }), + })); const sampleOnce = Effect.gen(function* () { const sampledAt = yield* DateTime.now; @@ -261,15 +292,16 @@ export const make = Effect.gen(function* () { }); yield* Ref.update(state, (current) => ({ samples: trimSamples([...current.samples, ...samples], sampledAtMs), - lastError: null, + lastFailure: null, })); }).pipe( - Effect.catch((error: unknown) => - Ref.update(state, (current) => ({ - ...current, - lastError: error instanceof Error ? error.message : "Failed to sample process resources.", - })), - ), + Effect.catchTags({ + ProcessDiagnosticsQueryTimeoutError: recordSamplingFailure, + ProcessDiagnosticsQueryFailedError: recordSamplingFailure, + ProcessDiagnosticsServerProcessSignalError: recordSamplingFailure, + ProcessDiagnosticsNotDescendantError: recordSamplingFailure, + ProcessDiagnosticsSignalFailedError: recordSamplingFailure, + }), ); yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe( @@ -287,7 +319,7 @@ export const make = Effect.gen(function* () { readAtMs, windowMs: input.windowMs, bucketMs: input.bucketMs, - lastError: current.lastError, + lastFailure: current.lastFailure, }); }); diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 1aa280ad63b25515c60390507d1219f5bda66f61..b76ea965afe7cab91371aa79718759253cc8b8e1 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -364,6 +364,16 @@ export const ServerProcessResourceHistorySummary = Schema.Struct({ }); export type ServerProcessResourceHistorySummary = typeof ServerProcessResourceHistorySummary.Type; +export const ServerProcessResourceHistoryFailureTag = Schema.Literals([ + "ProcessDiagnosticsQueryTimeoutError", + "ProcessDiagnosticsQueryFailedError", + "ProcessDiagnosticsServerProcessSignalError", + "ProcessDiagnosticsNotDescendantError", + "ProcessDiagnosticsSignalFailedError", +]); +export type ServerProcessResourceHistoryFailureTag = + typeof ServerProcessResourceHistoryFailureTag.Type; + export const ServerProcessResourceHistoryResult = Schema.Struct({ readAt: Schema.DateTimeUtc, windowMs: NonNegativeInt, @@ -375,6 +385,7 @@ export const ServerProcessResourceHistoryResult = Schema.Struct({ topProcesses: Schema.Array(ServerProcessResourceHistorySummary), error: Schema.Option( Schema.Struct({ + failureTag: ServerProcessResourceHistoryFailureTag, message: TrimmedNonEmptyString, }), ), -- 2.54.0