authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 04:55:28-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 04:55:28-07:00
logcdba9a038e41c6c807a2b58bd43e83dcf98d74b7
tree246ef475668942b87e9ce9fcce61e68f4fbcbf8b
parent8e018ff91f64d6caedc029e498eca446ff0fad20
signature Signed by PGP key B5690EEEBB952194

perf(server): idle servers no longer scan every process every 5 seconds (#19399)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

4 files changed, 130 insertions(+), 21 deletions(-)

apps/server/src/resourceTelemetry/NativeTelemetryClient.test.ts+108-9
......@@ -1,12 +1,30 @@
1import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2import * as NodeServices from "@effect/platform-node/NodeServices";
13import type { HostPowerSnapshot } from "@t3tools/contracts";
4import {
5 RESOURCE_MONITOR_PROTOCOL_VERSION,
6 ResourceMonitorCommand,
7 type ResourceMonitorHelloEvent,
8} from "@t3tools/contracts";
29import { describe, expect, it } from "@effect/vitest";
310import * as DateTime from "effect/DateTime";
411import * as Deferred from "effect/Deferred";
512import * as Effect from "effect/Effect";
13import * as Exit from "effect/Exit";
614import * as Fiber from "effect/Fiber";
15import * as Layer from "effect/Layer";
16import * as Queue from "effect/Queue";
717import * as Ref from "effect/Ref";
18import * as Schema from "effect/Schema";
19import * as Scope from "effect/Scope";
820import * as Semaphore from "effect/Semaphore";
21import * as Sink from "effect/Sink";
22import * as Stream from "effect/Stream";
23import { ChildProcessSpawner } from "effect/process";
924
25import * as ServerConfig from "../config.ts";
26import * as NativeTelemetryClient from "./NativeTelemetryClient.ts";
27import * as ResourceMonitorBinary from "./ResourceMonitorBinary.ts";
1028import {
1129 canCommandNativeTelemetrySidecar,
1230 canRequestNativeTelemetryRetry,
......@@ -40,25 +58,106 @@ describe("resolveNativeSampleIntervalMs", () => {
4058 expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 1)).toBe(5_000);
4159 });
4260
43 it("slows background telemetry and serves live diagnostics at 1Hz", () => {
61 it("stops timed sampling without a live subscriber and serves live diagnostics at 1Hz", () => {
4462 const unknown: HostPowerSnapshot = {
4563 ...basePower,
4664 source: "unknown",
4765 stale: true,
4866 };
49 expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(5_000);
67 expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(0);
5068 expect(resolveNativeSampleIntervalMs(unknown, 1)).toBe(1_000);
51 expect(
52 resolveNativeSampleIntervalMs(
53 { ...basePower, stale: true, locked: "true", suspended: true },
54 0,
55 ),
56 ).toBe(5_000);
57 expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(5_000);
69 expect(resolveNativeSampleIntervalMs({ ...basePower, locked: "true" }, 0)).toBe(0);
70 expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 0)).toBe(0);
71 expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(0);
5872 expect(resolveNativeSampleIntervalMs(basePower, 1)).toBe(1_000);
5973 });
6074});
6175
76const decodeCommandLine = Schema.decodeUnknownSync(Schema.fromJsonString(ResourceMonitorCommand));
77
78const hello: ResourceMonitorHelloEvent = {
79 version: RESOURCE_MONITOR_PROTOCOL_VERSION,
80 type: "hello",
81 sidecarVersion: "test",
82 sidecarPid: 4242,
83 platform: "linux",
84 arch: "x64",
85 capabilities: {
86 cumulativeCpuTime: true,
87 currentCpuPercent: true,
88 residentMemory: true,
89 virtualMemory: true,
90 ioBytes: true,
91 processStartTime: true,
92 processTree: true,
93 },
94};
95
96describe("NativeTelemetryClient", () => {
97 it.effect("asks the sidecar for timed samples only while a live subscriber exists", () =>
98 Effect.gen(function* () {
99 const commands = yield* Queue.unbounded<ResourceMonitorCommand>();
100 // A sidecar that says hello, stays alive, and records every command it receives.
101 const spawner = ChildProcessSpawner.make(() =>
102 Effect.succeed(
103 ChildProcessSpawner.makeHandle({
104 pid: ChildProcessSpawner.ProcessId(hello.sidecarPid),
105 exitCode: Effect.never,
106 isRunning: Effect.succeed(true),
107 kill: () => Effect.void,
108 unref: Effect.succeed(Effect.void),
109 stdin: Sink.forEach((chunk: Uint8Array) =>
110 Effect.forEach(new TextDecoder().decode(chunk).split("\n").filter(Boolean), (line) =>
111 Queue.offer(commands, decodeCommandLine(line)),
112 ),
113 ),
114 stdout: Stream.encodeText(Stream.make(`${JSON.stringify(hello)}\n`)).pipe(
115 Stream.concat(Stream.never),
116 ),
117 stderr: Stream.empty,
118 all: Stream.empty,
119 getInputFd: () => Sink.drain,
120 getOutputFd: () => Stream.empty,
121 }),
122 ),
123 );
124 const client = yield* NativeTelemetryClient.make().pipe(
125 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
126 Effect.provideService(ResourceMonitorBinary.ResourceMonitorBinary, {
127 resolve: Effect.succeed("/test/t3-resource-monitor"),
128 }),
129 );
130
131 const configure = yield* Queue.take(commands);
132 expect(configure).toMatchObject({ type: "configure", sampleIntervalMs: 0 });
133 expect(yield* Queue.take(commands)).toMatchObject({ type: "setExternalProcesses" });
134
135 const live = yield* Scope.make();
136 yield* client.snapshots.pipe(Stream.runDrain, Effect.forkIn(live));
137 expect(yield* Queue.take(commands)).toMatchObject({
138 type: "setSampleInterval",
139 sampleIntervalMs: 1_000,
140 });
141 expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: true });
142
143 yield* Scope.close(live, Exit.void);
144 expect(yield* Queue.take(commands)).toMatchObject({
145 type: "setSampleInterval",
146 sampleIntervalMs: 0,
147 });
148 expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: false });
149 expect((yield* client.health).sampleIntervalMs).toBe(0);
150 }).pipe(
151 Effect.provide(
152 Layer.mergeAll(
153 ServerConfig.layerTest(process.cwd(), { prefix: "t3-native-telemetry-test-" }),
154 NodeCrypto.layer,
155 ).pipe(Layer.provideMerge(NodeServices.layer)),
156 ),
157 ),
158 );
159});
160
62161describe("canRequestNativeTelemetryRetry", () => {
63162 it("only accepts retry while the supervisor is waiting without a live sidecar", () => {
64163 expect(canRequestNativeTelemetryRetry("degraded", false)).toBe(true);
apps/server/src/resourceTelemetry/NativeTelemetryClient.ts+9-7
......@@ -40,7 +40,8 @@ import * as ServerConfig from "../config.ts";
4040import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts";
4141
4242const SAMPLE_INTERVAL_MS = 1_000;
43const UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS = 5_000;
43// The sidecar treats 0 as "sample only on request".
44const IDLE_SAMPLE_INTERVAL_MS = 0;
4445const BATTERY_SAMPLE_INTERVAL_MS = 5_000;
4546const CONSTRAINED_SAMPLE_INTERVAL_MS = 15_000;
4647const HANDSHAKE_TIMEOUT = Duration.seconds(5);
......@@ -258,13 +259,14 @@ function isThermallyConstrained(snapshot: HostPowerSnapshot): boolean {
258259 return snapshot.thermalState === "serious" || snapshot.thermalState === "critical";
259260}
260261
262// Without a live subscriber the sidecar does not sample on a timer, so an idle server
263// never walks the process table. History then covers only time with a subscriber.
261264export function resolveNativeSampleIntervalMs(
262265 snapshot: HostPowerSnapshot,
263266 liveSubscriberCount: number,
264267): number {
265 if (snapshot.stale || snapshot.source === "unknown") {
266 return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;
267 }
268 if (liveSubscriberCount === 0) return IDLE_SAMPLE_INTERVAL_MS;
269 if (snapshot.stale || snapshot.source === "unknown") return SAMPLE_INTERVAL_MS;
268270 if (
269271 snapshot.suspended ||
270272 snapshot.locked === "true" ||
......@@ -274,7 +276,7 @@ export function resolveNativeSampleIntervalMs(
274276 return CONSTRAINED_SAMPLE_INTERVAL_MS;
275277 }
276278 if (snapshot.onBattery === "true") return BATTERY_SAMPLE_INTERVAL_MS;
277 return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;
279 return SAMPLE_INTERVAL_MS;
278280}
279281
280282export function commitCollectionControlUpdate<E, R>(
......@@ -382,7 +384,7 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
382384 updatedAt: initializedAt,
383385 },
384386 liveSubscriberCount: 0,
385 sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,
387 sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
386388 });
387389 const appliedCollectionControl = yield* Ref.make(yield* Ref.get(collectionControl));
388390 const externalProcesses = yield* Ref.make<ReadonlyArray<ResourceMonitorExternalProcess>>([]);
......@@ -1071,7 +1073,7 @@ export const layerTest = (
10711073 lastSampleAt: Option.none<DateTime.Utc>(),
10721074 lastError: Option.some("Resource monitor test implementation is unavailable."),
10731075 restartCount: 0,
1074 sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,
1076 sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
10751077 });
10761078 return Layer.succeed(
10771079 NativeTelemetryClient,
apps/web/src/components/settings/DiagnosticsSettings.tsx+8-2
......@@ -1038,11 +1038,17 @@ export function DiagnosticsSettingsPanel() {
10381038 <StatBlock
10391039 label="Samples"
10401040 value={resourceData ? formatCount(resourceData.retainedSampleCount) : "..."}
1041 tooltip="In-memory process samples retained by the server. This resets when the server restarts."
1041 tooltip="In-memory process samples retained by the server. The server samples only while diagnostics is open, and this resets when the server restarts."
10421042 />
10431043 <StatBlock
10441044 label="Interval"
1045 value={resourceData ? formatDuration(resourceData.sampleIntervalMs) : "..."}
1045 value={
1046 resourceData
1047 ? resourceData.sampleIntervalMs === 0
1048 ? "On demand"
1049 : formatDuration(resourceData.sampleIntervalMs)
1050 : "..."
1051 }
10461052 />
10471053 <StatBlock
10481054 label="Processes"
docs/internals/resource-telemetry.md+5-3
......@@ -8,9 +8,11 @@ protocol. A missing or failed collector leaves the server running.
88
99## Collection cost
1010
11The native child owns sampling and bounded in-memory history. The server requests
12continuous snapshots only while diagnostics has live subscribers and fetches
13history on demand. Consuming host power for background scheduling must not retain
11The native child owns sampling and bounded in-memory history. It samples on a timer
12only while diagnostics has live subscribers. With none, the server sends a sample
13interval of 0 and the child samples only on request, so an idle server never walks
14the process table. History therefore covers only time with a live subscriber, plus
15on-demand samples. Consuming host power for background scheduling must not retain
1416live diagnostics. There is no telemetry database or recurring shell-probe fallback.
1517
1618History has independent bounds for age, snapshot count, process rows, and retained