Commit 4befc863f54a3341911c346c8822286ec37e54af | the-knife

apps/server/src/orchestration-v2/EventSink.ts+6-1

...	...	@@ -399,7 +399,12 @@ const layerBase: Layer.Layer<
399	399	yield* applyStoredEvents(committed);
400	400	yield* effectOutbox.enqueue(input.effects);
401	401	return committed;
402		}),
	402	}).pipe(
	403	// Every streamed agent event is a write, so its statements would
	404	// dominate the trace file. This span and its sql.transaction span
	405	// still time each write, including the commit.
	406	Effect.withTracerEnabled(false),
	407	),
403	408	(storedEvents) =>
404	409	Effect.gen(function* () {
405	410	if (input.effects.length > 0) {

apps/server/src/orchestration-v2/FoundationPersistence.test.ts+26

...	...	@@ -447,6 +447,32 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
447	447	),
448	448	);
449	449
	450	it.effect("traces an event write and its transaction without its SQL statements", () =>
	451	Effect.gen(function* () {
	452	const sink = yield* EventSink.EventSinkV2;
	453	const now = yield* DateTime.now;
	454	const thread = makeThread(ThreadId.make("thread:untraced-write-statements"), now);
	455	const names: Array<string> = [];
	456	const tracer = Tracer.make({
	457	span: (options) => {
	458	names.push(options.name);
	459	return new Tracer.NativeSpan(options);
	460	},
	461	});
	462
	463	yield* sink
	464	.write({
	465	events: [
	466	threadCreatedEvent({ id: "event:untraced-write-statements:created", thread, now }),
	467	],
	468	})
	469	.pipe(Effect.withTracer(tracer));
	470
	471	// These two spans time a slow commit; the statements inside are not traced.
	472	assert.deepEqual(names, ["orchestrationV2.EventSink.write", "sql.transaction"]);
	473	}),
	474	);
	475
450	476	it.effect.each(["high-water", "replay"] as const)(
451	477	"bounds live events while the V2 %s query is blocked",
452	478	(phase) =>

apps/server/src/provider/EventNdjsonLogger.test.ts+93-29

...	...	@@ -4,9 +4,13 @@ import * as NodeOS from "node:os";
4	4	import * as NodePath from "node:path";
5	5
6	6	import { ThreadId } from "@t3tools/contracts";
7		import { assert, describe, it } from "@effect/vitest";
	7	import * as NodeServices from "@effect/platform-node/NodeServices";
	8	import { assert, it } from "@effect/vitest";
	9	import { RotatingFileSinkError } from "@t3tools/shared/logging";
8	10	import * as Clock from "effect/Clock";
9	11	import * as Effect from "effect/Effect";
	12	import * as Deferred from "effect/Deferred";
	13	import * as FileSystem from "effect/FileSystem";
10	14	import * as Fiber from "effect/Fiber";
11	15	import * as Logger from "effect/Logger";
12	16	import * as Schema from "effect/Schema";
...	...	@@ -49,7 +53,7 @@ function parseLogLine(line: string) {
49	53	};
50	54	}
51	55
52		describe("EventNdjsonLogger", () => {
	56	it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => {
53	57	it.effect("summarizes circular events without exposing their contents in diagnostics", () => {
54	58	const messages: Array<unknown> = [];
55	59	const logCapture = Logger.make<unknown, void>(({ message }) => {
...	...	@@ -256,8 +260,8 @@ describe("EventNdjsonLogger", () => {
256	260
257	261	assert.equal(NodeFS.existsSync(threadPath), false);
258	262	yield* TestClock.adjust(1_000);
259		assert.equal(NodeFS.existsSync(threadPath), true);
260	263	yield* store.close();
	264	assert.equal(NodeFS.existsSync(threadPath), true);
261	265	} finally {
262	266	NodeFS.rmSync(tempDir, { recursive: true, force: true });
263	267	}
...	...	@@ -281,10 +285,10 @@ describe("EventNdjsonLogger", () => {
281	285	yield* logger.write({ id: "accepted" }, ThreadId.make("thread-interrupted"));
282	286
283	287	yield* TestClock.adjust(1_000);
	288	yield* store.close();
284	289
285	290	assert.equal(NodeFS.existsSync(threadPath), true);
286	291	assert.include(NodeFS.readFileSync(threadPath, "utf8"), '{"id":"accepted"}');
287		yield* store.close();
288	292	} finally {
289	293	NodeFS.rmSync(tempDir, { recursive: true, force: true });
290	294	}
...	...	@@ -788,39 +792,99 @@ describe("EventNdjsonLogger", () => {
788	792	}),
789	793	);
790	794
791		it("attributes batches that were written before a later chunk fails", () => {
792		const records: ReadonlyArray<PendingRecord> = [
793		{
794		stream: "native",
795		threadSegment: "thread",
796		line: "first",
797		bytes: 5,
798		},
799		{
800		stream: "canonical",
801		threadSegment: "thread",
802		line: "second",
803		bytes: 6,
804		},
805		];
806		const attributed: Array<PendingRecord> = [];
807		let writes = 0;
808
809		assert.throws(() =>
810		writeBatchedMessages(
	795	it.effect("attributes batches that were written before a later chunk fails", () =>
	796	Effect.gen(function* () {
	797	const records: ReadonlyArray<PendingRecord> = [
	798	{
	799	stream: "native",
	800	threadSegment: "thread",
	801	line: "first",
	802	bytes: 5,
	803	},
	804	{
	805	stream: "canonical",
	806	threadSegment: "thread",
	807	line: "second",
	808	bytes: 6,
	809	},
	810	];
	811	const attributed: Array<PendingRecord> = [];
	812	let writes = 0;
	813
	814	const result = yield* writeBatchedMessages(
811	815	{
812	816	write: () => {
813	817	writes += 1;
814		if (writes === 2) throw new Error("simulated disk exhaustion");
	818	return writes === 2
	819	? Effect.fail(
	820	new RotatingFileSinkError({
	821	operation: "write",
	822	filePath: "test.log",
	823	cause: "simulated disk exhaustion",
	824	}),
	825	)
	826	: Effect.void;
815	827	},
816	828	},
817	829	records,
818	830	5,
819	831	(written) => attributed.push(...written),
820		),
821		);
822		assert.deepEqual(attributed, [records[0]]);
823		});
	832	).pipe(Effect.result);
	833	assert.equal(result._tag, "Failure");
	834	assert.deepEqual(attributed, [records[0]]);
	835	}),
	836	);
	837
	838	it.effect(
	839	"keeps unrelated work responsive and drains before close while a disk write is held",
	840	() =>
	841	Effect.gen(function* () {
	842	const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
	843	const basePath = NodePath.join(tempDir, "events.log");
	844	const fs = yield* FileSystem.FileSystem;
	845	const started = yield* Deferred.make<void>();
	846	const release = yield* Deferred.make<void>();
	847	let writes = 0;
	848	try {
	849	const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }).pipe(
	850	Effect.provideService(FileSystem.FileSystem, {
	851	...fs,
	852	writeFile: (filePath, content, options) =>
	853	Effect.gen(function* () {
	854	if (++writes === 1) {
	855	yield* Deferred.succeed(started, undefined);
	856	yield* Deferred.await(release);
	857	}
	858	yield* fs.writeFile(filePath, content, options);
	859	}),
	860	}),
	861	);
	862	const first = yield* store
	863	.logger("native")
	864	.write({ id: "first" }, ThreadId.make("held"))
	865	.pipe(Effect.forkChild);
	866	yield* Deferred.await(started);
	867	const closed = yield* Deferred.make<void>();
	868	const close = yield* store
	869	.close()
	870	.pipe(Effect.andThen(Deferred.succeed(closed, undefined)), Effect.forkChild);
	871	assert.equal(yield* Deferred.isDone(closed), false);
	872	// A separate fiber runs while the file operation remains pending.
	873	const heartbeat = yield* Deferred.make<void>();
	874	yield* Effect.forkChild(Deferred.succeed(heartbeat, undefined));
	875	yield* Deferred.await(heartbeat);
	876	yield* Deferred.succeed(release, undefined);
	877	yield* Fiber.join(first);
	878	yield* Fiber.join(close);
	879	assert.include(
	880	NodeFS.readFileSync(ownedLogPath(basePath, "held"), "utf8"),
	881	'{"id":"first"}',
	882	);
	883	} finally {
	884	NodeFS.rmSync(tempDir, { recursive: true, force: true });
	885	}
	886	}),
	887	);
824	888
825	889	it.effect("reports logical provider log writes to resource attribution", () =>
826	890	Effect.gen(function* () {

apps/server/src/provider/EventNdjsonLogger.ts+106-76

...	...	@@ -10,11 +10,14 @@ import * as NodePath from "node:path";
10	10
11	11	import type { ThreadId } from "@t3tools/contracts";
12	12	import type * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers";
13		import { RotatingFileSink } from "@t3tools/shared/logging";
	13	import { makeAsyncRotatingFileSink } from "@t3tools/shared/logging";
14	14	import { errorTag } from "@t3tools/shared/observability";
15	15	import * as Clock from "effect/Clock";
16	16	import * as DateTime from "effect/DateTime";
17	17	import * as Effect from "effect/Effect";
	18	import * as FileSystem from "effect/FileSystem";
	19	import * as Path from "effect/Path";
	20	import * as Result from "effect/Result";
18	21	import * as Exit from "effect/Exit";
19	22	import * as Schema from "effect/Schema";
20	23	import * as Scope from "effect/Scope";
...	...	@@ -135,10 +138,12 @@ export interface PendingRecord {
135	138	readonly bytes: number;
136	139	}
137	140
	141	type AsyncFileSink = Effect.Success<ReturnType<typeof makeAsyncRotatingFileSink>>;
	142
138	143	interface StoreState {
139	144	readonly pending: Array<PendingRecord>;
140	145	readonly pendingBytes: number;
141		readonly sinks: ReadonlyMap<string, RotatingFileSink>;
	146	readonly sinks: ReadonlyMap<string, AsyncFileSink>;
142	147	readonly flushScheduled: boolean;
143	148	readonly closed: boolean;
144	149	readonly lastRetentionAt: number;
...	...	@@ -369,65 +374,69 @@ export function boundProviderEventForLogging(event: unknown): unknown {
369	374	return summarizeProviderEvent(event);
370	375	}
371	376
372		export function writeBatchedMessages(
373		sink: Pick<RotatingFileSink, "write">,
	377	export const writeBatchedMessages = Effect.fnUntraced(function* (
	378	sink: AsyncFileSink,
374	379	records: ReadonlyArray<PendingRecord>,
375	380	maxBytes: number,
376	381	onWritten: (records: ReadonlyArray<PendingRecord>) => void,
377		): void {
	382	) {
378	383	let pendingRecords: Array<PendingRecord> = [];
379	384	let pendingBytes = 0;
380	385
381		const flush = () => {
	386	const flush = Effect.gen(function* () {
382	387	if (pendingRecords.length === 0) return;
383	388	const writtenRecords = pendingRecords;
384		sink.write(writtenRecords.map((record) => record.line).join(""));
	389	yield* sink.write(writtenRecords.map((record) => record.line).join(""));
385	390	onWritten(writtenRecords);
386	391	pendingRecords = [];
387	392	pendingBytes = 0;
388		};
	393	});
389	394
390	395	for (const record of records) {
391	396	if (pendingBytes > 0 && pendingBytes + record.bytes > maxBytes) {
392		flush();
	397	yield* flush;
393	398	}
394	399	pendingRecords.push(record);
395	400	pendingBytes += record.bytes;
396	401	if (pendingBytes >= maxBytes) {
397		flush();
	402	yield* flush;
398	403	}
399	404	}
400		flush();
401		}
	405	yield* flush;
	406	});
402	407
403		function isProviderLogFile(filePath: string, fileName: string, filePrefix: string): boolean {
	408	async function isProviderLogFile(
	409	filePath: string,
	410	fileName: string,
	411	filePrefix: string,
	412	): Promise<boolean> {
404	413	if (!/\.log(?:\.\d+)?$/u.test(fileName)) return false;
405	414	if (fileName.startsWith(filePrefix)) return true;
406	415
407		const descriptor = NodeFS.openSync(filePath, "r");
	416	const descriptor = await NodeFS.promises.open(filePath, "r");
408	417	try {
409	418	const header = Buffer.alloc(256);
410		const bytesRead = NodeFS.readSync(descriptor, header, 0, header.byteLength, 0);
	419	const { bytesRead } = await descriptor.read(header, 0, header.byteLength, 0);
411	420	return /^\[[^\]\r\n]+\] (?:NTIVE\|CANON\|ORCH): /u.test(header.toString("utf8", 0, bytesRead));
412	421	} finally {
413		NodeFS.closeSync(descriptor);
	422	await descriptor.close();
414	423	}
415	424	}
416	425
417		function enforceRetention(input: {
	426	async function enforceRetention(input: {
418	427	readonly directory: string;
419	428	readonly maxTotalBytes: number;
420	429	readonly maxAgeMs: number;
421	430	readonly activeFilePaths: ReadonlySet<string>;
422	431	readonly filePrefix: string;
423	432	readonly now: number;
424		}): RetentionResult {
	433	}): Promise<RetentionResult> {
425	434	const failures: Array<FileOperationFailure> = [];
426	435	const files: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
427	436
428	437	let entries: ReadonlyArray<NodeFS.Dirent>;
429	438	try {
430		entries = NodeFS.readdirSync(input.directory, { withFileTypes: true });
	439	entries = await NodeFS.promises.readdir(input.directory, { withFileTypes: true });
431	440	} catch (cause) {
432	441	return { failures: [{ filePath: input.directory, cause }] };
433	442	}
...	...	@@ -436,8 +445,8 @@ function enforceRetention(input: {
436	445	if (!entry.isFile()) continue;
437	446	const filePath = NodePath.join(input.directory, entry.name);
438	447	try {
439		if (!isProviderLogFile(filePath, entry.name, input.filePrefix)) continue;
440		const stat = NodeFS.statSync(filePath);
	448	if (!(await isProviderLogFile(filePath, entry.name, input.filePrefix))) continue;
	449	const stat = await NodeFS.promises.stat(filePath);
441	450	files.push({ filePath, mtimeMs: stat.mtimeMs, size: stat.size });
442	451	} catch (cause) {
443	452	failures.push({ filePath, cause });
...	...	@@ -445,10 +454,10 @@ function enforceRetention(input: {
445	454	}
446	455
447	456	let totalBytes = files.reduce((total, file) => total + file.size, 0);
448		const remove = (file: (typeof files)[number]) => {
	457	const remove = async (file: (typeof files)[number]) => {
449	458	if (input.activeFilePaths.has(file.filePath)) return false;
450	459	try {
451		NodeFS.rmSync(file.filePath, { force: true });
	460	await NodeFS.promises.rm(file.filePath, { force: true });
452	461	totalBytes -= file.size;
453	462	return true;
454	463	} catch (cause) {
...	...	@@ -457,16 +466,16 @@ function enforceRetention(input: {
457	466	}
458	467	};
459	468
460		const retained = files.filter((file) => {
461		if (input.now - file.mtimeMs <= input.maxAgeMs) return true;
462		return !remove(file);
463		});
	469	const retained: typeof files = [];
	470	for (const file of files) {
	471	if (input.now - file.mtimeMs <= input.maxAgeMs \|\| !(await remove(file))) retained.push(file);
	472	}
464	473
465	474	for (const file of retained.toSorted(
466	475	(left, right) => left.mtimeMs - right.mtimeMs \|\| left.filePath.localeCompare(right.filePath),
467	476	)) {
468	477	if (totalBytes <= input.maxTotalBytes) break;
469		remove(file);
	478	await remove(file);
470	479	}
471	480
472	481	return { failures };
...	...	@@ -517,7 +526,7 @@ function resolveOptions(
517	526	return Effect.succeed(resolved);
518	527	}
519	528
520		function drainPending(input: {
	529	const drainPending = Effect.fnUntraced(function* (input: {
521	530	readonly directory: string;
522	531	readonly options: ResolvedOptions;
523	532	readonly state: StoreState;
...	...	@@ -525,7 +534,10 @@ function drainPending(input: {
525	534	readonly now: number;
526	535	readonly timerFired: boolean;
527	536	readonly close: boolean;
528		}): readonly [DrainResult, StoreState] {
	537	readonly makeSink: (
	538	filePath: string,
	539	) => Effect.Effect<AsyncFileSink, Effect.Error<ReturnType<typeof makeAsyncRotatingFileSink>>>;
	540	}): Effect.fn.Return<readonly [DrainResult, StoreState]> {
529	541	if (input.state.closed) {
530	542	return [{ attributions: [], failures: [] }, input.state];
531	543	}
...	...	@@ -548,22 +560,20 @@ function drainPending(input: {
548	560	const filePath = providerLogPath(input.directory, input.filePrefix, threadSegment);
549	561	let sink = sinks.get(threadSegment);
550	562	if (!sink) {
551		try {
552		sink = new RotatingFileSink({
553		filePath,
554		maxBytes: input.options.maxBytes,
555		maxFiles: input.options.maxFiles,
556		throwOnError: true,
557		});
558		sinks.set(threadSegment, sink);
559		} catch (cause) {
560		failures.push({ filePath, cause });
	563	const created = yield* input.makeSink(filePath).pipe(Effect.result);
	564	if (Result.isFailure(created)) {
	565	failures.push({ filePath, cause: created.failure });
561	566	continue;
562	567	}
	568	sink = created.success;
	569	sinks.set(threadSegment, sink);
563	570	}
564	571
565		try {
566		writeBatchedMessages(sink, records, input.options.maxBytes, (writtenRecords) => {
	572	const written = yield* writeBatchedMessages(
	573	sink,
	574	records,
	575	input.options.maxBytes,
	576	(writtenRecords) => {
567	577	for (const record of writtenRecords) {
568	578	const current = attributionByStream.get(record.stream) ?? {
569	579	count: 0,
...	...	@@ -574,28 +584,31 @@ function drainPending(input: {
574	584	logicalWriteBytes: current.logicalWriteBytes + record.bytes,
575	585	});
576	586	}
577		});
578		} catch (cause) {
	587	},
	588	).pipe(Effect.result);
	589	if (Result.isFailure(written)) {
579	590	sinks.delete(threadSegment);
580		failures.push({ filePath, cause });
	591	failures.push({ filePath, cause: written.failure });
581	592	}
582	593	}
583	594
584	595	const retentionDue =
585	596	input.now - input.state.lastRetentionAt >= input.options.retentionCheckIntervalMs;
586	597	const retention = retentionDue
587		? enforceRetention({
588		directory: input.directory,
589		maxTotalBytes: input.options.maxTotalBytes,
590		maxAgeMs: input.options.maxAgeMs,
591		activeFilePaths: new Set(
592		Array.from(sinks.keys(), (threadSegment) =>
593		providerLogPath(input.directory, input.filePrefix, threadSegment),
	598	? yield* Effect.promise(() =>
	599	enforceRetention({
	600	directory: input.directory,
	601	maxTotalBytes: input.options.maxTotalBytes,
	602	maxAgeMs: input.options.maxAgeMs,
	603	activeFilePaths: new Set(
	604	Array.from(sinks.keys(), (threadSegment) =>
	605	providerLogPath(input.directory, input.filePrefix, threadSegment),
	606	),
594	607	),
595		),
596		filePrefix: input.filePrefix,
597		now: input.now,
598		})
	608	filePrefix: input.filePrefix,
	609	now: input.now,
	610	}),
	611	)
599	612	: { failures: [] };
600	613
601	614	return [
...	...	@@ -615,7 +628,7 @@ function drainPending(input: {
615	628	lastRetentionAt: retentionDue ? input.now : input.state.lastRetentionAt,
616	629	},
617	630	];
618		}
	631	});
619	632
620	633	const serializeEvent = Effect.fnUntraced(function* (event: unknown) {
621	634	return yield* encodeUnknownJsonString(event).pipe(
...	...	@@ -630,18 +643,32 @@ const serializeEvent = Effect.fnUntraced(function* (event: unknown) {
630	643	export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
631	644	filePath: string,
632	645	options: EventNdjsonLogStoreOptions = {},
633		): Effect.fn.Return<EventNdjsonLogStore, EventNdjsonLogStoreError> {
	646	): Effect.fn.Return<
	647	EventNdjsonLogStore,
	648	EventNdjsonLogStoreError,
	649	FileSystem.FileSystem \| Path.Path
	650	> {
	651	const fs = yield* FileSystem.FileSystem;
	652	const path = yield* Path.Path;
634	653	const resolved = yield* resolveOptions(filePath, options);
635	654	const directory = NodePath.dirname(filePath);
636	655	const filePrefix = providerLogPrefix(filePath);
637	656
638		yield* Effect.try({
639		try: () => NodeFS.mkdirSync(directory, { recursive: true }),
640		catch: (cause) => new EventNdjsonLogDirectoryError({ directory, cause }),
641		});
	657	yield* fs
	658	.makeDirectory(directory, { recursive: true })
	659	.pipe(Effect.mapError((cause) => new EventNdjsonLogDirectoryError({ directory, cause })));
	660	const makeSink = (filePath: string) =>
	661	makeAsyncRotatingFileSink({
	662	filePath,
	663	maxBytes: resolved.maxBytes,
	664	maxFiles: resolved.maxFiles,
	665	}).pipe(
	666	Effect.provideService(FileSystem.FileSystem, fs),
	667	Effect.provideService(Path.Path, path),
	668	);
642	669
643	670	const initializedAt = yield* Clock.currentTimeMillis;
644		const initialRetention = yield* Effect.sync(() =>
	671	const initialRetention = yield* Effect.promise(() =>
645	672	enforceRetention({
646	673	directory,
647	674	maxTotalBytes: resolved.maxTotalBytes,
...	...	@@ -671,17 +698,16 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
671	698	const flush = Effect.fnUntraced(function* (timerFired: boolean, close: boolean) {
672	699	const startedAt = yield* Clock.currentTimeMillis;
673	700	const result = yield* SynchronizedRef.modifyEffect(stateRef, (state) =>
674		Effect.sync(() =>
675		drainPending({
676		directory,
677		options: resolved,
678		state,
679		filePrefix,
680		now: startedAt,
681		timerFired,
682		close,
683		}),
684		),
	701	drainPending({
	702	directory,
	703	options: resolved,
	704	state,
	705	filePrefix,
	706	now: startedAt,
	707	timerFired,
	708	close,
	709	makeSink,
	710	}),
685	711	);
686	712
687	713	for (const failure of result.failures) {
...	...	@@ -714,7 +740,7 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
714	740	{ discard: true },
715	741	);
716	742	}
717		});
	743	}, Effect.uninterruptible);
718	744
719	745	const scheduleFlush = Effect.fnUntraced(function* () {
720	746	yield* Effect.forkIn(
...	...	@@ -791,7 +817,11 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
791	817	export const makeEventNdjsonLogger = Effect.fnUntraced(function* (
792	818	filePath: string,
793	819	options: EventNdjsonLoggerOptions,
794		): Effect.fn.Return<ProviderEventLoggers.EventNdjsonLogger \| undefined> {
	820	): Effect.fn.Return<
	821	ProviderEventLoggers.EventNdjsonLogger \| undefined,
	822	never,
	823	FileSystem.FileSystem \| Path.Path
	824	> {
795	825	const store = yield* makeEventNdjsonLogStore(filePath, options).pipe(
796	826	Effect.catch((error) =>
797	827	logWarning(error.message, { error }).pipe(

apps/server/src/serverRuntimeStartup.ts+15-9

...	...	@@ -346,8 +346,10 @@ const maybeOpenBrowser = (target: string) =>
346	346
347	347	const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>
348	348	effect.pipe(
349		Effect.annotateSpans({ "startup.phase": phase }),
350		Effect.withSpan(`server.startup.${phase}`),
	349	// Attributes on the span itself, not annotateSpans: an annotation follows
	350	// every fiber forked inside the phase, such as the effect worker, for the
	351	// life of the server.
	352	Effect.withSpan(`server.startup.${phase}`, { attributes: { "startup.phase": phase } }),
351	353	);
352	354
353	355	interface StartupOptions {
...	...	@@ -581,8 +583,9 @@ const make = (options?: StartupOptions) =>
581	583	Effect.gen(function* () {
582	584	yield* Effect.logDebug("startup phase: recording startup heartbeat");
583	585	yield* recordStartupHeartbeat.pipe(
584		Effect.annotateSpans({ "startup.phase": "heartbeat.record" }),
585		Effect.withSpan("server.startup.heartbeat.record"),
	586	Effect.withSpan("server.startup.heartbeat.record", {
	587	attributes: { "startup.phase": "heartbeat.record" },
	588	}),
586	589	Effect.ignoreCause({ log: true }),
587	590	);
588	591	if (serverConfig.startupPresentation === "headless") {
...	...	@@ -653,12 +656,15 @@ const make = (options?: StartupOptions) =>
653	656	yield* Effect.logDebug("startup phase: complete");
654	657	yield* flushCompileCache;
655	658	}).pipe(
656		Effect.annotateSpans({
657		"server.mode": serverConfig.mode,
658		"server.port": serverConfig.port,
659		"server.host": serverConfig.host ?? "default",
	659	Effect.withSpan("server.startup", {
	660	kind: "server",
	661	root: true,
	662	attributes: {
	663	"server.mode": serverConfig.mode,
	664	"server.port": serverConfig.port,
	665	"server.host": serverConfig.host ?? "default",
	666	},
660	667	}),
661		Effect.withSpan("server.startup", { kind: "server", root: true }),
662	668	);
663	669
664	670	yield* Effect.forkScoped(

docs/internals/performance-regressions.md+6

...	...	@@ -90,6 +90,12 @@ counts keep the full status path. Cached fetch failures share the existing backo
90	90	the actual failed attempt. Unchanged desktop environment bootstrap reads retain their array
91	91	identity so polling does not invalidate subscribers.
92	92
	93	Local trace records and provider event logs use asynchronous, serialized file writes and rotation.
	94	Batching synchronous appends still stalls the server event loop on slow storage, especially on
	95	Windows. While a trace write is slow, producers keep up to one trace file of records, so the spans
	96	that time a stall survive it; records drop only while the disk stays stuck. Provider logging
	97	retains backpressure. Shutdown drains accepted writes before closing their scope.
	98
93	99	Relay awareness queues one pending publication per thread and retains a follow-up when the thread
94	100	changes during a send. Transcript deltas and tool output do not enqueue awareness work. Run,
95	101	request, and relevant thread metadata events do. Configuration is checked before shell reads, and

docs/operations/observability.md+2-1

...	...	@@ -646,7 +646,8 @@ Current high-value span and metric boundaries include:
646	646	- provider session and turn operations
647	647	- git command execution and git hook events
648	648	- terminal session lifecycle
649		- sqlite query execution
	649	- sqlite query execution, except the statements inside `orchestrationV2.EventSink.write`, which
	650	its `sql.transaction` span times as one
650	651	- event loop stalls (`server.eventLoop.stall`)
651	652
652	653	### Current Constraints

packages/shared/src/logging.test.ts+61

...	...	@@ -3,12 +3,17 @@ import * as NodeFS from "node:fs";
3	3	import * as NodeOS from "node:os";
4	4	import * as NodePath from "node:path";
5	5	import { afterEach, describe, expect, it } from "vite-plus/test";
	6	import * as NodeServices from "@effect/platform-node/NodeServices";
	7	import { it as effectIt } from "@effect/vitest";
	8	import * as Effect from "effect/Effect";
	9	import * as FileSystem from "effect/FileSystem";
6	10	import * as HostProcess from "./HostProcess.ts";
7	11
8	12	import {
9	13	RotatingFileSink,
10	14	RotatingFileSinkConfigurationError,
11	15	RotatingFileSinkError,
	16	makeAsyncRotatingFileSink,
12	17	} from "./logging.ts";
13	18
14	19	const windowsHost = HostProcess.Platform.defaultValue() === "win32";
...	...	@@ -35,6 +40,62 @@ afterEach(() => {
35	40	}
36	41	});
37	42
	43	effectIt.layer(NodeServices.layer)("asynchronous rotating file sink", (it) => {
	44	it.effect("serializes concurrent appends across rotation and resumes an existing file", () =>
	45	Effect.gen(function* () {
	46	const directory = makeTempDirectory();
	47	const filePath = NodePath.join(directory, "async.log");
	48	NodeFS.writeFileSync(`${filePath}.50`, "obsolete");
	49	const lines = Array.from(
	50	{ length: 100 },
	51	(_, index) => `${index.toString().padStart(3, "0")}\n`,
	52	);
	53	const sink = yield* makeAsyncRotatingFileSink({ filePath, maxBytes: 32, maxFiles: 20 });
	54	yield* Effect.forEach(lines, (line) => sink.write(line), {
	55	concurrency: "unbounded",
	56	discard: true,
	57	});
	58	const resumed = yield* makeAsyncRotatingFileSink({ filePath, maxBytes: 32, maxFiles: 20 });
	59	yield* resumed.write("100\n");
	60	expect(NodeFS.existsSync(`${filePath}.50`)).toBe(false);
	61	const backups = NodeFS.readdirSync(directory)
	62	.filter((name) => name.startsWith("async.log."))
	63	.sort((a, b) => Number(b.slice(10)) - Number(a.slice(10)));
	64	const contents = [...backups, "async.log"].map((name) =>
	65	NodeFS.readFileSync(NodePath.join(directory, name), "utf8"),
	66	);
	67	expect(contents.every((chunk) => Buffer.byteLength(chunk) <= 32)).toBe(true);
	68	expect(contents.join("")).toBe(lines.join("") + "100\n");
	69	}),
	70	);
	71
	72	it.effect("reports a failed append and recovers its size before the next rotation", () =>
	73	Effect.gen(function* () {
	74	const directory = makeTempDirectory();
	75	const filePath = NodePath.join(directory, "recovery.log");
	76	NodeFS.writeFileSync(filePath, "old\n");
	77	const fs = yield* FileSystem.FileSystem;
	78	let attempts = 0;
	79	const sink = yield* makeAsyncRotatingFileSink({ filePath, maxBytes: 8, maxFiles: 2 }).pipe(
	80	Effect.provideService(FileSystem.FileSystem, {
	81	...fs,
	82	writeFile: (path, bytes, options) =>
	83	++attempts === 1
	84	? fs.writeFile(directory, bytes, options)
	85	: fs.writeFile(path, bytes, options),
	86	}),
	87	);
	88	const failed = yield* sink.write("new\n").pipe(Effect.result);
	89	expect(failed._tag).toBe("Failure");
	90	if (failed._tag === "Failure") expect(failed.failure.operation).toBe("write");
	91	yield* sink.write("new\n");
	92	yield* sink.write("end\n");
	93	expect(NodeFS.readFileSync(`${filePath}.1`, "utf8")).toBe("old\nnew\n");
	94	expect(NodeFS.readFileSync(filePath, "utf8")).toBe("end\n");
	95	}),
	96	);
	97	});
	98
38	99	describe("RotatingFileSink", () => {
39	100	it.each([
40	101	{ option: "maxBytes" as const, maxBytes: 0, maxFiles: 1 },

packages/shared/src/logging.ts+83

...	...	@@ -2,6 +2,10 @@
2	2	import * as NodeFS from "node:fs";
3	3	import * as NodePath from "node:path";
4	4	import * as Schema from "effect/Schema";
	5	import * as Effect from "effect/Effect";
	6	import * as FileSystem from "effect/FileSystem";
	7	import * as Path from "effect/Path";
	8	import * as Semaphore from "effect/Semaphore";
5	9
6	10	export interface RotatingFileSinkOptions {
7	11	readonly filePath: string;
...	...	@@ -180,3 +184,82 @@ export class RotatingFileSink {
180	184	return `${this.filePath}.${index}`;
181	185	}
182	186	}
	187
	188	/** Serialized asynchronous writes and rotation without blocking the event loop. */
	189	export const makeAsyncRotatingFileSink = Effect.fnUntraced(function* (
	190	options: Omit<RotatingFileSinkOptions, "throwOnError">,
	191	) {
	192	const fs = yield* FileSystem.FileSystem;
	193	const path = yield* Path.Path;
	194	if (options.maxBytes < 1)
	195	return yield* new RotatingFileSinkConfigurationError({
	196	option: "maxBytes",
	197	received: options.maxBytes,
	198	minimum: 1,
	199	});
	200	if (options.maxFiles < 1)
	201	return yield* new RotatingFileSinkConfigurationError({
	202	option: "maxFiles",
	203	received: options.maxFiles,
	204	minimum: 1,
	205	});
	206	const failure = (operation: RotatingFileSinkError["operation"]) => (cause: unknown) =>
	207	new RotatingFileSinkError({ operation, filePath: options.filePath, cause });
	208	const directory = path.dirname(options.filePath);
	209	const basename = path.basename(options.filePath);
	210	yield* fs
	211	.makeDirectory(directory, { recursive: true })
	212	.pipe(Effect.mapError(failure("initialize")));
	213	yield* Effect.gen(function* () {
	214	for (const entry of yield* fs.readDirectory(directory)) {
	215	if (!entry.startsWith(`${basename}.`)) continue;
	216	const suffix = Number(entry.slice(basename.length + 1));
	217	if (!Number.isInteger(suffix) \|\| suffix <= options.maxFiles) continue;
	218	yield* fs.remove(path.join(directory, entry), { force: true });
	219	}
	220	}).pipe(Effect.mapError(failure("prune")));
	221	const readSize = fs.stat(options.filePath).pipe(
	222	Effect.map((stat) => Number(stat.size)),
	223	Effect.catch((cause) =>
	224	cause.reason._tag === "NotFound" ? Effect.succeed(0) : Effect.fail(failure("read")(cause)),
	225	),
	226	);
	227	let currentSize = yield* readSize;
	228	const semaphore = yield* Semaphore.make(1);
	229	const rotate = Effect.gen(function* () {
	230	yield* fs.remove(`${options.filePath}.${options.maxFiles}`, { force: true });
	231	for (let index = options.maxFiles - 1; index >= 1; index--) {
	232	const source = `${options.filePath}.${index}`;
	233	if (yield* fs.exists(source)) yield* fs.rename(source, `${options.filePath}.${index + 1}`);
	234	}
	235	if (yield* fs.exists(options.filePath))
	236	yield* fs.rename(options.filePath, `${options.filePath}.1`);
	237	currentSize = 0;
	238	}).pipe(Effect.mapError(failure("rotate")));
	239	const write = (chunk: string \| Buffer) =>
	240	semaphore
	241	.withPermits(1)(
	242	Effect.gen(function* () {
	243	const buffer = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
	244	if (buffer.length === 0) return;
	245	if (currentSize > 0 && currentSize + buffer.length > options.maxBytes) yield* rotate;
	246	yield* fs
	247	.writeFile(options.filePath, buffer, { flag: "a" })
	248	.pipe(Effect.mapError(failure("write")));
	249	currentSize += buffer.length;
	250	}).pipe(
	251	Effect.tapError(() =>
	252	readSize.pipe(
	253	Effect.tap((size) =>
	254	Effect.sync(() => {
	255	currentSize = size;
	256	}),
	257	),
	258	Effect.ignore,
	259	),
	260	),
	261	),
	262	)
	263	.pipe(Effect.uninterruptible);
	264	return { write };
	265	});

packages/shared/src/observability.test.ts+70-6

...	...	@@ -3,6 +3,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
3	3	import * as Arr from "effect/Array";
4	4	import * as Cause from "effect/Cause";
5	5	import * as Effect from "effect/Effect";
	6	import * as Deferred from "effect/Deferred";
	7	import * as Fiber from "effect/Fiber";
6	8	import * as Exit from "effect/Exit";
7	9	import * as FileSystem from "effect/FileSystem";
8	10	import * as Layer from "effect/Layer";
...	...	@@ -16,7 +18,6 @@ import * as Tracer from "effect/Tracer";
16	18	import * as TestClock from "effect/testing/TestClock";
17	19	import { vi } from "vite-plus/test";
18	20
19		import { RotatingFileSink } from "./logging.ts";
20	21	import {
21	22	causeErrorTag,
22	23	compactTraceAttributes,
...	...	@@ -242,7 +243,7 @@ describe("observability", () => {
242	243	sampled ? [{ _tag: "Success" }] : [],
243	244	);
244	245	}
245		}),
	246	}).pipe(Effect.provide(NodeServices.layer)),
246	247	);
247	248
248	249	it.effect("preserves failure and interruption causes in spans and exported traces", () =>
...	...	@@ -290,7 +291,7 @@ describe("observability", () => {
290	291	}
291	292	assert.equal(records[0]?.exit._tag, Cause.hasInterrupts(cause) ? "Interrupted" : "Failure");
292	293	}
293		}),
	294	}).pipe(Effect.provide(NodeServices.layer)),
294	295	);
295	296
296	297	it("normalizes circular arrays, maps, and sets without recursing forever", () => {
...	...	@@ -331,6 +332,64 @@ describe("observability", () => {
331	332	});
332	333
333	334	nodeServicesIt("node services", (it) => {
	335	it.effect(
	336	"keeps trace producers responsive during a slow write, bounds the backlog, and drains on close",
	337	() =>
	338	Effect.scoped(
	339	Effect.gen(function* () {
	340	const fs = yield* FileSystem.FileSystem;
	341	const path = yield* Path.Path;
	342	const directory = yield* fs.makeTempDirectoryScoped({
	343	prefix: "t3-trace-backpressure-",
	344	});
	345	const tracePath = path.join(directory, "trace.ndjson");
	346	const started = yield* Deferred.make<void>();
	347	const release = yield* Deferred.make<void>();
	348	const closed = yield* Ref.make(false);
	349	const sink = yield* makeTraceSink({
	350	filePath: tracePath,
	351	maxBytes: 5_000_000,
	352	maxFiles: 2,
	353	batchWindowMs: 10_000,
	354	}).pipe(
	355	Effect.provideService(FileSystem.FileSystem, {
	356	...fs,
	357	writeFile: (file, data, options) =>
	358	Effect.gen(function* () {
	359	yield* Deferred.succeed(started, undefined);
	360	yield* Deferred.await(release);
	361	yield* fs.writeFile(file, data, options);
	362	}),
	363	}),
	364	);
	365	sink.push(makeRecord("first"));
	366	const flushing = yield* Effect.forkChild(sink.flush);
	367	yield* Deferred.await(started);
	368	// These pushes return while the write is held. The backlog keeps one
	369	// trace file of records: every small one, then large ones until the
	370	// next would not fit.
	371	for (let index = 0; index < 2_000; index++)
	372	sink.push(makeRecord("small", String(index)));
	373	for (let index = 0; index < 3; index++)
	374	sink.push({ ...makeRecord("large"), attributes: { payload: "x".repeat(2_000_000) } });
	375	const closing = yield* Effect.forkChild(
	376	sink.close().pipe(Effect.tap(() => Ref.set(closed, true))),
	377	);
	378	yield* Effect.yieldNow;
	379	assert.isFalse(yield* Ref.get(closed));
	380	yield* Deferred.succeed(release, undefined);
	381	yield* Fiber.join(flushing);
	382	yield* Fiber.join(closing);
	383	const records = yield* readTraceRecords(tracePath);
	384	assert.deepStrictEqual(
	385	records.map((record) => record.name),
	386	["first", ...Array.from({ length: 2_000 }, () => "small"), "large", "large"],
	387	);
	388	assert.isTrue(yield* Ref.get(closed));
	389	}),
	390	),
	391	);
	392
334	393	it.effect("flushes buffered trace records on close", () =>
335	394	Effect.scoped(
336	395	Effect.gen(function* () {
...	...	@@ -551,7 +610,7 @@ describe("observability", () => {
551	610	const tracePath = path.join(tempDir, "shared.trace.ndjson");
552	611	// A directory at the trace path fails every append, like a full disk.
553	612	yield* fileSystem.makeDirectory(tracePath);
554		const write = vi.spyOn(RotatingFileSink.prototype, "write");
	613	const write = vi.spyOn(fileSystem, "writeFile");
555	614	yield* Effect.addFinalizer(() => Effect.sync(() => write.mockRestore()));
556	615
557	616	const sink = yield* makeTraceSink({
...	...	@@ -564,16 +623,18 @@ describe("observability", () => {
564	623	for (let index = 0; index < 1_024; index += 1) {
565	624	sink.push(makeRecord("lost", String(index)));
566	625	}
	626	yield* sink.flush;
567	627	// Timed flushes run in the fiber forked inside the makeTraceSink span.
568	628	for (let index = 0; index < 5; index += 1) {
569	629	sink.push(makeRecord("lost"));
570	630	yield* TestClock.adjust("1 second");
	631	yield* sink.flush;
571	632	}
572	633
573	634	// One write per batch, never a growing backlog.
574	635	assert.deepStrictEqual(
575		write.mock.calls.map(([chunk]) => String(chunk).split("\n").length - 1),
576		[256, 256, 256, 256, 1, 1, 1, 1, 1],
	636	write.mock.calls.map(([, chunk]) => Buffer.from(chunk).toString().split("\n").length - 1),
	637	[1024, 1, 1, 1, 1, 1],
577	638	);
578	639	expect(logs).toEqual([
579	640	{ logLevel: "Warn", message: [expect.any(String), { filePath: tracePath }] },
...	...	@@ -583,6 +644,7 @@ describe("observability", () => {
583	644	yield* fileSystem.remove(tracePath, { recursive: true });
584	645	sink.push(makeRecord("recovered"));
585	646	yield* TestClock.adjust("1 second");
	647	yield* sink.flush;
586	648
587	649	const records = yield* readTraceRecords(tracePath);
588	650	assert.deepStrictEqual(
...	...	@@ -597,6 +659,7 @@ describe("observability", () => {
597	659	// Healthy flushes after the recovery log nothing.
598	660	sink.push(makeRecord("healthy"));
599	661	yield* TestClock.adjust("1 second");
	662	yield* sink.flush;
600	663	expect(logs).toHaveLength(2);
601	664
602	665	// A new failure episode warns again.
...	...	@@ -604,6 +667,7 @@ describe("observability", () => {
604	667	yield* fileSystem.makeDirectory(tracePath);
605	668	sink.push(makeRecord("lost-again"));
606	669	yield* TestClock.adjust("1 second");
	670	yield* sink.flush;
607	671	assert.deepStrictEqual(
608	672	logs.map((log) => log.logLevel),
609	673	["Warn", "Info", "Warn"],

packages/shared/src/observability.ts+84-54

...	...	@@ -4,13 +4,16 @@ import * as Effect from "effect/Effect";
4	4	import type * as Exit from "effect/Exit";
5	5	import * as ExitRuntime from "effect/Exit";
6	6	import * as Option from "effect/Option";
	7	import * as Queue from "effect/Queue";
	8	import * as Result from "effect/Result";
	9	import * as Semaphore from "effect/Semaphore";
7	10	import * as Schema from "effect/Schema";
8	11	import * as SchemaIssue from "effect/SchemaIssue";
9	12	import * as SchemaTransformation from "effect/SchemaTransformation";
10	13	import * as Tracer from "effect/Tracer";
11	14	import { OtlpResource, OtlpTracer, OtlpSerialization } from "effect/observability";
12	15
13		import { RotatingFileSink } from "./logging.ts";
	16	import { makeAsyncRotatingFileSink } from "./logging.ts";
14	17
15	18	export const OtlpProtocol = Schema.Literals(["http/json", "http/protobuf"]);
16	19	export type OtlpProtocol = typeof OtlpProtocol.Type;
...	...	@@ -37,7 +40,6 @@ export const DEFAULT_SIGNAL_EXPORT: SignalExport = {
37	40	};
38	41
39	42	const FLUSH_BUFFER_THRESHOLD = 256;
40		const textEncoder = new TextEncoder();
41	43
42	44	export type TraceAttributes = Readonly<Record<string, unknown>>;
43	45
...	...	@@ -364,14 +366,20 @@ function spanToTraceRecord(span: SerializableSpan): EffectTraceRecord {
364	366	}
365	367
366	368	export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {
367		const sink = new RotatingFileSink({
	369	const sink = yield* makeAsyncRotatingFileSink({
368	370	filePath: options.filePath,
369	371	maxBytes: options.maxBytes,
370	372	maxFiles: options.maxFiles,
371		throwOnError: true,
372		});
373
374		let buffer: Array<string> = [];
	373	}).pipe(Effect.orDie);
	374
	375	let buffer: Array<{ readonly text: string; readonly bytes: number }> = [];
	376	let bufferBytes = 0;
	377	// While a write is slow, push keeps up to one trace file of records (at least
	378	// 4 MiB), so the spans that time a stall still reach the file. Only a disk
	379	// that stays stuck drops records, instead of growing memory or blocking push.
	380	const maxBufferBytes = Math.max(options.maxBytes, 4 * 1024 * 1024);
	381	const flushRequests = yield* Queue.sliding<void>(1);
	382	const flushSemaphore = yield* Semaphore.make(1);
375	383	// A failure episode starts at the first dropped record and ends when a flush
376	384	// sees that the latest write succeeded. Flush checks once per window, so a
377	385	// disk that flaps inside one window stays one episode.
...	...	@@ -384,36 +392,35 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Trac
384	392	durationMs: 0,
385	393	};
386	394
387		const flushUnsafe = () => {
	395	const flushBuffered = Effect.gen(function* () {
388	396	if (buffer.length === 0) {
389	397	return;
390	398	}
391	399
392	400	const records = buffer;
393	401	buffer = [];
	402	bufferBytes = 0;
394	403	let persistedCount = 0;
395	404
396	405	while (persistedCount < records.length) {
397		const firstRecordBytes = textEncoder.encode(records[persistedCount]).byteLength;
398		if (firstRecordBytes > options.maxBytes) {
399		persistedCount += 1;
400		continue;
401		}
402
	406	// push drops any record larger than a file, so the first one always fits.
	407	const firstRecordBytes = records[persistedCount]!.bytes;
403	408	let nextIndex = persistedCount + 1;
404	409	let chunkBytes = firstRecordBytes;
405	410	while (nextIndex < records.length) {
406		const nextRecordBytes = textEncoder.encode(records[nextIndex]).byteLength;
	411	const nextRecordBytes = records[nextIndex]!.bytes;
407	412	if (chunkBytes + nextRecordBytes > options.maxBytes) break;
408	413	chunkBytes += nextRecordBytes;
409	414	nextIndex += 1;
410	415	}
411	416
412		const chunk = records.slice(persistedCount, nextIndex).join("");
	417	const chunk = records
	418	.slice(persistedCount, nextIndex)
	419	.map((record) => record.text)
	420	.join("");
413	421	const startedAt = performance.now();
414		try {
415		sink.write(chunk);
416		} catch {
	422	const written = yield* sink.write(chunk).pipe(Effect.result);
	423	if (Result.isFailure(written)) {
417	424	// A failing disk (ENOSPC, EACCES, EIO) drops the rest of the batch.
418	425	// Retrying it would grow the backlog, and every later push would
419	426	// retry all of it.
...	...	@@ -429,54 +436,77 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Trac
429	436	};
430	437	persistedCount = nextIndex;
431	438	}
432		};
	439	});
433	440
434	441	// Logs once when writes start failing and once when they recover, so a disk
435	442	// that stays broken costs one line, not one per flush.
436		const flush = Effect.gen(function* () {
437		flushUnsafe();
438		const stats = pendingFlushStats;
439		pendingFlushStats = {
440		logicalWriteBytes: 0,
441		count: 0,
442		durationMs: 0,
443		};
444		if (stats.count > 0 && options.onFlush) {
445		yield* options.onFlush(stats).pipe(Effect.ignore);
446		}
447		if (droppedCount > 0 && !failureReported) {
448		failureReported = true;
449		yield* Effect.logWarning("Trace writes are failing, dropping records until they recover", {
450		filePath: options.filePath,
451		});
452		}
453		if (failureReported && !writeFailing) {
454		yield* Effect.logInfo("Trace writes recovered", {
455		filePath: options.filePath,
456		droppedCount,
457		});
458		failureReported = false;
459		droppedCount = 0;
460		}
461		}).pipe(
462		Effect.withTracerEnabled(false),
463		// The timed fiber inherits the makeTraceSink span. That span has ended but
464		// lives as long as the sink, so the tracer logger must not add logs to it.
465		Effect.updateContext(Context.omit(Tracer.ParentSpan)<never>),
466		);
	443	const flush = flushSemaphore
	444	.withPermits(1)(
	445	Effect.gen(function* () {
	446	const droppedAtStart = droppedCount;
	447	yield* flushBuffered;
	448	const stats = pendingFlushStats;
	449	pendingFlushStats = {
	450	logicalWriteBytes: 0,
	451	count: 0,
	452	durationMs: 0,
	453	};
	454	if (stats.count > 0 && options.onFlush) {
	455	yield* options.onFlush(stats).pipe(Effect.ignore);
	456	}
	457	if (droppedCount > 0 && !failureReported) {
	458	failureReported = true;
	459	yield* Effect.logWarning(
	460	"Trace writer could not keep up, dropping records until it recovers",
	461	{
	462	filePath: options.filePath,
	463	},
	464	);
	465	} else if (failureReported && !writeFailing && droppedCount === droppedAtStart) {
	466	yield* Effect.logInfo("Trace writes recovered", {
	467	filePath: options.filePath,
	468	droppedCount,
	469	});
	470	failureReported = false;
	471	droppedCount = 0;
	472	}
	473	}),
	474	)
	475	.pipe(
	476	// A cancelled write still completes in the OS. Finish it before another
	477	// flush rotates the file or the scope's finalizer drains the remaining records.
	478	Effect.uninterruptible,
	479	Effect.withTracerEnabled(false),
	480	// The timed fiber inherits the makeTraceSink span. That span has ended but
	481	// lives as long as the sink, so the tracer logger must not add logs to it.
	482	Effect.updateContext(Context.omit(Tracer.ParentSpan)<never>),
	483	);
467	484
468	485	yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));
469	486	yield* Effect.forkScoped(
470		Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),
	487	// Flush when push fills a batch or the batch window passes.
	488	Queue.take(flushRequests).pipe(
	489	Effect.timeoutOption(`${options.batchWindowMs} millis`),
	490	Effect.andThen(flush),
	491	Effect.forever,
	492	),
471	493	);
472	494
473	495	return {
474	496	filePath: options.filePath,
475	497	push(record) {
476	498	try {
477		buffer.push(`${JSON.stringify(record)}\n`);
	499	const text = `${JSON.stringify(record)}\n`;
	500	const bytes = Buffer.byteLength(text);
	501	if (bytes > options.maxBytes) return;
	502	if (bufferBytes + bytes > maxBufferBytes) {
	503	droppedCount++;
	504	return;
	505	}
	506	buffer.push({ text, bytes });
	507	bufferBytes += bytes;
478	508	if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {
479		flushUnsafe();
	509	Queue.offerUnsafe(flushRequests, undefined);
480	510	}
481	511	} catch {
482	512	return;

author	Theo Browne <me@t3.gg> 2026-10-14 05:37:03-07:00
committer	GitHub <noreply@github.com> 2026-10-14 05:37:03-07:00
log	4befc863f54a3341911c346c8822286ec37e54af
tree	e410dc10e49587f03d09bc8eb15e58f8049ba733
parent	8414d168191599dec08588080d0b77506c849424
signature	Signed by PGP key B5690EEEBB952194

perf(server): traces keep ~40% more history and no longer block the event loop (#19430)