Commit 752f96e9ae72be9d56686902454eb7a3c7d8686a | the-knife

README.md+2

...	...	@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs.
45	45
46	46	We are not accepting contributions yet.
47	47
	48	Observability guide: [docs/observability.md](./docs/observability.md)
	49
48	50	## If you REALLY want to contribute still.... read this first
49	51
50	52	Read [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.

apps/server/src/cli-config.test.ts+17

...	...	@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";
9	9	import { resolveServerConfig } from "./cli";
10	10
11	11	it.layer(NodeServices.layer)("cli config resolution", (it) => {
	12	const defaultObservabilityConfig = {
	13	traceMinLevel: "Info",
	14	traceTimingEnabled: true,
	15	traceBatchWindowMs: 200,
	16	traceMaxBytes: 10 * 1024 * 1024,
	17	traceMaxFiles: 10,
	18	otlpTracesUrl: undefined,
	19	otlpMetricsUrl: undefined,
	20	otlpExportIntervalMs: 10_000,
	21	otlpServiceName: "t3-server",
	22	} as const;
	23
12	24	const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {
13	25	const fs = yield* FileSystem.FileSystem;
14	26	const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });
...	...	@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
62	74
63	75	expect(resolved).toEqual({
64	76	logLevel: "Warn",
	77	...defaultObservabilityConfig,
65	78	mode: "desktop",
66	79	port: 4001,
67	80	cwd: process.cwd(),
...	...	@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
123	136
124	137	expect(resolved).toEqual({
125	138	logLevel: "Debug",
	139	...defaultObservabilityConfig,
126	140	mode: "web",
127	141	port: 8788,
128	142	cwd: process.cwd(),
...	...	@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
187	201
188	202	expect(resolved).toEqual({
189	203	logLevel: "Info",
	204	...defaultObservabilityConfig,
190	205	mode: "desktop",
191	206	port: 4888,
192	207	cwd: process.cwd(),
...	...	@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
241	256	resolved.attachmentsDir,
242	257	resolved.worktreesDir,
243	258	path.dirname(resolved.serverLogPath),
	259	path.dirname(resolved.serverTracePath),
244	260	]) {
245	261	expect(yield* fs.exists(directory)).toBe(true);
246	262	}
...	...	@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
300	316
301	317	expect(resolved).toEqual({
302	318	logLevel: "Debug",
	319	...defaultObservabilityConfig,
303	320	mode: "web",
304	321	port: 8788,
305	322	cwd: process.cwd(),

apps/server/src/cli.ts+36-1

...	...	@@ -1,5 +1,5 @@
1	1	import { NetService } from "@t3tools/shared/Net";
2		import { Config, Effect, LogLevel, Option, Schema } from "effect";
	2	import { Config, Effect, FileSystem, LogLevel, Option, Path, Schema } from "effect";
3	3	import { Command, Flag, GlobalFlag } from "effect/unstable/cli";
4	4
5	5	import {
...	...	@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(
81	81
82	82	const EnvServerConfig = Config.all({
83	83	logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),
	84	traceMinLevel: Config.logLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")),
	85	traceTimingEnabled: Config.boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)),
	86	traceFile: Config.string("T3CODE_TRACE_FILE").pipe(
	87	Config.option,
	88	Config.map(Option.getOrUndefined),
	89	),
	90	traceMaxBytes: Config.int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)),
	91	traceMaxFiles: Config.int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)),
	92	traceBatchWindowMs: Config.int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(200)),
	93	otlpTracesUrl: Config.string("T3CODE_OTLP_TRACES_URL").pipe(
	94	Config.option,
	95	Config.map(Option.getOrUndefined),
	96	),
	97	otlpMetricsUrl: Config.string("T3CODE_OTLP_METRICS_URL").pipe(
	98	Config.option,
	99	Config.map(Option.getOrUndefined),
	100	),
	101	otlpExportIntervalMs: Config.int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe(
	102	Config.withDefault(10_000),
	103	),
	104	otlpServiceName: Config.string("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")),
84	105	mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(
85	106	Config.option,
86	107	Config.map(Option.getOrUndefined),
...	...	@@ -137,6 +158,8 @@ export const resolveServerConfig = (
137	158	) =>
138	159	Effect.gen(function* () {
139	160	const { findAvailablePort } = yield* NetService;
	161	const path = yield* Path.Path;
	162	const fs = yield* FileSystem.FileSystem;
140	163	const env = yield* EnvServerConfig;
141	164	const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;
142	165	const bootstrapEnvelope =
...	...	@@ -190,6 +213,8 @@ export const resolveServerConfig = (
190	213	);
191	214	const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
192	215	yield* ensureServerDirectories(derivedPaths);
	216	const serverTracePath = env.traceFile ?? derivedPaths.serverTracePath;
	217	yield* fs.makeDirectory(path.dirname(serverTracePath), { recursive: true });
193	218	const noBrowser = resolveBooleanFlag(
194	219	flags.noBrowser,
195	220	Option.getOrElse(
...	...	@@ -248,11 +273,21 @@ export const resolveServerConfig = (
248	273
249	274	const config: ServerConfigShape = {
250	275	logLevel,
	276	traceMinLevel: env.traceMinLevel,
	277	traceTimingEnabled: env.traceTimingEnabled,
	278	traceBatchWindowMs: env.traceBatchWindowMs,
	279	traceMaxBytes: env.traceMaxBytes,
	280	traceMaxFiles: env.traceMaxFiles,
	281	otlpTracesUrl: env.otlpTracesUrl,
	282	otlpMetricsUrl: env.otlpMetricsUrl,
	283	otlpExportIntervalMs: env.otlpExportIntervalMs,
	284	otlpServiceName: env.otlpServiceName,
251	285	mode,
252	286	port,
253	287	cwd: process.cwd(),
254	288	baseDir,
255	289	...derivedPaths,
	290	serverTracePath,
256	291	host,
257	292	staticDir,
258	293	devUrl,

apps/server/src/config.ts+20

...	...	@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {
25	25	readonly attachmentsDir: string;
26	26	readonly logsDir: string;
27	27	readonly serverLogPath: string;
	28	readonly serverTracePath: string;
28	29	readonly providerLogsDir: string;
29	30	readonly providerEventLogPath: string;
30	31	readonly terminalLogsDir: string;
...	...	@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {
36	37	*/
37	38	export interface ServerConfigShape extends ServerDerivedPaths {
38	39	readonly logLevel: LogLevel.LogLevel;
	40	readonly traceMinLevel: LogLevel.LogLevel;
	41	readonly traceTimingEnabled: boolean;
	42	readonly traceBatchWindowMs: number;
	43	readonly traceMaxBytes: number;
	44	readonly traceMaxFiles: number;
	45	readonly otlpTracesUrl: string \| undefined;
	46	readonly otlpMetricsUrl: string \| undefined;
	47	readonly otlpExportIntervalMs: number;
	48	readonly otlpServiceName: string;
39	49	readonly mode: RuntimeMode;
40	50	readonly port: number;
41	51	readonly host: string \| undefined;
...	...	@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (
68	78	attachmentsDir,
69	79	logsDir,
70	80	serverLogPath: join(logsDir, "server.log"),
	81	serverTracePath: join(logsDir, "server.trace.ndjson"),
71	82	providerLogsDir,
72	83	providerEventLogPath: join(providerLogsDir, "events.log"),
73	84	terminalLogsDir: join(logsDir, "terminals"),
...	...	@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigS
117	128
118	129	return {
119	130	logLevel: "Error",
	131	traceMinLevel: "Info",
	132	traceTimingEnabled: true,
	133	traceBatchWindowMs: 200,
	134	traceMaxBytes: 10 * 1024 * 1024,
	135	traceMaxFiles: 10,
	136	otlpTracesUrl: undefined,
	137	otlpMetricsUrl: undefined,
	138	otlpExportIntervalMs: 10_000,
	139	otlpServiceName: "t3-server",
120	140	cwd,
121	141	baseDir,
122	142	...derivedPaths,

apps/server/src/git/Layers/GitCore.ts+49-6

...	...	@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
20	20
21	21	import { GitCommandError, type GitBranch } from "@t3tools/contracts";
22	22	import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";
	23	import { compactTraceAttributes } from "../../observability/Attributes.ts";
	24	import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../../observability/Metrics.ts";
23	25	import {
24	26	GitCore,
25	27	type ExecuteGitProgress,
...	...	@@ -372,6 +374,18 @@ interface Trace2Monitor {
372	374	readonly flush: Effect.Effect<void, never>;
373	375	}
374	376
	377	const nowUnixNano = (): bigint => BigInt(Date.now()) * 1_000_000n;
	378
	379	const addCurrentSpanEvent = (name: string, attributes: Record<string, unknown>) =>
	380	Effect.currentSpan.pipe(
	381	Effect.tap((span) =>
	382	Effect.sync(() => {
	383	span.event(name, nowUnixNano(), compactTraceAttributes(attributes));
	384	}),
	385	),
	386	Effect.catch(() => Effect.void),
	387	);
	388
375	389	function trace2ChildKey(record: Record<string, unknown>): string \| null {
376	390	const childId = record.child_id;
377	391	if (typeof childId === "number" \|\| typeof childId === "string") {
...	...	@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
444	458
445	459	if (event === "child_start") {
446	460	hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });
	461	yield* addCurrentSpanEvent("git.hook.started", {
	462	hookName,
	463	});
447	464	if (progress.onHookStarted) {
448	465	yield* progress.onHookStarted(hookName);
449	466	}
...	...	@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
452	469
453	470	if (event === "child_exit") {
454	471	hookStartByChildKey.delete(childKey);
	472	const code = traceRecord.success.code;
	473	const exitCode = typeof code === "number" && Number.isInteger(code) ? code : null;
	474	const durationMs = started ? Math.max(0, Date.now() - started.startedAtMs) : null;
	475	yield* addCurrentSpanEvent("git.hook.finished", {
	476	hookName: started?.hookName ?? hookName,
	477	exitCode,
	478	durationMs,
	479	});
455	480	if (progress.onHookFinished) {
456		const code = traceRecord.success.code;
457	481	yield* progress.onHookFinished({
458	482	hookName: started?.hookName ?? hookName,
459		exitCode: typeof code === "number" && Number.isInteger(code) ? code : null,
460		durationMs: started ? Math.max(0, Date.now() - started.startedAtMs) : null,
	483	exitCode,
	484	durationMs,
461	485	});
462	486	}
463	487	}
...	...	@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
609	633	const path = yield* Path.Path;
610	634	const { worktreesDir } = yield* ServerConfig;
611	635
612		let execute: GitCoreShape["execute"];
	636	let executeRaw: GitCoreShape["execute"];
613	637
614	638	if (options?.executeOverride) {
615		execute = options.executeOverride;
	639	executeRaw = options.executeOverride;
616	640	} else {
617	641	const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
618		execute = Effect.fnUntraced(function* (input) {
	642	executeRaw = Effect.fnUntraced(function* (input) {
619	643	const commandInput = {
620	644	...input,
621	645	args: [...input.args],
...	...	@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
716	740	});
717	741	}
718	742
	743	const execute: GitCoreShape["execute"] = (input) =>
	744	executeRaw(input).pipe(
	745	withMetrics({
	746	counter: gitCommandsTotal,
	747	timer: gitCommandDuration,
	748	attributes: {
	749	operation: input.operation,
	750	},
	751	}),
	752	Effect.withSpan(input.operation, {
	753	kind: "client",
	754	attributes: {
	755	"git.operation": input.operation,
	756	"git.cwd": input.cwd,
	757	"git.args_count": input.args.length,
	758	},
	759	}),
	760	);
	761
719	762	const executeGit = (
720	763	operation: string,
721	764	cwd: string,

apps/server/src/observability/Attributes.test.ts created+46

...	...	@@ -0,0 +1,46 @@
	1	import { assert, describe, it } from "@effect/vitest";
	2
	3	import { compactTraceAttributes, normalizeModelMetricLabel } from "./Attributes.ts";
	4
	5	describe("Attributes", () => {
	6	it("normalizes circular arrays, maps, and sets without recursing forever", () => {
	7	const array: Array<unknown> = ["alpha"];
	8	array.push(array);
	9
	10	const map = new Map<string, unknown>();
	11	map.set("self", map);
	12
	13	const set = new Set<unknown>();
	14	set.add(set);
	15
	16	assert.deepStrictEqual(
	17	compactTraceAttributes({
	18	array,
	19	map,
	20	set,
	21	}),
	22	{
	23	array: ["alpha", "[Circular]"],
	24	map: { self: "[Circular]" },
	25	set: ["[Circular]"],
	26	},
	27	);
	28	});
	29
	30	it("normalizes invalid dates without throwing", () => {
	31	assert.deepStrictEqual(
	32	compactTraceAttributes({
	33	invalidDate: new Date("not-a-real-date"),
	34	}),
	35	{
	36	invalidDate: "Invalid Date",
	37	},
	38	);
	39	});
	40
	41	it("groups GPT-family models under a shared metric label", () => {
	42	assert.strictEqual(normalizeModelMetricLabel("gpt-4o"), "gpt");
	43	assert.strictEqual(normalizeModelMetricLabel("gpt-5.4"), "gpt");
	44	assert.strictEqual(normalizeModelMetricLabel("claude-sonnet-4"), "claude");
	45	});
	46	});

apps/server/src/observability/Attributes.ts created+128

...	...	@@ -0,0 +1,128 @@
	1	import { Cause, Exit } from "effect";
	2
	3	export type MetricAttributeValue = string;
	4	export type MetricAttributes = Readonly<Record<string, MetricAttributeValue>>;
	5	export type TraceAttributes = Readonly<Record<string, unknown>>;
	6	export type ObservabilityOutcome = "success" \| "failure" \| "interrupt";
	7
	8	function isPlainObject(value: unknown): value is Record<string, unknown> {
	9	return typeof value === "object" && value !== null && !Array.isArray(value);
	10	}
	11
	12	function markSeen(value: object, seen: WeakSet<object>): boolean {
	13	if (seen.has(value)) {
	14	return true;
	15	}
	16	seen.add(value);
	17	return false;
	18	}
	19
	20	function normalizeJsonValue(value: unknown, seen: WeakSet<object> = new WeakSet()): unknown {
	21	if (
	22	value === null \|\|
	23	value === undefined \|\|
	24	typeof value === "string" \|\|
	25	typeof value === "number" \|\|
	26	typeof value === "boolean"
	27	) {
	28	return value ?? null;
	29	}
	30	if (typeof value === "bigint") {
	31	return value.toString();
	32	}
	33	if (value instanceof Date) {
	34	return Number.isNaN(value.getTime()) ? "Invalid Date" : value.toISOString();
	35	}
	36	if (value instanceof Error) {
	37	return {
	38	name: value.name,
	39	message: value.message,
	40	...(value.stack ? { stack: value.stack } : {}),
	41	};
	42	}
	43	if (Array.isArray(value)) {
	44	if (markSeen(value, seen)) {
	45	return "[Circular]";
	46	}
	47	return value.map((entry) => normalizeJsonValue(entry, seen));
	48	}
	49	if (value instanceof Map) {
	50	if (markSeen(value, seen)) {
	51	return "[Circular]";
	52	}
	53	return Object.fromEntries(
	54	Array.from(value.entries(), ([key, entryValue]) => [
	55	String(key),
	56	normalizeJsonValue(entryValue, seen),
	57	]),
	58	);
	59	}
	60	if (value instanceof Set) {
	61	if (markSeen(value, seen)) {
	62	return "[Circular]";
	63	}
	64	return Array.from(value.values(), (entry) => normalizeJsonValue(entry, seen));
	65	}
	66	if (!isPlainObject(value)) {
	67	return String(value);
	68	}
	69	if (markSeen(value, seen)) {
	70	return "[Circular]";
	71	}
	72	return Object.fromEntries(
	73	Object.entries(value).map(([key, entryValue]) => [key, normalizeJsonValue(entryValue, seen)]),
	74	);
	75	}
	76
	77	export function compactTraceAttributes(
	78	attributes: Readonly<Record<string, unknown>>,
	79	): TraceAttributes {
	80	return Object.fromEntries(
	81	Object.entries(attributes)
	82	.filter(([, value]) => value !== undefined)
	83	.map(([key, value]) => [key, normalizeJsonValue(value)]),
	84	);
	85	}
	86
	87	export function compactMetricAttributes(
	88	attributes: Readonly<Record<string, unknown>>,
	89	): MetricAttributes {
	90	return Object.fromEntries(
	91	Object.entries(attributes).flatMap(([key, value]) => {
	92	if (value === undefined \|\| value === null) {
	93	return [];
	94	}
	95	if (typeof value === "string") {
	96	return [[key, value]];
	97	}
	98	if (typeof value === "number" \|\| typeof value === "boolean" \|\| typeof value === "bigint") {
	99	return [[key, String(value)]];
	100	}
	101	return [];
	102	}),
	103	);
	104	}
	105
	106	export function outcomeFromExit(exit: Exit.Exit<unknown, unknown>): ObservabilityOutcome {
	107	if (Exit.isSuccess(exit)) {
	108	return "success";
	109	}
	110	return Cause.hasInterruptsOnly(exit.cause) ? "interrupt" : "failure";
	111	}
	112
	113	export function normalizeModelMetricLabel(model: string \| null \| undefined): string \| undefined {
	114	const normalized = model?.trim().toLowerCase();
	115	if (!normalized) {
	116	return undefined;
	117	}
	118	if (normalized.includes("gpt")) {
	119	return "gpt";
	120	}
	121	if (normalized.includes("claude")) {
	122	return "claude";
	123	}
	124	if (normalized.includes("gemini")) {
	125	return "gemini";
	126	}
	127	return "other";
	128	}

apps/server/src/observability/Layers/Observability.ts created+64

...	...	@@ -0,0 +1,64 @@
	1	import { Effect, Layer, References, Tracer } from "effect";
	2	import { OtlpMetrics, OtlpSerialization, OtlpTracer } from "effect/unstable/observability";
	3
	4	import { ServerConfig } from "../../config.ts";
	5	import { ServerLoggerLive } from "../../serverLogger.ts";
	6	import { makeLocalFileTracer } from "../LocalFileTracer.ts";
	7
	8	const otlpSerializationLayer = OtlpSerialization.layerJson;
	9
	10	export const ObservabilityLive = Layer.unwrap(
	11	Effect.gen(function* () {
	12	const config = yield* ServerConfig;
	13
	14	const traceReferencesLayer = Layer.mergeAll(
	15	Layer.succeed(Tracer.MinimumTraceLevel, config.traceMinLevel),
	16	Layer.succeed(References.TracerTimingEnabled, config.traceTimingEnabled),
	17	);
	18
	19	const tracerLayer = Layer.effect(
	20	Tracer.Tracer,
	21	Effect.gen(function* () {
	22	const delegate =
	23	config.otlpTracesUrl === undefined
	24	? undefined
	25	: yield* OtlpTracer.make({
	26	url: config.otlpTracesUrl,
	27	exportInterval: `${config.otlpExportIntervalMs} millis`,
	28	resource: {
	29	serviceName: config.otlpServiceName,
	30	attributes: {
	31	"service.runtime": "t3-server",
	32	"service.mode": config.mode,
	33	},
	34	},
	35	});
	36
	37	return yield* makeLocalFileTracer({
	38	filePath: config.serverTracePath,
	39	maxBytes: config.traceMaxBytes,
	40	maxFiles: config.traceMaxFiles,
	41	batchWindowMs: config.traceBatchWindowMs,
	42	...(delegate ? { delegate } : {}),
	43	});
	44	}),
	45	).pipe(Layer.provideMerge(otlpSerializationLayer));
	46
	47	const metricsLayer =
	48	config.otlpMetricsUrl === undefined
	49	? Layer.empty
	50	: OtlpMetrics.layer({
	51	url: config.otlpMetricsUrl,
	52	exportInterval: `${config.otlpExportIntervalMs} millis`,
	53	resource: {
	54	serviceName: config.otlpServiceName,
	55	attributes: {
	56	"service.runtime": "t3-server",
	57	"service.mode": config.mode,
	58	},
	59	},
	60	}).pipe(Layer.provideMerge(otlpSerializationLayer));
	61
	62	return Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer);
	63	}),
	64	);

apps/server/src/observability/LocalFileTracer.test.ts created+117

...	...	@@ -0,0 +1,117 @@
	1	import fs from "node:fs";
	2	import os from "node:os";
	3	import path from "node:path";
	4
	5	import { assert, describe, it } from "@effect/vitest";
	6	import { Effect, Layer, Logger, References, Tracer } from "effect";
	7
	8	import type { TraceRecord } from "./TraceRecord.ts";
	9	import { makeLocalFileTracer } from "./LocalFileTracer.ts";
	10
	11	const makeTestLayer = (tracePath: string) =>
	12	Layer.mergeAll(
	13	Layer.effect(
	14	Tracer.Tracer,
	15	makeLocalFileTracer({
	16	filePath: tracePath,
	17	maxBytes: 1024 * 1024,
	18	maxFiles: 2,
	19	batchWindowMs: 10_000,
	20	}),
	21	),
	22	Logger.layer([Logger.tracerLogger], { mergeWithExisting: false }),
	23	Layer.succeed(References.MinimumLogLevel, "Info"),
	24	);
	25
	26	const readTraceRecords = (tracePath: string): Array<TraceRecord> =>
	27	fs
	28	.readFileSync(tracePath, "utf8")
	29	.trim()
	30	.split("\n")
	31	.filter((line) => line.length > 0)
	32	.map((line) => JSON.parse(line) as TraceRecord);
	33
	34	describe("LocalFileTracer", () => {
	35	it.effect("writes nested spans to disk and captures log messages as span events", () =>
	36	Effect.gen(function* () {
	37	const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
	38	const tracePath = path.join(tempDir, "server.trace.ndjson");
	39
	40	try {
	41	yield* Effect.scoped(
	42	Effect.gen(function* () {
	43	const program = Effect.gen(function* () {
	44	yield* Effect.annotateCurrentSpan({
	45	"demo.parent": true,
	46	});
	47	yield* Effect.logInfo("parent event");
	48	yield* Effect.gen(function* () {
	49	yield* Effect.annotateCurrentSpan({
	50	"demo.child": true,
	51	});
	52	yield* Effect.logInfo("child event");
	53	}).pipe(Effect.withSpan("child-span"));
	54	}).pipe(Effect.withSpan("parent-span"));
	55
	56	yield* program.pipe(Effect.provide(makeTestLayer(tracePath)));
	57	}),
	58	);
	59
	60	const records = readTraceRecords(tracePath);
	61	assert.equal(records.length, 2);
	62
	63	const parent = records.find((record) => record.name === "parent-span");
	64	const child = records.find((record) => record.name === "child-span");
	65
	66	assert.notEqual(parent, undefined);
	67	assert.notEqual(child, undefined);
	68	if (!parent \|\| !child) {
	69	return;
	70	}
	71
	72	assert.equal(child.parentSpanId, parent.spanId);
	73	assert.equal(parent.attributes["demo.parent"], true);
	74	assert.equal(child.attributes["demo.child"], true);
	75	assert.equal(
	76	parent.events.some((event) => event.name === "parent event"),
	77	true,
	78	);
	79	assert.equal(
	80	child.events.some((event) => event.name === "child event"),
	81	true,
	82	);
	83	assert.equal(
	84	child.events.some((event) => event.attributes["effect.logLevel"] === "INFO"),
	85	true,
	86	);
	87	} finally {
	88	fs.rmSync(tempDir, { recursive: true, force: true });
	89	}
	90	}),
	91	);
	92
	93	it.effect("serializes interrupted spans with an interrupted exit status", () =>
	94	Effect.gen(function* () {
	95	const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
	96	const tracePath = path.join(tempDir, "server.trace.ndjson");
	97
	98	try {
	99	yield* Effect.scoped(
	100	Effect.exit(
	101	Effect.interrupt.pipe(
	102	Effect.withSpan("interrupt-span"),
	103	Effect.provide(makeTestLayer(tracePath)),
	104	),
	105	),
	106	);
	107
	108	const records = readTraceRecords(tracePath);
	109	assert.equal(records.length, 1);
	110	assert.equal(records[0]?.name, "interrupt-span");
	111	assert.equal(records[0]?.exit._tag, "Interrupted");
	112	} finally {
	113	fs.rmSync(tempDir, { recursive: true, force: true });
	114	}
	115	}),
	116	);
	117	});

apps/server/src/observability/LocalFileTracer.ts created+104

...	...	@@ -0,0 +1,104 @@
	1	import type * as Exit from "effect/Exit";
	2	import { Effect, Option, Tracer } from "effect";
	3
	4	import { spanToTraceRecord } from "./TraceRecord.ts";
	5	import { makeTraceSink } from "./TraceSink.ts";
	6
	7	export interface LocalFileTracerOptions {
	8	readonly filePath: string;
	9	readonly maxBytes: number;
	10	readonly maxFiles: number;
	11	readonly batchWindowMs: number;
	12	readonly delegate?: Tracer.Tracer;
	13	}
	14
	15	class LocalFileSpan implements Tracer.Span {
	16	readonly _tag = "Span";
	17	readonly name: string;
	18	readonly spanId: string;
	19	readonly traceId: string;
	20	readonly parent: Option.Option<Tracer.AnySpan>;
	21	readonly annotations: Tracer.Span["annotations"];
	22	readonly links: Array<Tracer.SpanLink>;
	23	readonly sampled: boolean;
	24	readonly kind: Tracer.SpanKind;
	25
	26	status: Tracer.SpanStatus;
	27	attributes: Map<string, unknown>;
	28	events: Array<[name: string, startTime: bigint, attributes: Record<string, unknown>]>;
	29
	30	constructor(
	31	options: Parameters<Tracer.Tracer["span"]>[0],
	32	private readonly delegate: Tracer.Span,
	33	private readonly push: (record: ReturnType<typeof spanToTraceRecord>) => void,
	34	) {
	35	this.name = delegate.name;
	36	this.spanId = delegate.spanId;
	37	this.traceId = delegate.traceId;
	38	this.parent = options.parent;
	39	this.annotations = options.annotations;
	40	this.links = [...options.links];
	41	this.sampled = delegate.sampled;
	42	this.kind = delegate.kind;
	43	this.status = {
	44	_tag: "Started",
	45	startTime: options.startTime,
	46	};
	47	this.attributes = new Map();
	48	this.events = [];
	49	}
	50
	51	end(endTime: bigint, exit: Exit.Exit<unknown, unknown>): void {
	52	this.status = {
	53	_tag: "Ended",
	54	startTime: this.status.startTime,
	55	endTime,
	56	exit,
	57	};
	58	this.delegate.end(endTime, exit);
	59
	60	if (this.sampled) {
	61	this.push(spanToTraceRecord(this));
	62	}
	63	}
	64
	65	attribute(key: string, value: unknown): void {
	66	this.attributes.set(key, value);
	67	this.delegate.attribute(key, value);
	68	}
	69
	70	event(name: string, startTime: bigint, attributes?: Record<string, unknown>): void {
	71	const nextAttributes = attributes ?? {};
	72	this.events.push([name, startTime, nextAttributes]);
	73	this.delegate.event(name, startTime, nextAttributes);
	74	}
	75
	76	addLinks(links: ReadonlyArray<Tracer.SpanLink>): void {
	77	this.links.push(...links);
	78	this.delegate.addLinks(links);
	79	}
	80	}
	81
	82	export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (
	83	options: LocalFileTracerOptions,
	84	) {
	85	const sink = yield* makeTraceSink({
	86	filePath: options.filePath,
	87	maxBytes: options.maxBytes,
	88	maxFiles: options.maxFiles,
	89	batchWindowMs: options.batchWindowMs,
	90	});
	91
	92	const delegate =
	93	options.delegate ??
	94	Tracer.make({
	95	span: (spanOptions) => new Tracer.NativeSpan(spanOptions),
	96	});
	97
	98	return Tracer.make({
	99	span(spanOptions) {
	100	return new LocalFileSpan(spanOptions, delegate.span(spanOptions), sink.push);
	101	},
	102	...(delegate.context ? { context: delegate.context } : {}),
	103	});
	104	});

apps/server/src/observability/Metrics.test.ts created+111

...	...	@@ -0,0 +1,111 @@
	1	import { assert, describe, it } from "@effect/vitest";
	2	import { Effect, Metric } from "effect";
	3
	4	import { withMetrics } from "./Metrics.ts";
	5
	6	const hasMetricSnapshot = (
	7	snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
	8	id: string,
	9	attributes: Readonly<Record<string, string>>,
	10	) =>
	11	snapshots.some(
	12	(snapshot) =>
	13	snapshot.id === id &&
	14	Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
	15	);
	16
	17	describe("withMetrics", () => {
	18	it.effect("supports pipe-style usage", () =>
	19	Effect.gen(function* () {
	20	const counter = Metric.counter("with_metrics_pipe_total");
	21	const timer = Metric.timer("with_metrics_pipe_duration");
	22
	23	const result = yield* Effect.succeed("ok").pipe(
	24	withMetrics({
	25	counter,
	26	timer,
	27	attributes: {
	28	operation: "pipe",
	29	},
	30	}),
	31	);
	32
	33	assert.equal(result, "ok");
	34
	35	const snapshots = yield* Metric.snapshot;
	36	assert.equal(
	37	hasMetricSnapshot(snapshots, "with_metrics_pipe_total", {
	38	operation: "pipe",
	39	outcome: "success",
	40	}),
	41	true,
	42	);
	43	assert.equal(
	44	hasMetricSnapshot(snapshots, "with_metrics_pipe_duration", {
	45	operation: "pipe",
	46	}),
	47	true,
	48	);
	49	}),
	50	);
	51
	52	it.effect("supports direct invocation", () =>
	53	Effect.gen(function* () {
	54	const counter = Metric.counter("with_metrics_direct_total");
	55
	56	yield* withMetrics(Effect.fail("boom"), {
	57	counter,
	58	attributes: {
	59	operation: "direct",
	60	},
	61	}).pipe(Effect.exit);
	62
	63	const snapshots = yield* Metric.snapshot;
	64	assert.equal(
	65	hasMetricSnapshot(snapshots, "with_metrics_direct_total", {
	66	operation: "direct",
	67	outcome: "failure",
	68	}),
	69	true,
	70	);
	71	}),
	72	);
	73
	74	it.effect("evaluates attributes lazily after the wrapped effect runs", () =>
	75	Effect.gen(function* () {
	76	const counter = Metric.counter("with_metrics_lazy_total");
	77	const timer = Metric.timer("with_metrics_lazy_duration");
	78	let provider = "unknown";
	79
	80	yield* Effect.sync(() => {
	81	provider = "codex";
	82	}).pipe(
	83	withMetrics({
	84	counter,
	85	timer,
	86	attributes: () => ({
	87	provider,
	88	operation: "lazy",
	89	}),
	90	}),
	91	);
	92
	93	const snapshots = yield* Metric.snapshot;
	94	assert.equal(
	95	hasMetricSnapshot(snapshots, "with_metrics_lazy_total", {
	96	provider: "codex",
	97	operation: "lazy",
	98	outcome: "success",
	99	}),
	100	true,
	101	);
	102	assert.equal(
	103	hasMetricSnapshot(snapshots, "with_metrics_lazy_duration", {
	104	provider: "codex",
	105	operation: "lazy",
	106	}),
	107	true,
	108	);
	109	}),
	110	);
	111	});

apps/server/src/observability/Metrics.ts created+160

...	...	@@ -0,0 +1,160 @@
	1	import { Duration, Effect, Exit, Metric } from "effect";
	2	import { dual } from "effect/Function";
	3
	4	import {
	5	compactMetricAttributes,
	6	normalizeModelMetricLabel,
	7	outcomeFromExit,
	8	} from "./Attributes.ts";
	9
	10	export const rpcRequestsTotal = Metric.counter("t3_rpc_requests_total", {
	11	description: "Total RPC requests handled by the websocket RPC server.",
	12	});
	13
	14	export const rpcRequestDuration = Metric.timer("t3_rpc_request_duration", {
	15	description: "RPC request handling duration.",
	16	});
	17
	18	export const orchestrationCommandsTotal = Metric.counter("t3_orchestration_commands_total", {
	19	description: "Total orchestration commands dispatched.",
	20	});
	21
	22	export const orchestrationCommandDuration = Metric.timer("t3_orchestration_command_duration", {
	23	description: "Orchestration command dispatch duration.",
	24	});
	25
	26	export const orchestrationCommandAckDuration = Metric.timer(
	27	"t3_orchestration_command_ack_duration",
	28	{
	29	description:
	30	"Time from orchestration command dispatch to the first committed domain event emitted for that command.",
	31	},
	32	);
	33
	34	export const orchestrationEventsProcessedTotal = Metric.counter(
	35	"t3_orchestration_events_processed_total",
	36	{
	37	description: "Total orchestration intent events processed by runtime reactors.",
	38	},
	39	);
	40
	41	export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
	42	description: "Total provider session lifecycle operations.",
	43	});
	44
	45	export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", {
	46	description: "Total provider turn lifecycle operations.",
	47	});
	48
	49	export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
	50	description: "Provider turn request duration.",
	51	});
	52
	53	export const providerRuntimeEventsTotal = Metric.counter("t3_provider_runtime_events_total", {
	54	description: "Total canonical provider runtime events processed.",
	55	});
	56
	57	export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
	58	description: "Total git commands executed by the server runtime.",
	59	});
	60
	61	export const gitCommandDuration = Metric.timer("t3_git_command_duration", {
	62	description: "Git command execution duration.",
	63	});
	64
	65	export const terminalSessionsTotal = Metric.counter("t3_terminal_sessions_total", {
	66	description: "Total terminal sessions started.",
	67	});
	68
	69	export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total", {
	70	description: "Total terminal restart requests handled.",
	71	});
	72
	73	export const metricAttributes = (
	74	attributes: Readonly<Record<string, unknown>>,
	75	): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));
	76
	77	export const increment = (
	78	metric: Metric.Metric<number, unknown>,
	79	attributes: Readonly<Record<string, unknown>>,
	80	amount = 1,
	81	) => Metric.update(Metric.withAttributes(metric, metricAttributes(attributes)), amount);
	82
	83	export interface WithMetricsOptions {
	84	readonly counter?: Metric.Metric<number, unknown>;
	85	readonly timer?: Metric.Metric<Duration.Duration, unknown>;
	86	readonly attributes?:
	87	\| Readonly<Record<string, unknown>>
	88	\| (() => Readonly<Record<string, unknown>>);
	89	readonly outcomeAttributes?: (
	90	outcome: ReturnType<typeof outcomeFromExit>,
	91	) => Readonly<Record<string, unknown>>;
	92	}
	93
	94	const withMetricsImpl = <A, E, R>(
	95	effect: Effect.Effect<A, E, R>,
	96	options: WithMetricsOptions,
	97	): Effect.Effect<A, E, R> =>
	98	Effect.gen(function* () {
	99	const startedAt = Date.now();
	100	const exit = yield* Effect.exit(effect);
	101	const duration = Duration.millis(Math.max(0, Date.now() - startedAt));
	102	const baseAttributes =
	103	typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
	104
	105	if (options.timer) {
	106	yield* Metric.update(
	107	Metric.withAttributes(options.timer, metricAttributes(baseAttributes)),
	108	duration,
	109	);
	110	}
	111
	112	if (options.counter) {
	113	const outcome = outcomeFromExit(exit);
	114	yield* Metric.update(
	115	Metric.withAttributes(
	116	options.counter,
	117	metricAttributes({
	118	...baseAttributes,
	119	outcome,
	120	...(options.outcomeAttributes ? options.outcomeAttributes(outcome) : {}),
	121	}),
	122	),
	123	1,
	124	);
	125	}
	126
	127	if (Exit.isSuccess(exit)) {
	128	return exit.value;
	129	}
	130	return yield* Effect.failCause(exit.cause);
	131	});
	132
	133	export const withMetrics: {
	134	<A, E, R>(
	135	options: WithMetricsOptions,
	136	): (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
	137	<A, E, R>(effect: Effect.Effect<A, E, R>, options: WithMetricsOptions): Effect.Effect<A, E, R>;
	138	} = dual(2, withMetricsImpl);
	139
	140	export const providerMetricAttributes = (
	141	provider: string,
	142	extra?: Readonly<Record<string, unknown>>,
	143	) =>
	144	compactMetricAttributes({
	145	provider,
	146	...extra,
	147	});
	148
	149	export const providerTurnMetricAttributes = (input: {
	150	readonly provider: string;
	151	readonly model: string \| null \| undefined;
	152	readonly extra?: Readonly<Record<string, unknown>>;
	153	}) => {
	154	const modelFamily = normalizeModelMetricLabel(input.model);
	155	return compactMetricAttributes({
	156	provider: input.provider,
	157	...(modelFamily ? { modelFamily } : {}),
	158	...input.extra,
	159	});
	160	};

apps/server/src/observability/RpcInstrumentation.test.ts created+161

...	...	@@ -0,0 +1,161 @@
	1	import { assert, describe, it } from "@effect/vitest";
	2	import { Effect, Exit, Metric, Stream } from "effect";
	3
	4	import {
	5	observeRpcEffect,
	6	observeRpcStream,
	7	observeRpcStreamEffect,
	8	} from "./RpcInstrumentation.ts";
	9
	10	const hasMetricSnapshot = (
	11	snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
	12	id: string,
	13	attributes: Readonly<Record<string, string>>,
	14	) =>
	15	snapshots.some(
	16	(snapshot) =>
	17	snapshot.id === id &&
	18	Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
	19	);
	20
	21	describe("RpcInstrumentation", () => {
	22	it.effect("records success metrics for unary RPC handlers", () =>
	23	Effect.gen(function* () {
	24	yield* observeRpcEffect("rpc.instrumentation.success", Effect.succeed("ok"), {
	25	"rpc.aggregate": "test",
	26	}).pipe(Effect.withSpan("rpc.instrumentation.success.span"));
	27
	28	const snapshots = yield* Metric.snapshot;
	29
	30	assert.equal(
	31	hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
	32	method: "rpc.instrumentation.success",
	33	outcome: "success",
	34	}),
	35	true,
	36	);
	37	assert.equal(
	38	hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
	39	method: "rpc.instrumentation.success",
	40	}),
	41	true,
	42	);
	43	}),
	44	);
	45
	46	it.effect("records failure outcomes for unary RPC handlers", () =>
	47	Effect.gen(function* () {
	48	yield* Effect.exit(
	49	observeRpcEffect("rpc.instrumentation.failure", Effect.fail("boom"), {
	50	"rpc.aggregate": "test",
	51	}).pipe(Effect.withSpan("rpc.instrumentation.failure.span")),
	52	);
	53
	54	const snapshots = yield* Metric.snapshot;
	55
	56	assert.equal(
	57	hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
	58	method: "rpc.instrumentation.failure",
	59	outcome: "failure",
	60	}),
	61	true,
	62	);
	63	assert.equal(
	64	hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
	65	method: "rpc.instrumentation.failure",
	66	}),
	67	true,
	68	);
	69	}),
	70	);
	71
	72	it.effect("records subscription activation metrics for stream RPC handlers", () =>
	73	Effect.gen(function* () {
	74	const events = yield* Stream.runCollect(
	75	observeRpcStreamEffect(
	76	"rpc.instrumentation.stream",
	77	Effect.succeed(Stream.make("a", "b")),
	78	{ "rpc.aggregate": "test" },
	79	).pipe(Stream.withSpan("rpc.instrumentation.stream.span")),
	80	);
	81
	82	assert.deepStrictEqual(Array.from(events), ["a", "b"]);
	83
	84	const snapshots = yield* Metric.snapshot;
	85
	86	assert.equal(
	87	hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
	88	method: "rpc.instrumentation.stream",
	89	outcome: "success",
	90	}),
	91	true,
	92	);
	93	assert.equal(
	94	hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
	95	method: "rpc.instrumentation.stream",
	96	}),
	97	true,
	98	);
	99	}),
	100	);
	101
	102	it.effect("records failure outcomes for direct stream RPC handlers during consumption", () =>
	103	Effect.gen(function* () {
	104	const exit = yield* Stream.runCollect(
	105	observeRpcStream(
	106	"rpc.instrumentation.stream.failure",
	107	Stream.make("a").pipe(Stream.concat(Stream.fail("boom"))),
	108	{ "rpc.aggregate": "test" },
	109	).pipe(Stream.withSpan("rpc.instrumentation.stream.failure.span")),
	110	).pipe(Effect.exit);
	111
	112	assert.equal(Exit.isFailure(exit), true);
	113
	114	const snapshots = yield* Metric.snapshot;
	115
	116	assert.equal(
	117	hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
	118	method: "rpc.instrumentation.stream.failure",
	119	outcome: "failure",
	120	}),
	121	true,
	122	);
	123	assert.equal(
	124	hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
	125	method: "rpc.instrumentation.stream.failure",
	126	}),
	127	true,
	128	);
	129	}),
	130	);
	131
	132	it.effect("records failure outcomes when a stream RPC effect produces a failing stream", () =>
	133	Effect.gen(function* () {
	134	const exit = yield* Stream.runCollect(
	135	observeRpcStreamEffect(
	136	"rpc.instrumentation.stream.effect.failure",
	137	Effect.succeed(Stream.fail("boom")),
	138	{ "rpc.aggregate": "test" },
	139	).pipe(Stream.withSpan("rpc.instrumentation.stream.effect.failure.span")),
	140	).pipe(Effect.exit);
	141
	142	assert.equal(Exit.isFailure(exit), true);
	143
	144	const snapshots = yield* Metric.snapshot;
	145
	146	assert.equal(
	147	hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
	148	method: "rpc.instrumentation.stream.effect.failure",
	149	outcome: "failure",
	150	}),
	151	true,
	152	);
	153	assert.equal(
	154	hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
	155	method: "rpc.instrumentation.stream.effect.failure",
	156	}),
	157	true,
	158	);
	159	}),
	160	);
	161	});

apps/server/src/observability/RpcInstrumentation.ts created+89

...	...	@@ -0,0 +1,89 @@
	1	import { Duration, Effect, Exit, Metric, Stream } from "effect";
	2
	3	import { outcomeFromExit } from "./Attributes.ts";
	4	import { metricAttributes, rpcRequestDuration, rpcRequestsTotal, withMetrics } from "./Metrics.ts";
	5
	6	const annotateRpcSpan = (
	7	method: string,
	8	traceAttributes?: Readonly<Record<string, unknown>>,
	9	): Effect.Effect<void, never, never> =>
	10	Effect.annotateCurrentSpan({
	11	"rpc.method": method,
	12	...traceAttributes,
	13	});
	14
	15	const recordRpcStreamMetrics = <E>(
	16	method: string,
	17	startedAt: number,
	18	exit: Exit.Exit<unknown, E>,
	19	): Effect.Effect<void, never, never> =>
	20	Effect.gen(function* () {
	21	yield* Metric.update(
	22	Metric.withAttributes(rpcRequestDuration, metricAttributes({ method })),
	23	Duration.millis(Math.max(0, Date.now() - startedAt)),
	24	);
	25	yield* Metric.update(
	26	Metric.withAttributes(
	27	rpcRequestsTotal,
	28	metricAttributes({
	29	method,
	30	outcome: outcomeFromExit(exit),
	31	}),
	32	),
	33	1,
	34	);
	35	});
	36
	37	export const observeRpcEffect = <A, E, R>(
	38	method: string,
	39	effect: Effect.Effect<A, E, R>,
	40	traceAttributes?: Readonly<Record<string, unknown>>,
	41	): Effect.Effect<A, E, R> =>
	42	Effect.gen(function* () {
	43	yield* annotateRpcSpan(method, traceAttributes);
	44
	45	return yield* effect.pipe(
	46	withMetrics({
	47	counter: rpcRequestsTotal,
	48	timer: rpcRequestDuration,
	49	attributes: {
	50	method,
	51	},
	52	}),
	53	);
	54	});
	55
	56	export const observeRpcStream = <A, E, R>(
	57	method: string,
	58	stream: Stream.Stream<A, E, R>,
	59	traceAttributes?: Readonly<Record<string, unknown>>,
	60	): Stream.Stream<A, E, R> =>
	61	Stream.unwrap(
	62	Effect.gen(function* () {
	63	yield* annotateRpcSpan(method, traceAttributes);
	64	const startedAt = Date.now();
	65	return stream.pipe(Stream.onExit((exit) => recordRpcStreamMetrics(method, startedAt, exit)));
	66	}),
	67	);
	68
	69	export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectError, EffectContext>(
	70	method: string,
	71	effect: Effect.Effect<Stream.Stream<A, StreamError, StreamContext>, EffectError, EffectContext>,
	72	traceAttributes?: Readonly<Record<string, unknown>>,
	73	): Stream.Stream<A, StreamError \| EffectError, StreamContext \| EffectContext> =>
	74	Stream.unwrap(
	75	Effect.gen(function* () {
	76	yield* annotateRpcSpan(method, traceAttributes);
	77	const startedAt = Date.now();
	78	const exit = yield* Effect.exit(effect);
	79
	80	if (Exit.isFailure(exit)) {
	81	yield* recordRpcStreamMetrics(method, startedAt, exit);
	82	return yield* Effect.failCause(exit.cause);
	83	}
	84
	85	return exit.value.pipe(
	86	Stream.onExit((streamExit) => recordRpcStreamMetrics(method, startedAt, streamExit)),
	87	);
	88	}),
	89	);

apps/server/src/observability/TraceRecord.ts created+100

...	...	@@ -0,0 +1,100 @@
	1	import { Cause, Exit, Option, Tracer } from "effect";
	2
	3	import { compactTraceAttributes } from "./Attributes.ts";
	4
	5	export interface TraceRecord {
	6	readonly type: "effect-span";
	7	readonly name: string;
	8	readonly traceId: string;
	9	readonly spanId: string;
	10	readonly parentSpanId?: string;
	11	readonly sampled: boolean;
	12	readonly kind: Tracer.SpanKind;
	13	readonly startTimeUnixNano: string;
	14	readonly endTimeUnixNano: string;
	15	readonly durationMs: number;
	16	readonly attributes: Readonly<Record<string, unknown>>;
	17	readonly events: ReadonlyArray<{
	18	readonly name: string;
	19	readonly timeUnixNano: string;
	20	readonly attributes: Readonly<Record<string, unknown>>;
	21	}>;
	22	readonly links: ReadonlyArray<{
	23	readonly traceId: string;
	24	readonly spanId: string;
	25	readonly attributes: Readonly<Record<string, unknown>>;
	26	}>;
	27	readonly exit:
	28	\| {
	29	readonly _tag: "Success";
	30	}
	31	\| {
	32	readonly _tag: "Interrupted";
	33	readonly cause: string;
	34	}
	35	\| {
	36	readonly _tag: "Failure";
	37	readonly cause: string;
	38	};
	39	}
	40
	41	interface SerializableSpan {
	42	readonly name: string;
	43	readonly traceId: string;
	44	readonly spanId: string;
	45	readonly parent: Option.Option<Tracer.AnySpan>;
	46	readonly status: Tracer.SpanStatus;
	47	readonly sampled: boolean;
	48	readonly kind: Tracer.SpanKind;
	49	readonly attributes: ReadonlyMap<string, unknown>;
	50	readonly links: ReadonlyArray<Tracer.SpanLink>;
	51	readonly events: ReadonlyArray<
	52	readonly [name: string, startTime: bigint, attributes: Record<string, unknown>]
	53	>;
	54	}
	55
	56	function formatTraceExit(exit: Exit.Exit<unknown, unknown>): TraceRecord["exit"] {
	57	if (Exit.isSuccess(exit)) {
	58	return { _tag: "Success" };
	59	}
	60	if (Cause.hasInterruptsOnly(exit.cause)) {
	61	return {
	62	_tag: "Interrupted",
	63	cause: Cause.pretty(exit.cause),
	64	};
	65	}
	66	return {
	67	_tag: "Failure",
	68	cause: Cause.pretty(exit.cause),
	69	};
	70	}
	71
	72	export function spanToTraceRecord(span: SerializableSpan): TraceRecord {
	73	const status = span.status as Extract<Tracer.SpanStatus, { _tag: "Ended" }>;
	74	const parentSpanId = Option.getOrUndefined(span.parent)?.spanId;
	75
	76	return {
	77	type: "effect-span",
	78	name: span.name,
	79	traceId: span.traceId,
	80	spanId: span.spanId,
	81	...(parentSpanId ? { parentSpanId } : {}),
	82	sampled: span.sampled,
	83	kind: span.kind,
	84	startTimeUnixNano: String(status.startTime),
	85	endTimeUnixNano: String(status.endTime),
	86	durationMs: Number(status.endTime - status.startTime) / 1_000_000,
	87	attributes: compactTraceAttributes(Object.fromEntries(span.attributes)),
	88	events: span.events.map(([name, startTime, attributes]) => ({
	89	name,
	90	timeUnixNano: String(startTime),
	91	attributes: compactTraceAttributes(attributes),
	92	})),
	93	links: span.links.map((link) => ({
	94	traceId: link.span.traceId,
	95	spanId: link.span.spanId,
	96	attributes: compactTraceAttributes(link.attributes),
	97	})),
	98	exit: formatTraceExit(status.exit),
	99	};
	100	}

apps/server/src/observability/TraceSink.test.ts created+152

...	...	@@ -0,0 +1,152 @@
	1	import fs from "node:fs";
	2	import os from "node:os";
	3	import path from "node:path";
	4
	5	import { assert, describe, it } from "@effect/vitest";
	6	import { Effect } from "effect";
	7
	8	import type { TraceRecord } from "./TraceRecord.ts";
	9	import { makeTraceSink } from "./TraceSink.ts";
	10
	11	const makeRecord = (name: string, suffix = ""): TraceRecord => ({
	12	type: "effect-span",
	13	name,
	14	traceId: `trace-${name}-${suffix}`,
	15	spanId: `span-${name}-${suffix}`,
	16	sampled: true,
	17	kind: "internal",
	18	startTimeUnixNano: "1",
	19	endTimeUnixNano: "2",
	20	durationMs: 1,
	21	attributes: {
	22	payload: suffix,
	23	},
	24	events: [],
	25	links: [],
	26	exit: {
	27	_tag: "Success",
	28	},
	29	});
	30
	31	describe("TraceSink", () => {
	32	it.effect("flushes buffered records on close", () =>
	33	Effect.scoped(
	34	Effect.gen(function* () {
	35	const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
	36	const tracePath = path.join(tempDir, "server.trace.ndjson");
	37
	38	try {
	39	const sink = yield* makeTraceSink({
	40	filePath: tracePath,
	41	maxBytes: 1024,
	42	maxFiles: 2,
	43	batchWindowMs: 10_000,
	44	});
	45
	46	sink.push(makeRecord("alpha"));
	47	sink.push(makeRecord("beta"));
	48	yield* sink.close();
	49
	50	const lines = fs
	51	.readFileSync(tracePath, "utf8")
	52	.trim()
	53	.split("\n")
	54	.map((line) => JSON.parse(line) as TraceRecord);
	55
	56	assert.equal(lines.length, 2);
	57	assert.equal(lines[0]?.name, "alpha");
	58	assert.equal(lines[1]?.name, "beta");
	59	} finally {
	60	fs.rmSync(tempDir, { recursive: true, force: true });
	61	}
	62	}),
	63	),
	64	);
	65
	66	it.effect("rotates the trace file when the configured max size is exceeded", () =>
	67	Effect.scoped(
	68	Effect.gen(function* () {
	69	const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
	70	const tracePath = path.join(tempDir, "server.trace.ndjson");
	71
	72	try {
	73	const sink = yield* makeTraceSink({
	74	filePath: tracePath,
	75	maxBytes: 180,
	76	maxFiles: 2,
	77	batchWindowMs: 10_000,
	78	});
	79
	80	for (let index = 0; index < 8; index += 1) {
	81	sink.push(makeRecord("rotate", `${index}-${"x".repeat(48)}`));
	82	yield* sink.flush;
	83	}
	84	yield* sink.close();
	85
	86	const matchingFiles = fs
	87	.readdirSync(tempDir)
	88	.filter(
	89	(entry) =>
	90	entry === "server.trace.ndjson" \|\| entry.startsWith("server.trace.ndjson."),
	91	)
	92	.toSorted();
	93
	94	assert.equal(
	95	matchingFiles.some((entry) => entry === "server.trace.ndjson.1"),
	96	true,
	97	);
	98	assert.equal(
	99	matchingFiles.some((entry) => entry === "server.trace.ndjson.3"),
	100	false,
	101	);
	102	} finally {
	103	fs.rmSync(tempDir, { recursive: true, force: true });
	104	}
	105	}),
	106	),
	107	);
	108
	109	it.effect("drops only the invalid record when serialization fails", () =>
	110	Effect.scoped(
	111	Effect.gen(function* () {
	112	const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
	113	const tracePath = path.join(tempDir, "server.trace.ndjson");
	114
	115	try {
	116	const sink = yield* makeTraceSink({
	117	filePath: tracePath,
	118	maxBytes: 1024,
	119	maxFiles: 2,
	120	batchWindowMs: 10_000,
	121	});
	122
	123	const circular: Array<unknown> = [];
	124	circular.push(circular);
	125
	126	sink.push(makeRecord("alpha"));
	127	sink.push({
	128	...makeRecord("invalid"),
	129	attributes: {
	130	circular,
	131	},
	132	} as TraceRecord);
	133	sink.push(makeRecord("beta"));
	134	yield* sink.close();
	135
	136	const lines = fs
	137	.readFileSync(tracePath, "utf8")
	138	.trim()
	139	.split("\n")
	140	.map((line) => JSON.parse(line) as TraceRecord);
	141
	142	assert.deepStrictEqual(
	143	lines.map((line) => line.name),
	144	["alpha", "beta"],
	145	);
	146	} finally {
	147	fs.rmSync(tempDir, { recursive: true, force: true });
	148	}
	149	}),
	150	),
	151	);
	152	});

apps/server/src/observability/TraceSink.ts created+68

...	...	@@ -0,0 +1,68 @@
	1	import { RotatingFileSink } from "@t3tools/shared/logging";
	2	import { Effect } from "effect";
	3
	4	import type { TraceRecord } from "./TraceRecord.ts";
	5
	6	const FLUSH_BUFFER_THRESHOLD = 32;
	7
	8	export interface TraceSinkOptions {
	9	readonly filePath: string;
	10	readonly maxBytes: number;
	11	readonly maxFiles: number;
	12	readonly batchWindowMs: number;
	13	}
	14
	15	export interface TraceSink {
	16	readonly filePath: string;
	17	push: (record: TraceRecord) => void;
	18	flush: Effect.Effect<void>;
	19	close: () => Effect.Effect<void>;
	20	}
	21
	22	export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {
	23	const sink = new RotatingFileSink({
	24	filePath: options.filePath,
	25	maxBytes: options.maxBytes,
	26	maxFiles: options.maxFiles,
	27	});
	28
	29	let buffer: Array<string> = [];
	30
	31	const flushUnsafe = () => {
	32	if (buffer.length === 0) {
	33	return;
	34	}
	35
	36	const chunk = buffer.join("");
	37	buffer = [];
	38
	39	try {
	40	sink.write(chunk);
	41	} catch {
	42	buffer.unshift(chunk);
	43	}
	44	};
	45
	46	const flush = Effect.sync(flushUnsafe).pipe(Effect.withTracerEnabled(false));
	47
	48	yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));
	49	yield* Effect.forkScoped(
	50	Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),
	51	);
	52
	53	return {
	54	filePath: options.filePath,
	55	push(record) {
	56	try {
	57	buffer.push(`${JSON.stringify(record)}\n`);
	58	if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {
	59	flushUnsafe();
	60	}
	61	} catch {
	62	return;
	63	}
	64	},
	65	flush,
	66	close: () => flush,
	67	} satisfies TraceSink;
	68	});

apps/server/src/open.ts+6-1

...	...	@@ -233,10 +233,15 @@ export class Open extends ServiceMap.Service<Open, OpenShape>()("t3/open") {}
233	233	// Implementations
234	234	// ==============================
235	235
236		export const resolveEditorLaunch = Effect.fnUntraced(function* (
	236	export const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* (
237	237	input: OpenInEditorInput,
238	238	platform: NodeJS.Platform = process.platform,
239	239	): Effect.fn.Return<EditorLaunch, OpenError> {
	240	yield* Effect.annotateCurrentSpan({
	241	"open.editor": input.editor,
	242	"open.cwd": input.cwd,
	243	"open.platform": platform,
	244	});
240	245	const editorDef = EDITORS.find((editor) => editor.id === input.editor);
241	246	if (!editorDef) {
242	247	return yield* new OpenError({ message: `Unknown editor: ${input.editor}` });

apps/server/src/orchestration/Layers/CheckpointReactor.ts+116-112

...	...	@@ -119,7 +119,7 @@ const make = Effect.gen(function* () {
119	119	createdAt: input.createdAt,
120	120	});
121	121
122		const resolveSessionRuntimeForThread = Effect.fnUntraced(function* (
	122	const resolveSessionRuntimeForThread = Effect.fn("resolveSessionRuntimeForThread")(function* (
123	123	threadId: ThreadId,
124	124	): Effect.fn.Return<Option.Option<{ readonly threadId: ThreadId; readonly cwd: string }>> {
125	125	const readModel = yield* orchestrationEngine.getReadModel();
...	...	@@ -153,7 +153,7 @@ const make = Effect.gen(function* () {
153	153	// active provider session CWD and falling back to the thread/project config.
154	154	// Returns undefined when no CWD can be determined or the workspace is not
155	155	// a git repository.
156		const resolveCheckpointCwd = Effect.fnUntraced(function* (input: {
	156	const resolveCheckpointCwd = Effect.fn("resolveCheckpointCwd")(function* (input: {
157	157	readonly threadId: ThreadId;
158	158	readonly thread: { readonly projectId: ProjectId; readonly worktreePath: string \| null };
159	159	readonly projects: ReadonlyArray<{ readonly id: ProjectId; readonly workspaceRoot: string }>;
...	...	@@ -188,7 +188,7 @@ const make = Effect.gen(function* () {
188	188	// Shared tail for both capture paths: creates the git checkpoint ref, diffs
189	189	// it against the previous turn, then dispatches the domain events to update
190	190	// the orchestration read model.
191		const captureAndDispatchCheckpoint = Effect.fnUntraced(function* (input: {
	191	const captureAndDispatchCheckpoint = Effect.fn("captureAndDispatchCheckpoint")(function* (input: {
192	192	readonly threadId: ThreadId;
193	193	readonly turnId: TurnId;
194	194	readonly thread: {
...	...	@@ -321,70 +321,70 @@ const make = Effect.gen(function* () {
321	321	});
322	322
323	323	// Captures a real git checkpoint when a turn completes via a runtime event.
324		const captureCheckpointFromTurnCompletion = Effect.fnUntraced(function* (
325		event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>,
326		) {
327		const turnId = toTurnId(event.turnId);
328		if (!turnId) {
329		return;
330		}
331
332		const readModel = yield* orchestrationEngine.getReadModel();
333		const thread = readModel.threads.find((entry) => entry.id === event.threadId);
334		if (!thread) {
335		return;
336		}
	324	const captureCheckpointFromTurnCompletion = Effect.fn("captureCheckpointFromTurnCompletion")(
	325	function* (event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>) {
	326	const turnId = toTurnId(event.turnId);
	327	if (!turnId) {
	328	return;
	329	}
337	330
338		// When a primary turn is active, only that turn may produce completion checkpoints.
339		if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {
340		return;
341		}
	331	const readModel = yield* orchestrationEngine.getReadModel();
	332	const thread = readModel.threads.find((entry) => entry.id === event.threadId);
	333	if (!thread) {
	334	return;
	335	}
342	336
343		// Only skip if a real (non-placeholder) checkpoint already exists for this turn.
344		// ProviderRuntimeIngestion may insert placeholder entries with status "missing"
345		// before this reactor runs; those must not prevent real git capture.
346		if (
347		thread.checkpoints.some(
348		(checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
349		)
350		) {
351		return;
352		}
	337	// When a primary turn is active, only that turn may produce completion checkpoints.
	338	if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {
	339	return;
	340	}
353	341
354		const checkpointCwd = yield* resolveCheckpointCwd({
355		threadId: thread.id,
356		thread,
357		projects: readModel.projects,
358		preferSessionRuntime: true,
359		});
360		if (!checkpointCwd) {
361		return;
362		}
	342	// Only skip if a real (non-placeholder) checkpoint already exists for this turn.
	343	// ProviderRuntimeIngestion may insert placeholder entries with status "missing"
	344	// before this reactor runs; those must not prevent real git capture.
	345	if (
	346	thread.checkpoints.some(
	347	(checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
	348	)
	349	) {
	350	return;
	351	}
363	352
364		// If a placeholder checkpoint exists for this turn, reuse its turn count
365		// instead of incrementing past it.
366		const existingPlaceholder = thread.checkpoints.find(
367		(checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",
368		);
369		const currentTurnCount = thread.checkpoints.reduce(
370		(maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
371		0,
372		);
373		const nextTurnCount = existingPlaceholder
374		? existingPlaceholder.checkpointTurnCount
375		: currentTurnCount + 1;
	353	const checkpointCwd = yield* resolveCheckpointCwd({
	354	threadId: thread.id,
	355	thread,
	356	projects: readModel.projects,
	357	preferSessionRuntime: true,
	358	});
	359	if (!checkpointCwd) {
	360	return;
	361	}
376	362
377		yield* captureAndDispatchCheckpoint({
378		threadId: thread.id,
379		turnId,
380		thread,
381		cwd: checkpointCwd,
382		turnCount: nextTurnCount,
383		status: checkpointStatusFromRuntime(event.payload.state),
384		assistantMessageId: undefined,
385		createdAt: event.createdAt,
386		});
387		});
	363	// If a placeholder checkpoint exists for this turn, reuse its turn count
	364	// instead of incrementing past it.
	365	const existingPlaceholder = thread.checkpoints.find(
	366	(checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",
	367	);
	368	const currentTurnCount = thread.checkpoints.reduce(
	369	(maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
	370	0,
	371	);
	372	const nextTurnCount = existingPlaceholder
	373	? existingPlaceholder.checkpointTurnCount
	374	: currentTurnCount + 1;
	375
	376	yield* captureAndDispatchCheckpoint({
	377	threadId: thread.id,
	378	turnId,
	379	thread,
	380	cwd: checkpointCwd,
	381	turnCount: nextTurnCount,
	382	status: checkpointStatusFromRuntime(event.payload.state),
	383	assistantMessageId: undefined,
	384	createdAt: event.createdAt,
	385	});
	386	},
	387	);
388	388
389	389	// Captures a real git checkpoint when a placeholder checkpoint (status "missing")
390	390	// is detected via a domain event. This replaces the placeholder with a real
...	...	@@ -394,7 +394,7 @@ const make = Effect.gen(function* () {
394	394	// events from the Codex runtime. This handler fires when the corresponding
395	395	// domain event arrives, allowing the reactor to capture the actual filesystem
396	396	// state into a git ref and dispatch a replacement checkpoint.
397		const captureCheckpointFromPlaceholder = Effect.fnUntraced(function* (
	397	const captureCheckpointFromPlaceholder = Effect.fn("captureCheckpointFromPlaceholder")(function* (
398	398	event: Extract<OrchestrationEvent, { type: "thread.turn-diff-completed" }>,
399	399	) {
400	400	const { threadId, turnId, checkpointTurnCount, status } = event.payload;
...	...	@@ -448,57 +448,59 @@ const make = Effect.gen(function* () {
448	448	});
449	449	});
450	450
451		const ensurePreTurnBaselineFromTurnStart = Effect.fnUntraced(function* (
452		event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>,
453		) {
454		const turnId = toTurnId(event.turnId);
455		if (!turnId) {
456		return;
457		}
	451	const ensurePreTurnBaselineFromTurnStart = Effect.fn("ensurePreTurnBaselineFromTurnStart")(
	452	function* (event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>) {
	453	const turnId = toTurnId(event.turnId);
	454	if (!turnId) {
	455	return;
	456	}
458	457
459		const readModel = yield* orchestrationEngine.getReadModel();
460		const thread = readModel.threads.find((entry) => entry.id === event.threadId);
461		if (!thread) {
462		return;
463		}
	458	const readModel = yield* orchestrationEngine.getReadModel();
	459	const thread = readModel.threads.find((entry) => entry.id === event.threadId);
	460	if (!thread) {
	461	return;
	462	}
464	463
465		const checkpointCwd = yield* resolveCheckpointCwd({
466		threadId: thread.id,
467		thread,
468		projects: readModel.projects,
469		preferSessionRuntime: false,
470		});
471		if (!checkpointCwd) {
472		return;
473		}
	464	const checkpointCwd = yield* resolveCheckpointCwd({
	465	threadId: thread.id,
	466	thread,
	467	projects: readModel.projects,
	468	preferSessionRuntime: false,
	469	});
	470	if (!checkpointCwd) {
	471	return;
	472	}
474	473
475		const currentTurnCount = thread.checkpoints.reduce(
476		(maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
477		0,
478		);
479		const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);
480		const baselineExists = yield* checkpointStore.hasCheckpointRef({
481		cwd: checkpointCwd,
482		checkpointRef: baselineCheckpointRef,
483		});
484		if (baselineExists) {
485		return;
486		}
	474	const currentTurnCount = thread.checkpoints.reduce(
	475	(maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
	476	0,
	477	);
	478	const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);
	479	const baselineExists = yield* checkpointStore.hasCheckpointRef({
	480	cwd: checkpointCwd,
	481	checkpointRef: baselineCheckpointRef,
	482	});
	483	if (baselineExists) {
	484	return;
	485	}
487	486
488		yield* checkpointStore.captureCheckpoint({
489		cwd: checkpointCwd,
490		checkpointRef: baselineCheckpointRef,
491		});
492		yield* receiptBus.publish({
493		type: "checkpoint.baseline.captured",
494		threadId: thread.id,
495		checkpointTurnCount: currentTurnCount,
496		checkpointRef: baselineCheckpointRef,
497		createdAt: event.createdAt,
498		});
499		});
	487	yield* checkpointStore.captureCheckpoint({
	488	cwd: checkpointCwd,
	489	checkpointRef: baselineCheckpointRef,
	490	});
	491	yield* receiptBus.publish({
	492	type: "checkpoint.baseline.captured",
	493	threadId: thread.id,
	494	checkpointTurnCount: currentTurnCount,
	495	checkpointRef: baselineCheckpointRef,
	496	createdAt: event.createdAt,
	497	});
	498	},
	499	);
500	500
501		const ensurePreTurnBaselineFromDomainTurnStart = Effect.fnUntraced(function* (
	501	const ensurePreTurnBaselineFromDomainTurnStart = Effect.fn(
	502	"ensurePreTurnBaselineFromDomainTurnStart",
	503	)(function* (
502	504	event: Extract<
503	505	OrchestrationEvent,
504	506	{ type: "thread.turn-start-requested" \| "thread.message-sent" }
...	...	@@ -557,7 +559,7 @@ const make = Effect.gen(function* () {
557	559	});
558	560	});
559	561
560		const handleRevertRequested = Effect.fnUntraced(function* (
	562	const handleRevertRequested = Effect.fn("handleRevertRequested")(function* (
561	563	event: Extract<OrchestrationEvent, { type: "thread.checkpoint-revert-requested" }>,
562	564	) {
563	565	const now = new Date().toISOString();
...	...	@@ -685,7 +687,7 @@ const make = Effect.gen(function* () {
685	687	);
686	688	});
687	689
688		const processDomainEvent = Effect.fnUntraced(function* (event: OrchestrationEvent) {
	690	const processDomainEvent = Effect.fn("processDomainEvent")(function* (event: OrchestrationEvent) {
689	691	if (event.type === "thread.turn-start-requested" \|\| event.type === "thread.message-sent") {
690	692	yield* ensurePreTurnBaselineFromDomainTurnStart(event);
691	693	return;
...	...	@@ -724,7 +726,9 @@ const make = Effect.gen(function* () {
724	726	}
725	727	});
726	728
727		const processRuntimeEvent = Effect.fnUntraced(function* (event: ProviderRuntimeEvent) {
	729	const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
	730	event: ProviderRuntimeEvent,
	731	) {
728	732	if (event.type === "turn.started") {
729	733	yield* ensurePreTurnBaselineFromTurnStart(event);
730	734	return;

apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+101-1

...	...	@@ -8,7 +8,7 @@ import {
8	8	TurnId,
9	9	type OrchestrationEvent,
10	10	} from "@t3tools/contracts";
11		import { Effect, Layer, ManagedRuntime, Option, Queue, Stream } from "effect";
	11	import { Effect, Layer, ManagedRuntime, Metric, Option, Queue, Stream } from "effect";
12	12	import { describe, expect, it } from "vitest";
13	13
14	14	import { PersistenceSqlError } from "../../persistence/Errors.ts";
...	...	@@ -62,6 +62,17 @@ function now() {
62	62	return new Date().toISOString();
63	63	}
64	64
	65	const hasMetricSnapshot = (
	66	snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
	67	id: string,
	68	attributes: Readonly<Record<string, string>>,
	69	) =>
	70	snapshots.some(
	71	(snapshot) =>
	72	snapshot.id === id &&
	73	Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
	74	);
	75
65	76	describe("OrchestrationEngine", () => {
66	77	it("bootstraps the in-memory read model from persisted projections", async () => {
67	78	const failOnHistoricalReplayStore: OrchestrationEventStoreShape = {
...	...	@@ -408,6 +419,95 @@ describe("OrchestrationEngine", () => {
408	419	await system.dispose();
409	420	});
410	421
	422	it("records command ack duration using the first committed event type", async () => {
	423	const system = await createOrchestrationSystem();
	424	const { engine } = system;
	425	const createdAt = now();
	426
	427	await system.run(
	428	engine.dispatch({
	429	type: "project.create",
	430	commandId: CommandId.makeUnsafe("cmd-project-ack-create"),
	431	projectId: asProjectId("project-ack"),
	432	title: "Ack Project",
	433	workspaceRoot: "/tmp/project-ack",
	434	defaultModelSelection: {
	435	provider: "codex",
	436	model: "gpt-5-codex",
	437	},
	438	createdAt,
	439	}),
	440	);
	441
	442	await system.run(
	443	engine.dispatch({
	444	type: "thread.create",
	445	commandId: CommandId.makeUnsafe("cmd-thread-ack-create"),
	446	threadId: ThreadId.makeUnsafe("thread-ack"),
	447	projectId: asProjectId("project-ack"),
	448	title: "Ack Thread",
	449	modelSelection: {
	450	provider: "codex",
	451	model: "gpt-5-codex",
	452	},
	453	interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
	454	runtimeMode: "full-access",
	455	branch: null,
	456	worktreePath: null,
	457	createdAt,
	458	}),
	459	);
	460
	461	const snapshots = await system.run(Metric.snapshot);
	462	expect(
	463	hasMetricSnapshot(snapshots, "t3_orchestration_command_ack_duration", {
	464	commandType: "thread.create",
	465	aggregateKind: "thread",
	466	ackEventType: "thread.created",
	467	}),
	468	).toBe(true);
	469
	470	await system.dispose();
	471	});
	472
	473	it("records failed command dispatches as metric failures", async () => {
	474	const system = await createOrchestrationSystem();
	475	const { engine } = system;
	476	const createdAt = now();
	477
	478	await expect(
	479	system.run(
	480	engine.dispatch({
	481	type: "thread.create",
	482	commandId: CommandId.makeUnsafe("cmd-thread-missing-project"),
	483	threadId: ThreadId.makeUnsafe("thread-missing-project"),
	484	projectId: asProjectId("project-missing"),
	485	title: "Missing Project Thread",
	486	modelSelection: {
	487	provider: "codex",
	488	model: "gpt-5-codex",
	489	},
	490	interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
	491	runtimeMode: "full-access",
	492	branch: null,
	493	worktreePath: null,
	494	createdAt,
	495	}),
	496	),
	497	).rejects.toThrow("does not exist");
	498
	499	const snapshots = await system.run(Metric.snapshot);
	500	expect(
	501	hasMetricSnapshot(snapshots, "t3_orchestration_commands_total", {
	502	commandType: "thread.create",
	503	aggregateKind: "thread",
	504	outcome: "failure",
	505	}),
	506	).toBe(true);
	507
	508	await system.dispose();
	509	});
	510
411	511	it("stores completed checkpoint summaries even when no files changed", async () => {
412	512	const system = await createOrchestrationSystem();
413	513	const { engine } = system;

apps/server/src/orchestration/Layers/OrchestrationEngine.ts+168-95

...	...	@@ -5,9 +5,28 @@ import type {
5	5	ThreadId,
6	6	} from "@t3tools/contracts";
7	7	import { OrchestrationCommand } from "@t3tools/contracts";
8		import { Deferred, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect";
	8	import {
	9	Cause,
	10	Deferred,
	11	Duration,
	12	Effect,
	13	Exit,
	14	Layer,
	15	Metric,
	16	Option,
	17	PubSub,
	18	Queue,
	19	Schema,
	20	Stream,
	21	} from "effect";
9	22	import * as SqlClient from "effect/unstable/sql/SqlClient";
10	23
	24	import {
	25	metricAttributes,
	26	orchestrationCommandAckDuration,
	27	orchestrationCommandsTotal,
	28	orchestrationCommandDuration,
	29	} from "../../observability/Metrics.ts";
11	30	import { toPersistenceSqlError } from "../../persistence/Errors.ts";
12	31	import { OrchestrationEventStore } from "../../persistence/Services/OrchestrationEventStore.ts";
13	32	import { OrchestrationCommandReceiptRepository } from "../../persistence/Services/OrchestrationCommandReceipts.ts";
...	...	@@ -28,6 +47,7 @@ import {
28	47	interface CommandEnvelope {
29	48	command: OrchestrationCommand;
30	49	result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;
	50	startedAtMs: number;
31	51	}
32	52
33	53	function commandToAggregateRef(command: OrchestrationCommand): {
...	...	@@ -64,6 +84,12 @@ const makeOrchestrationEngine = Effect.gen(function* () {
64	84
65	85	const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {
66	86	const dispatchStartSequence = readModel.snapshotSequence;
	87	const processingStartedAtMs = Date.now();
	88	const aggregateRef = commandToAggregateRef(envelope.command);
	89	const baseMetricAttributes = {
	90	commandType: envelope.command.type,
	91	aggregateKind: aggregateRef.aggregateKind,
	92	} as const;
67	93	const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {
68	94	const persistedEvents = yield* Stream.runCollect(
69	95	eventStore.readFromSequence(dispatchStartSequence),
...	...	@@ -83,113 +109,160 @@ const makeOrchestrationEngine = Effect.gen(function* () {
83	109	}
84	110	});
85	111
86		return Effect.gen(function* () {
87		const existingReceipt = yield* commandReceiptRepository.getByCommandId({
88		commandId: envelope.command.commandId,
89		});
90		if (Option.isSome(existingReceipt)) {
91		if (existingReceipt.value.status === "accepted") {
92		yield* Deferred.succeed(envelope.result, {
93		sequence: existingReceipt.value.resultSequence,
94		});
95		return;
96		}
97		yield* Deferred.fail(
98		envelope.result,
99		new OrchestrationCommandPreviouslyRejectedError({
	112	return Effect.exit(
	113	Effect.gen(function* () {
	114	yield* Effect.annotateCurrentSpan({
	115	"orchestration.command_id": envelope.command.commandId,
	116	"orchestration.command_type": envelope.command.type,
	117	"orchestration.aggregate_kind": aggregateRef.aggregateKind,
	118	"orchestration.aggregate_id": aggregateRef.aggregateId,
	119	});
	120
	121	const existingReceipt = yield* commandReceiptRepository.getByCommandId({
	122	commandId: envelope.command.commandId,
	123	});
	124	if (Option.isSome(existingReceipt)) {
	125	if (existingReceipt.value.status === "accepted") {
	126	return {
	127	sequence: existingReceipt.value.resultSequence,
	128	};
	129	}
	130	return yield* new OrchestrationCommandPreviouslyRejectedError({
100	131	commandId: envelope.command.commandId,
101	132	detail: existingReceipt.value.error ?? "Previously rejected.",
102		}),
103		);
104		return;
105		}
	133	});
	134	}
106	135
107		const eventBase = yield* decideOrchestrationCommand({
108		command: envelope.command,
109		readModel,
110		});
111		const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];
112		const committedCommand = yield* sql
113		.withTransaction(
114		Effect.gen(function* () {
115		const committedEvents: OrchestrationEvent[] = [];
116		let nextReadModel = readModel;
117
118		for (const nextEvent of eventBases) {
119		const savedEvent = yield* eventStore.append(nextEvent);
120		nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
121		yield* projectionPipeline.projectEvent(savedEvent);
122		committedEvents.push(savedEvent);
123		}
	136	const eventBase = yield* decideOrchestrationCommand({
	137	command: envelope.command,
	138	readModel,
	139	});
	140	const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];
	141	const committedCommand = yield* sql
	142	.withTransaction(
	143	Effect.gen(function* () {
	144	const committedEvents: OrchestrationEvent[] = [];
	145	let nextReadModel = readModel;
124	146
125		const lastSavedEvent = committedEvents.at(-1) ?? null;
126		if (lastSavedEvent === null) {
127		return yield* new OrchestrationCommandInvariantError({
128		commandType: envelope.command.type,
129		detail: "Command produced no events.",
130		});
131		}
	147	for (const nextEvent of eventBases) {
	148	const savedEvent = yield* eventStore.append(nextEvent);
	149	nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
	150	yield* projectionPipeline.projectEvent(savedEvent);
	151	committedEvents.push(savedEvent);
	152	}
132	153
133		yield* commandReceiptRepository.upsert({
134		commandId: envelope.command.commandId,
135		aggregateKind: lastSavedEvent.aggregateKind,
136		aggregateId: lastSavedEvent.aggregateId,
137		acceptedAt: lastSavedEvent.occurredAt,
138		resultSequence: lastSavedEvent.sequence,
139		status: "accepted",
140		error: null,
141		});
	154	const lastSavedEvent = committedEvents.at(-1) ?? null;
	155	if (lastSavedEvent === null) {
	156	return yield* new OrchestrationCommandInvariantError({
	157	commandType: envelope.command.type,
	158	detail: "Command produced no events.",
	159	});
	160	}
142	161
143		return {
144		committedEvents,
145		lastSequence: lastSavedEvent.sequence,
146		nextReadModel,
147		} as const;
148		}),
149		)
150		.pipe(
151		Effect.catchTag("SqlError", (sqlError) =>
152		Effect.fail(
153		toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),
	162	yield* commandReceiptRepository.upsert({
	163	commandId: envelope.command.commandId,
	164	aggregateKind: lastSavedEvent.aggregateKind,
	165	aggregateId: lastSavedEvent.aggregateId,
	166	acceptedAt: lastSavedEvent.occurredAt,
	167	resultSequence: lastSavedEvent.sequence,
	168	status: "accepted",
	169	error: null,
	170	});
	171
	172	return {
	173	committedEvents,
	174	lastSequence: lastSavedEvent.sequence,
	175	nextReadModel,
	176	} as const;
	177	}),
	178	)
	179	.pipe(
	180	Effect.catchTag("SqlError", (sqlError) =>
	181	Effect.fail(
	182	toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),
	183	),
154	184	),
155		),
156		);
	185	);
157	186
158		readModel = committedCommand.nextReadModel;
159		for (const event of committedCommand.committedEvents) {
160		yield* PubSub.publish(eventPubSub, event);
161		}
162		yield* Deferred.succeed(envelope.result, { sequence: committedCommand.lastSequence });
163		}).pipe(
164		Effect.catch((error) =>
165		Effect.gen(function* () {
166		yield* reconcileReadModelAfterDispatchFailure.pipe(
167		Effect.catch(() =>
168		Effect.logWarning(
169		"failed to reconcile orchestration read model after dispatch failure",
170		).pipe(
171		Effect.annotateLogs({
172		commandId: envelope.command.commandId,
173		snapshotSequence: readModel.snapshotSequence,
	187	readModel = committedCommand.nextReadModel;
	188	for (const [index, event] of committedCommand.committedEvents.entries()) {
	189	yield* PubSub.publish(eventPubSub, event);
	190	if (index === 0) {
	191	yield* Metric.update(
	192	Metric.withAttributes(
	193	orchestrationCommandAckDuration,
	194	metricAttributes({
	195	...baseMetricAttributes,
	196	ackEventType: event.type,
174	197	}),
175	198	),
	199	Duration.millis(Math.max(0, Date.now() - envelope.startedAtMs)),
	200	);
	201	}
	202	}
	203	return { sequence: committedCommand.lastSequence };
	204	}).pipe(Effect.withSpan(`orchestration.command.${envelope.command.type}`)),
	205	).pipe(
	206	Effect.flatMap((exit) =>
	207	Effect.gen(function* () {
	208	const outcome = Exit.isSuccess(exit)
	209	? "success"
	210	: Cause.hasInterruptsOnly(exit.cause)
	211	? "interrupt"
	212	: "failure";
	213	yield* Metric.update(
	214	Metric.withAttributes(
	215	orchestrationCommandDuration,
	216	metricAttributes(baseMetricAttributes),
	217	),
	218	Duration.millis(Math.max(0, Date.now() - processingStartedAtMs)),
	219	);
	220	yield* Metric.update(
	221	Metric.withAttributes(
	222	orchestrationCommandsTotal,
	223	metricAttributes({
	224	...baseMetricAttributes,
	225	outcome,
	226	}),
176	227	),
	228	1,
177	229	);
178	230
179		if (Schema.is(OrchestrationCommandInvariantError)(error)) {
180		const aggregateRef = commandToAggregateRef(envelope.command);
181		yield* commandReceiptRepository
182		.upsert({
183		commandId: envelope.command.commandId,
184		aggregateKind: aggregateRef.aggregateKind,
185		aggregateId: aggregateRef.aggregateId,
186		acceptedAt: new Date().toISOString(),
187		resultSequence: readModel.snapshotSequence,
188		status: "rejected",
189		error: error.message,
190		})
191		.pipe(Effect.catch(() => Effect.void));
	231	if (Exit.isSuccess(exit)) {
	232	yield* Deferred.succeed(envelope.result, exit.value);
	233	return;
192	234	}
	235
	236	const error = Cause.squash(exit.cause) as OrchestrationDispatchError;
	237	if (!Schema.is(OrchestrationCommandPreviouslyRejectedError)(error)) {
	238	yield* reconcileReadModelAfterDispatchFailure.pipe(
	239	Effect.catch(() =>
	240	Effect.logWarning(
	241	"failed to reconcile orchestration read model after dispatch failure",
	242	).pipe(
	243	Effect.annotateLogs({
	244	commandId: envelope.command.commandId,
	245	snapshotSequence: readModel.snapshotSequence,
	246	}),
	247	),
	248	),
	249	);
	250
	251	if (Schema.is(OrchestrationCommandInvariantError)(error)) {
	252	yield* commandReceiptRepository
	253	.upsert({
	254	commandId: envelope.command.commandId,
	255	aggregateKind: aggregateRef.aggregateKind,
	256	aggregateId: aggregateRef.aggregateId,
	257	acceptedAt: new Date().toISOString(),
	258	resultSequence: readModel.snapshotSequence,
	259	status: "rejected",
	260	error: error.message,
	261	})
	262	.pipe(Effect.catch(() => Effect.void));
	263	}
	264	}
	265
193	266	yield* Deferred.fail(envelope.result, error);
194	267	}),
195	268	),
...	...	@@ -214,7 +287,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
214	287	const dispatch: OrchestrationEngineShape["dispatch"] = (command) =>
215	288	Effect.gen(function* () {
216	289	const result = yield* Deferred.make<{ sequence: number }, OrchestrationDispatchError>();
217		yield* Queue.offer(commandQueue, { command, result });
	290	yield* Queue.offer(commandQueue, { command, result, startedAtMs: Date.now() });
218	291	return yield* Deferred.await(result);
219	292	});
220	293

apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+103-88

...	...	@@ -16,6 +16,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
16	16
17	17	import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";
18	18	import { GitCore } from "../../git/Services/GitCore.ts";
	19	import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts";
19	20	import { ProviderAdapterRequestError, ProviderServiceError } from "../../provider/Errors.ts";
20	21	import { TextGeneration } from "../../git/Services/TextGeneration.ts";
21	22	import { ProviderService } from "../../provider/Services/ProviderService.ts";
...	...	@@ -212,12 +213,12 @@ const make = Effect.gen(function* () {
212	213	createdAt: input.createdAt,
213	214	});
214	215
215		const resolveThread = Effect.fnUntraced(function* (threadId: ThreadId) {
	216	const resolveThread = Effect.fn("resolveThread")(function* (threadId: ThreadId) {
216	217	const readModel = yield* orchestrationEngine.getReadModel();
217	218	return readModel.threads.find((entry) => entry.id === threadId);
218	219	});
219	220
220		const ensureSessionForThread = Effect.fnUntraced(function* (
	221	const ensureSessionForThread = Effect.fn("ensureSessionForThread")(function* (
221	222	threadId: ThreadId,
222	223	createdAt: string,
223	224	options?: {
...	...	@@ -357,7 +358,7 @@ const make = Effect.gen(function* () {
357	358	return startedSession.threadId;
358	359	});
359	360
360		const sendTurnForThread = Effect.fnUntraced(function* (input: {
	361	const sendTurnForThread = Effect.fn("sendTurnForThread")(function* (input: {
361	362	readonly threadId: ThreadId;
362	363	readonly messageText: string;
363	364	readonly attachments?: ReadonlyArray<ChatAttachment>;
...	...	@@ -409,7 +410,9 @@ const make = Effect.gen(function* () {
409	410	});
410	411	});
411	412
412		const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fnUntraced(function* (input: {
	413	const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fn(
	414	"maybeGenerateAndRenameWorktreeBranchForFirstTurn",
	415	)(function* (input: {
413	416	readonly threadId: ThreadId;
414	417	readonly branch: string \| null;
415	418	readonly worktreePath: string \| null;
...	...	@@ -461,50 +464,52 @@ const make = Effect.gen(function* () {
461	464	);
462	465	});
463	466
464		const maybeGenerateThreadTitleForFirstTurn = Effect.fnUntraced(function* (input: {
465		readonly threadId: ThreadId;
466		readonly cwd: string;
467		readonly messageText: string;
468		readonly attachments?: ReadonlyArray<ChatAttachment>;
469		readonly titleSeed?: string;
470		}) {
471		const attachments = input.attachments ?? [];
472		yield* Effect.gen(function* () {
473		const { textGenerationModelSelection: modelSelection } =
474		yield* serverSettingsService.getSettings;
475
476		const generated = yield* textGeneration.generateThreadTitle({
477		cwd: input.cwd,
478		message: input.messageText,
479		...(attachments.length > 0 ? { attachments } : {}),
480		modelSelection,
481		});
482		if (!generated) return;
	467	const maybeGenerateThreadTitleForFirstTurn = Effect.fn("maybeGenerateThreadTitleForFirstTurn")(
	468	function* (input: {
	469	readonly threadId: ThreadId;
	470	readonly cwd: string;
	471	readonly messageText: string;
	472	readonly attachments?: ReadonlyArray<ChatAttachment>;
	473	readonly titleSeed?: string;
	474	}) {
	475	const attachments = input.attachments ?? [];
	476	yield* Effect.gen(function* () {
	477	const { textGenerationModelSelection: modelSelection } =
	478	yield* serverSettingsService.getSettings;
	479
	480	const generated = yield* textGeneration.generateThreadTitle({
	481	cwd: input.cwd,
	482	message: input.messageText,
	483	...(attachments.length > 0 ? { attachments } : {}),
	484	modelSelection,
	485	});
	486	if (!generated) return;
483	487
484		const thread = yield* resolveThread(input.threadId);
485		if (!thread) return;
486		if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {
487		return;
488		}
	488	const thread = yield* resolveThread(input.threadId);
	489	if (!thread) return;
	490	if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {
	491	return;
	492	}
489	493
490		yield* orchestrationEngine.dispatch({
491		type: "thread.meta.update",
492		commandId: serverCommandId("thread-title-rename"),
493		threadId: input.threadId,
494		title: generated.title,
495		});
496		}).pipe(
497		Effect.catchCause((cause) =>
498		Effect.logWarning("provider command reactor failed to generate or rename thread title", {
	494	yield* orchestrationEngine.dispatch({
	495	type: "thread.meta.update",
	496	commandId: serverCommandId("thread-title-rename"),
499	497	threadId: input.threadId,
500		cwd: input.cwd,
501		cause: Cause.pretty(cause),
502		}),
503		),
504		);
505		});
	498	title: generated.title,
	499	});
	500	}).pipe(
	501	Effect.catchCause((cause) =>
	502	Effect.logWarning("provider command reactor failed to generate or rename thread title", {
	503	threadId: input.threadId,
	504	cwd: input.cwd,
	505	cause: Cause.pretty(cause),
	506	}),
	507	),
	508	);
	509	},
	510	);
506	511
507		const processTurnStartRequested = Effect.fnUntraced(function* (
	512	const processTurnStartRequested = Effect.fn("processTurnStartRequested")(function* (
508	513	event: Extract<ProviderIntentEvent, { type: "thread.turn-start-requested" }>,
509	514	) {
510	515	const key = turnStartKeyForEvent(event);
...	...	@@ -583,7 +588,7 @@ const make = Effect.gen(function* () {
583	588	);
584	589	});
585	590
586		const processTurnInterruptRequested = Effect.fnUntraced(function* (
	591	const processTurnInterruptRequested = Effect.fn("processTurnInterruptRequested")(function* (
587	592	event: Extract<ProviderIntentEvent, { type: "thread.turn-interrupt-requested" }>,
588	593	) {
589	594	const thread = yield* resolveThread(event.payload.threadId);
...	...	@@ -606,7 +611,7 @@ const make = Effect.gen(function* () {
606	611	yield* providerService.interruptTurn({ threadId: event.payload.threadId });
607	612	});
608	613
609		const processApprovalResponseRequested = Effect.fnUntraced(function* (
	614	const processApprovalResponseRequested = Effect.fn("processApprovalResponseRequested")(function* (
610	615	event: Extract<ProviderIntentEvent, { type: "thread.approval-response-requested" }>,
611	616	) {
612	617	const thread = yield* resolveThread(event.payload.threadId);
...	...	@@ -653,50 +658,52 @@ const make = Effect.gen(function* () {
653	658	);
654	659	});
655	660
656		const processUserInputResponseRequested = Effect.fnUntraced(function* (
657		event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,
658		) {
659		const thread = yield* resolveThread(event.payload.threadId);
660		if (!thread) {
661		return;
662		}
663		const hasSession = thread.session && thread.session.status !== "stopped";
664		if (!hasSession) {
665		return yield* appendProviderFailureActivity({
666		threadId: event.payload.threadId,
667		kind: "provider.user-input.respond.failed",
668		summary: "Provider user input response failed",
669		detail: "No active provider session is bound to this thread.",
670		turnId: null,
671		createdAt: event.payload.createdAt,
672		requestId: event.payload.requestId,
673		});
674		}
	661	const processUserInputResponseRequested = Effect.fn("processUserInputResponseRequested")(
	662	function* (
	663	event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,
	664	) {
	665	const thread = yield* resolveThread(event.payload.threadId);
	666	if (!thread) {
	667	return;
	668	}
	669	const hasSession = thread.session && thread.session.status !== "stopped";
	670	if (!hasSession) {
	671	return yield* appendProviderFailureActivity({
	672	threadId: event.payload.threadId,
	673	kind: "provider.user-input.respond.failed",
	674	summary: "Provider user input response failed",
	675	detail: "No active provider session is bound to this thread.",
	676	turnId: null,
	677	createdAt: event.payload.createdAt,
	678	requestId: event.payload.requestId,
	679	});
	680	}
675	681
676		yield* providerService
677		.respondToUserInput({
678		threadId: event.payload.threadId,
679		requestId: event.payload.requestId,
680		answers: event.payload.answers,
681		})
682		.pipe(
683		Effect.catchCause((cause) =>
684		appendProviderFailureActivity({
685		threadId: event.payload.threadId,
686		kind: "provider.user-input.respond.failed",
687		summary: "Provider user input response failed",
688		detail: isUnknownPendingUserInputRequestError(cause)
689		? stalePendingRequestDetail("user-input", event.payload.requestId)
690		: Cause.pretty(cause),
691		turnId: null,
692		createdAt: event.payload.createdAt,
693		requestId: event.payload.requestId,
694		}),
695		),
696		);
697		});
	682	yield* providerService
	683	.respondToUserInput({
	684	threadId: event.payload.threadId,
	685	requestId: event.payload.requestId,
	686	answers: event.payload.answers,
	687	})
	688	.pipe(
	689	Effect.catchCause((cause) =>
	690	appendProviderFailureActivity({
	691	threadId: event.payload.threadId,
	692	kind: "provider.user-input.respond.failed",
	693	summary: "Provider user input response failed",
	694	detail: isUnknownPendingUserInputRequestError(cause)
	695	? stalePendingRequestDetail("user-input", event.payload.requestId)
	696	: Cause.pretty(cause),
	697	turnId: null,
	698	createdAt: event.payload.createdAt,
	699	requestId: event.payload.requestId,
	700	}),
	701	),
	702	);
	703	},
	704	);
698	705
699		const processSessionStopRequested = Effect.fnUntraced(function* (
	706	const processSessionStopRequested = Effect.fn("processSessionStopRequested")(function* (
700	707	event: Extract<ProviderIntentEvent, { type: "thread.session-stop-requested" }>,
701	708	) {
702	709	const thread = yield* resolveThread(event.payload.threadId);
...	...	@@ -727,6 +734,14 @@ const make = Effect.gen(function* () {
727	734	const processDomainEvent = Effect.fn("processDomainEvent")(function* (
728	735	event: ProviderIntentEvent,
729	736	) {
	737	yield* Effect.annotateCurrentSpan({
	738	"orchestration.event_type": event.type,
	739	"orchestration.thread_id": event.payload.threadId,
	740	...(event.commandId ? { "orchestration.command_id": event.commandId } : {}),
	741	});
	742	yield* increment(orchestrationEventsProcessedTotal, {
	743	eventType: event.type,
	744	});
730	745	switch (event.type) {
731	746	case "thread.runtime-mode-set": {
732	747	const thread = yield* resolveThread(event.payload.threadId);

apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+44-41

...	...	@@ -524,7 +524,7 @@ const make = Effect.fn("make")(function* () {
524	524	lookup: () => Effect.succeed({ text: "", createdAt: "" }),
525	525	});
526	526
527		const isGitRepoForThread = Effect.fnUntraced(function* (threadId: ThreadId) {
	527	const isGitRepoForThread = Effect.fn("isGitRepoForThread")(function* (threadId: ThreadId) {
528	528	const readModel = yield* orchestrationEngine.getReadModel();
529	529	const thread = readModel.threads.find((entry) => entry.id === threadId);
530	530	if (!thread) {
...	...	@@ -796,9 +796,9 @@ const make = Effect.fn("make")(function* () {
796	796	).pipe(Effect.asVoid);
797	797	});
798	798
799		const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fnUntraced(function* (
800		threadId: ThreadId,
801		) {
	799	const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fn(
	800	"getSourceProposedPlanReferenceForPendingTurnStart",
	801	)(function* (threadId: ThreadId) {
802	802	const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({
803	803	threadId,
804	804	});
...	...	@@ -818,16 +818,17 @@ const make = Effect.fn("make")(function* () {
818	818	} as const;
819	819	});
820	820
821		const getExpectedProviderTurnIdForThread = Effect.fnUntraced(function* (threadId: ThreadId) {
822		const sessions = yield* providerService.listSessions();
823		const session = sessions.find((entry) => entry.threadId === threadId);
824		return session?.activeTurnId;
825		});
826
827		const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fnUntraced(function* (
828		threadId: ThreadId,
829		eventTurnId: TurnId \| undefined,
830		) {
	821	const getExpectedProviderTurnIdForThread = Effect.fn("getExpectedProviderTurnIdForThread")(
	822	function* (threadId: ThreadId) {
	823	const sessions = yield* providerService.listSessions();
	824	const session = sessions.find((entry) => entry.threadId === threadId);
	825	return session?.activeTurnId;
	826	},
	827	);
	828
	829	const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fn(
	830	"getSourceProposedPlanReferenceForAcceptedTurnStart",
	831	)(function* (threadId: ThreadId, eventTurnId: TurnId \| undefined) {
831	832	if (eventTurnId === undefined) {
832	833	return null;
833	834	}
...	...	@@ -840,34 +841,36 @@ const make = Effect.fn("make")(function* () {
840	841	return yield* getSourceProposedPlanReferenceForPendingTurnStart(threadId);
841	842	});
842	843
843		const markSourceProposedPlanImplemented = Effect.fnUntraced(function* (
844		sourceThreadId: ThreadId,
845		sourcePlanId: OrchestrationProposedPlanId,
846		implementationThreadId: ThreadId,
847		implementedAt: string,
848		) {
849		const readModel = yield* orchestrationEngine.getReadModel();
850		const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);
851		const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);
852		if (!sourceThread \|\| !sourcePlan \|\| sourcePlan.implementedAt !== null) {
853		return;
854		}
	844	const markSourceProposedPlanImplemented = Effect.fn("markSourceProposedPlanImplemented")(
	845	function* (
	846	sourceThreadId: ThreadId,
	847	sourcePlanId: OrchestrationProposedPlanId,
	848	implementationThreadId: ThreadId,
	849	implementedAt: string,
	850	) {
	851	const readModel = yield* orchestrationEngine.getReadModel();
	852	const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);
	853	const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);
	854	if (!sourceThread \|\| !sourcePlan \|\| sourcePlan.implementedAt !== null) {
	855	return;
	856	}
855	857
856		yield* orchestrationEngine.dispatch({
857		type: "thread.proposed-plan.upsert",
858		commandId: CommandId.makeUnsafe(
859		`provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,
860		),
861		threadId: sourceThread.id,
862		proposedPlan: {
863		...sourcePlan,
864		implementedAt,
865		implementationThreadId,
866		updatedAt: implementedAt,
867		},
868		createdAt: implementedAt,
869		});
870		});
	858	yield* orchestrationEngine.dispatch({
	859	type: "thread.proposed-plan.upsert",
	860	commandId: CommandId.makeUnsafe(
	861	`provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,
	862	),
	863	threadId: sourceThread.id,
	864	proposedPlan: {
	865	...sourcePlan,
	866	implementedAt,
	867	implementationThreadId,
	868	updatedAt: implementedAt,
	869	},
	870	createdAt: implementedAt,
	871	});
	872	},
	873	);
871	874
872	875	const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
873	876	event: ProviderRuntimeEvent,

apps/server/src/persistence/Layers/Sqlite.ts+11-1

...	...	@@ -6,6 +6,7 @@ import { ServerConfig } from "../../config.ts";
6	6
7	7	type RuntimeSqliteLayerConfig = {
8	8	readonly filename: string;
	9	readonly spanAttributes?: Record<string, unknown>;
9	10	};
10	11
11	12	type Loader = {
...	...	@@ -41,7 +42,16 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(
41	42	const path = yield* Path.Path;
42	43	yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
43	44
44		return Layer.provideMerge(setup, makeRuntimeSqliteLayer({ filename: dbPath }));
	45	return Layer.provideMerge(
	46	setup,
	47	makeRuntimeSqliteLayer({
	48	filename: dbPath,
	49	spanAttributes: {
	50	"db.name": path.basename(dbPath),
	51	"service.name": "t3-server",
	52	},
	53	}),
	54	);
45	55	}, Layer.unwrap);
46	56
47	57	export const SqlitePersistenceMemory = Layer.provideMerge(

apps/server/src/provider/Layers/ProviderService.test.ts+126-1

...	...	@@ -20,7 +20,7 @@ import {
20	20	import { it, assert, vi } from "@effect/vitest";
21	21	import { assertFailure } from "@effect/vitest/utils";
22	22
23		import { Effect, Fiber, Layer, Option, PubSub, Ref, Stream } from "effect";
	23	import { Effect, Fiber, Layer, Metric, Option, PubSub, Ref, Stream } from "effect";
24	24	import * as SqlClient from "effect/unstable/sql/SqlClient";
25	25
26	26	import {
...	...	@@ -230,6 +230,17 @@ function makeFakeCodexAdapter(provider: ProviderKind = "codex") {
230	230	const sleep = (ms: number) =>
231	231	Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, ms)));
232	232
	233	const hasMetricSnapshot = (
	234	snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
	235	id: string,
	236	attributes: Readonly<Record<string, string>>,
	237	) =>
	238	snapshots.some(
	239	(snapshot) =>
	240	snapshot.id === id &&
	241	Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
	242	);
	243
233	244	function makeProviderServiceLayer() {
234	245	const codex = makeFakeCodexAdapter();
235	246	const claude = makeFakeCodexAdapter("claudeAgent");
...	...	@@ -1060,6 +1071,120 @@ fanout.layer("ProviderServiceLive fanout", (it) => {
1060	1071	);
1061	1072	}),
1062	1073	);
	1074
	1075	it.effect("records provider metrics with the routed provider label", () =>
	1076	Effect.gen(function* () {
	1077	const provider = yield* ProviderService;
	1078
	1079	const session = yield* provider.startSession(asThreadId("thread-metrics"), {
	1080	provider: "claudeAgent",
	1081	threadId: asThreadId("thread-metrics"),
	1082	cwd: "/tmp/project",
	1083	runtimeMode: "full-access",
	1084	});
	1085
	1086	yield* provider.interruptTurn({ threadId: session.threadId });
	1087	yield* provider.respondToRequest({
	1088	threadId: session.threadId,
	1089	requestId: asRequestId("req-metrics-1"),
	1090	decision: "accept",
	1091	});
	1092	yield* provider.respondToUserInput({
	1093	threadId: session.threadId,
	1094	requestId: asRequestId("req-metrics-2"),
	1095	answers: {
	1096	sandbox_mode: "workspace-write",
	1097	},
	1098	});
	1099	yield* provider.rollbackConversation({
	1100	threadId: session.threadId,
	1101	numTurns: 1,
	1102	});
	1103	yield* provider.stopSession({ threadId: session.threadId });
	1104
	1105	const snapshots = yield* Metric.snapshot;
	1106
	1107	assert.equal(
	1108	hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
	1109	provider: "claudeAgent",
	1110	operation: "interrupt",
	1111	outcome: "success",
	1112	}),
	1113	true,
	1114	);
	1115	assert.equal(
	1116	hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
	1117	provider: "claudeAgent",
	1118	operation: "approval-response",
	1119	outcome: "success",
	1120	}),
	1121	true,
	1122	);
	1123	assert.equal(
	1124	hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
	1125	provider: "claudeAgent",
	1126	operation: "user-input-response",
	1127	outcome: "success",
	1128	}),
	1129	true,
	1130	);
	1131	assert.equal(
	1132	hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
	1133	provider: "claudeAgent",
	1134	operation: "rollback",
	1135	outcome: "success",
	1136	}),
	1137	true,
	1138	);
	1139	assert.equal(
	1140	hasMetricSnapshot(snapshots, "t3_provider_sessions_total", {
	1141	provider: "claudeAgent",
	1142	operation: "stop",
	1143	outcome: "success",
	1144	}),
	1145	true,
	1146	);
	1147	}),
	1148	);
	1149
	1150	it.effect(
	1151	"records sendTurn metrics with the resolved provider when modelSelection is omitted",
	1152	() =>
	1153	Effect.gen(function* () {
	1154	const provider = yield* ProviderService;
	1155
	1156	const session = yield* provider.startSession(asThreadId("thread-send-metrics"), {
	1157	provider: "claudeAgent",
	1158	threadId: asThreadId("thread-send-metrics"),
	1159	cwd: "/tmp/project-send-metrics",
	1160	runtimeMode: "full-access",
	1161	});
	1162
	1163	yield* provider.sendTurn({
	1164	threadId: session.threadId,
	1165	input: "hello",
	1166	attachments: [],
	1167	});
	1168
	1169	const snapshots = yield* Metric.snapshot;
	1170
	1171	assert.equal(
	1172	hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
	1173	provider: "claudeAgent",
	1174	operation: "send",
	1175	outcome: "success",
	1176	}),
	1177	true,
	1178	);
	1179	assert.equal(
	1180	hasMetricSnapshot(snapshots, "t3_provider_turn_duration", {
	1181	provider: "claudeAgent",
	1182	operation: "send",
	1183	}),
	1184	true,
	1185	);
	1186	}),
	1187	);
1063	1188	});
1064	1189
1065	1190	const validation = makeProviderServiceLayer();

apps/server/src/provider/Layers/ProviderService.ts+323-161

...	...	@@ -24,6 +24,16 @@ import {
24	24	} from "@t3tools/contracts";
25	25	import { Effect, Layer, Option, PubSub, Queue, Schema, SchemaIssue, Stream } from "effect";
26	26
	27	import {
	28	increment,
	29	providerMetricAttributes,
	30	providerRuntimeEventsTotal,
	31	providerSessionsTotal,
	32	providerTurnDuration,
	33	providerTurnsTotal,
	34	providerTurnMetricAttributes,
	35	withMetrics,
	36	} from "../../observability/Metrics.ts";
27	37	import { ProviderValidationError } from "../Errors.ts";
28	38	import { ProviderAdapterRegistry } from "../Services/ProviderAdapterRegistry.ts";
29	39	import { ProviderService, type ProviderServiceShape } from "../Services/ProviderService.ts";
...	...	@@ -179,7 +189,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
179	189	const providers = yield* registry.listProviders();
180	190	const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider));
181	191	const processRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect<void> =>
182		publishRuntimeEvent(event);
	192	increment(providerRuntimeEventsTotal, {
	193	provider: event.provider,
	194	eventType: event.type,
	195	}).pipe(Effect.andThen(publishRuntimeEvent(event)));
183	196
184	197	const worker = Effect.forever(
185	198	Queue.take(runtimeEventQueue).pipe(Effect.flatMap(processRuntimeEvent)),
...	...	@@ -196,58 +209,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
196	209	readonly binding: ProviderRuntimeBinding;
197	210	readonly operation: string;
198	211	}) {
199		const adapter = yield* registry.getByProvider(input.binding.provider);
200		const hasResumeCursor =
201		input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;
202		const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);
203		if (hasActiveSession) {
204		const activeSessions = yield* adapter.listSessions();
205		const existing = activeSessions.find(
206		(session) => session.threadId === input.binding.threadId,
207		);
208		if (existing) {
209		yield* upsertSessionBinding(existing, input.binding.threadId);
210		yield* analytics.record("provider.session.recovered", {
211		provider: existing.provider,
212		strategy: "adopt-existing",
213		hasResumeCursor: existing.resumeCursor !== undefined,
214		});
215		return { adapter, session: existing } as const;
	212	yield* Effect.annotateCurrentSpan({
	213	"provider.operation": "recover-session",
	214	"provider.kind": input.binding.provider,
	215	"provider.thread_id": input.binding.threadId,
	216	});
	217	return yield* Effect.gen(function* () {
	218	const adapter = yield* registry.getByProvider(input.binding.provider);
	219	const hasResumeCursor =
	220	input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;
	221	const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);
	222	if (hasActiveSession) {
	223	const activeSessions = yield* adapter.listSessions();
	224	const existing = activeSessions.find(
	225	(session) => session.threadId === input.binding.threadId,
	226	);
	227	if (existing) {
	228	yield* upsertSessionBinding(existing, input.binding.threadId);
	229	yield* analytics.record("provider.session.recovered", {
	230	provider: existing.provider,
	231	strategy: "adopt-existing",
	232	hasResumeCursor: existing.resumeCursor !== undefined,
	233	});
	234	return { adapter, session: existing } as const;
	235	}
216	236	}
217		}
218	237
219		if (!hasResumeCursor) {
220		return yield* toValidationError(
221		input.operation,
222		`Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,
223		);
224		}
	238	if (!hasResumeCursor) {
	239	return yield* toValidationError(
	240	input.operation,
	241	`Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,
	242	);
	243	}
225	244
226		const persistedCwd = readPersistedCwd(input.binding.runtimePayload);
227		const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);
	245	const persistedCwd = readPersistedCwd(input.binding.runtimePayload);
	246	const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);
228	247
229		const resumed = yield* adapter.startSession({
230		threadId: input.binding.threadId,
231		provider: input.binding.provider,
232		...(persistedCwd ? { cwd: persistedCwd } : {}),
233		...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),
234		...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),
235		runtimeMode: input.binding.runtimeMode ?? "full-access",
236		});
237		if (resumed.provider !== adapter.provider) {
238		return yield* toValidationError(
239		input.operation,
240		`Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,
241		);
242		}
	248	const resumed = yield* adapter.startSession({
	249	threadId: input.binding.threadId,
	250	provider: input.binding.provider,
	251	...(persistedCwd ? { cwd: persistedCwd } : {}),
	252	...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),
	253	...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),
	254	runtimeMode: input.binding.runtimeMode ?? "full-access",
	255	});
	256	if (resumed.provider !== adapter.provider) {
	257	return yield* toValidationError(
	258	input.operation,
	259	`Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,
	260	);
	261	}
243	262
244		yield* upsertSessionBinding(resumed, input.binding.threadId);
245		yield* analytics.record("provider.session.recovered", {
246		provider: resumed.provider,
247		strategy: "resume-thread",
248		hasResumeCursor: resumed.resumeCursor !== undefined,
249		});
250		return { adapter, session: resumed } as const;
	263	yield* upsertSessionBinding(resumed, input.binding.threadId);
	264	yield* analytics.record("provider.session.recovered", {
	265	provider: resumed.provider,
	266	strategy: "resume-thread",
	267	hasResumeCursor: resumed.resumeCursor !== undefined,
	268	});
	269	return { adapter, session: resumed } as const;
	270	}).pipe(
	271	withMetrics({
	272	counter: providerSessionsTotal,
	273	attributes: providerMetricAttributes(input.binding.provider, {
	274	operation: "recover",
	275	}),
	276	}),
	277	);
251	278	});
252	279
253	280	const resolveRoutableSession = Effect.fn("resolveRoutableSession")(function* (input: {
...	...	@@ -291,52 +318,69 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
291	318	threadId,
292	319	provider: parsed.provider ?? "codex",
293	320	};
294		const settings = yield* serverSettings.getSettings.pipe(
295		Effect.mapError((error) =>
296		toValidationError(
297		"ProviderService.startSession",
298		`Failed to load provider settings: ${error.message}`,
299		error,
	321	yield* Effect.annotateCurrentSpan({
	322	"provider.operation": "start-session",
	323	"provider.kind": input.provider,
	324	"provider.thread_id": threadId,
	325	"provider.runtime_mode": input.runtimeMode,
	326	});
	327	return yield* Effect.gen(function* () {
	328	const settings = yield* serverSettings.getSettings.pipe(
	329	Effect.mapError((error) =>
	330	toValidationError(
	331	"ProviderService.startSession",
	332	`Failed to load provider settings: ${error.message}`,
	333	error,
	334	),
300	335	),
301		),
302		);
303		if (!settings.providers[input.provider].enabled) {
304		return yield* toValidationError(
305		"ProviderService.startSession",
306		`Provider '${input.provider}' is disabled in T3 Code settings.`,
307	336	);
308		}
309		const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));
310		const effectiveResumeCursor =
311		input.resumeCursor ??
312		(persistedBinding?.provider === input.provider ? persistedBinding.resumeCursor : undefined);
313		const adapter = yield* registry.getByProvider(input.provider);
314		const session = yield* adapter.startSession({
315		...input,
316		...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),
317		});
	337	if (!settings.providers[input.provider].enabled) {
	338	return yield* toValidationError(
	339	"ProviderService.startSession",
	340	`Provider '${input.provider}' is disabled in T3 Code settings.`,
	341	);
	342	}
	343	const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));
	344	const effectiveResumeCursor =
	345	input.resumeCursor ??
	346	(persistedBinding?.provider === input.provider
	347	? persistedBinding.resumeCursor
	348	: undefined);
	349	const adapter = yield* registry.getByProvider(input.provider);
	350	const session = yield* adapter.startSession({
	351	...input,
	352	...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),
	353	});
318	354
319		if (session.provider !== adapter.provider) {
320		return yield* toValidationError(
321		"ProviderService.startSession",
322		`Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,
323		);
324		}
	355	if (session.provider !== adapter.provider) {
	356	return yield* toValidationError(
	357	"ProviderService.startSession",
	358	`Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,
	359	);
	360	}
325	361
326		yield* upsertSessionBinding(session, threadId, {
327		modelSelection: input.modelSelection,
328		});
329		yield* analytics.record("provider.session.started", {
330		provider: session.provider,
331		runtimeMode: input.runtimeMode,
332		hasResumeCursor: session.resumeCursor !== undefined,
333		hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,
334		hasModel:
335		typeof input.modelSelection?.model === "string" &&
336		input.modelSelection.model.trim().length > 0,
337		});
	362	yield* upsertSessionBinding(session, threadId, {
	363	modelSelection: input.modelSelection,
	364	});
	365	yield* analytics.record("provider.session.started", {
	366	provider: session.provider,
	367	runtimeMode: input.runtimeMode,
	368	hasResumeCursor: session.resumeCursor !== undefined,
	369	hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,
	370	hasModel:
	371	typeof input.modelSelection?.model === "string" &&
	372	input.modelSelection.model.trim().length > 0,
	373	});
338	374
339		return session;
	375	return session;
	376	}).pipe(
	377	withMetrics({
	378	counter: providerSessionsTotal,
	379	attributes: providerMetricAttributes(input.provider, {
	380	operation: "start",
	381	}),
	382	}),
	383	);
340	384	},
341	385	);
342	386
...	...	@@ -357,32 +401,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
357	401	"Either input text or at least one attachment is required",
358	402	);
359	403	}
360		const routed = yield* resolveRoutableSession({
361		threadId: input.threadId,
362		operation: "ProviderService.sendTurn",
363		allowRecovery: true,
364		});
365		const turn = yield* routed.adapter.sendTurn(input);
366		yield* directory.upsert({
367		threadId: input.threadId,
368		provider: routed.adapter.provider,
369		status: "running",
370		...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
371		runtimePayload: {
372		...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
373		activeTurnId: turn.turnId,
374		lastRuntimeEvent: "provider.sendTurn",
375		lastRuntimeEventAt: new Date().toISOString(),
376		},
377		});
378		yield* analytics.record("provider.turn.sent", {
379		provider: routed.adapter.provider,
380		model: input.modelSelection?.model,
381		interactionMode: input.interactionMode,
382		attachmentCount: input.attachments.length,
383		hasInput: typeof input.input === "string" && input.input.trim().length > 0,
	404	yield* Effect.annotateCurrentSpan({
	405	"provider.operation": "send-turn",
	406	"provider.thread_id": input.threadId,
	407	"provider.interaction_mode": input.interactionMode,
	408	"provider.attachment_count": input.attachments.length,
384	409	});
385		return turn;
	410	let metricProvider = "unknown";
	411	let metricModel = input.modelSelection?.model;
	412	return yield* Effect.gen(function* () {
	413	const routed = yield* resolveRoutableSession({
	414	threadId: input.threadId,
	415	operation: "ProviderService.sendTurn",
	416	allowRecovery: true,
	417	});
	418	metricProvider = routed.adapter.provider;
	419	metricModel = input.modelSelection?.model;
	420	yield* Effect.annotateCurrentSpan({
	421	"provider.kind": routed.adapter.provider,
	422	...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}),
	423	});
	424	const turn = yield* routed.adapter.sendTurn(input);
	425	yield* directory.upsert({
	426	threadId: input.threadId,
	427	provider: routed.adapter.provider,
	428	status: "running",
	429	...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
	430	runtimePayload: {
	431	...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
	432	activeTurnId: turn.turnId,
	433	lastRuntimeEvent: "provider.sendTurn",
	434	lastRuntimeEventAt: new Date().toISOString(),
	435	},
	436	});
	437	yield* analytics.record("provider.turn.sent", {
	438	provider: routed.adapter.provider,
	439	model: input.modelSelection?.model,
	440	interactionMode: input.interactionMode,
	441	attachmentCount: input.attachments.length,
	442	hasInput: typeof input.input === "string" && input.input.trim().length > 0,
	443	});
	444	return turn;
	445	}).pipe(
	446	withMetrics({
	447	counter: providerTurnsTotal,
	448	timer: providerTurnDuration,
	449	attributes: () =>
	450	providerTurnMetricAttributes({
	451	provider: metricProvider,
	452	model: metricModel,
	453	extra: {
	454	operation: "send",
	455	},
	456	}),
	457	}),
	458	);
386	459	});
387	460
388	461	const interruptTurn: ProviderServiceShape["interruptTurn"] = Effect.fn("interruptTurn")(
...	...	@@ -392,15 +465,33 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
392	465	schema: ProviderInterruptTurnInput,
393	466	payload: rawInput,
394	467	});
395		const routed = yield* resolveRoutableSession({
396		threadId: input.threadId,
397		operation: "ProviderService.interruptTurn",
398		allowRecovery: true,
399		});
400		yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);
401		yield* analytics.record("provider.turn.interrupted", {
402		provider: routed.adapter.provider,
403		});
	468	let metricProvider = "unknown";
	469	return yield* Effect.gen(function* () {
	470	const routed = yield* resolveRoutableSession({
	471	threadId: input.threadId,
	472	operation: "ProviderService.interruptTurn",
	473	allowRecovery: true,
	474	});
	475	metricProvider = routed.adapter.provider;
	476	yield* Effect.annotateCurrentSpan({
	477	"provider.operation": "interrupt-turn",
	478	"provider.kind": routed.adapter.provider,
	479	"provider.thread_id": input.threadId,
	480	"provider.turn_id": input.turnId,
	481	});
	482	yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);
	483	yield* analytics.record("provider.turn.interrupted", {
	484	provider: routed.adapter.provider,
	485	});
	486	}).pipe(
	487	withMetrics({
	488	counter: providerTurnsTotal,
	489	outcomeAttributes: () =>
	490	providerMetricAttributes(metricProvider, {
	491	operation: "interrupt",
	492	}),
	493	}),
	494	);
404	495	},
405	496	);
406	497
...	...	@@ -411,16 +502,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
411	502	schema: ProviderRespondToRequestInput,
412	503	payload: rawInput,
413	504	});
414		const routed = yield* resolveRoutableSession({
415		threadId: input.threadId,
416		operation: "ProviderService.respondToRequest",
417		allowRecovery: true,
418		});
419		yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);
420		yield* analytics.record("provider.request.responded", {
421		provider: routed.adapter.provider,
422		decision: input.decision,
423		});
	505	let metricProvider = "unknown";
	506	return yield* Effect.gen(function* () {
	507	const routed = yield* resolveRoutableSession({
	508	threadId: input.threadId,
	509	operation: "ProviderService.respondToRequest",
	510	allowRecovery: true,
	511	});
	512	metricProvider = routed.adapter.provider;
	513	yield* Effect.annotateCurrentSpan({
	514	"provider.operation": "respond-to-request",
	515	"provider.kind": routed.adapter.provider,
	516	"provider.thread_id": input.threadId,
	517	"provider.request_id": input.requestId,
	518	});
	519	yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);
	520	yield* analytics.record("provider.request.responded", {
	521	provider: routed.adapter.provider,
	522	decision: input.decision,
	523	});
	524	}).pipe(
	525	withMetrics({
	526	counter: providerTurnsTotal,
	527	outcomeAttributes: () =>
	528	providerMetricAttributes(metricProvider, {
	529	operation: "approval-response",
	530	}),
	531	}),
	532	);
424	533	},
425	534	);
426	535
...	...	@@ -432,12 +541,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
432	541	schema: ProviderRespondToUserInputInput,
433	542	payload: rawInput,
434	543	});
435		const routed = yield* resolveRoutableSession({
436		threadId: input.threadId,
437		operation: "ProviderService.respondToUserInput",
438		allowRecovery: true,
439		});
440		yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);
	544	let metricProvider = "unknown";
	545	return yield* Effect.gen(function* () {
	546	const routed = yield* resolveRoutableSession({
	547	threadId: input.threadId,
	548	operation: "ProviderService.respondToUserInput",
	549	allowRecovery: true,
	550	});
	551	metricProvider = routed.adapter.provider;
	552	yield* Effect.annotateCurrentSpan({
	553	"provider.operation": "respond-to-user-input",
	554	"provider.kind": routed.adapter.provider,
	555	"provider.thread_id": input.threadId,
	556	"provider.request_id": input.requestId,
	557	});
	558	yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);
	559	}).pipe(
	560	withMetrics({
	561	counter: providerTurnsTotal,
	562	outcomeAttributes: () =>
	563	providerMetricAttributes(metricProvider, {
	564	operation: "user-input-response",
	565	}),
	566	}),
	567	);
441	568	});
442	569
443	570	const stopSession: ProviderServiceShape["stopSession"] = Effect.fn("stopSession")(
...	...	@@ -447,18 +574,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
447	574	schema: ProviderStopSessionInput,
448	575	payload: rawInput,
449	576	});
450		const routed = yield* resolveRoutableSession({
451		threadId: input.threadId,
452		operation: "ProviderService.stopSession",
453		allowRecovery: false,
454		});
455		if (routed.isActive) {
456		yield* routed.adapter.stopSession(routed.threadId);
457		}
458		yield* directory.remove(input.threadId);
459		yield* analytics.record("provider.session.stopped", {
460		provider: routed.adapter.provider,
461		});
	577	let metricProvider = "unknown";
	578	return yield* Effect.gen(function* () {
	579	const routed = yield* resolveRoutableSession({
	580	threadId: input.threadId,
	581	operation: "ProviderService.stopSession",
	582	allowRecovery: false,
	583	});
	584	metricProvider = routed.adapter.provider;
	585	yield* Effect.annotateCurrentSpan({
	586	"provider.operation": "stop-session",
	587	"provider.kind": routed.adapter.provider,
	588	"provider.thread_id": input.threadId,
	589	});
	590	if (routed.isActive) {
	591	yield* routed.adapter.stopSession(routed.threadId);
	592	}
	593	yield* directory.remove(input.threadId);
	594	yield* analytics.record("provider.session.stopped", {
	595	provider: routed.adapter.provider,
	596	});
	597	}).pipe(
	598	withMetrics({
	599	counter: providerSessionsTotal,
	600	outcomeAttributes: () =>
	601	providerMetricAttributes(metricProvider, {
	602	operation: "stop",
	603	}),
	604	}),
	605	);
462	606	},
463	607	);
464	608
...	...	@@ -524,16 +668,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
524	668	if (input.numTurns === 0) {
525	669	return;
526	670	}
527		const routed = yield* resolveRoutableSession({
528		threadId: input.threadId,
529		operation: "ProviderService.rollbackConversation",
530		allowRecovery: true,
531		});
532		yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);
533		yield* analytics.record("provider.conversation.rolled_back", {
534		provider: routed.adapter.provider,
535		turns: input.numTurns,
536		});
	671	let metricProvider = "unknown";
	672	return yield* Effect.gen(function* () {
	673	const routed = yield* resolveRoutableSession({
	674	threadId: input.threadId,
	675	operation: "ProviderService.rollbackConversation",
	676	allowRecovery: true,
	677	});
	678	metricProvider = routed.adapter.provider;
	679	yield* Effect.annotateCurrentSpan({
	680	"provider.operation": "rollback-conversation",
	681	"provider.kind": routed.adapter.provider,
	682	"provider.thread_id": input.threadId,
	683	"provider.rollback_turns": input.numTurns,
	684	});
	685	yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);
	686	yield* analytics.record("provider.conversation.rolled_back", {
	687	provider: routed.adapter.provider,
	688	turns: input.numTurns,
	689	});
	690	}).pipe(
	691	withMetrics({
	692	counter: providerTurnsTotal,
	693	outcomeAttributes: () =>
	694	providerMetricAttributes(metricProvider, {
	695	operation: "rollback",
	696	}),
	697	}),
	698	);
537	699	});
538	700
539	701	const runStopAll = Effect.fn("runStopAll")(function* () {

apps/server/src/server.test.ts+21-2

...	...	@@ -139,8 +139,17 @@ const buildAppUnderTest = (options?: {
139	139	const baseDir = options?.config?.baseDir ?? tempBaseDir;
140	140	const devUrl = options?.config?.devUrl;
141	141	const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
142		const config = {
	142	const config: ServerConfigShape = {
143	143	logLevel: "Info",
	144	traceMinLevel: "Info",
	145	traceTimingEnabled: true,
	146	traceBatchWindowMs: 200,
	147	traceMaxBytes: 10 * 1024 * 1024,
	148	traceMaxFiles: 10,
	149	otlpTracesUrl: undefined,
	150	otlpMetricsUrl: undefined,
	151	otlpExportIntervalMs: 10_000,
	152	otlpServiceName: "t3-server",
144	153	mode: "web",
145	154	port: 0,
146	155	host: "127.0.0.1",
...	...	@@ -154,7 +163,7 @@ const buildAppUnderTest = (options?: {
154	163	autoBootstrapProjectFromCwd: false,
155	164	logWebSocketEvents: false,
156	165	...options?.config,
157		} satisfies ServerConfigShape;
	166	};
158	167	const layerConfig = Layer.succeed(ServerConfig, config);
159	168
160	169	const appLayer = HttpRouter.serve(makeRoutesLayer, {
...	...	@@ -533,6 +542,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
533	542	} as const;
534	543
535	544	yield* buildAppUnderTest({
	545	config: {
	546	otlpTracesUrl: "http://localhost:4318/v1/traces",
	547	otlpMetricsUrl: "http://localhost:4318/v1/metrics",
	548	},
536	549	layers: {
537	550	keybindings: {
538	551	loadConfigState: Effect.succeed({
...	...	@@ -561,6 +574,12 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
561	574	assert.deepEqual(first.config.keybindings, []);
562	575	assert.deepEqual(first.config.issues, []);
563	576	assert.deepEqual(first.config.providers, providers);
	577	assert.equal(first.config.observability.logsDirectoryPath.endsWith("/logs"), true);
	578	assert.equal(first.config.observability.localTracingEnabled, true);
	579	assert.equal(first.config.observability.otlpTracesUrl, "http://localhost:4318/v1/traces");
	580	assert.equal(first.config.observability.otlpTracesEnabled, true);
	581	assert.equal(first.config.observability.otlpMetricsUrl, "http://localhost:4318/v1/metrics");
	582	assert.equal(first.config.observability.otlpMetricsEnabled, true);
564	583	assert.deepEqual(first.config.settings, DEFAULT_SERVER_SETTINGS);
565	584	}
566	585	assert.deepEqual(second, {

apps/server/src/server.ts+2-2

...	...	@@ -29,7 +29,6 @@ import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration";
29	29	import { TerminalManagerLive } from "./terminal/Layers/Manager";
30	30	import { GitManagerLive } from "./git/Layers/GitManager";
31	31	import { KeybindingsLive } from "./keybindings";
32		import { ServerLoggerLive } from "./serverLogger";
33	32	import { ServerRuntimeStartup, ServerRuntimeStartupLive } from "./serverRuntimeStartup";
34	33	import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor";
35	34	import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus";
...	...	@@ -42,6 +41,7 @@ import { ProjectFaviconResolverLive } from "./project/Layers/ProjectFaviconResol
42	41	import { WorkspaceEntriesLive } from "./workspace/Layers/WorkspaceEntries";
43	42	import { WorkspaceFileSystemLive } from "./workspace/Layers/WorkspaceFileSystem";
44	43	import { WorkspacePathsLive } from "./workspace/Layers/WorkspacePaths";
	44	import { ObservabilityLive } from "./observability/Layers/Observability";
45	45
46	46	const PtyAdapterLive = Layer.unwrap(
47	47	Effect.gen(function* () {
...	...	@@ -231,7 +231,7 @@ export const makeServerLayer = Layer.unwrap(
231	231	return serverApplicationLayer.pipe(
232	232	Layer.provideMerge(RuntimeServicesLive),
233	233	Layer.provideMerge(HttpServerLive),
234		Layer.provide(ServerLoggerLive),
	234	Layer.provide(ObservabilityLive),
235	235	Layer.provideMerge(FetchHttpClient.layer),
236	236	Layer.provideMerge(PlatformServicesLive),
237	237	);

apps/server/src/serverLogger.ts+1-4

...	...	@@ -4,11 +4,8 @@ import { ServerConfig } from "./config";
4	4
5	5	export const ServerLoggerLive = Effect.gen(function* () {
6	6	const config = yield* ServerConfig;
7		const { serverLogPath } = config;
8
9		const fileLogger = Logger.formatSimple.pipe(Logger.toFile(serverLogPath));
10	7	const minimumLogLevelLayer = Layer.succeed(References.MinimumLogLevel, config.logLevel);
11		const loggerLayer = Logger.layer([Logger.consolePretty(), fileLogger], {
	8	const loggerLayer = Logger.layer([Logger.consolePretty(), Logger.tracerLogger], {
12	9	mergeWithExisting: false,
13	10	});
14	11

apps/server/src/serverRuntimeStartup.ts+62-31

...	...	@@ -141,6 +141,8 @@ export const recordStartupHeartbeat = Effect.gen(function* () {
141	141	});
142	142
143	143	export const launchStartupHeartbeat = recordStartupHeartbeat.pipe(
	144	Effect.annotateSpans({ "startup.phase": "heartbeat.record" }),
	145	Effect.withSpan("server.startup.heartbeat.record"),
144	146	Effect.ignoreCause({ log: true }),
145	147	Effect.forkScoped,
146	148	Effect.asVoid,
...	...	@@ -248,7 +250,14 @@ const maybeOpenBrowser = Effect.gen(function* () {
248	250	);
249	251	});
250	252
	253	const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>
	254	effect.pipe(
	255	Effect.annotateSpans({ "startup.phase": phase }),
	256	Effect.withSpan(`server.startup.${phase}`),
	257	);
	258
251	259	const makeServerRuntimeStartup = Effect.gen(function* () {
	260	const serverConfig = yield* ServerConfig;
252	261	const keybindings = yield* Keybindings;
253	262	const orchestrationReactor = yield* OrchestrationReactor;
254	263	const lifecycleEvents = yield* ServerLifecycleEvents;
...	...	@@ -262,46 +271,65 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
262	271
263	272	const startup = Effect.gen(function* () {
264	273	yield* Effect.logDebug("startup phase: starting keybindings runtime");
265		yield* keybindings.start.pipe(
266		Effect.catch((error) =>
267		Effect.logWarning("failed to start keybindings runtime", {
268		path: error.configPath,
269		detail: error.detail,
270		cause: error.cause,
271		}),
	274	yield* runStartupPhase(
	275	"keybindings.start",
	276	keybindings.start.pipe(
	277	Effect.catch((error) =>
	278	Effect.logWarning("failed to start keybindings runtime", {
	279	path: error.configPath,
	280	detail: error.detail,
	281	cause: error.cause,
	282	}),
	283	),
	284	Effect.forkScoped,
272	285	),
273		Effect.forkScoped,
274	286	);
275	287
276	288	yield* Effect.logDebug("startup phase: starting server settings runtime");
277		yield* serverSettings.start.pipe(
278		Effect.catch((error) =>
279		Effect.logWarning("failed to start server settings runtime", {
280		path: error.settingsPath,
281		detail: error.detail,
282		cause: error.cause,
283		}),
	289	yield* runStartupPhase(
	290	"settings.start",
	291	serverSettings.start.pipe(
	292	Effect.catch((error) =>
	293	Effect.logWarning("failed to start server settings runtime", {
	294	path: error.settingsPath,
	295	detail: error.detail,
	296	cause: error.cause,
	297	}),
	298	),
	299	Effect.forkScoped,
284	300	),
285		Effect.forkScoped,
286	301	);
287	302
288	303	yield* Effect.logDebug("startup phase: starting orchestration reactors");
289		yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope));
	304	yield* runStartupPhase(
	305	"reactors.start",
	306	orchestrationReactor.start().pipe(Scope.provide(reactorScope)),
	307	);
290	308
291	309	yield* Effect.logDebug("startup phase: preparing welcome payload");
292		const welcome = yield* autoBootstrapWelcome;
	310	const welcome = yield* runStartupPhase("welcome.prepare", autoBootstrapWelcome);
293	311	yield* Effect.logDebug("startup phase: publishing welcome event", {
294	312	cwd: welcome.cwd,
295	313	projectName: welcome.projectName,
296	314	bootstrapProjectId: welcome.bootstrapProjectId,
297	315	bootstrapThreadId: welcome.bootstrapThreadId,
298	316	});
299		yield* lifecycleEvents.publish({
300		version: 1,
301		type: "welcome",
302		payload: welcome,
303		});
304		});
	317	yield* runStartupPhase(
	318	"welcome.publish",
	319	lifecycleEvents.publish({
	320	version: 1,
	321	type: "welcome",
	322	payload: welcome,
	323	}),
	324	);
	325	}).pipe(
	326	Effect.annotateSpans({
	327	"server.mode": serverConfig.mode,
	328	"server.port": serverConfig.port,
	329	"server.host": serverConfig.host ?? "default",
	330	}),
	331	Effect.withSpan("server.startup", { kind: "server", root: true }),
	332	);
305	333
306	334	yield* Effect.forkScoped(
307	335	Effect.gen(function* () {
...	...	@@ -319,18 +347,21 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
319	347	yield* Effect.logDebug("Accepting commands");
320	348	yield* commandGate.signalCommandReady;
321	349	yield* Effect.logDebug("startup phase: waiting for http listener");
322		yield* Deferred.await(httpListening);
	350	yield* runStartupPhase("http.wait", Deferred.await(httpListening));
323	351	yield* Effect.logDebug("startup phase: publishing ready event");
324		yield* lifecycleEvents.publish({
325		version: 1,
326		type: "ready",
327		payload: { at: new Date().toISOString() },
328		});
	352	yield* runStartupPhase(
	353	"ready.publish",
	354	lifecycleEvents.publish({
	355	version: 1,
	356	type: "ready",
	357	payload: { at: new Date().toISOString() },
	358	}),
	359	);
329	360
330	361	yield* Effect.logDebug("startup phase: recording startup heartbeat");
331	362	yield* launchStartupHeartbeat;
332	363	yield* Effect.logDebug("startup phase: browser open check");
333		yield* maybeOpenBrowser;
	364	yield* runStartupPhase("browser.open", maybeOpenBrowser);
334	365	yield* Effect.logDebug("startup phase: complete");
335	366	}),
336	367	);

apps/server/src/telemetry/Layers/AnalyticsService.ts+13-11

...	...	@@ -125,17 +125,19 @@ const makeAnalyticsService = Effect.gen(function* () {
125	125	}
126	126	}).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));
127	127
128		const record: AnalyticsServiceShape["record"] = Effect.fnUntraced(function* (event, properties) {
129		if (!telemetryConfig.enabled \|\| !identifier) return;
130
131		const enqueueResult = yield* enqueueBufferedEvent(event, properties);
132		if (enqueueResult.dropped) {
133		yield* Effect.logDebug("analytics buffer full; dropping oldest event", {
134		size: enqueueResult.size,
135		event,
136		});
137		}
138		});
	128	const record: AnalyticsServiceShape["record"] = Effect.fn("record")(
	129	function* (event, properties) {
	130	if (!telemetryConfig.enabled \|\| !identifier) return;
	131
	132	const enqueueResult = yield* enqueueBufferedEvent(event, properties);
	133	if (enqueueResult.dropped) {
	134	yield* Effect.logDebug("analytics buffer full; dropping oldest event", {
	135	size: enqueueResult.size,
	136	event,
	137	});
	138	}
	139	},
	140	);
139	141
140	142	yield* Effect.forever(Effect.sleep(1000).pipe(Effect.flatMap(() => flush)), {
141	143	disableYield: true,

apps/server/src/terminal/Layers/Manager.ts+55-39

...	...	@@ -23,6 +23,11 @@ import {
23	23	} from "effect";
24	24
25	25	import { ServerConfig } from "../../config";
	26	import {
	27	increment,
	28	terminalRestartsTotal,
	29	terminalSessionsTotal,
	30	} from "../../observability/Metrics";
26	31	import { runProcess } from "../../processRunner";
27	32	import {
28	33	TerminalCwdError,
...	...	@@ -1294,6 +1299,12 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1294	1299	eventType: "started" \| "restarted",
1295	1300	) {
1296	1301	yield* stopProcess(session);
	1302	yield* Effect.annotateCurrentSpan({
	1303	"terminal.thread_id": session.threadId,
	1304	"terminal.id": session.terminalId,
	1305	"terminal.event_type": eventType,
	1306	"terminal.cwd": input.cwd,
	1307	});
1297	1308
1298	1309	yield* modifyManagerState((state) => {
1299	1310	session.status = "starting";
...	...	@@ -1314,45 +1325,49 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1314	1325	let startedShell: string \| null = null;
1315	1326
1316	1327	const startResult = yield* Effect.result(
1317		Effect.gen(function* () {
1318		const shellCandidates = resolveShellCandidates(shellResolver);
1319		const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);
1320		const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);
1321		ptyProcess = spawnResult.process;
1322		startedShell = spawnResult.shellLabel;
1323
1324		const processPid = ptyProcess.pid;
1325		const unsubscribeData = ptyProcess.onData((data) => {
1326		if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {
1327		return;
1328		}
1329		runFork(drainProcessEvents(session, processPid));
1330		});
1331		const unsubscribeExit = ptyProcess.onExit((event) => {
1332		if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {
1333		return;
1334		}
1335		runFork(drainProcessEvents(session, processPid));
1336		});
1337
1338		yield* modifyManagerState((state) => {
1339		session.process = ptyProcess;
1340		session.pid = processPid;
1341		session.status = "running";
1342		session.updatedAt = new Date().toISOString();
1343		session.unsubscribeData = unsubscribeData;
1344		session.unsubscribeExit = unsubscribeExit;
1345		return [undefined, state] as const;
1346		});
1347
1348		yield* publishEvent({
1349		type: eventType,
1350		threadId: session.threadId,
1351		terminalId: session.terminalId,
1352		createdAt: new Date().toISOString(),
1353		snapshot: snapshot(session),
1354		});
1355		}),
	1328	increment(terminalSessionsTotal, { lifecycle: eventType }).pipe(
	1329	Effect.andThen(
	1330	Effect.gen(function* () {
	1331	const shellCandidates = resolveShellCandidates(shellResolver);
	1332	const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);
	1333	const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);
	1334	ptyProcess = spawnResult.process;
	1335	startedShell = spawnResult.shellLabel;
	1336
	1337	const processPid = ptyProcess.pid;
	1338	const unsubscribeData = ptyProcess.onData((data) => {
	1339	if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {
	1340	return;
	1341	}
	1342	runFork(drainProcessEvents(session, processPid));
	1343	});
	1344	const unsubscribeExit = ptyProcess.onExit((event) => {
	1345	if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {
	1346	return;
	1347	}
	1348	runFork(drainProcessEvents(session, processPid));
	1349	});
	1350
	1351	yield* modifyManagerState((state) => {
	1352	session.process = ptyProcess;
	1353	session.pid = processPid;
	1354	session.status = "running";
	1355	session.updatedAt = new Date().toISOString();
	1356	session.unsubscribeData = unsubscribeData;
	1357	session.unsubscribeExit = unsubscribeExit;
	1358	return [undefined, state] as const;
	1359	});
	1360
	1361	yield* publishEvent({
	1362	type: eventType,
	1363	threadId: session.threadId,
	1364	terminalId: session.terminalId,
	1365	createdAt: new Date().toISOString(),
	1366	snapshot: snapshot(session),
	1367	});
	1368	}),
	1369	),
	1370	),
1356	1371	);
1357	1372
1358	1373	if (startResult._tag === "Success") {
...	...	@@ -1722,6 +1737,7 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1722	1737	withThreadLock(
1723	1738	input.threadId,
1724	1739	Effect.gen(function* () {
	1740	yield* increment(terminalRestartsTotal, { scope: "thread" });
1725	1741	const terminalId = input.terminalId ?? DEFAULT_TERMINAL_ID;
1726	1742	yield* assertValidCwd(input.cwd);
1727	1743

apps/server/src/ws.ts+247-121

...	...	@@ -28,6 +28,11 @@ import { Open, resolveAvailableEditors } from "./open";
28	28	import { normalizeDispatchCommand } from "./orchestration/Normalizer";
29	29	import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine";
30	30	import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery";
	31	import {
	32	observeRpcEffect,
	33	observeRpcStream,
	34	observeRpcStreamEffect,
	35	} from "./observability/RpcInstrumentation";
31	36	import { ProviderRegistry } from "./provider/Services/ProviderRegistry";
32	37	import { ServerLifecycleEvents } from "./serverLifecycleEvents";
33	38	import { ServerRuntimeStartup } from "./serverRuntimeStartup";
...	...	@@ -67,85 +72,114 @@ const WsRpcLayer = WsRpcGroup.toLayer(
67	72	issues: keybindingsConfig.issues,
68	73	providers,
69	74	availableEditors: resolveAvailableEditors(),
	75	observability: {
	76	logsDirectoryPath: config.logsDir,
	77	localTracingEnabled: true,
	78	...(config.otlpTracesUrl !== undefined ? { otlpTracesUrl: config.otlpTracesUrl } : {}),
	79	otlpTracesEnabled: config.otlpTracesUrl !== undefined,
	80	...(config.otlpMetricsUrl !== undefined ? { otlpMetricsUrl: config.otlpMetricsUrl } : {}),
	81	otlpMetricsEnabled: config.otlpMetricsUrl !== undefined,
	82	},
70	83	settings,
71	84	};
72	85	});
73	86
74	87	return WsRpcGroup.of({
75	88	[ORCHESTRATION_WS_METHODS.getSnapshot]: (_input) =>
76		projectionSnapshotQuery.getSnapshot().pipe(
77		Effect.mapError(
78		(cause) =>
79		new OrchestrationGetSnapshotError({
80		message: "Failed to load orchestration snapshot",
81		cause,
82		}),
	89	observeRpcEffect(
	90	ORCHESTRATION_WS_METHODS.getSnapshot,
	91	projectionSnapshotQuery.getSnapshot().pipe(
	92	Effect.mapError(
	93	(cause) =>
	94	new OrchestrationGetSnapshotError({
	95	message: "Failed to load orchestration snapshot",
	96	cause,
	97	}),
	98	),
83	99	),
	100	{ "rpc.aggregate": "orchestration" },
84	101	),
85	102	[ORCHESTRATION_WS_METHODS.dispatchCommand]: (command) =>
86		Effect.gen(function* () {
87		const normalizedCommand = yield* normalizeDispatchCommand(command);
88		const result = yield* startup.enqueueCommand(
89		orchestrationEngine.dispatch(normalizedCommand),
90		);
91		if (normalizedCommand.type === "thread.archive") {
92		yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
93		Effect.catch((error) =>
94		Effect.logWarning("failed to close thread terminals after archive", {
95		threadId: normalizedCommand.threadId,
96		error: error.message,
97		}),
98		),
	103	observeRpcEffect(
	104	ORCHESTRATION_WS_METHODS.dispatchCommand,
	105	Effect.gen(function* () {
	106	const normalizedCommand = yield* normalizeDispatchCommand(command);
	107	const result = yield* startup.enqueueCommand(
	108	orchestrationEngine.dispatch(normalizedCommand),
99	109	);
100		}
101		return result;
102		}).pipe(
103		Effect.mapError((cause) =>
104		Schema.is(OrchestrationDispatchCommandError)(cause)
105		? cause
106		: new OrchestrationDispatchCommandError({
107		message: "Failed to dispatch orchestration command",
108		cause,
109		}),
	110	if (normalizedCommand.type === "thread.archive") {
	111	yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
	112	Effect.catch((error) =>
	113	Effect.logWarning("failed to close thread terminals after archive", {
	114	threadId: normalizedCommand.threadId,
	115	error: error.message,
	116	}),
	117	),
	118	);
	119	}
	120	return result;
	121	}).pipe(
	122	Effect.mapError((cause) =>
	123	Schema.is(OrchestrationDispatchCommandError)(cause)
	124	? cause
	125	: new OrchestrationDispatchCommandError({
	126	message: "Failed to dispatch orchestration command",
	127	cause,
	128	}),
	129	),
110	130	),
	131	{ "rpc.aggregate": "orchestration" },
111	132	),
112	133	[ORCHESTRATION_WS_METHODS.getTurnDiff]: (input) =>
113		checkpointDiffQuery.getTurnDiff(input).pipe(
114		Effect.mapError(
115		(cause) =>
116		new OrchestrationGetTurnDiffError({
117		message: "Failed to load turn diff",
118		cause,
119		}),
	134	observeRpcEffect(
	135	ORCHESTRATION_WS_METHODS.getTurnDiff,
	136	checkpointDiffQuery.getTurnDiff(input).pipe(
	137	Effect.mapError(
	138	(cause) =>
	139	new OrchestrationGetTurnDiffError({
	140	message: "Failed to load turn diff",
	141	cause,
	142	}),
	143	),
120	144	),
	145	{ "rpc.aggregate": "orchestration" },
121	146	),
122	147	[ORCHESTRATION_WS_METHODS.getFullThreadDiff]: (input) =>
123		checkpointDiffQuery.getFullThreadDiff(input).pipe(
124		Effect.mapError(
125		(cause) =>
126		new OrchestrationGetFullThreadDiffError({
127		message: "Failed to load full thread diff",
128		cause,
129		}),
	148	observeRpcEffect(
	149	ORCHESTRATION_WS_METHODS.getFullThreadDiff,
	150	checkpointDiffQuery.getFullThreadDiff(input).pipe(
	151	Effect.mapError(
	152	(cause) =>
	153	new OrchestrationGetFullThreadDiffError({
	154	message: "Failed to load full thread diff",
	155	cause,
	156	}),
	157	),
130	158	),
	159	{ "rpc.aggregate": "orchestration" },
131	160	),
132	161	[ORCHESTRATION_WS_METHODS.replayEvents]: (input) =>
133		Stream.runCollect(
134		orchestrationEngine.readEvents(
135		clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),
136		),
137		).pipe(
138		Effect.map((events) => Array.from(events)),
139		Effect.mapError(
140		(cause) =>
141		new OrchestrationReplayEventsError({
142		message: "Failed to replay orchestration events",
143		cause,
144		}),
	162	observeRpcEffect(
	163	ORCHESTRATION_WS_METHODS.replayEvents,
	164	Stream.runCollect(
	165	orchestrationEngine.readEvents(
	166	clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),
	167	),
	168	).pipe(
	169	Effect.map((events) => Array.from(events)),
	170	Effect.mapError(
	171	(cause) =>
	172	new OrchestrationReplayEventsError({
	173	message: "Failed to replay orchestration events",
	174	cause,
	175	}),
	176	),
145	177	),
	178	{ "rpc.aggregate": "orchestration" },
146	179	),
147	180	[WS_METHODS.subscribeOrchestrationDomainEvents]: (_input) =>
148		Stream.unwrap(
	181	observeRpcStreamEffect(
	182	WS_METHODS.subscribeOrchestrationDomainEvents,
149	183	Effect.gen(function* () {
150	184	const snapshot = yield* orchestrationEngine.getReadModel();
151	185	const fromSequenceExclusive = snapshot.snapshotSequence;
...	...	@@ -200,82 +234,167 @@ const WsRpcLayer = WsRpcGroup.toLayer(
200	234	Stream.flatMap((events) => Stream.fromIterable(events)),
201	235	);
202	236	}),
	237	{ "rpc.aggregate": "orchestration" },
203	238	),
204		[WS_METHODS.serverGetConfig]: (_input) => loadServerConfig,
	239	[WS_METHODS.serverGetConfig]: (_input) =>
	240	observeRpcEffect(WS_METHODS.serverGetConfig, loadServerConfig, {
	241	"rpc.aggregate": "server",
	242	}),
205	243	[WS_METHODS.serverRefreshProviders]: (_input) =>
206		providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),
	244	observeRpcEffect(
	245	WS_METHODS.serverRefreshProviders,
	246	providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),
	247	{ "rpc.aggregate": "server" },
	248	),
207	249	[WS_METHODS.serverUpsertKeybinding]: (rule) =>
208		Effect.gen(function* () {
209		const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);
210		return { keybindings: keybindingsConfig, issues: [] };
	250	observeRpcEffect(
	251	WS_METHODS.serverUpsertKeybinding,
	252	Effect.gen(function* () {
	253	const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);
	254	return { keybindings: keybindingsConfig, issues: [] };
	255	}),
	256	{ "rpc.aggregate": "server" },
	257	),
	258	[WS_METHODS.serverGetSettings]: (_input) =>
	259	observeRpcEffect(WS_METHODS.serverGetSettings, serverSettings.getSettings, {
	260	"rpc.aggregate": "server",
	261	}),
	262	[WS_METHODS.serverUpdateSettings]: ({ patch }) =>
	263	observeRpcEffect(WS_METHODS.serverUpdateSettings, serverSettings.updateSettings(patch), {
	264	"rpc.aggregate": "server",
211	265	}),
212		[WS_METHODS.serverGetSettings]: (_input) => serverSettings.getSettings,
213		[WS_METHODS.serverUpdateSettings]: ({ patch }) => serverSettings.updateSettings(patch),
214	266	[WS_METHODS.projectsSearchEntries]: (input) =>
215		workspaceEntries.search(input).pipe(
216		Effect.mapError(
217		(cause) =>
218		new ProjectSearchEntriesError({
219		message: `Failed to search workspace entries: ${cause.detail}`,
220		cause,
221		}),
	267	observeRpcEffect(
	268	WS_METHODS.projectsSearchEntries,
	269	workspaceEntries.search(input).pipe(
	270	Effect.mapError(
	271	(cause) =>
	272	new ProjectSearchEntriesError({
	273	message: `Failed to search workspace entries: ${cause.detail}`,
	274	cause,
	275	}),
	276	),
222	277	),
	278	{ "rpc.aggregate": "workspace" },
223	279	),
224	280	[WS_METHODS.projectsWriteFile]: (input) =>
225		workspaceFileSystem.writeFile(input).pipe(
226		Effect.mapError((cause) => {
227		const message = Schema.is(WorkspacePathOutsideRootError)(cause)
228		? "Workspace file path must stay within the project root."
229		: "Failed to write workspace file";
230		return new ProjectWriteFileError({
231		message,
232		cause,
233		});
234		}),
	281	observeRpcEffect(
	282	WS_METHODS.projectsWriteFile,
	283	workspaceFileSystem.writeFile(input).pipe(
	284	Effect.mapError((cause) => {
	285	const message = Schema.is(WorkspacePathOutsideRootError)(cause)
	286	? "Workspace file path must stay within the project root."
	287	: "Failed to write workspace file";
	288	return new ProjectWriteFileError({
	289	message,
	290	cause,
	291	});
	292	}),
	293	),
	294	{ "rpc.aggregate": "workspace" },
235	295	),
236		[WS_METHODS.shellOpenInEditor]: (input) => open.openInEditor(input),
237		[WS_METHODS.gitStatus]: (input) => gitManager.status(input),
238		[WS_METHODS.gitPull]: (input) => git.pullCurrentBranch(input.cwd),
	296	[WS_METHODS.shellOpenInEditor]: (input) =>
	297	observeRpcEffect(WS_METHODS.shellOpenInEditor, open.openInEditor(input), {
	298	"rpc.aggregate": "workspace",
	299	}),
	300	[WS_METHODS.gitStatus]: (input) =>
	301	observeRpcEffect(WS_METHODS.gitStatus, gitManager.status(input), {
	302	"rpc.aggregate": "git",
	303	}),
	304	[WS_METHODS.gitPull]: (input) =>
	305	observeRpcEffect(WS_METHODS.gitPull, git.pullCurrentBranch(input.cwd), {
	306	"rpc.aggregate": "git",
	307	}),
239	308	[WS_METHODS.gitRunStackedAction]: (input) =>
240		Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>
241		gitManager
242		.runStackedAction(input, {
243		actionId: input.actionId,
244		progressReporter: {
245		publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),
246		},
247		})
248		.pipe(
249		Effect.matchCauseEffect({
250		onFailure: (cause) => Queue.failCause(queue, cause),
251		onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),
252		}),
253		),
	309	observeRpcStream(
	310	WS_METHODS.gitRunStackedAction,
	311	Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>
	312	gitManager
	313	.runStackedAction(input, {
	314	actionId: input.actionId,
	315	progressReporter: {
	316	publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),
	317	},
	318	})
	319	.pipe(
	320	Effect.matchCauseEffect({
	321	onFailure: (cause) => Queue.failCause(queue, cause),
	322	onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),
	323	}),
	324	),
	325	),
	326	{ "rpc.aggregate": "git" },
254	327	),
255		[WS_METHODS.gitResolvePullRequest]: (input) => gitManager.resolvePullRequest(input),
	328	[WS_METHODS.gitResolvePullRequest]: (input) =>
	329	observeRpcEffect(WS_METHODS.gitResolvePullRequest, gitManager.resolvePullRequest(input), {
	330	"rpc.aggregate": "git",
	331	}),
256	332	[WS_METHODS.gitPreparePullRequestThread]: (input) =>
257		gitManager.preparePullRequestThread(input),
258		[WS_METHODS.gitListBranches]: (input) => git.listBranches(input),
259		[WS_METHODS.gitCreateWorktree]: (input) => git.createWorktree(input),
260		[WS_METHODS.gitRemoveWorktree]: (input) => git.removeWorktree(input),
261		[WS_METHODS.gitCreateBranch]: (input) => git.createBranch(input),
262		[WS_METHODS.gitCheckout]: (input) => Effect.scoped(git.checkoutBranch(input)),
263		[WS_METHODS.gitInit]: (input) => git.initRepo(input),
264		[WS_METHODS.terminalOpen]: (input) => terminalManager.open(input),
265		[WS_METHODS.terminalWrite]: (input) => terminalManager.write(input),
266		[WS_METHODS.terminalResize]: (input) => terminalManager.resize(input),
267		[WS_METHODS.terminalClear]: (input) => terminalManager.clear(input),
268		[WS_METHODS.terminalRestart]: (input) => terminalManager.restart(input),
269		[WS_METHODS.terminalClose]: (input) => terminalManager.close(input),
	333	observeRpcEffect(
	334	WS_METHODS.gitPreparePullRequestThread,
	335	gitManager.preparePullRequestThread(input),
	336	{ "rpc.aggregate": "git" },
	337	),
	338	[WS_METHODS.gitListBranches]: (input) =>
	339	observeRpcEffect(WS_METHODS.gitListBranches, git.listBranches(input), {
	340	"rpc.aggregate": "git",
	341	}),
	342	[WS_METHODS.gitCreateWorktree]: (input) =>
	343	observeRpcEffect(WS_METHODS.gitCreateWorktree, git.createWorktree(input), {
	344	"rpc.aggregate": "git",
	345	}),
	346	[WS_METHODS.gitRemoveWorktree]: (input) =>
	347	observeRpcEffect(WS_METHODS.gitRemoveWorktree, git.removeWorktree(input), {
	348	"rpc.aggregate": "git",
	349	}),
	350	[WS_METHODS.gitCreateBranch]: (input) =>
	351	observeRpcEffect(WS_METHODS.gitCreateBranch, git.createBranch(input), {
	352	"rpc.aggregate": "git",
	353	}),
	354	[WS_METHODS.gitCheckout]: (input) =>
	355	observeRpcEffect(WS_METHODS.gitCheckout, Effect.scoped(git.checkoutBranch(input)), {
	356	"rpc.aggregate": "git",
	357	}),
	358	[WS_METHODS.gitInit]: (input) =>
	359	observeRpcEffect(WS_METHODS.gitInit, git.initRepo(input), { "rpc.aggregate": "git" }),
	360	[WS_METHODS.terminalOpen]: (input) =>
	361	observeRpcEffect(WS_METHODS.terminalOpen, terminalManager.open(input), {
	362	"rpc.aggregate": "terminal",
	363	}),
	364	[WS_METHODS.terminalWrite]: (input) =>
	365	observeRpcEffect(WS_METHODS.terminalWrite, terminalManager.write(input), {
	366	"rpc.aggregate": "terminal",
	367	}),
	368	[WS_METHODS.terminalResize]: (input) =>
	369	observeRpcEffect(WS_METHODS.terminalResize, terminalManager.resize(input), {
	370	"rpc.aggregate": "terminal",
	371	}),
	372	[WS_METHODS.terminalClear]: (input) =>
	373	observeRpcEffect(WS_METHODS.terminalClear, terminalManager.clear(input), {
	374	"rpc.aggregate": "terminal",
	375	}),
	376	[WS_METHODS.terminalRestart]: (input) =>
	377	observeRpcEffect(WS_METHODS.terminalRestart, terminalManager.restart(input), {
	378	"rpc.aggregate": "terminal",
	379	}),
	380	[WS_METHODS.terminalClose]: (input) =>
	381	observeRpcEffect(WS_METHODS.terminalClose, terminalManager.close(input), {
	382	"rpc.aggregate": "terminal",
	383	}),
270	384	[WS_METHODS.subscribeTerminalEvents]: (_input) =>
271		Stream.callback<TerminalEvent>((queue) =>
272		Effect.acquireRelease(
273		terminalManager.subscribe((event) => Queue.offer(queue, event)),
274		(unsubscribe) => Effect.sync(unsubscribe),
	385	observeRpcStream(
	386	WS_METHODS.subscribeTerminalEvents,
	387	Stream.callback<TerminalEvent>((queue) =>
	388	Effect.acquireRelease(
	389	terminalManager.subscribe((event) => Queue.offer(queue, event)),
	390	(unsubscribe) => Effect.sync(unsubscribe),
	391	),
275	392	),
	393	{ "rpc.aggregate": "terminal" },
276	394	),
277	395	[WS_METHODS.subscribeServerConfig]: (_input) =>
278		Stream.unwrap(
	396	observeRpcStreamEffect(
	397	WS_METHODS.subscribeServerConfig,
279	398	Effect.gen(function* () {
280	399	const keybindingsUpdates = keybindings.streamChanges.pipe(
281	400	Stream.map((event) => ({
...	...	@@ -310,9 +429,11 @@ const WsRpcLayer = WsRpcGroup.toLayer(
310	429	Stream.merge(keybindingsUpdates, Stream.merge(providerStatuses, settingsUpdates)),
311	430	);
312	431	}),
	432	{ "rpc.aggregate": "server" },
313	433	),
314	434	[WS_METHODS.subscribeServerLifecycle]: (_input) =>
315		Stream.unwrap(
	435	observeRpcStreamEffect(
	436	WS_METHODS.subscribeServerLifecycle,
316	437	Effect.gen(function* () {
317	438	const snapshot = yield* lifecycleEvents.snapshot;
318	439	const snapshotEvents = Array.from(snapshot.events).toSorted(
...	...	@@ -323,6 +444,7 @@ const WsRpcLayer = WsRpcGroup.toLayer(
323	444	);
324	445	return Stream.concat(Stream.fromIterable(snapshotEvents), liveEvents);
325	446	}),
	447	{ "rpc.aggregate": "server" },
326	448	),
327	449	});
328	450	}),
...	...	@@ -330,9 +452,13 @@ const WsRpcLayer = WsRpcGroup.toLayer(
330	452
331	453	export const websocketRpcRouteLayer = Layer.unwrap(
332	454	Effect.gen(function* () {
333		const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup).pipe(
334		Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)),
335		);
	455	const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, {
	456	spanPrefix: "ws.rpc",
	457	spanAttributes: {
	458	"rpc.transport": "websocket",
	459	"rpc.system": "effect-rpc",
	460	},
	461	}).pipe(Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)));
336	462	return HttpRouter.add(
337	463	"GET",
338	464	"/ws",

apps/web/src/components/ChatView.browser.tsx+6

...	...	@@ -135,6 +135,12 @@ function createBaseServerConfig(): ServerConfig {
135	135	},
136	136	],
137	137	availableEditors: [],
	138	observability: {
	139	logsDirectoryPath: "/repo/project/.t3/logs",
	140	localTracingEnabled: true,
	141	otlpTracesEnabled: false,
	142	otlpMetricsEnabled: false,
	143	},
138	144	settings: {
139	145	...DEFAULT_SERVER_SETTINGS,
140	146	...DEFAULT_CLIENT_SETTINGS,

apps/web/src/components/KeybindingsToast.browser.tsx+6

...	...	@@ -56,6 +56,12 @@ function createBaseServerConfig(): ServerConfig {
56	56	},
57	57	],
58	58	availableEditors: [],
	59	observability: {
	60	logsDirectoryPath: "/repo/project/.t3/logs",
	61	localTracingEnabled: true,
	62	otlpTracesEnabled: false,
	63	otlpMetricsEnabled: false,
	64	},
59	65	settings: {
60	66	enableAssistantStreaming: false,
61	67	defaultThreadEnvMode: "local" as const,

apps/web/src/components/settings/SettingsPanels.browser.tsx created+91

...	...	@@ -0,0 +1,91 @@
	1	import "../../index.css";
	2
	3	import { DEFAULT_SERVER_SETTINGS, type NativeApi, type ServerConfig } from "@t3tools/contracts";
	4	import { page } from "vitest/browser";
	5	import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
	6	import { render } from "vitest-browser-react";
	7
	8	import { __resetNativeApiForTests } from "../../nativeApi";
	9	import { AppAtomRegistryProvider } from "../../rpc/atomRegistry";
	10	import { resetServerStateForTests, setServerConfigSnapshot } from "../../rpc/serverState";
	11	import { GeneralSettingsPanel } from "./SettingsPanels";
	12
	13	function createBaseServerConfig(): ServerConfig {
	14	return {
	15	cwd: "/repo/project",
	16	keybindingsConfigPath: "/repo/project/.t3code-keybindings.json",
	17	keybindings: [],
	18	issues: [],
	19	providers: [],
	20	availableEditors: ["cursor"],
	21	observability: {
	22	logsDirectoryPath: "/repo/project/.t3/logs",
	23	localTracingEnabled: true,
	24	otlpTracesUrl: "http://localhost:4318/v1/traces",
	25	otlpTracesEnabled: true,
	26	otlpMetricsEnabled: false,
	27	},
	28	settings: DEFAULT_SERVER_SETTINGS,
	29	};
	30	}
	31
	32	describe("GeneralSettingsPanel observability", () => {
	33	beforeEach(() => {
	34	resetServerStateForTests();
	35	__resetNativeApiForTests();
	36	localStorage.clear();
	37	document.body.innerHTML = "";
	38	});
	39
	40	afterEach(() => {
	41	resetServerStateForTests();
	42	__resetNativeApiForTests();
	43	document.body.innerHTML = "";
	44	});
	45
	46	it("shows diagnostics inside About with a single logs-folder action", async () => {
	47	setServerConfigSnapshot(createBaseServerConfig());
	48
	49	await render(
	50	<AppAtomRegistryProvider>
	51	<GeneralSettingsPanel />
	52	</AppAtomRegistryProvider>,
	53	);
	54
	55	await expect.element(page.getByText("About")).toBeInTheDocument();
	56	await expect.element(page.getByText("Diagnostics")).toBeInTheDocument();
	57	await expect.element(page.getByText("Open logs folder")).toBeInTheDocument();
	58	await expect
	59	.element(page.getByText("/repo/project/.t3/logs", { exact: true }))
	60	.toBeInTheDocument();
	61	await expect
	62	.element(
	63	page.getByText(
	64	"Local trace file. OTLP exporting traces to http://localhost:4318/v1/traces.",
	65	),
	66	)
	67	.toBeInTheDocument();
	68	});
	69
	70	it("opens the logs folder in the preferred editor", async () => {
	71	const openInEditor = vi.fn<NativeApi["shell"]["openInEditor"]>().mockResolvedValue(undefined);
	72	window.nativeApi = {
	73	shell: {
	74	openInEditor,
	75	},
	76	} as unknown as NativeApi;
	77
	78	setServerConfigSnapshot(createBaseServerConfig());
	79
	80	await render(
	81	<AppAtomRegistryProvider>
	82	<GeneralSettingsPanel />
	83	</AppAtomRegistryProvider>,
	84	);
	85
	86	const openLogsButton = page.getByText("Open logs folder");
	87	await openLogsButton.click();
	88
	89	expect(openInEditor).toHaveBeenCalledWith("/repo/project/.t3/logs", "cursor");
	90	});
	91	});

apps/web/src/components/settings/SettingsPanels.tsx+87-22

...	...	@@ -61,6 +61,7 @@ import { ProjectFavicon } from "../ProjectFavicon";
61	61	import {
62	62	useServerAvailableEditors,
63	63	useServerKeybindingsConfigPath,
	64	useServerObservability,
64	65	useServerProviders,
65	66	} from "../../rpc/serverState";
66	67
...	...	@@ -518,8 +519,13 @@ export function GeneralSettingsPanel() {
518	519	const { theme, setTheme } = useTheme();
519	520	const settings = useSettings();
520	521	const { updateSettings } = useUpdateSettings();
521		const [isOpeningKeybindings, setIsOpeningKeybindings] = useState(false);
522		const [openKeybindingsError, setOpenKeybindingsError] = useState<string \| null>(null);
	522	const [openingPathByTarget, setOpeningPathByTarget] = useState({
	523	keybindings: false,
	524	logsDirectory: false,
	525	});
	526	const [openPathErrorByTarget, setOpenPathErrorByTarget] = useState<
	527	Partial<Record<"keybindings" \| "logsDirectory", string \| null>>
	528	>({});
523	529	const [openProviderDetails, setOpenProviderDetails] = useState<Record<ProviderKind, boolean>>({
524	530	codex: Boolean(
525	531	settings.providers.codex.binaryPath !== DEFAULT_UNIFIED_SETTINGS.providers.codex.binaryPath \|\|
...	...	@@ -561,8 +567,21 @@ export function GeneralSettingsPanel() {
561	567
562	568	const keybindingsConfigPath = useServerKeybindingsConfigPath();
563	569	const availableEditors = useServerAvailableEditors();
	570	const observability = useServerObservability();
564	571	const serverProviders = useServerProviders();
565	572	const codexHomePath = settings.providers.codex.homePath;
	573	const logsDirectoryPath = observability?.logsDirectoryPath ?? null;
	574	const diagnosticsDescription = (() => {
	575	const exports: string[] = [];
	576	if (observability?.otlpTracesEnabled && observability.otlpTracesUrl) {
	577	exports.push(`traces to ${observability.otlpTracesUrl}`);
	578	}
	579	if (observability?.otlpMetricsEnabled && observability.otlpMetricsUrl) {
	580	exports.push(`metrics to ${observability.otlpMetricsUrl}`);
	581	}
	582	const mode = observability?.localTracingEnabled ? "Local trace file" : "Terminal logs only";
	583	return exports.length > 0 ? `${mode}. OTLP exporting ${exports.join(" and ")}.` : `${mode}.`;
	584	})();
566	585
567	586	const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);
568	587	const textGenProvider = textGenerationModelSelection.provider;
...	...	@@ -579,27 +598,49 @@ export function GeneralSettingsPanel() {
579	598	DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,
580	599	);
581	600
	601	const openInPreferredEditor = useCallback(
	602	(target: "keybindings" \| "logsDirectory", path: string \| null, failureMessage: string) => {
	603	if (!path) return;
	604	setOpenPathErrorByTarget((existing) => ({ ...existing, [target]: null }));
	605	setOpeningPathByTarget((existing) => ({ ...existing, [target]: true }));
	606
	607	const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
	608	if (!editor) {
	609	setOpenPathErrorByTarget((existing) => ({
	610	...existing,
	611	[target]: "No available editors found.",
	612	}));
	613	setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
	614	return;
	615	}
	616
	617	void ensureNativeApi()
	618	.shell.openInEditor(path, editor)
	619	.catch((error) => {
	620	setOpenPathErrorByTarget((existing) => ({
	621	...existing,
	622	[target]: error instanceof Error ? error.message : failureMessage,
	623	}));
	624	})
	625	.finally(() => {
	626	setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
	627	});
	628	},
	629	[availableEditors],
	630	);
	631
582	632	const openKeybindingsFile = useCallback(() => {
583		if (!keybindingsConfigPath) return;
584		setOpenKeybindingsError(null);
585		setIsOpeningKeybindings(true);
586		const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
587		if (!editor) {
588		setOpenKeybindingsError("No available editors found.");
589		setIsOpeningKeybindings(false);
590		return;
591		}
592		void ensureNativeApi()
593		.shell.openInEditor(keybindingsConfigPath, editor)
594		.catch((error) => {
595		setOpenKeybindingsError(
596		error instanceof Error ? error.message : "Unable to open keybindings file.",
597		);
598		})
599		.finally(() => {
600		setIsOpeningKeybindings(false);
601		});
602		}, [availableEditors, keybindingsConfigPath]);
	633	openInPreferredEditor("keybindings", keybindingsConfigPath, "Unable to open keybindings file.");
	634	}, [keybindingsConfigPath, openInPreferredEditor]);
	635
	636	const openLogsDirectory = useCallback(() => {
	637	openInPreferredEditor("logsDirectory", logsDirectoryPath, "Unable to open logs folder.");
	638	}, [logsDirectoryPath, openInPreferredEditor]);
	639
	640	const openKeybindingsError = openPathErrorByTarget.keybindings ?? null;
	641	const openDiagnosticsError = openPathErrorByTarget.logsDirectory ?? null;
	642	const isOpeningKeybindings = openingPathByTarget.keybindings;
	643	const isOpeningLogsDirectory = openingPathByTarget.logsDirectory;
603	644
604	645	const addCustomModel = useCallback(
605	646	(provider: ProviderKind) => {
...	...	@@ -1408,6 +1449,30 @@ export function GeneralSettingsPanel() {
1408	1449	description="Current version of the application."
1409	1450	/>
1410	1451	)}
	1452	<SettingsRow
	1453	title="Diagnostics"
	1454	description={diagnosticsDescription}
	1455	status={
	1456	<>
	1457	<span className="block break-all font-mono text-[11px] text-foreground">
	1458	{logsDirectoryPath ?? "Resolving logs directory..."}
	1459	</span>
	1460	{openDiagnosticsError ? (
	1461	<span className="mt-1 block text-destructive">{openDiagnosticsError}</span>
	1462	) : null}
	1463	</>
	1464	}
	1465	control={
	1466	<Button
	1467	size="xs"
	1468	variant="outline"
	1469	disabled={!logsDirectoryPath \|\| isOpeningLogsDirectory}
	1470	onClick={openLogsDirectory}
	1471	>
	1472	{isOpeningLogsDirectory ? "Opening..." : "Open logs folder"}
	1473	</Button>
	1474	}
	1475	/>
1411	1476	</SettingsSection>
1412	1477	</SettingsPageContainer>
1413	1478	);

apps/web/src/rpc/serverState.test.ts+6

...	...	@@ -57,6 +57,12 @@ const baseServerConfig: ServerConfig = {
57	57	issues: [],
58	58	providers: defaultProviders,
59	59	availableEditors: ["cursor"],
	60	observability: {
	61	logsDirectoryPath: "/tmp/workspace/.config/logs",
	62	localTracingEnabled: true,
	63	otlpTracesEnabled: false,
	64	otlpMetricsEnabled: false,
	65	},
60	66	settings: DEFAULT_SERVER_SETTINGS,
61	67	};
62	68

apps/web/src/rpc/serverState.ts+5

...	...	@@ -50,6 +50,7 @@ const selectAvailableEditors = (config: ServerConfig \| null): ReadonlyArray<Edit
50	50	const selectKeybindings = (config: ServerConfig \| null) => config?.keybindings ?? EMPTY_KEYBINDINGS;
51	51	const selectKeybindingsConfigPath = (config: ServerConfig \| null) =>
52	52	config?.keybindingsConfigPath ?? null;
	53	const selectObservability = (config: ServerConfig \| null) => config?.observability ?? null;
53	54	const selectProviders = (config: ServerConfig \| null) =>
54	55	config?.providers ?? EMPTY_SERVER_PROVIDERS;
55	56	const selectSettings = (config: ServerConfig \| null): ServerSettings =>
...	...	@@ -281,6 +282,10 @@ export function useServerKeybindingsConfigPath(): string \| null {
281	282	return useAtomValue(serverConfigAtom, selectKeybindingsConfigPath);
282	283	}
283	284
	285	export function useServerObservability(): ServerConfig["observability"] \| null {
	286	return useAtomValue(serverConfigAtom, selectObservability);
	287	}
	288
284	289	export function useServerWelcomeSubscription(
285	290	listener: (payload: ServerLifecycleWelcomePayload) => void,
286	291	): void {

apps/web/src/wsNativeApi.test.ts+6

...	...	@@ -159,6 +159,12 @@ const baseServerConfig: ServerConfig = {
159	159	issues: [],
160	160	providers: defaultProviders,
161	161	availableEditors: ["cursor"],
	162	observability: {
	163	logsDirectoryPath: "/tmp/workspace/.config/logs",
	164	localTracingEnabled: true,
	165	otlpTracesEnabled: false,
	166	otlpMetricsEnabled: false,
	167	},
162	168	settings: DEFAULT_SERVER_SETTINGS,
163	169	};
164	170

docs/observability.md created+519

...	...	@@ -0,0 +1,519 @@
	1	# Observability
	2
	3	T3 Code has one server-side observability model:
	4
	5	- pretty logs go to stdout for humans
	6	- completed spans go to a local NDJSON trace file
	7	- traces and metrics can also be exported over OTLP to a real backend like Grafana LGTM
	8
	9	The local trace file is the persisted source of truth. There is no separate persisted server log file anymore.
	10
	11	## Where To Find Things
	12
	13	### Logs
	14
	15	Logs are human-facing only:
	16
	17	- destination: stdout
	18	- format: `Logger.consolePretty()`
	19	- persistence: none
	20
	21	If you want a log message to show up in the trace file, emit it inside an active span with `Effect.log...`. `Logger.tracerLogger` will attach it as a span event.
	22
	23	### Traces
	24
	25	Completed spans are written as NDJSON records to `serverTracePath` (by default, `~/.t3/userdata/logs/server.trace.ndjson`).
	26
	27	Important fields in each record:
	28
	29	- `name`: span name
	30	- `traceId`, `spanId`, `parentSpanId`: correlation
	31	- `durationMs`: elapsed time
	32	- `attributes`: structured context
	33	- `events`: embedded logs and custom events
	34	- `exit`: `Success`, `Failure`, or `Interrupted`
	35
	36	The schema lives in `apps/server/src/observability/TraceRecord.ts`.
	37
	38	### Metrics
	39
	40	Metrics are not written to a local file.
	41
	42	- local persistence: none
	43	- remote export: OTLP only, when configured
	44	- current definitions: `apps/server/src/observability/Metrics.ts`
	45
	46	If OTLP is not configured, metrics still exist in-process, but you will not have a local artifact to inspect.
	47
	48	### Related Artifacts
	49
	50	Provider event NDJSON files still exist for provider runtime streams. Those are separate from the main server trace file.
	51
	52	## Run The Server In Instrumented Mode
	53
	54	There are two useful modes:
	55
	56	- local-only: stdout + local `server.trace.ndjson`
	57	- full local observability: stdout + local trace file + OTLP export to Grafana/Tempo/Prometheus
	58
	59	The local trace file is always on. OTLP export is opt-in.
	60
	61	### Option 1: Local Traces Only
	62
	63	You do not need any extra env vars. Just run the app normally and inspect `server.trace.ndjson`.
	64
	65	Examples:
	66
	67	```bash
	68	npx t3
	69	```
	70
	71	```bash
	72	bun dev
	73	```
	74
	75	```bash
	76	bun dev:desktop
	77	```
	78
	79	### Option 2: Run With A Local LGTM Stack
	80
	81	#### 1. Start Grafana LGTM
	82
	83	```bash
	84	docker run --name lgtm \
	85	-p 3000:3000 \
	86	-p 4317:4317 \
	87	-p 4318:4318 \
	88	--rm -ti \
	89	grafana/otel-lgtm
	90	```
	91
	92	Then open `http://localhost:3000`.
	93
	94	Default Grafana login:
	95
	96	- username: `admin`
	97	- password: `admin`
	98
	99	#### 2. Export OTLP env vars
	100
	101	```bash
	102	export T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces
	103	export T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics
	104	export T3CODE_OTLP_SERVICE_NAME=t3-local
	105	```
	106
	107	Optional:
	108
	109	```bash
	110	export T3CODE_TRACE_MIN_LEVEL=Info
	111	export T3CODE_TRACE_TIMING_ENABLED=true
	112	```
	113
	114	#### 3. Launch the app from that same shell
	115
	116	CLI:
	117
	118	```bash
	119	npx t3
	120	```
	121
	122	Monorepo web/server dev:
	123
	124	```bash
	125	bun dev
	126	```
	127
	128	Monorepo desktop dev:
	129
	130	```bash
	131	bun dev:desktop
	132	```
	133
	134	Packaged desktop app:
	135
	136	Launch the actual app executable from the same shell so the desktop app and embedded backend inherit `T3CODE_OTLP_*`.
	137
	138	macOS app bundle example:
	139
	140	```bash
	141	T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
	142	T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
	143	T3CODE_OTLP_SERVICE_NAME=t3-desktop \
	144	"/Applications/T3 Code.app/Contents/MacOS/T3 Code"
	145	```
	146
	147	Direct binary example:
	148
	149	```bash
	150	T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
	151	T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
	152	T3CODE_OTLP_SERVICE_NAME=t3-desktop \
	153	./path/to/your/desktop-app-binary
	154	```
	155
	156	Do not rely on launching from Finder, Spotlight, the dock, or the Start menu after setting shell env vars. Those launches usually will not pick them up.
	157
	158	#### 4. Fully restart after changing env
	159
	160	The backend reads observability config at process start. If you change OTLP env vars, stop the app completely and start it again.
	161
	162	## How To Use Traces And Metrics To Debug The Server
	163
	164	### Start With The Local Trace File
	165
	166	The trace file is the fastest way to inspect raw span data.
	167
	168	Tail it:
	169
	170	```bash
	171	tail -f "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	172	```
	173
	174	In monorepo dev, use:
	175
	176	```bash
	177	tail -f ./dev/logs/server.trace.ndjson
	178	```
	179
	180	Show failed spans:
	181
	182	```bash
	183	jq -c 'select(.exit._tag != "Success") \| {
	184	name,
	185	durationMs,
	186	exit,
	187	attributes
	188	}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	189	```
	190
	191	Show slow spans:
	192
	193	```bash
	194	jq -c 'select(.durationMs > 1000) \| {
	195	name,
	196	durationMs,
	197	traceId,
	198	spanId
	199	}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	200	```
	201
	202	Inspect embedded log events:
	203
	204	```bash
	205	jq -c 'select(any(.events[]?; .attributes["effect.logLevel"] != null)) \| {
	206	name,
	207	durationMs,
	208	events: [
	209	.events[]
	210	\| select(.attributes["effect.logLevel"] != null)
	211	\| {
	212	message: .name,
	213	level: .attributes["effect.logLevel"]
	214	}
	215	]
	216	}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	217	```
	218
	219	Follow one trace:
	220
	221	```bash
	222	jq -r 'select(.traceId == "TRACE_ID_HERE") \| [
	223	.name,
	224	.spanId,
	225	(.parentSpanId // "-"),
	226	.durationMs
	227	] \| @tsv' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	228	```
	229
	230	Filter orchestration commands:
	231
	232	```bash
	233	jq -c 'select(.attributes["orchestration.command_type"] != null) \| {
	234	name,
	235	durationMs,
	236	commandType: .attributes["orchestration.command_type"],
	237	aggregateKind: .attributes["orchestration.aggregate_kind"]
	238	}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	239	```
	240
	241	Filter git activity:
	242
	243	```bash
	244	jq -c 'select(.attributes["git.operation"] != null) \| {
	245	name,
	246	durationMs,
	247	operation: .attributes["git.operation"],
	248	cwd: .attributes["git.cwd"],
	249	hookEvents: [
	250	.events[]
	251	\| select(.name == "git.hook.started" or .name == "git.hook.finished")
	252	]
	253	}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
	254	```
	255
	256	### Use Tempo When You Need A Real Trace Viewer
	257
	258	Tempo is better than raw NDJSON when you want to:
	259
	260	- search across many traces
	261	- inspect parent/child relationships visually
	262	- compare many slow traces
	263	- drill into one failing request without hand-joining by `traceId`
	264
	265	Recommended flow in Grafana:
	266
	267	1. Open `Explore`.
	268	2. Pick the `Tempo` data source.
	269	3. Set the time range to something recent like `Last 15 minutes`.
	270	4. Start broad. Do not begin with a very narrow query.
	271	5. Look for spans from your configured service name, then narrow by span name or attributes.
	272
	273	Good first searches:
	274
	275	- service name such as `t3-local`, `t3-dev`, or `t3-desktop`
	276	- span names like `sql.execute`, `git.runCommand`, `provider.sendTurn`
	277	- orchestration spans with attributes like `orchestration.command_type`
	278
	279	Once you know traces are arriving, narrower TraceQL queries like `name = "sql.execute"` become useful.
	280
	281	### Use Metrics To See Systemic Problems
	282
	283	Traces are best for one request. Metrics are best for trends.
	284
	285	Good metric families to watch:
	286
	287	- `t3_rpc_request_duration`
	288	- `t3_orchestration_command_duration`
	289	- `t3_orchestration_command_ack_duration`
	290	- `t3_provider_turn_duration`
	291	- `t3_git_command_duration`
	292	- `t3_db_query_duration`
	293
	294	Counters tell you volume and failure rate:
	295
	296	- `t3_rpc_requests_total`
	297	- `t3_orchestration_commands_total`
	298	- `t3_provider_turns_total`
	299	- `t3_git_commands_total`
	300	- `t3_db_queries_total`
	301
	302	Use metrics when the question is:
	303
	304	- "is this always slow?"
	305	- "did this get worse after a change?"
	306	- "which command type is failing most often?"
	307
	308	Use traces when the question is:
	309
	310	- "what happened in this specific request?"
	311	- "which child span caused this one slow interaction?"
	312	- "what logs were emitted inside the failing flow?"
	313
	314	### What The New Ack Metric Means
	315
	316	`t3_orchestration_command_ack_duration` measures:
	317
	318	- start: command dispatch enters the orchestration engine
	319	- end: the first committed domain event for that command is published by the server
	320
	321	That is a server-side acknowledgment metric. It does not measure:
	322
	323	- websocket transit to the browser
	324	- client receipt
	325	- React render time
	326
	327	If you need those later, add client-side instrumentation or a dedicated server fanout metric.
	328
	329	## Common Workflows
	330
	331	### "Why did this request fail?"
	332
	333	1. Start with the local NDJSON file.
	334	2. Find spans where `exit._tag != "Success"`.
	335	3. Group by `traceId`.
	336	4. Inspect sibling spans and span events.
	337	5. If needed, move to Tempo for the full trace tree.
	338
	339	### "Why is the UI feeling slow?"
	340
	341	1. Search for slow top-level spans in the trace file or Tempo.
	342	2. Check child spans for sqlite, git, provider, or terminal work.
	343	3. Look at the matching duration metrics to see whether the slowness is systemic.
	344
	345	### "Did this command take too long to acknowledge?"
	346
	347	1. Check `t3_orchestration_command_ack_duration` by `commandType`.
	348	2. If it is high, inspect the corresponding orchestration trace.
	349	3. Look at child spans for projection, sqlite, provider, or git work.
	350
	351	### "Are git hooks causing latency?"
	352
	353	1. Filter `git.operation` spans.
	354	2. Inspect `git.hook.started` and `git.hook.finished` events.
	355	3. Compare hook timing to the enclosing git span duration.
	356
	357	### "Why do I have spans locally but nothing in Grafana?"
	358
	359	Usually one of these is true:
	360
	361	- `T3CODE_OTLP_TRACES_URL` was not set
	362	- the app was launched from a different environment than the one where you exported the vars
	363	- the app was not fully restarted after changing env
	364	- Grafana is looking at the wrong time range or service name
	365
	366	If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.
	367
	368	## How To Think About Adding Tracing To Future Code
	369
	370	### Prefer Boundaries Over Tiny Helpers
	371
	372	Good span boundaries:
	373
	374	- RPC methods
	375	- orchestration command handling
	376	- provider adapter calls
	377	- external process calls
	378	- persistence writes
	379	- queue handoffs
	380
	381	Avoid tracing every tiny helper. Most helpers should inherit the active span rather than create a new one.
	382
	383	### Reuse `Effect.fn(...)` Where It Already Exists
	384
	385	The codebase already uses `Effect.fn("name")` heavily. That should usually be your first tracing boundary.
	386
	387	For ad hoc work:
	388
	389	```ts
	390	import { Effect } from "effect";
	391
	392	const runThing = Effect.gen(function* () {
	393	yield* Effect.annotateCurrentSpan({
	394	"thing.id": "abc123",
	395	"thing.kind": "example",
	396	});
	397
	398	yield* Effect.logInfo("starting thing");
	399	return yield* doWork();
	400	}).pipe(Effect.withSpan("thing.run"));
	401	```
	402
	403	### Put High-Cardinality Detail On Spans
	404
	405	Use span annotations for IDs, paths, and other detailed context:
	406
	407	```ts
	408	yield *
	409	Effect.annotateCurrentSpan({
	410	"provider.thread_id": input.threadId,
	411	"provider.request_id": input.requestId,
	412	"git.cwd": input.cwd,
	413	});
	414	```
	415
	416	### Keep Metric Labels Low Cardinality
	417
	418	Good metric labels:
	419
	420	- operation kind
	421	- method name
	422	- provider kind
	423	- aggregate kind
	424	- outcome
	425
	426	Bad metric labels:
	427
	428	- raw thread IDs
	429	- command IDs
	430	- file paths
	431	- cwd
	432	- full prompts
	433	- full model strings when a normalized family label would do
	434
	435	Detailed context belongs on spans, not metrics.
	436
	437	### Use Logs As Span Events
	438
	439	Logs inside a span become part of the trace story:
	440
	441	```ts
	442	yield * Effect.logInfo("starting provider turn");
	443	yield * Effect.logDebug("waiting for approval response");
	444	```
	445
	446	Those messages show up as span events because `Logger.tracerLogger` is installed.
	447
	448	### Use The Pipeable Metrics API
	449
	450	`withMetrics(...)` is the default way to attach a counter and timer to an effect:
	451
	452	```ts
	453	import { someCounter, someDuration, withMetrics } from "../observability/Metrics.ts";
	454
	455	const program = doWork().pipe(
	456	withMetrics({
	457	counter: someCounter,
	458	timer: someDuration,
	459	attributes: {
	460	operation: "work",
	461	},
	462	}),
	463	);
	464	```
	465
	466	## Detailed API Reference
	467
	468	### Runtime Wiring
	469
	470	The server observability layer is assembled in `apps/server/src/observability/Layers/Observability.ts`.
	471
	472	It provides:
	473
	474	- pretty stdout logger
	475	- `Logger.tracerLogger`
	476	- local NDJSON tracer
	477	- optional OTLP trace exporter
	478	- optional OTLP metrics exporter
	479	- Effect trace-level and timing refs
	480
	481	### Env Vars
	482
	483	Local trace file:
	484
	485	- `T3CODE_TRACE_FILE`: override trace file path
	486	- `T3CODE_TRACE_MAX_BYTES`: per-file rotation size, default `10485760`
	487	- `T3CODE_TRACE_MAX_FILES`: rotated file count, default `10`
	488	- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `200`
	489	- `T3CODE_TRACE_MIN_LEVEL`: minimum trace level, default `Info`
	490	- `T3CODE_TRACE_TIMING_ENABLED`: enable timing metadata, default `true`
	491
	492	OTLP export:
	493
	494	- `T3CODE_OTLP_TRACES_URL`: OTLP trace endpoint
	495	- `T3CODE_OTLP_METRICS_URL`: OTLP metric endpoint
	496	- `T3CODE_OTLP_EXPORT_INTERVAL_MS`: export interval, default `10000`
	497	- `T3CODE_OTLP_SERVICE_NAME`: service name, default `t3-server`
	498
	499	If the OTLP URLs are unset, local tracing still works and metrics stay in-process only.
	500
	501	### What Is Instrumented Today
	502
	503	Current high-value span and metric boundaries include:
	504
	505	- Effect RPC websocket request spans from `effect/rpc`
	506	- RPC request metrics in `apps/server/src/observability/RpcInstrumentation.ts`
	507	- startup phases
	508	- orchestration command processing
	509	- orchestration command acknowledgment latency
	510	- provider session and turn operations
	511	- git command execution and git hook events
	512	- terminal session lifecycle
	513	- sqlite query execution
	514
	515	### Current Constraints
	516
	517	- logs outside spans are not persisted
	518	- metrics are not snapshotted locally
	519	- the old `serverLogPath` still exists in config for compatibility, but the trace file is the persisted artifact that matters

packages/contracts/src/server.ts+11

...	...	@@ -72,6 +72,16 @@ export type ServerProvider = typeof ServerProvider.Type;
72	72	export const ServerProviders = Schema.Array(ServerProvider);
73	73	export type ServerProviders = typeof ServerProviders.Type;
74	74
	75	export const ServerObservability = Schema.Struct({
	76	logsDirectoryPath: TrimmedNonEmptyString,
	77	localTracingEnabled: Schema.Boolean,
	78	otlpTracesUrl: Schema.optional(TrimmedNonEmptyString),
	79	otlpTracesEnabled: Schema.Boolean,
	80	otlpMetricsUrl: Schema.optional(TrimmedNonEmptyString),
	81	otlpMetricsEnabled: Schema.Boolean,
	82	});
	83	export type ServerObservability = typeof ServerObservability.Type;
	84
75	85	export const ServerConfig = Schema.Struct({
76	86	cwd: TrimmedNonEmptyString,
77	87	keybindingsConfigPath: TrimmedNonEmptyString,
...	...	@@ -79,6 +89,7 @@ export const ServerConfig = Schema.Struct({
79	89	issues: ServerConfigIssues,
80	90	providers: ServerProviders,
81	91	availableEditors: Schema.Array(EditorId),
	92	observability: ServerObservability,
82	93	settings: ServerSettings,
83	94	});
84	95	export type ServerConfig = typeof ServerConfig.Type;

turbo.json+11-1

...	...	@@ -11,7 +11,17 @@
11	11	"T3CODE_NO_BROWSER",
12	12	"T3CODE_HOME",
13	13	"T3CODE_AUTH_TOKEN",
14		"T3CODE_DESKTOP_WS_URL"
	14	"T3CODE_DESKTOP_WS_URL",
	15	"T3CODE_TRACE_MIN_LEVEL",
	16	"T3CODE_TRACE_TIMING_ENABLED",
	17	"T3CODE_TRACE_FILE",
	18	"T3CODE_TRACE_MAX_BYTES",
	19	"T3CODE_TRACE_MAX_FILES",
	20	"T3CODE_TRACE_BATCH_WINDOW_MS",
	21	"T3CODE_OTLP_TRACES_URL",
	22	"T3CODE_OTLP_METRICS_URL",
	23	"T3CODE_OTLP_EXPORT_INTERVAL_MS",
	24	"T3CODE_OTLP_SERVICE_NAME"
15	25	],
16	26	"tasks": {
17	27	"build": {

author	Julius Marminge <julius0216@outlook.com> 2026-04-02 19:30:51-07:00
committer	GitHub <noreply@github.com> 2026-04-03 02:30:51+00:00
log	752f96e9ae72be9d56686902454eb7a3c7d8686a
tree	737810cabed4b5b571b235045790cb1694423491
parent	1b272fd70992ac349791dd6b6c084ef1d042d700
signature	Signed by PGP key B5690EEEBB952194

Add server observability tracing and metrics (#1697)