authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-04-02 19:30:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-04-03 02:30:51+00:00
log752f96e9ae72be9d56686902454eb7a3c7d8686a
tree737810cabed4b5b571b235045790cb1694423491
parent1b272fd70992ac349791dd6b6c084ef1d042d700
signature Signed by PGP key B5690EEEBB952194

Add server observability tracing and metrics (#1697)

Co-authored-by: codex <codex@users.noreply.github.com>

43 files changed, 3571 insertions(+), 741 deletions(-)

README.md+2
...@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs....@@ -45,6 +45,8 @@ We are very very early in this project. Expect bugs.
4545
46We are not accepting contributions yet.46We are not accepting contributions yet.
4747
48Observability guide: [docs/observability.md](./docs/observability.md)
49
48## If you REALLY want to contribute still.... read this first50## If you REALLY want to contribute still.... read this first
4951
50Read [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.52Read [CONTRIBUTING.md](./CONTRIBUTING.md) before opening an issue or PR.
apps/server/src/cli-config.test.ts+17
...@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";...@@ -9,6 +9,18 @@ import { deriveServerPaths } from "./config";
9import { resolveServerConfig } from "./cli";9import { resolveServerConfig } from "./cli";
1010
11it.layer(NodeServices.layer)("cli config resolution", (it) => {11it.layer(NodeServices.layer)("cli config resolution", (it) => {
12 const defaultObservabilityConfig = {
13 traceMinLevel: "Info",
14 traceTimingEnabled: true,
15 traceBatchWindowMs: 200,
16 traceMaxBytes: 10 * 1024 * 1024,
17 traceMaxFiles: 10,
18 otlpTracesUrl: undefined,
19 otlpMetricsUrl: undefined,
20 otlpExportIntervalMs: 10_000,
21 otlpServiceName: "t3-server",
22 } as const;
23
12 const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {24 const openBootstrapFd = Effect.fn(function* (payload: Record<string, unknown>) {
13 const fs = yield* FileSystem.FileSystem;25 const fs = yield* FileSystem.FileSystem;
14 const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });26 const filePath = yield* fs.makeTempFileScoped({ prefix: "t3-bootstrap-", suffix: ".ndjson" });
...@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {...@@ -62,6 +74,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
6274
63 expect(resolved).toEqual({75 expect(resolved).toEqual({
64 logLevel: "Warn",76 logLevel: "Warn",
77 ...defaultObservabilityConfig,
65 mode: "desktop",78 mode: "desktop",
66 port: 4001,79 port: 4001,
67 cwd: process.cwd(),80 cwd: process.cwd(),
...@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {...@@ -123,6 +136,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
123136
124 expect(resolved).toEqual({137 expect(resolved).toEqual({
125 logLevel: "Debug",138 logLevel: "Debug",
139 ...defaultObservabilityConfig,
126 mode: "web",140 mode: "web",
127 port: 8788,141 port: 8788,
128 cwd: process.cwd(),142 cwd: process.cwd(),
...@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {...@@ -187,6 +201,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
187201
188 expect(resolved).toEqual({202 expect(resolved).toEqual({
189 logLevel: "Info",203 logLevel: "Info",
204 ...defaultObservabilityConfig,
190 mode: "desktop",205 mode: "desktop",
191 port: 4888,206 port: 4888,
192 cwd: process.cwd(),207 cwd: process.cwd(),
...@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {...@@ -241,6 +256,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
241 resolved.attachmentsDir,256 resolved.attachmentsDir,
242 resolved.worktreesDir,257 resolved.worktreesDir,
243 path.dirname(resolved.serverLogPath),258 path.dirname(resolved.serverLogPath),
259 path.dirname(resolved.serverTracePath),
244 ]) {260 ]) {
245 expect(yield* fs.exists(directory)).toBe(true);261 expect(yield* fs.exists(directory)).toBe(true);
246 }262 }
...@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {...@@ -300,6 +316,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => {
300316
301 expect(resolved).toEqual({317 expect(resolved).toEqual({
302 logLevel: "Debug",318 logLevel: "Debug",
319 ...defaultObservabilityConfig,
303 mode: "web",320 mode: "web",
304 port: 8788,321 port: 8788,
305 cwd: process.cwd(),322 cwd: process.cwd(),
apps/server/src/cli.ts+36-1
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1import { NetService } from "@t3tools/shared/Net";1import { NetService } from "@t3tools/shared/Net";
2import { Config, Effect, LogLevel, Option, Schema } from "effect";2import { Config, Effect, FileSystem, LogLevel, Option, Path, Schema } from "effect";
3import { Command, Flag, GlobalFlag } from "effect/unstable/cli";3import { Command, Flag, GlobalFlag } from "effect/unstable/cli";
44
5import {5import {
...@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(...@@ -81,6 +81,27 @@ const logWebSocketEventsFlag = Flag.boolean("log-websocket-events").pipe(
8181
82const EnvServerConfig = Config.all({82const EnvServerConfig = Config.all({
83 logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),83 logLevel: Config.logLevel("T3CODE_LOG_LEVEL").pipe(Config.withDefault("Info")),
84 traceMinLevel: Config.logLevel("T3CODE_TRACE_MIN_LEVEL").pipe(Config.withDefault("Info")),
85 traceTimingEnabled: Config.boolean("T3CODE_TRACE_TIMING_ENABLED").pipe(Config.withDefault(true)),
86 traceFile: Config.string("T3CODE_TRACE_FILE").pipe(
87 Config.option,
88 Config.map(Option.getOrUndefined),
89 ),
90 traceMaxBytes: Config.int("T3CODE_TRACE_MAX_BYTES").pipe(Config.withDefault(10 * 1024 * 1024)),
91 traceMaxFiles: Config.int("T3CODE_TRACE_MAX_FILES").pipe(Config.withDefault(10)),
92 traceBatchWindowMs: Config.int("T3CODE_TRACE_BATCH_WINDOW_MS").pipe(Config.withDefault(200)),
93 otlpTracesUrl: Config.string("T3CODE_OTLP_TRACES_URL").pipe(
94 Config.option,
95 Config.map(Option.getOrUndefined),
96 ),
97 otlpMetricsUrl: Config.string("T3CODE_OTLP_METRICS_URL").pipe(
98 Config.option,
99 Config.map(Option.getOrUndefined),
100 ),
101 otlpExportIntervalMs: Config.int("T3CODE_OTLP_EXPORT_INTERVAL_MS").pipe(
102 Config.withDefault(10_000),
103 ),
104 otlpServiceName: Config.string("T3CODE_OTLP_SERVICE_NAME").pipe(Config.withDefault("t3-server")),
84 mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(105 mode: Config.schema(RuntimeMode, "T3CODE_MODE").pipe(
85 Config.option,106 Config.option,
86 Config.map(Option.getOrUndefined),107 Config.map(Option.getOrUndefined),
...@@ -137,6 +158,8 @@ export const resolveServerConfig = (...@@ -137,6 +158,8 @@ export const resolveServerConfig = (
137) =>158) =>
138 Effect.gen(function* () {159 Effect.gen(function* () {
139 const { findAvailablePort } = yield* NetService;160 const { findAvailablePort } = yield* NetService;
161 const path = yield* Path.Path;
162 const fs = yield* FileSystem.FileSystem;
140 const env = yield* EnvServerConfig;163 const env = yield* EnvServerConfig;
141 const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;164 const bootstrapFd = Option.getOrUndefined(flags.bootstrapFd) ?? env.bootstrapFd;
142 const bootstrapEnvelope =165 const bootstrapEnvelope =
...@@ -190,6 +213,8 @@ export const resolveServerConfig = (...@@ -190,6 +213,8 @@ export const resolveServerConfig = (
190 );213 );
191 const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);214 const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
192 yield* ensureServerDirectories(derivedPaths);215 yield* ensureServerDirectories(derivedPaths);
216 const serverTracePath = env.traceFile ?? derivedPaths.serverTracePath;
217 yield* fs.makeDirectory(path.dirname(serverTracePath), { recursive: true });
193 const noBrowser = resolveBooleanFlag(218 const noBrowser = resolveBooleanFlag(
194 flags.noBrowser,219 flags.noBrowser,
195 Option.getOrElse(220 Option.getOrElse(
...@@ -248,11 +273,21 @@ export const resolveServerConfig = (...@@ -248,11 +273,21 @@ export const resolveServerConfig = (
248273
249 const config: ServerConfigShape = {274 const config: ServerConfigShape = {
250 logLevel,275 logLevel,
276 traceMinLevel: env.traceMinLevel,
277 traceTimingEnabled: env.traceTimingEnabled,
278 traceBatchWindowMs: env.traceBatchWindowMs,
279 traceMaxBytes: env.traceMaxBytes,
280 traceMaxFiles: env.traceMaxFiles,
281 otlpTracesUrl: env.otlpTracesUrl,
282 otlpMetricsUrl: env.otlpMetricsUrl,
283 otlpExportIntervalMs: env.otlpExportIntervalMs,
284 otlpServiceName: env.otlpServiceName,
251 mode,285 mode,
252 port,286 port,
253 cwd: process.cwd(),287 cwd: process.cwd(),
254 baseDir,288 baseDir,
255 ...derivedPaths,289 ...derivedPaths,
290 serverTracePath,
256 host,291 host,
257 staticDir,292 staticDir,
258 devUrl,293 devUrl,
apps/server/src/config.ts+20
...@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {...@@ -25,6 +25,7 @@ export interface ServerDerivedPaths {
25 readonly attachmentsDir: string;25 readonly attachmentsDir: string;
26 readonly logsDir: string;26 readonly logsDir: string;
27 readonly serverLogPath: string;27 readonly serverLogPath: string;
28 readonly serverTracePath: string;
28 readonly providerLogsDir: string;29 readonly providerLogsDir: string;
29 readonly providerEventLogPath: string;30 readonly providerEventLogPath: string;
30 readonly terminalLogsDir: string;31 readonly terminalLogsDir: string;
...@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {...@@ -36,6 +37,15 @@ export interface ServerDerivedPaths {
36 */37 */
37export interface ServerConfigShape extends ServerDerivedPaths {38export interface ServerConfigShape extends ServerDerivedPaths {
38 readonly logLevel: LogLevel.LogLevel;39 readonly logLevel: LogLevel.LogLevel;
40 readonly traceMinLevel: LogLevel.LogLevel;
41 readonly traceTimingEnabled: boolean;
42 readonly traceBatchWindowMs: number;
43 readonly traceMaxBytes: number;
44 readonly traceMaxFiles: number;
45 readonly otlpTracesUrl: string | undefined;
46 readonly otlpMetricsUrl: string | undefined;
47 readonly otlpExportIntervalMs: number;
48 readonly otlpServiceName: string;
39 readonly mode: RuntimeMode;49 readonly mode: RuntimeMode;
40 readonly port: number;50 readonly port: number;
41 readonly host: string | undefined;51 readonly host: string | undefined;
...@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (...@@ -68,6 +78,7 @@ export const deriveServerPaths = Effect.fn(function* (
68 attachmentsDir,78 attachmentsDir,
69 logsDir,79 logsDir,
70 serverLogPath: join(logsDir, "server.log"),80 serverLogPath: join(logsDir, "server.log"),
81 serverTracePath: join(logsDir, "server.trace.ndjson"),
71 providerLogsDir,82 providerLogsDir,
72 providerEventLogPath: join(providerLogsDir, "events.log"),83 providerEventLogPath: join(providerLogsDir, "events.log"),
73 terminalLogsDir: join(logsDir, "terminals"),84 terminalLogsDir: join(logsDir, "terminals"),
...@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigS...@@ -117,6 +128,15 @@ export class ServerConfig extends ServiceMap.Service<ServerConfig, ServerConfigS
117128
118 return {129 return {
119 logLevel: "Error",130 logLevel: "Error",
131 traceMinLevel: "Info",
132 traceTimingEnabled: true,
133 traceBatchWindowMs: 200,
134 traceMaxBytes: 10 * 1024 * 1024,
135 traceMaxFiles: 10,
136 otlpTracesUrl: undefined,
137 otlpMetricsUrl: undefined,
138 otlpExportIntervalMs: 10_000,
139 otlpServiceName: "t3-server",
120 cwd,140 cwd,
121 baseDir,141 baseDir,
122 ...derivedPaths,142 ...derivedPaths,
apps/server/src/git/Layers/GitCore.ts+49-6
...@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";...@@ -20,6 +20,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
2020
21import { GitCommandError, type GitBranch } from "@t3tools/contracts";21import { GitCommandError, type GitBranch } from "@t3tools/contracts";
22import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";22import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git";
23import { compactTraceAttributes } from "../../observability/Attributes.ts";
24import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../../observability/Metrics.ts";
23import {25import {
24 GitCore,26 GitCore,
25 type ExecuteGitProgress,27 type ExecuteGitProgress,
...@@ -372,6 +374,18 @@ interface Trace2Monitor {...@@ -372,6 +374,18 @@ interface Trace2Monitor {
372 readonly flush: Effect.Effect<void, never>;374 readonly flush: Effect.Effect<void, never>;
373}375}
374376
377const nowUnixNano = (): bigint => BigInt(Date.now()) * 1_000_000n;
378
379const addCurrentSpanEvent = (name: string, attributes: Record<string, unknown>) =>
380 Effect.currentSpan.pipe(
381 Effect.tap((span) =>
382 Effect.sync(() => {
383 span.event(name, nowUnixNano(), compactTraceAttributes(attributes));
384 }),
385 ),
386 Effect.catch(() => Effect.void),
387 );
388
375function trace2ChildKey(record: Record<string, unknown>): string | null {389function trace2ChildKey(record: Record<string, unknown>): string | null {
376 const childId = record.child_id;390 const childId = record.child_id;
377 if (typeof childId === "number" || typeof childId === "string") {391 if (typeof childId === "number" || typeof childId === "string") {
...@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (...@@ -444,6 +458,9 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
444458
445 if (event === "child_start") {459 if (event === "child_start") {
446 hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });460 hookStartByChildKey.set(childKey, { hookName, startedAtMs: Date.now() });
461 yield* addCurrentSpanEvent("git.hook.started", {
462 hookName,
463 });
447 if (progress.onHookStarted) {464 if (progress.onHookStarted) {
448 yield* progress.onHookStarted(hookName);465 yield* progress.onHookStarted(hookName);
449 }466 }
...@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (...@@ -452,12 +469,19 @@ const createTrace2Monitor = Effect.fn("createTrace2Monitor")(function* (
452469
453 if (event === "child_exit") {470 if (event === "child_exit") {
454 hookStartByChildKey.delete(childKey);471 hookStartByChildKey.delete(childKey);
472 const code = traceRecord.success.code;
473 const exitCode = typeof code === "number" && Number.isInteger(code) ? code : null;
474 const durationMs = started ? Math.max(0, Date.now() - started.startedAtMs) : null;
475 yield* addCurrentSpanEvent("git.hook.finished", {
476 hookName: started?.hookName ?? hookName,
477 exitCode,
478 durationMs,
479 });
455 if (progress.onHookFinished) {480 if (progress.onHookFinished) {
456 const code = traceRecord.success.code;
457 yield* progress.onHookFinished({481 yield* progress.onHookFinished({
458 hookName: started?.hookName ?? hookName,482 hookName: started?.hookName ?? hookName,
459 exitCode: typeof code === "number" && Number.isInteger(code) ? code : null,483 exitCode,
460 durationMs: started ? Math.max(0, Date.now() - started.startedAtMs) : null,484 durationMs,
461 });485 });
462 }486 }
463 }487 }
...@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {...@@ -609,13 +633,13 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
609 const path = yield* Path.Path;633 const path = yield* Path.Path;
610 const { worktreesDir } = yield* ServerConfig;634 const { worktreesDir } = yield* ServerConfig;
611635
612 let execute: GitCoreShape["execute"];636 let executeRaw: GitCoreShape["execute"];
613637
614 if (options?.executeOverride) {638 if (options?.executeOverride) {
615 execute = options.executeOverride;639 executeRaw = options.executeOverride;
616 } else {640 } else {
617 const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;641 const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
618 execute = Effect.fnUntraced(function* (input) {642 executeRaw = Effect.fnUntraced(function* (input) {
619 const commandInput = {643 const commandInput = {
620 ...input,644 ...input,
621 args: [...input.args],645 args: [...input.args],
...@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {...@@ -716,6 +740,25 @@ export const makeGitCore = Effect.fn("makeGitCore")(function* (options?: {
716 });740 });
717 }741 }
718742
743 const execute: GitCoreShape["execute"] = (input) =>
744 executeRaw(input).pipe(
745 withMetrics({
746 counter: gitCommandsTotal,
747 timer: gitCommandDuration,
748 attributes: {
749 operation: input.operation,
750 },
751 }),
752 Effect.withSpan(input.operation, {
753 kind: "client",
754 attributes: {
755 "git.operation": input.operation,
756 "git.cwd": input.cwd,
757 "git.args_count": input.args.length,
758 },
759 }),
760 );
761
719 const executeGit = (762 const executeGit = (
720 operation: string,763 operation: string,
721 cwd: string,764 cwd: string,
apps/server/src/observability/Attributes.test.ts created+46
...@@ -0,0 +1,46 @@
1import { assert, describe, it } from "@effect/vitest";
2
3import { compactTraceAttributes, normalizeModelMetricLabel } from "./Attributes.ts";
4
5describe("Attributes", () => {
6 it("normalizes circular arrays, maps, and sets without recursing forever", () => {
7 const array: Array<unknown> = ["alpha"];
8 array.push(array);
9
10 const map = new Map<string, unknown>();
11 map.set("self", map);
12
13 const set = new Set<unknown>();
14 set.add(set);
15
16 assert.deepStrictEqual(
17 compactTraceAttributes({
18 array,
19 map,
20 set,
21 }),
22 {
23 array: ["alpha", "[Circular]"],
24 map: { self: "[Circular]" },
25 set: ["[Circular]"],
26 },
27 );
28 });
29
30 it("normalizes invalid dates without throwing", () => {
31 assert.deepStrictEqual(
32 compactTraceAttributes({
33 invalidDate: new Date("not-a-real-date"),
34 }),
35 {
36 invalidDate: "Invalid Date",
37 },
38 );
39 });
40
41 it("groups GPT-family models under a shared metric label", () => {
42 assert.strictEqual(normalizeModelMetricLabel("gpt-4o"), "gpt");
43 assert.strictEqual(normalizeModelMetricLabel("gpt-5.4"), "gpt");
44 assert.strictEqual(normalizeModelMetricLabel("claude-sonnet-4"), "claude");
45 });
46});
apps/server/src/observability/Attributes.ts created+128
...@@ -0,0 +1,128 @@
1import { Cause, Exit } from "effect";
2
3export type MetricAttributeValue = string;
4export type MetricAttributes = Readonly<Record<string, MetricAttributeValue>>;
5export type TraceAttributes = Readonly<Record<string, unknown>>;
6export type ObservabilityOutcome = "success" | "failure" | "interrupt";
7
8function isPlainObject(value: unknown): value is Record<string, unknown> {
9 return typeof value === "object" && value !== null && !Array.isArray(value);
10}
11
12function markSeen(value: object, seen: WeakSet<object>): boolean {
13 if (seen.has(value)) {
14 return true;
15 }
16 seen.add(value);
17 return false;
18}
19
20function normalizeJsonValue(value: unknown, seen: WeakSet<object> = new WeakSet()): unknown {
21 if (
22 value === null ||
23 value === undefined ||
24 typeof value === "string" ||
25 typeof value === "number" ||
26 typeof value === "boolean"
27 ) {
28 return value ?? null;
29 }
30 if (typeof value === "bigint") {
31 return value.toString();
32 }
33 if (value instanceof Date) {
34 return Number.isNaN(value.getTime()) ? "Invalid Date" : value.toISOString();
35 }
36 if (value instanceof Error) {
37 return {
38 name: value.name,
39 message: value.message,
40 ...(value.stack ? { stack: value.stack } : {}),
41 };
42 }
43 if (Array.isArray(value)) {
44 if (markSeen(value, seen)) {
45 return "[Circular]";
46 }
47 return value.map((entry) => normalizeJsonValue(entry, seen));
48 }
49 if (value instanceof Map) {
50 if (markSeen(value, seen)) {
51 return "[Circular]";
52 }
53 return Object.fromEntries(
54 Array.from(value.entries(), ([key, entryValue]) => [
55 String(key),
56 normalizeJsonValue(entryValue, seen),
57 ]),
58 );
59 }
60 if (value instanceof Set) {
61 if (markSeen(value, seen)) {
62 return "[Circular]";
63 }
64 return Array.from(value.values(), (entry) => normalizeJsonValue(entry, seen));
65 }
66 if (!isPlainObject(value)) {
67 return String(value);
68 }
69 if (markSeen(value, seen)) {
70 return "[Circular]";
71 }
72 return Object.fromEntries(
73 Object.entries(value).map(([key, entryValue]) => [key, normalizeJsonValue(entryValue, seen)]),
74 );
75}
76
77export function compactTraceAttributes(
78 attributes: Readonly<Record<string, unknown>>,
79): TraceAttributes {
80 return Object.fromEntries(
81 Object.entries(attributes)
82 .filter(([, value]) => value !== undefined)
83 .map(([key, value]) => [key, normalizeJsonValue(value)]),
84 );
85}
86
87export function compactMetricAttributes(
88 attributes: Readonly<Record<string, unknown>>,
89): MetricAttributes {
90 return Object.fromEntries(
91 Object.entries(attributes).flatMap(([key, value]) => {
92 if (value === undefined || value === null) {
93 return [];
94 }
95 if (typeof value === "string") {
96 return [[key, value]];
97 }
98 if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
99 return [[key, String(value)]];
100 }
101 return [];
102 }),
103 );
104}
105
106export function outcomeFromExit(exit: Exit.Exit<unknown, unknown>): ObservabilityOutcome {
107 if (Exit.isSuccess(exit)) {
108 return "success";
109 }
110 return Cause.hasInterruptsOnly(exit.cause) ? "interrupt" : "failure";
111}
112
113export function normalizeModelMetricLabel(model: string | null | undefined): string | undefined {
114 const normalized = model?.trim().toLowerCase();
115 if (!normalized) {
116 return undefined;
117 }
118 if (normalized.includes("gpt")) {
119 return "gpt";
120 }
121 if (normalized.includes("claude")) {
122 return "claude";
123 }
124 if (normalized.includes("gemini")) {
125 return "gemini";
126 }
127 return "other";
128}
apps/server/src/observability/Layers/Observability.ts created+64
...@@ -0,0 +1,64 @@
1import { Effect, Layer, References, Tracer } from "effect";
2import { OtlpMetrics, OtlpSerialization, OtlpTracer } from "effect/unstable/observability";
3
4import { ServerConfig } from "../../config.ts";
5import { ServerLoggerLive } from "../../serverLogger.ts";
6import { makeLocalFileTracer } from "../LocalFileTracer.ts";
7
8const otlpSerializationLayer = OtlpSerialization.layerJson;
9
10export const ObservabilityLive = Layer.unwrap(
11 Effect.gen(function* () {
12 const config = yield* ServerConfig;
13
14 const traceReferencesLayer = Layer.mergeAll(
15 Layer.succeed(Tracer.MinimumTraceLevel, config.traceMinLevel),
16 Layer.succeed(References.TracerTimingEnabled, config.traceTimingEnabled),
17 );
18
19 const tracerLayer = Layer.effect(
20 Tracer.Tracer,
21 Effect.gen(function* () {
22 const delegate =
23 config.otlpTracesUrl === undefined
24 ? undefined
25 : yield* OtlpTracer.make({
26 url: config.otlpTracesUrl,
27 exportInterval: `${config.otlpExportIntervalMs} millis`,
28 resource: {
29 serviceName: config.otlpServiceName,
30 attributes: {
31 "service.runtime": "t3-server",
32 "service.mode": config.mode,
33 },
34 },
35 });
36
37 return yield* makeLocalFileTracer({
38 filePath: config.serverTracePath,
39 maxBytes: config.traceMaxBytes,
40 maxFiles: config.traceMaxFiles,
41 batchWindowMs: config.traceBatchWindowMs,
42 ...(delegate ? { delegate } : {}),
43 });
44 }),
45 ).pipe(Layer.provideMerge(otlpSerializationLayer));
46
47 const metricsLayer =
48 config.otlpMetricsUrl === undefined
49 ? Layer.empty
50 : OtlpMetrics.layer({
51 url: config.otlpMetricsUrl,
52 exportInterval: `${config.otlpExportIntervalMs} millis`,
53 resource: {
54 serviceName: config.otlpServiceName,
55 attributes: {
56 "service.runtime": "t3-server",
57 "service.mode": config.mode,
58 },
59 },
60 }).pipe(Layer.provideMerge(otlpSerializationLayer));
61
62 return Layer.mergeAll(ServerLoggerLive, traceReferencesLayer, tracerLayer, metricsLayer);
63 }),
64);
apps/server/src/observability/LocalFileTracer.test.ts created+117
...@@ -0,0 +1,117 @@
1import fs from "node:fs";
2import os from "node:os";
3import path from "node:path";
4
5import { assert, describe, it } from "@effect/vitest";
6import { Effect, Layer, Logger, References, Tracer } from "effect";
7
8import type { TraceRecord } from "./TraceRecord.ts";
9import { makeLocalFileTracer } from "./LocalFileTracer.ts";
10
11const makeTestLayer = (tracePath: string) =>
12 Layer.mergeAll(
13 Layer.effect(
14 Tracer.Tracer,
15 makeLocalFileTracer({
16 filePath: tracePath,
17 maxBytes: 1024 * 1024,
18 maxFiles: 2,
19 batchWindowMs: 10_000,
20 }),
21 ),
22 Logger.layer([Logger.tracerLogger], { mergeWithExisting: false }),
23 Layer.succeed(References.MinimumLogLevel, "Info"),
24 );
25
26const readTraceRecords = (tracePath: string): Array<TraceRecord> =>
27 fs
28 .readFileSync(tracePath, "utf8")
29 .trim()
30 .split("\n")
31 .filter((line) => line.length > 0)
32 .map((line) => JSON.parse(line) as TraceRecord);
33
34describe("LocalFileTracer", () => {
35 it.effect("writes nested spans to disk and captures log messages as span events", () =>
36 Effect.gen(function* () {
37 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
38 const tracePath = path.join(tempDir, "server.trace.ndjson");
39
40 try {
41 yield* Effect.scoped(
42 Effect.gen(function* () {
43 const program = Effect.gen(function* () {
44 yield* Effect.annotateCurrentSpan({
45 "demo.parent": true,
46 });
47 yield* Effect.logInfo("parent event");
48 yield* Effect.gen(function* () {
49 yield* Effect.annotateCurrentSpan({
50 "demo.child": true,
51 });
52 yield* Effect.logInfo("child event");
53 }).pipe(Effect.withSpan("child-span"));
54 }).pipe(Effect.withSpan("parent-span"));
55
56 yield* program.pipe(Effect.provide(makeTestLayer(tracePath)));
57 }),
58 );
59
60 const records = readTraceRecords(tracePath);
61 assert.equal(records.length, 2);
62
63 const parent = records.find((record) => record.name === "parent-span");
64 const child = records.find((record) => record.name === "child-span");
65
66 assert.notEqual(parent, undefined);
67 assert.notEqual(child, undefined);
68 if (!parent || !child) {
69 return;
70 }
71
72 assert.equal(child.parentSpanId, parent.spanId);
73 assert.equal(parent.attributes["demo.parent"], true);
74 assert.equal(child.attributes["demo.child"], true);
75 assert.equal(
76 parent.events.some((event) => event.name === "parent event"),
77 true,
78 );
79 assert.equal(
80 child.events.some((event) => event.name === "child event"),
81 true,
82 );
83 assert.equal(
84 child.events.some((event) => event.attributes["effect.logLevel"] === "INFO"),
85 true,
86 );
87 } finally {
88 fs.rmSync(tempDir, { recursive: true, force: true });
89 }
90 }),
91 );
92
93 it.effect("serializes interrupted spans with an interrupted exit status", () =>
94 Effect.gen(function* () {
95 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-local-tracer-"));
96 const tracePath = path.join(tempDir, "server.trace.ndjson");
97
98 try {
99 yield* Effect.scoped(
100 Effect.exit(
101 Effect.interrupt.pipe(
102 Effect.withSpan("interrupt-span"),
103 Effect.provide(makeTestLayer(tracePath)),
104 ),
105 ),
106 );
107
108 const records = readTraceRecords(tracePath);
109 assert.equal(records.length, 1);
110 assert.equal(records[0]?.name, "interrupt-span");
111 assert.equal(records[0]?.exit._tag, "Interrupted");
112 } finally {
113 fs.rmSync(tempDir, { recursive: true, force: true });
114 }
115 }),
116 );
117});
apps/server/src/observability/LocalFileTracer.ts created+104
...@@ -0,0 +1,104 @@
1import type * as Exit from "effect/Exit";
2import { Effect, Option, Tracer } from "effect";
3
4import { spanToTraceRecord } from "./TraceRecord.ts";
5import { makeTraceSink } from "./TraceSink.ts";
6
7export interface LocalFileTracerOptions {
8 readonly filePath: string;
9 readonly maxBytes: number;
10 readonly maxFiles: number;
11 readonly batchWindowMs: number;
12 readonly delegate?: Tracer.Tracer;
13}
14
15class LocalFileSpan implements Tracer.Span {
16 readonly _tag = "Span";
17 readonly name: string;
18 readonly spanId: string;
19 readonly traceId: string;
20 readonly parent: Option.Option<Tracer.AnySpan>;
21 readonly annotations: Tracer.Span["annotations"];
22 readonly links: Array<Tracer.SpanLink>;
23 readonly sampled: boolean;
24 readonly kind: Tracer.SpanKind;
25
26 status: Tracer.SpanStatus;
27 attributes: Map<string, unknown>;
28 events: Array<[name: string, startTime: bigint, attributes: Record<string, unknown>]>;
29
30 constructor(
31 options: Parameters<Tracer.Tracer["span"]>[0],
32 private readonly delegate: Tracer.Span,
33 private readonly push: (record: ReturnType<typeof spanToTraceRecord>) => void,
34 ) {
35 this.name = delegate.name;
36 this.spanId = delegate.spanId;
37 this.traceId = delegate.traceId;
38 this.parent = options.parent;
39 this.annotations = options.annotations;
40 this.links = [...options.links];
41 this.sampled = delegate.sampled;
42 this.kind = delegate.kind;
43 this.status = {
44 _tag: "Started",
45 startTime: options.startTime,
46 };
47 this.attributes = new Map();
48 this.events = [];
49 }
50
51 end(endTime: bigint, exit: Exit.Exit<unknown, unknown>): void {
52 this.status = {
53 _tag: "Ended",
54 startTime: this.status.startTime,
55 endTime,
56 exit,
57 };
58 this.delegate.end(endTime, exit);
59
60 if (this.sampled) {
61 this.push(spanToTraceRecord(this));
62 }
63 }
64
65 attribute(key: string, value: unknown): void {
66 this.attributes.set(key, value);
67 this.delegate.attribute(key, value);
68 }
69
70 event(name: string, startTime: bigint, attributes?: Record<string, unknown>): void {
71 const nextAttributes = attributes ?? {};
72 this.events.push([name, startTime, nextAttributes]);
73 this.delegate.event(name, startTime, nextAttributes);
74 }
75
76 addLinks(links: ReadonlyArray<Tracer.SpanLink>): void {
77 this.links.push(...links);
78 this.delegate.addLinks(links);
79 }
80}
81
82export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (
83 options: LocalFileTracerOptions,
84) {
85 const sink = yield* makeTraceSink({
86 filePath: options.filePath,
87 maxBytes: options.maxBytes,
88 maxFiles: options.maxFiles,
89 batchWindowMs: options.batchWindowMs,
90 });
91
92 const delegate =
93 options.delegate ??
94 Tracer.make({
95 span: (spanOptions) => new Tracer.NativeSpan(spanOptions),
96 });
97
98 return Tracer.make({
99 span(spanOptions) {
100 return new LocalFileSpan(spanOptions, delegate.span(spanOptions), sink.push);
101 },
102 ...(delegate.context ? { context: delegate.context } : {}),
103 });
104});
apps/server/src/observability/Metrics.test.ts created+111
...@@ -0,0 +1,111 @@
1import { assert, describe, it } from "@effect/vitest";
2import { Effect, Metric } from "effect";
3
4import { withMetrics } from "./Metrics.ts";
5
6const hasMetricSnapshot = (
7 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
8 id: string,
9 attributes: Readonly<Record<string, string>>,
10) =>
11 snapshots.some(
12 (snapshot) =>
13 snapshot.id === id &&
14 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
15 );
16
17describe("withMetrics", () => {
18 it.effect("supports pipe-style usage", () =>
19 Effect.gen(function* () {
20 const counter = Metric.counter("with_metrics_pipe_total");
21 const timer = Metric.timer("with_metrics_pipe_duration");
22
23 const result = yield* Effect.succeed("ok").pipe(
24 withMetrics({
25 counter,
26 timer,
27 attributes: {
28 operation: "pipe",
29 },
30 }),
31 );
32
33 assert.equal(result, "ok");
34
35 const snapshots = yield* Metric.snapshot;
36 assert.equal(
37 hasMetricSnapshot(snapshots, "with_metrics_pipe_total", {
38 operation: "pipe",
39 outcome: "success",
40 }),
41 true,
42 );
43 assert.equal(
44 hasMetricSnapshot(snapshots, "with_metrics_pipe_duration", {
45 operation: "pipe",
46 }),
47 true,
48 );
49 }),
50 );
51
52 it.effect("supports direct invocation", () =>
53 Effect.gen(function* () {
54 const counter = Metric.counter("with_metrics_direct_total");
55
56 yield* withMetrics(Effect.fail("boom"), {
57 counter,
58 attributes: {
59 operation: "direct",
60 },
61 }).pipe(Effect.exit);
62
63 const snapshots = yield* Metric.snapshot;
64 assert.equal(
65 hasMetricSnapshot(snapshots, "with_metrics_direct_total", {
66 operation: "direct",
67 outcome: "failure",
68 }),
69 true,
70 );
71 }),
72 );
73
74 it.effect("evaluates attributes lazily after the wrapped effect runs", () =>
75 Effect.gen(function* () {
76 const counter = Metric.counter("with_metrics_lazy_total");
77 const timer = Metric.timer("with_metrics_lazy_duration");
78 let provider = "unknown";
79
80 yield* Effect.sync(() => {
81 provider = "codex";
82 }).pipe(
83 withMetrics({
84 counter,
85 timer,
86 attributes: () => ({
87 provider,
88 operation: "lazy",
89 }),
90 }),
91 );
92
93 const snapshots = yield* Metric.snapshot;
94 assert.equal(
95 hasMetricSnapshot(snapshots, "with_metrics_lazy_total", {
96 provider: "codex",
97 operation: "lazy",
98 outcome: "success",
99 }),
100 true,
101 );
102 assert.equal(
103 hasMetricSnapshot(snapshots, "with_metrics_lazy_duration", {
104 provider: "codex",
105 operation: "lazy",
106 }),
107 true,
108 );
109 }),
110 );
111});
apps/server/src/observability/Metrics.ts created+160
...@@ -0,0 +1,160 @@
1import { Duration, Effect, Exit, Metric } from "effect";
2import { dual } from "effect/Function";
3
4import {
5 compactMetricAttributes,
6 normalizeModelMetricLabel,
7 outcomeFromExit,
8} from "./Attributes.ts";
9
10export const rpcRequestsTotal = Metric.counter("t3_rpc_requests_total", {
11 description: "Total RPC requests handled by the websocket RPC server.",
12});
13
14export const rpcRequestDuration = Metric.timer("t3_rpc_request_duration", {
15 description: "RPC request handling duration.",
16});
17
18export const orchestrationCommandsTotal = Metric.counter("t3_orchestration_commands_total", {
19 description: "Total orchestration commands dispatched.",
20});
21
22export const orchestrationCommandDuration = Metric.timer("t3_orchestration_command_duration", {
23 description: "Orchestration command dispatch duration.",
24});
25
26export const orchestrationCommandAckDuration = Metric.timer(
27 "t3_orchestration_command_ack_duration",
28 {
29 description:
30 "Time from orchestration command dispatch to the first committed domain event emitted for that command.",
31 },
32);
33
34export const orchestrationEventsProcessedTotal = Metric.counter(
35 "t3_orchestration_events_processed_total",
36 {
37 description: "Total orchestration intent events processed by runtime reactors.",
38 },
39);
40
41export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
42 description: "Total provider session lifecycle operations.",
43});
44
45export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", {
46 description: "Total provider turn lifecycle operations.",
47});
48
49export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
50 description: "Provider turn request duration.",
51});
52
53export const providerRuntimeEventsTotal = Metric.counter("t3_provider_runtime_events_total", {
54 description: "Total canonical provider runtime events processed.",
55});
56
57export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
58 description: "Total git commands executed by the server runtime.",
59});
60
61export const gitCommandDuration = Metric.timer("t3_git_command_duration", {
62 description: "Git command execution duration.",
63});
64
65export const terminalSessionsTotal = Metric.counter("t3_terminal_sessions_total", {
66 description: "Total terminal sessions started.",
67});
68
69export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total", {
70 description: "Total terminal restart requests handled.",
71});
72
73export const metricAttributes = (
74 attributes: Readonly<Record<string, unknown>>,
75): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));
76
77export const increment = (
78 metric: Metric.Metric<number, unknown>,
79 attributes: Readonly<Record<string, unknown>>,
80 amount = 1,
81) => Metric.update(Metric.withAttributes(metric, metricAttributes(attributes)), amount);
82
83export interface WithMetricsOptions {
84 readonly counter?: Metric.Metric<number, unknown>;
85 readonly timer?: Metric.Metric<Duration.Duration, unknown>;
86 readonly attributes?:
87 | Readonly<Record<string, unknown>>
88 | (() => Readonly<Record<string, unknown>>);
89 readonly outcomeAttributes?: (
90 outcome: ReturnType<typeof outcomeFromExit>,
91 ) => Readonly<Record<string, unknown>>;
92}
93
94const withMetricsImpl = <A, E, R>(
95 effect: Effect.Effect<A, E, R>,
96 options: WithMetricsOptions,
97): Effect.Effect<A, E, R> =>
98 Effect.gen(function* () {
99 const startedAt = Date.now();
100 const exit = yield* Effect.exit(effect);
101 const duration = Duration.millis(Math.max(0, Date.now() - startedAt));
102 const baseAttributes =
103 typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
104
105 if (options.timer) {
106 yield* Metric.update(
107 Metric.withAttributes(options.timer, metricAttributes(baseAttributes)),
108 duration,
109 );
110 }
111
112 if (options.counter) {
113 const outcome = outcomeFromExit(exit);
114 yield* Metric.update(
115 Metric.withAttributes(
116 options.counter,
117 metricAttributes({
118 ...baseAttributes,
119 outcome,
120 ...(options.outcomeAttributes ? options.outcomeAttributes(outcome) : {}),
121 }),
122 ),
123 1,
124 );
125 }
126
127 if (Exit.isSuccess(exit)) {
128 return exit.value;
129 }
130 return yield* Effect.failCause(exit.cause);
131 });
132
133export const withMetrics: {
134 <A, E, R>(
135 options: WithMetricsOptions,
136 ): (effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
137 <A, E, R>(effect: Effect.Effect<A, E, R>, options: WithMetricsOptions): Effect.Effect<A, E, R>;
138} = dual(2, withMetricsImpl);
139
140export const providerMetricAttributes = (
141 provider: string,
142 extra?: Readonly<Record<string, unknown>>,
143) =>
144 compactMetricAttributes({
145 provider,
146 ...extra,
147 });
148
149export const providerTurnMetricAttributes = (input: {
150 readonly provider: string;
151 readonly model: string | null | undefined;
152 readonly extra?: Readonly<Record<string, unknown>>;
153}) => {
154 const modelFamily = normalizeModelMetricLabel(input.model);
155 return compactMetricAttributes({
156 provider: input.provider,
157 ...(modelFamily ? { modelFamily } : {}),
158 ...input.extra,
159 });
160};
apps/server/src/observability/RpcInstrumentation.test.ts created+161
...@@ -0,0 +1,161 @@
1import { assert, describe, it } from "@effect/vitest";
2import { Effect, Exit, Metric, Stream } from "effect";
3
4import {
5 observeRpcEffect,
6 observeRpcStream,
7 observeRpcStreamEffect,
8} from "./RpcInstrumentation.ts";
9
10const hasMetricSnapshot = (
11 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
12 id: string,
13 attributes: Readonly<Record<string, string>>,
14) =>
15 snapshots.some(
16 (snapshot) =>
17 snapshot.id === id &&
18 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
19 );
20
21describe("RpcInstrumentation", () => {
22 it.effect("records success metrics for unary RPC handlers", () =>
23 Effect.gen(function* () {
24 yield* observeRpcEffect("rpc.instrumentation.success", Effect.succeed("ok"), {
25 "rpc.aggregate": "test",
26 }).pipe(Effect.withSpan("rpc.instrumentation.success.span"));
27
28 const snapshots = yield* Metric.snapshot;
29
30 assert.equal(
31 hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
32 method: "rpc.instrumentation.success",
33 outcome: "success",
34 }),
35 true,
36 );
37 assert.equal(
38 hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
39 method: "rpc.instrumentation.success",
40 }),
41 true,
42 );
43 }),
44 );
45
46 it.effect("records failure outcomes for unary RPC handlers", () =>
47 Effect.gen(function* () {
48 yield* Effect.exit(
49 observeRpcEffect("rpc.instrumentation.failure", Effect.fail("boom"), {
50 "rpc.aggregate": "test",
51 }).pipe(Effect.withSpan("rpc.instrumentation.failure.span")),
52 );
53
54 const snapshots = yield* Metric.snapshot;
55
56 assert.equal(
57 hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
58 method: "rpc.instrumentation.failure",
59 outcome: "failure",
60 }),
61 true,
62 );
63 assert.equal(
64 hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
65 method: "rpc.instrumentation.failure",
66 }),
67 true,
68 );
69 }),
70 );
71
72 it.effect("records subscription activation metrics for stream RPC handlers", () =>
73 Effect.gen(function* () {
74 const events = yield* Stream.runCollect(
75 observeRpcStreamEffect(
76 "rpc.instrumentation.stream",
77 Effect.succeed(Stream.make("a", "b")),
78 { "rpc.aggregate": "test" },
79 ).pipe(Stream.withSpan("rpc.instrumentation.stream.span")),
80 );
81
82 assert.deepStrictEqual(Array.from(events), ["a", "b"]);
83
84 const snapshots = yield* Metric.snapshot;
85
86 assert.equal(
87 hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
88 method: "rpc.instrumentation.stream",
89 outcome: "success",
90 }),
91 true,
92 );
93 assert.equal(
94 hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
95 method: "rpc.instrumentation.stream",
96 }),
97 true,
98 );
99 }),
100 );
101
102 it.effect("records failure outcomes for direct stream RPC handlers during consumption", () =>
103 Effect.gen(function* () {
104 const exit = yield* Stream.runCollect(
105 observeRpcStream(
106 "rpc.instrumentation.stream.failure",
107 Stream.make("a").pipe(Stream.concat(Stream.fail("boom"))),
108 { "rpc.aggregate": "test" },
109 ).pipe(Stream.withSpan("rpc.instrumentation.stream.failure.span")),
110 ).pipe(Effect.exit);
111
112 assert.equal(Exit.isFailure(exit), true);
113
114 const snapshots = yield* Metric.snapshot;
115
116 assert.equal(
117 hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
118 method: "rpc.instrumentation.stream.failure",
119 outcome: "failure",
120 }),
121 true,
122 );
123 assert.equal(
124 hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
125 method: "rpc.instrumentation.stream.failure",
126 }),
127 true,
128 );
129 }),
130 );
131
132 it.effect("records failure outcomes when a stream RPC effect produces a failing stream", () =>
133 Effect.gen(function* () {
134 const exit = yield* Stream.runCollect(
135 observeRpcStreamEffect(
136 "rpc.instrumentation.stream.effect.failure",
137 Effect.succeed(Stream.fail("boom")),
138 { "rpc.aggregate": "test" },
139 ).pipe(Stream.withSpan("rpc.instrumentation.stream.effect.failure.span")),
140 ).pipe(Effect.exit);
141
142 assert.equal(Exit.isFailure(exit), true);
143
144 const snapshots = yield* Metric.snapshot;
145
146 assert.equal(
147 hasMetricSnapshot(snapshots, "t3_rpc_requests_total", {
148 method: "rpc.instrumentation.stream.effect.failure",
149 outcome: "failure",
150 }),
151 true,
152 );
153 assert.equal(
154 hasMetricSnapshot(snapshots, "t3_rpc_request_duration", {
155 method: "rpc.instrumentation.stream.effect.failure",
156 }),
157 true,
158 );
159 }),
160 );
161});
apps/server/src/observability/RpcInstrumentation.ts created+89
...@@ -0,0 +1,89 @@
1import { Duration, Effect, Exit, Metric, Stream } from "effect";
2
3import { outcomeFromExit } from "./Attributes.ts";
4import { metricAttributes, rpcRequestDuration, rpcRequestsTotal, withMetrics } from "./Metrics.ts";
5
6const annotateRpcSpan = (
7 method: string,
8 traceAttributes?: Readonly<Record<string, unknown>>,
9): Effect.Effect<void, never, never> =>
10 Effect.annotateCurrentSpan({
11 "rpc.method": method,
12 ...traceAttributes,
13 });
14
15const recordRpcStreamMetrics = <E>(
16 method: string,
17 startedAt: number,
18 exit: Exit.Exit<unknown, E>,
19): Effect.Effect<void, never, never> =>
20 Effect.gen(function* () {
21 yield* Metric.update(
22 Metric.withAttributes(rpcRequestDuration, metricAttributes({ method })),
23 Duration.millis(Math.max(0, Date.now() - startedAt)),
24 );
25 yield* Metric.update(
26 Metric.withAttributes(
27 rpcRequestsTotal,
28 metricAttributes({
29 method,
30 outcome: outcomeFromExit(exit),
31 }),
32 ),
33 1,
34 );
35 });
36
37export const observeRpcEffect = <A, E, R>(
38 method: string,
39 effect: Effect.Effect<A, E, R>,
40 traceAttributes?: Readonly<Record<string, unknown>>,
41): Effect.Effect<A, E, R> =>
42 Effect.gen(function* () {
43 yield* annotateRpcSpan(method, traceAttributes);
44
45 return yield* effect.pipe(
46 withMetrics({
47 counter: rpcRequestsTotal,
48 timer: rpcRequestDuration,
49 attributes: {
50 method,
51 },
52 }),
53 );
54 });
55
56export const observeRpcStream = <A, E, R>(
57 method: string,
58 stream: Stream.Stream<A, E, R>,
59 traceAttributes?: Readonly<Record<string, unknown>>,
60): Stream.Stream<A, E, R> =>
61 Stream.unwrap(
62 Effect.gen(function* () {
63 yield* annotateRpcSpan(method, traceAttributes);
64 const startedAt = Date.now();
65 return stream.pipe(Stream.onExit((exit) => recordRpcStreamMetrics(method, startedAt, exit)));
66 }),
67 );
68
69export const observeRpcStreamEffect = <A, StreamError, StreamContext, EffectError, EffectContext>(
70 method: string,
71 effect: Effect.Effect<Stream.Stream<A, StreamError, StreamContext>, EffectError, EffectContext>,
72 traceAttributes?: Readonly<Record<string, unknown>>,
73): Stream.Stream<A, StreamError | EffectError, StreamContext | EffectContext> =>
74 Stream.unwrap(
75 Effect.gen(function* () {
76 yield* annotateRpcSpan(method, traceAttributes);
77 const startedAt = Date.now();
78 const exit = yield* Effect.exit(effect);
79
80 if (Exit.isFailure(exit)) {
81 yield* recordRpcStreamMetrics(method, startedAt, exit);
82 return yield* Effect.failCause(exit.cause);
83 }
84
85 return exit.value.pipe(
86 Stream.onExit((streamExit) => recordRpcStreamMetrics(method, startedAt, streamExit)),
87 );
88 }),
89 );
apps/server/src/observability/TraceRecord.ts created+100
...@@ -0,0 +1,100 @@
1import { Cause, Exit, Option, Tracer } from "effect";
2
3import { compactTraceAttributes } from "./Attributes.ts";
4
5export interface TraceRecord {
6 readonly type: "effect-span";
7 readonly name: string;
8 readonly traceId: string;
9 readonly spanId: string;
10 readonly parentSpanId?: string;
11 readonly sampled: boolean;
12 readonly kind: Tracer.SpanKind;
13 readonly startTimeUnixNano: string;
14 readonly endTimeUnixNano: string;
15 readonly durationMs: number;
16 readonly attributes: Readonly<Record<string, unknown>>;
17 readonly events: ReadonlyArray<{
18 readonly name: string;
19 readonly timeUnixNano: string;
20 readonly attributes: Readonly<Record<string, unknown>>;
21 }>;
22 readonly links: ReadonlyArray<{
23 readonly traceId: string;
24 readonly spanId: string;
25 readonly attributes: Readonly<Record<string, unknown>>;
26 }>;
27 readonly exit:
28 | {
29 readonly _tag: "Success";
30 }
31 | {
32 readonly _tag: "Interrupted";
33 readonly cause: string;
34 }
35 | {
36 readonly _tag: "Failure";
37 readonly cause: string;
38 };
39}
40
41interface SerializableSpan {
42 readonly name: string;
43 readonly traceId: string;
44 readonly spanId: string;
45 readonly parent: Option.Option<Tracer.AnySpan>;
46 readonly status: Tracer.SpanStatus;
47 readonly sampled: boolean;
48 readonly kind: Tracer.SpanKind;
49 readonly attributes: ReadonlyMap<string, unknown>;
50 readonly links: ReadonlyArray<Tracer.SpanLink>;
51 readonly events: ReadonlyArray<
52 readonly [name: string, startTime: bigint, attributes: Record<string, unknown>]
53 >;
54}
55
56function formatTraceExit(exit: Exit.Exit<unknown, unknown>): TraceRecord["exit"] {
57 if (Exit.isSuccess(exit)) {
58 return { _tag: "Success" };
59 }
60 if (Cause.hasInterruptsOnly(exit.cause)) {
61 return {
62 _tag: "Interrupted",
63 cause: Cause.pretty(exit.cause),
64 };
65 }
66 return {
67 _tag: "Failure",
68 cause: Cause.pretty(exit.cause),
69 };
70}
71
72export function spanToTraceRecord(span: SerializableSpan): TraceRecord {
73 const status = span.status as Extract<Tracer.SpanStatus, { _tag: "Ended" }>;
74 const parentSpanId = Option.getOrUndefined(span.parent)?.spanId;
75
76 return {
77 type: "effect-span",
78 name: span.name,
79 traceId: span.traceId,
80 spanId: span.spanId,
81 ...(parentSpanId ? { parentSpanId } : {}),
82 sampled: span.sampled,
83 kind: span.kind,
84 startTimeUnixNano: String(status.startTime),
85 endTimeUnixNano: String(status.endTime),
86 durationMs: Number(status.endTime - status.startTime) / 1_000_000,
87 attributes: compactTraceAttributes(Object.fromEntries(span.attributes)),
88 events: span.events.map(([name, startTime, attributes]) => ({
89 name,
90 timeUnixNano: String(startTime),
91 attributes: compactTraceAttributes(attributes),
92 })),
93 links: span.links.map((link) => ({
94 traceId: link.span.traceId,
95 spanId: link.span.spanId,
96 attributes: compactTraceAttributes(link.attributes),
97 })),
98 exit: formatTraceExit(status.exit),
99 };
100}
apps/server/src/observability/TraceSink.test.ts created+152
...@@ -0,0 +1,152 @@
1import fs from "node:fs";
2import os from "node:os";
3import path from "node:path";
4
5import { assert, describe, it } from "@effect/vitest";
6import { Effect } from "effect";
7
8import type { TraceRecord } from "./TraceRecord.ts";
9import { makeTraceSink } from "./TraceSink.ts";
10
11const makeRecord = (name: string, suffix = ""): TraceRecord => ({
12 type: "effect-span",
13 name,
14 traceId: `trace-${name}-${suffix}`,
15 spanId: `span-${name}-${suffix}`,
16 sampled: true,
17 kind: "internal",
18 startTimeUnixNano: "1",
19 endTimeUnixNano: "2",
20 durationMs: 1,
21 attributes: {
22 payload: suffix,
23 },
24 events: [],
25 links: [],
26 exit: {
27 _tag: "Success",
28 },
29});
30
31describe("TraceSink", () => {
32 it.effect("flushes buffered records on close", () =>
33 Effect.scoped(
34 Effect.gen(function* () {
35 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
36 const tracePath = path.join(tempDir, "server.trace.ndjson");
37
38 try {
39 const sink = yield* makeTraceSink({
40 filePath: tracePath,
41 maxBytes: 1024,
42 maxFiles: 2,
43 batchWindowMs: 10_000,
44 });
45
46 sink.push(makeRecord("alpha"));
47 sink.push(makeRecord("beta"));
48 yield* sink.close();
49
50 const lines = fs
51 .readFileSync(tracePath, "utf8")
52 .trim()
53 .split("\n")
54 .map((line) => JSON.parse(line) as TraceRecord);
55
56 assert.equal(lines.length, 2);
57 assert.equal(lines[0]?.name, "alpha");
58 assert.equal(lines[1]?.name, "beta");
59 } finally {
60 fs.rmSync(tempDir, { recursive: true, force: true });
61 }
62 }),
63 ),
64 );
65
66 it.effect("rotates the trace file when the configured max size is exceeded", () =>
67 Effect.scoped(
68 Effect.gen(function* () {
69 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
70 const tracePath = path.join(tempDir, "server.trace.ndjson");
71
72 try {
73 const sink = yield* makeTraceSink({
74 filePath: tracePath,
75 maxBytes: 180,
76 maxFiles: 2,
77 batchWindowMs: 10_000,
78 });
79
80 for (let index = 0; index < 8; index += 1) {
81 sink.push(makeRecord("rotate", `${index}-${"x".repeat(48)}`));
82 yield* sink.flush;
83 }
84 yield* sink.close();
85
86 const matchingFiles = fs
87 .readdirSync(tempDir)
88 .filter(
89 (entry) =>
90 entry === "server.trace.ndjson" || entry.startsWith("server.trace.ndjson."),
91 )
92 .toSorted();
93
94 assert.equal(
95 matchingFiles.some((entry) => entry === "server.trace.ndjson.1"),
96 true,
97 );
98 assert.equal(
99 matchingFiles.some((entry) => entry === "server.trace.ndjson.3"),
100 false,
101 );
102 } finally {
103 fs.rmSync(tempDir, { recursive: true, force: true });
104 }
105 }),
106 ),
107 );
108
109 it.effect("drops only the invalid record when serialization fails", () =>
110 Effect.scoped(
111 Effect.gen(function* () {
112 const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "t3-trace-sink-"));
113 const tracePath = path.join(tempDir, "server.trace.ndjson");
114
115 try {
116 const sink = yield* makeTraceSink({
117 filePath: tracePath,
118 maxBytes: 1024,
119 maxFiles: 2,
120 batchWindowMs: 10_000,
121 });
122
123 const circular: Array<unknown> = [];
124 circular.push(circular);
125
126 sink.push(makeRecord("alpha"));
127 sink.push({
128 ...makeRecord("invalid"),
129 attributes: {
130 circular,
131 },
132 } as TraceRecord);
133 sink.push(makeRecord("beta"));
134 yield* sink.close();
135
136 const lines = fs
137 .readFileSync(tracePath, "utf8")
138 .trim()
139 .split("\n")
140 .map((line) => JSON.parse(line) as TraceRecord);
141
142 assert.deepStrictEqual(
143 lines.map((line) => line.name),
144 ["alpha", "beta"],
145 );
146 } finally {
147 fs.rmSync(tempDir, { recursive: true, force: true });
148 }
149 }),
150 ),
151 );
152});
apps/server/src/observability/TraceSink.ts created+68
...@@ -0,0 +1,68 @@
1import { RotatingFileSink } from "@t3tools/shared/logging";
2import { Effect } from "effect";
3
4import type { TraceRecord } from "./TraceRecord.ts";
5
6const FLUSH_BUFFER_THRESHOLD = 32;
7
8export interface TraceSinkOptions {
9 readonly filePath: string;
10 readonly maxBytes: number;
11 readonly maxFiles: number;
12 readonly batchWindowMs: number;
13}
14
15export interface TraceSink {
16 readonly filePath: string;
17 push: (record: TraceRecord) => void;
18 flush: Effect.Effect<void>;
19 close: () => Effect.Effect<void>;
20}
21
22export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: TraceSinkOptions) {
23 const sink = new RotatingFileSink({
24 filePath: options.filePath,
25 maxBytes: options.maxBytes,
26 maxFiles: options.maxFiles,
27 });
28
29 let buffer: Array<string> = [];
30
31 const flushUnsafe = () => {
32 if (buffer.length === 0) {
33 return;
34 }
35
36 const chunk = buffer.join("");
37 buffer = [];
38
39 try {
40 sink.write(chunk);
41 } catch {
42 buffer.unshift(chunk);
43 }
44 };
45
46 const flush = Effect.sync(flushUnsafe).pipe(Effect.withTracerEnabled(false));
47
48 yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));
49 yield* Effect.forkScoped(
50 Effect.sleep(`${options.batchWindowMs} millis`).pipe(Effect.andThen(flush), Effect.forever),
51 );
52
53 return {
54 filePath: options.filePath,
55 push(record) {
56 try {
57 buffer.push(`${JSON.stringify(record)}\n`);
58 if (buffer.length >= FLUSH_BUFFER_THRESHOLD) {
59 flushUnsafe();
60 }
61 } catch {
62 return;
63 }
64 },
65 flush,
66 close: () => flush,
67 } satisfies TraceSink;
68});
apps/server/src/open.ts+6-1
...@@ -233,10 +233,15 @@ export class Open extends ServiceMap.Service<Open, OpenShape>()("t3/open") {}...@@ -233,10 +233,15 @@ export class Open extends ServiceMap.Service<Open, OpenShape>()("t3/open") {}
233// Implementations233// Implementations
234// ==============================234// ==============================
235235
236export const resolveEditorLaunch = Effect.fnUntraced(function* (236export const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* (
237 input: OpenInEditorInput,237 input: OpenInEditorInput,
238 platform: NodeJS.Platform = process.platform,238 platform: NodeJS.Platform = process.platform,
239): Effect.fn.Return<EditorLaunch, OpenError> {239): Effect.fn.Return<EditorLaunch, OpenError> {
240 yield* Effect.annotateCurrentSpan({
241 "open.editor": input.editor,
242 "open.cwd": input.cwd,
243 "open.platform": platform,
244 });
240 const editorDef = EDITORS.find((editor) => editor.id === input.editor);245 const editorDef = EDITORS.find((editor) => editor.id === input.editor);
241 if (!editorDef) {246 if (!editorDef) {
242 return yield* new OpenError({ message: `Unknown editor: ${input.editor}` });247 return yield* new OpenError({ message: `Unknown editor: ${input.editor}` });
apps/server/src/orchestration/Layers/CheckpointReactor.ts+116-112
...@@ -119,7 +119,7 @@ const make = Effect.gen(function* () {...@@ -119,7 +119,7 @@ const make = Effect.gen(function* () {
119 createdAt: input.createdAt,119 createdAt: input.createdAt,
120 });120 });
121121
122 const resolveSessionRuntimeForThread = Effect.fnUntraced(function* (122 const resolveSessionRuntimeForThread = Effect.fn("resolveSessionRuntimeForThread")(function* (
123 threadId: ThreadId,123 threadId: ThreadId,
124 ): Effect.fn.Return<Option.Option<{ readonly threadId: ThreadId; readonly cwd: string }>> {124 ): Effect.fn.Return<Option.Option<{ readonly threadId: ThreadId; readonly cwd: string }>> {
125 const readModel = yield* orchestrationEngine.getReadModel();125 const readModel = yield* orchestrationEngine.getReadModel();
...@@ -153,7 +153,7 @@ const make = Effect.gen(function* () {...@@ -153,7 +153,7 @@ const make = Effect.gen(function* () {
153 // active provider session CWD and falling back to the thread/project config.153 // active provider session CWD and falling back to the thread/project config.
154 // Returns undefined when no CWD can be determined or the workspace is not154 // Returns undefined when no CWD can be determined or the workspace is not
155 // a git repository.155 // a git repository.
156 const resolveCheckpointCwd = Effect.fnUntraced(function* (input: {156 const resolveCheckpointCwd = Effect.fn("resolveCheckpointCwd")(function* (input: {
157 readonly threadId: ThreadId;157 readonly threadId: ThreadId;
158 readonly thread: { readonly projectId: ProjectId; readonly worktreePath: string | null };158 readonly thread: { readonly projectId: ProjectId; readonly worktreePath: string | null };
159 readonly projects: ReadonlyArray<{ readonly id: ProjectId; readonly workspaceRoot: string }>;159 readonly projects: ReadonlyArray<{ readonly id: ProjectId; readonly workspaceRoot: string }>;
...@@ -188,7 +188,7 @@ const make = Effect.gen(function* () {...@@ -188,7 +188,7 @@ const make = Effect.gen(function* () {
188 // Shared tail for both capture paths: creates the git checkpoint ref, diffs188 // Shared tail for both capture paths: creates the git checkpoint ref, diffs
189 // it against the previous turn, then dispatches the domain events to update189 // it against the previous turn, then dispatches the domain events to update
190 // the orchestration read model.190 // the orchestration read model.
191 const captureAndDispatchCheckpoint = Effect.fnUntraced(function* (input: {191 const captureAndDispatchCheckpoint = Effect.fn("captureAndDispatchCheckpoint")(function* (input: {
192 readonly threadId: ThreadId;192 readonly threadId: ThreadId;
193 readonly turnId: TurnId;193 readonly turnId: TurnId;
194 readonly thread: {194 readonly thread: {
...@@ -321,70 +321,70 @@ const make = Effect.gen(function* () {...@@ -321,70 +321,70 @@ const make = Effect.gen(function* () {
321 });321 });
322322
323 // Captures a real git checkpoint when a turn completes via a runtime event.323 // Captures a real git checkpoint when a turn completes via a runtime event.
324 const captureCheckpointFromTurnCompletion = Effect.fnUntraced(function* (324 const captureCheckpointFromTurnCompletion = Effect.fn("captureCheckpointFromTurnCompletion")(
325 event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>,325 function* (event: Extract<ProviderRuntimeEvent, { type: "turn.completed" }>) {
326 ) {326 const turnId = toTurnId(event.turnId);
327 const turnId = toTurnId(event.turnId);327 if (!turnId) {
328 if (!turnId) {328 return;
329 return;329 }
330 }
331
332 const readModel = yield* orchestrationEngine.getReadModel();
333 const thread = readModel.threads.find((entry) => entry.id === event.threadId);
334 if (!thread) {
335 return;
336 }
337330
338 // When a primary turn is active, only that turn may produce completion checkpoints.331 const readModel = yield* orchestrationEngine.getReadModel();
339 if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {332 const thread = readModel.threads.find((entry) => entry.id === event.threadId);
340 return;333 if (!thread) {
341 }334 return;
335 }
342336
343 // Only skip if a real (non-placeholder) checkpoint already exists for this turn.337 // When a primary turn is active, only that turn may produce completion checkpoints.
344 // ProviderRuntimeIngestion may insert placeholder entries with status "missing"338 if (thread.session?.activeTurnId && !sameId(thread.session.activeTurnId, turnId)) {
345 // before this reactor runs; those must not prevent real git capture.339 return;
346 if (340 }
347 thread.checkpoints.some(
348 (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
349 )
350 ) {
351 return;
352 }
353341
354 const checkpointCwd = yield* resolveCheckpointCwd({342 // Only skip if a real (non-placeholder) checkpoint already exists for this turn.
355 threadId: thread.id,343 // ProviderRuntimeIngestion may insert placeholder entries with status "missing"
356 thread,344 // before this reactor runs; those must not prevent real git capture.
357 projects: readModel.projects,345 if (
358 preferSessionRuntime: true,346 thread.checkpoints.some(
359 });347 (checkpoint) => checkpoint.turnId === turnId && checkpoint.status !== "missing",
360 if (!checkpointCwd) {348 )
361 return;349 ) {
362 }350 return;
351 }
363352
364 // If a placeholder checkpoint exists for this turn, reuse its turn count353 const checkpointCwd = yield* resolveCheckpointCwd({
365 // instead of incrementing past it.354 threadId: thread.id,
366 const existingPlaceholder = thread.checkpoints.find(355 thread,
367 (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",356 projects: readModel.projects,
368 );357 preferSessionRuntime: true,
369 const currentTurnCount = thread.checkpoints.reduce(358 });
370 (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),359 if (!checkpointCwd) {
371 0,360 return;
372 );361 }
373 const nextTurnCount = existingPlaceholder
374 ? existingPlaceholder.checkpointTurnCount
375 : currentTurnCount + 1;
376362
377 yield* captureAndDispatchCheckpoint({363 // If a placeholder checkpoint exists for this turn, reuse its turn count
378 threadId: thread.id,364 // instead of incrementing past it.
379 turnId,365 const existingPlaceholder = thread.checkpoints.find(
380 thread,366 (checkpoint) => checkpoint.turnId === turnId && checkpoint.status === "missing",
381 cwd: checkpointCwd,367 );
382 turnCount: nextTurnCount,368 const currentTurnCount = thread.checkpoints.reduce(
383 status: checkpointStatusFromRuntime(event.payload.state),369 (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
384 assistantMessageId: undefined,370 0,
385 createdAt: event.createdAt,371 );
386 });372 const nextTurnCount = existingPlaceholder
387 });373 ? existingPlaceholder.checkpointTurnCount
374 : currentTurnCount + 1;
375
376 yield* captureAndDispatchCheckpoint({
377 threadId: thread.id,
378 turnId,
379 thread,
380 cwd: checkpointCwd,
381 turnCount: nextTurnCount,
382 status: checkpointStatusFromRuntime(event.payload.state),
383 assistantMessageId: undefined,
384 createdAt: event.createdAt,
385 });
386 },
387 );
388388
389 // Captures a real git checkpoint when a placeholder checkpoint (status "missing")389 // Captures a real git checkpoint when a placeholder checkpoint (status "missing")
390 // is detected via a domain event. This replaces the placeholder with a real390 // is detected via a domain event. This replaces the placeholder with a real
...@@ -394,7 +394,7 @@ const make = Effect.gen(function* () {...@@ -394,7 +394,7 @@ const make = Effect.gen(function* () {
394 // events from the Codex runtime. This handler fires when the corresponding394 // events from the Codex runtime. This handler fires when the corresponding
395 // domain event arrives, allowing the reactor to capture the actual filesystem395 // domain event arrives, allowing the reactor to capture the actual filesystem
396 // state into a git ref and dispatch a replacement checkpoint.396 // state into a git ref and dispatch a replacement checkpoint.
397 const captureCheckpointFromPlaceholder = Effect.fnUntraced(function* (397 const captureCheckpointFromPlaceholder = Effect.fn("captureCheckpointFromPlaceholder")(function* (
398 event: Extract<OrchestrationEvent, { type: "thread.turn-diff-completed" }>,398 event: Extract<OrchestrationEvent, { type: "thread.turn-diff-completed" }>,
399 ) {399 ) {
400 const { threadId, turnId, checkpointTurnCount, status } = event.payload;400 const { threadId, turnId, checkpointTurnCount, status } = event.payload;
...@@ -448,57 +448,59 @@ const make = Effect.gen(function* () {...@@ -448,57 +448,59 @@ const make = Effect.gen(function* () {
448 });448 });
449 });449 });
450450
451 const ensurePreTurnBaselineFromTurnStart = Effect.fnUntraced(function* (451 const ensurePreTurnBaselineFromTurnStart = Effect.fn("ensurePreTurnBaselineFromTurnStart")(
452 event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>,452 function* (event: Extract<ProviderRuntimeEvent, { type: "turn.started" }>) {
453 ) {453 const turnId = toTurnId(event.turnId);
454 const turnId = toTurnId(event.turnId);454 if (!turnId) {
455 if (!turnId) {455 return;
456 return;456 }
457 }
458457
459 const readModel = yield* orchestrationEngine.getReadModel();458 const readModel = yield* orchestrationEngine.getReadModel();
460 const thread = readModel.threads.find((entry) => entry.id === event.threadId);459 const thread = readModel.threads.find((entry) => entry.id === event.threadId);
461 if (!thread) {460 if (!thread) {
462 return;461 return;
463 }462 }
464463
465 const checkpointCwd = yield* resolveCheckpointCwd({464 const checkpointCwd = yield* resolveCheckpointCwd({
466 threadId: thread.id,465 threadId: thread.id,
467 thread,466 thread,
468 projects: readModel.projects,467 projects: readModel.projects,
469 preferSessionRuntime: false,468 preferSessionRuntime: false,
470 });469 });
471 if (!checkpointCwd) {470 if (!checkpointCwd) {
472 return;471 return;
473 }472 }
474473
475 const currentTurnCount = thread.checkpoints.reduce(474 const currentTurnCount = thread.checkpoints.reduce(
476 (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),475 (maxTurnCount, checkpoint) => Math.max(maxTurnCount, checkpoint.checkpointTurnCount),
477 0,476 0,
478 );477 );
479 const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);478 const baselineCheckpointRef = checkpointRefForThreadTurn(thread.id, currentTurnCount);
480 const baselineExists = yield* checkpointStore.hasCheckpointRef({479 const baselineExists = yield* checkpointStore.hasCheckpointRef({
481 cwd: checkpointCwd,480 cwd: checkpointCwd,
482 checkpointRef: baselineCheckpointRef,481 checkpointRef: baselineCheckpointRef,
483 });482 });
484 if (baselineExists) {483 if (baselineExists) {
485 return;484 return;
486 }485 }
487486
488 yield* checkpointStore.captureCheckpoint({487 yield* checkpointStore.captureCheckpoint({
489 cwd: checkpointCwd,488 cwd: checkpointCwd,
490 checkpointRef: baselineCheckpointRef,489 checkpointRef: baselineCheckpointRef,
491 });490 });
492 yield* receiptBus.publish({491 yield* receiptBus.publish({
493 type: "checkpoint.baseline.captured",492 type: "checkpoint.baseline.captured",
494 threadId: thread.id,493 threadId: thread.id,
495 checkpointTurnCount: currentTurnCount,494 checkpointTurnCount: currentTurnCount,
496 checkpointRef: baselineCheckpointRef,495 checkpointRef: baselineCheckpointRef,
497 createdAt: event.createdAt,496 createdAt: event.createdAt,
498 });497 });
499 });498 },
499 );
500500
501 const ensurePreTurnBaselineFromDomainTurnStart = Effect.fnUntraced(function* (501 const ensurePreTurnBaselineFromDomainTurnStart = Effect.fn(
502 "ensurePreTurnBaselineFromDomainTurnStart",
503 )(function* (
502 event: Extract<504 event: Extract<
503 OrchestrationEvent,505 OrchestrationEvent,
504 { type: "thread.turn-start-requested" | "thread.message-sent" }506 { type: "thread.turn-start-requested" | "thread.message-sent" }
...@@ -557,7 +559,7 @@ const make = Effect.gen(function* () {...@@ -557,7 +559,7 @@ const make = Effect.gen(function* () {
557 });559 });
558 });560 });
559561
560 const handleRevertRequested = Effect.fnUntraced(function* (562 const handleRevertRequested = Effect.fn("handleRevertRequested")(function* (
561 event: Extract<OrchestrationEvent, { type: "thread.checkpoint-revert-requested" }>,563 event: Extract<OrchestrationEvent, { type: "thread.checkpoint-revert-requested" }>,
562 ) {564 ) {
563 const now = new Date().toISOString();565 const now = new Date().toISOString();
...@@ -685,7 +687,7 @@ const make = Effect.gen(function* () {...@@ -685,7 +687,7 @@ const make = Effect.gen(function* () {
685 );687 );
686 });688 });
687689
688 const processDomainEvent = Effect.fnUntraced(function* (event: OrchestrationEvent) {690 const processDomainEvent = Effect.fn("processDomainEvent")(function* (event: OrchestrationEvent) {
689 if (event.type === "thread.turn-start-requested" || event.type === "thread.message-sent") {691 if (event.type === "thread.turn-start-requested" || event.type === "thread.message-sent") {
690 yield* ensurePreTurnBaselineFromDomainTurnStart(event);692 yield* ensurePreTurnBaselineFromDomainTurnStart(event);
691 return;693 return;
...@@ -724,7 +726,9 @@ const make = Effect.gen(function* () {...@@ -724,7 +726,9 @@ const make = Effect.gen(function* () {
724 }726 }
725 });727 });
726728
727 const processRuntimeEvent = Effect.fnUntraced(function* (event: ProviderRuntimeEvent) {729 const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
730 event: ProviderRuntimeEvent,
731 ) {
728 if (event.type === "turn.started") {732 if (event.type === "turn.started") {
729 yield* ensurePreTurnBaselineFromTurnStart(event);733 yield* ensurePreTurnBaselineFromTurnStart(event);
730 return;734 return;
apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+101-1
...@@ -8,7 +8,7 @@ import {...@@ -8,7 +8,7 @@ import {
8 TurnId,8 TurnId,
9 type OrchestrationEvent,9 type OrchestrationEvent,
10} from "@t3tools/contracts";10} from "@t3tools/contracts";
11import { Effect, Layer, ManagedRuntime, Option, Queue, Stream } from "effect";11import { Effect, Layer, ManagedRuntime, Metric, Option, Queue, Stream } from "effect";
12import { describe, expect, it } from "vitest";12import { describe, expect, it } from "vitest";
1313
14import { PersistenceSqlError } from "../../persistence/Errors.ts";14import { PersistenceSqlError } from "../../persistence/Errors.ts";
...@@ -62,6 +62,17 @@ function now() {...@@ -62,6 +62,17 @@ function now() {
62 return new Date().toISOString();62 return new Date().toISOString();
63}63}
6464
65const hasMetricSnapshot = (
66 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
67 id: string,
68 attributes: Readonly<Record<string, string>>,
69) =>
70 snapshots.some(
71 (snapshot) =>
72 snapshot.id === id &&
73 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
74 );
75
65describe("OrchestrationEngine", () => {76describe("OrchestrationEngine", () => {
66 it("bootstraps the in-memory read model from persisted projections", async () => {77 it("bootstraps the in-memory read model from persisted projections", async () => {
67 const failOnHistoricalReplayStore: OrchestrationEventStoreShape = {78 const failOnHistoricalReplayStore: OrchestrationEventStoreShape = {
...@@ -408,6 +419,95 @@ describe("OrchestrationEngine", () => {...@@ -408,6 +419,95 @@ describe("OrchestrationEngine", () => {
408 await system.dispose();419 await system.dispose();
409 });420 });
410421
422 it("records command ack duration using the first committed event type", async () => {
423 const system = await createOrchestrationSystem();
424 const { engine } = system;
425 const createdAt = now();
426
427 await system.run(
428 engine.dispatch({
429 type: "project.create",
430 commandId: CommandId.makeUnsafe("cmd-project-ack-create"),
431 projectId: asProjectId("project-ack"),
432 title: "Ack Project",
433 workspaceRoot: "/tmp/project-ack",
434 defaultModelSelection: {
435 provider: "codex",
436 model: "gpt-5-codex",
437 },
438 createdAt,
439 }),
440 );
441
442 await system.run(
443 engine.dispatch({
444 type: "thread.create",
445 commandId: CommandId.makeUnsafe("cmd-thread-ack-create"),
446 threadId: ThreadId.makeUnsafe("thread-ack"),
447 projectId: asProjectId("project-ack"),
448 title: "Ack Thread",
449 modelSelection: {
450 provider: "codex",
451 model: "gpt-5-codex",
452 },
453 interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
454 runtimeMode: "full-access",
455 branch: null,
456 worktreePath: null,
457 createdAt,
458 }),
459 );
460
461 const snapshots = await system.run(Metric.snapshot);
462 expect(
463 hasMetricSnapshot(snapshots, "t3_orchestration_command_ack_duration", {
464 commandType: "thread.create",
465 aggregateKind: "thread",
466 ackEventType: "thread.created",
467 }),
468 ).toBe(true);
469
470 await system.dispose();
471 });
472
473 it("records failed command dispatches as metric failures", async () => {
474 const system = await createOrchestrationSystem();
475 const { engine } = system;
476 const createdAt = now();
477
478 await expect(
479 system.run(
480 engine.dispatch({
481 type: "thread.create",
482 commandId: CommandId.makeUnsafe("cmd-thread-missing-project"),
483 threadId: ThreadId.makeUnsafe("thread-missing-project"),
484 projectId: asProjectId("project-missing"),
485 title: "Missing Project Thread",
486 modelSelection: {
487 provider: "codex",
488 model: "gpt-5-codex",
489 },
490 interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
491 runtimeMode: "full-access",
492 branch: null,
493 worktreePath: null,
494 createdAt,
495 }),
496 ),
497 ).rejects.toThrow("does not exist");
498
499 const snapshots = await system.run(Metric.snapshot);
500 expect(
501 hasMetricSnapshot(snapshots, "t3_orchestration_commands_total", {
502 commandType: "thread.create",
503 aggregateKind: "thread",
504 outcome: "failure",
505 }),
506 ).toBe(true);
507
508 await system.dispose();
509 });
510
411 it("stores completed checkpoint summaries even when no files changed", async () => {511 it("stores completed checkpoint summaries even when no files changed", async () => {
412 const system = await createOrchestrationSystem();512 const system = await createOrchestrationSystem();
413 const { engine } = system;513 const { engine } = system;
apps/server/src/orchestration/Layers/OrchestrationEngine.ts+168-95
...@@ -5,9 +5,28 @@ import type {...@@ -5,9 +5,28 @@ import type {
5 ThreadId,5 ThreadId,
6} from "@t3tools/contracts";6} from "@t3tools/contracts";
7import { OrchestrationCommand } from "@t3tools/contracts";7import { OrchestrationCommand } from "@t3tools/contracts";
8import { Deferred, Effect, Layer, Option, PubSub, Queue, Schema, Stream } from "effect";8import {
9 Cause,
10 Deferred,
11 Duration,
12 Effect,
13 Exit,
14 Layer,
15 Metric,
16 Option,
17 PubSub,
18 Queue,
19 Schema,
20 Stream,
21} from "effect";
9import * as SqlClient from "effect/unstable/sql/SqlClient";22import * as SqlClient from "effect/unstable/sql/SqlClient";
1023
24import {
25 metricAttributes,
26 orchestrationCommandAckDuration,
27 orchestrationCommandsTotal,
28 orchestrationCommandDuration,
29} from "../../observability/Metrics.ts";
11import { toPersistenceSqlError } from "../../persistence/Errors.ts";30import { toPersistenceSqlError } from "../../persistence/Errors.ts";
12import { OrchestrationEventStore } from "../../persistence/Services/OrchestrationEventStore.ts";31import { OrchestrationEventStore } from "../../persistence/Services/OrchestrationEventStore.ts";
13import { OrchestrationCommandReceiptRepository } from "../../persistence/Services/OrchestrationCommandReceipts.ts";32import { OrchestrationCommandReceiptRepository } from "../../persistence/Services/OrchestrationCommandReceipts.ts";
...@@ -28,6 +47,7 @@ import {...@@ -28,6 +47,7 @@ import {
28interface CommandEnvelope {47interface CommandEnvelope {
29 command: OrchestrationCommand;48 command: OrchestrationCommand;
30 result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;49 result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;
50 startedAtMs: number;
31}51}
3252
33function commandToAggregateRef(command: OrchestrationCommand): {53function commandToAggregateRef(command: OrchestrationCommand): {
...@@ -64,6 +84,12 @@ const makeOrchestrationEngine = Effect.gen(function* () {...@@ -64,6 +84,12 @@ const makeOrchestrationEngine = Effect.gen(function* () {
6484
65 const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {85 const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {
66 const dispatchStartSequence = readModel.snapshotSequence;86 const dispatchStartSequence = readModel.snapshotSequence;
87 const processingStartedAtMs = Date.now();
88 const aggregateRef = commandToAggregateRef(envelope.command);
89 const baseMetricAttributes = {
90 commandType: envelope.command.type,
91 aggregateKind: aggregateRef.aggregateKind,
92 } as const;
67 const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {93 const reconcileReadModelAfterDispatchFailure = Effect.gen(function* () {
68 const persistedEvents = yield* Stream.runCollect(94 const persistedEvents = yield* Stream.runCollect(
69 eventStore.readFromSequence(dispatchStartSequence),95 eventStore.readFromSequence(dispatchStartSequence),
...@@ -83,113 +109,160 @@ const makeOrchestrationEngine = Effect.gen(function* () {...@@ -83,113 +109,160 @@ const makeOrchestrationEngine = Effect.gen(function* () {
83 }109 }
84 });110 });
85111
86 return Effect.gen(function* () {112 return Effect.exit(
87 const existingReceipt = yield* commandReceiptRepository.getByCommandId({113 Effect.gen(function* () {
88 commandId: envelope.command.commandId,114 yield* Effect.annotateCurrentSpan({
89 });115 "orchestration.command_id": envelope.command.commandId,
90 if (Option.isSome(existingReceipt)) {116 "orchestration.command_type": envelope.command.type,
91 if (existingReceipt.value.status === "accepted") {117 "orchestration.aggregate_kind": aggregateRef.aggregateKind,
92 yield* Deferred.succeed(envelope.result, {118 "orchestration.aggregate_id": aggregateRef.aggregateId,
93 sequence: existingReceipt.value.resultSequence,119 });
94 });120
95 return;121 const existingReceipt = yield* commandReceiptRepository.getByCommandId({
96 }122 commandId: envelope.command.commandId,
97 yield* Deferred.fail(123 });
98 envelope.result,124 if (Option.isSome(existingReceipt)) {
99 new OrchestrationCommandPreviouslyRejectedError({125 if (existingReceipt.value.status === "accepted") {
126 return {
127 sequence: existingReceipt.value.resultSequence,
128 };
129 }
130 return yield* new OrchestrationCommandPreviouslyRejectedError({
100 commandId: envelope.command.commandId,131 commandId: envelope.command.commandId,
101 detail: existingReceipt.value.error ?? "Previously rejected.",132 detail: existingReceipt.value.error ?? "Previously rejected.",
102 }),133 });
103 );134 }
104 return;
105 }
106135
107 const eventBase = yield* decideOrchestrationCommand({136 const eventBase = yield* decideOrchestrationCommand({
108 command: envelope.command,137 command: envelope.command,
109 readModel,138 readModel,
110 });139 });
111 const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];140 const eventBases = Array.isArray(eventBase) ? eventBase : [eventBase];
112 const committedCommand = yield* sql141 const committedCommand = yield* sql
113 .withTransaction(142 .withTransaction(
114 Effect.gen(function* () {143 Effect.gen(function* () {
115 const committedEvents: OrchestrationEvent[] = [];144 const committedEvents: OrchestrationEvent[] = [];
116 let nextReadModel = readModel;145 let nextReadModel = readModel;
117
118 for (const nextEvent of eventBases) {
119 const savedEvent = yield* eventStore.append(nextEvent);
120 nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
121 yield* projectionPipeline.projectEvent(savedEvent);
122 committedEvents.push(savedEvent);
123 }
124146
125 const lastSavedEvent = committedEvents.at(-1) ?? null;147 for (const nextEvent of eventBases) {
126 if (lastSavedEvent === null) {148 const savedEvent = yield* eventStore.append(nextEvent);
127 return yield* new OrchestrationCommandInvariantError({149 nextReadModel = yield* projectEvent(nextReadModel, savedEvent);
128 commandType: envelope.command.type,150 yield* projectionPipeline.projectEvent(savedEvent);
129 detail: "Command produced no events.",151 committedEvents.push(savedEvent);
130 });152 }
131 }
132153
133 yield* commandReceiptRepository.upsert({154 const lastSavedEvent = committedEvents.at(-1) ?? null;
134 commandId: envelope.command.commandId,155 if (lastSavedEvent === null) {
135 aggregateKind: lastSavedEvent.aggregateKind,156 return yield* new OrchestrationCommandInvariantError({
136 aggregateId: lastSavedEvent.aggregateId,157 commandType: envelope.command.type,
137 acceptedAt: lastSavedEvent.occurredAt,158 detail: "Command produced no events.",
138 resultSequence: lastSavedEvent.sequence,159 });
139 status: "accepted",160 }
140 error: null,
141 });
142161
143 return {162 yield* commandReceiptRepository.upsert({
144 committedEvents,163 commandId: envelope.command.commandId,
145 lastSequence: lastSavedEvent.sequence,164 aggregateKind: lastSavedEvent.aggregateKind,
146 nextReadModel,165 aggregateId: lastSavedEvent.aggregateId,
147 } as const;166 acceptedAt: lastSavedEvent.occurredAt,
148 }),167 resultSequence: lastSavedEvent.sequence,
149 )168 status: "accepted",
150 .pipe(169 error: null,
151 Effect.catchTag("SqlError", (sqlError) =>170 });
152 Effect.fail(171
153 toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),172 return {
173 committedEvents,
174 lastSequence: lastSavedEvent.sequence,
175 nextReadModel,
176 } as const;
177 }),
178 )
179 .pipe(
180 Effect.catchTag("SqlError", (sqlError) =>
181 Effect.fail(
182 toPersistenceSqlError("OrchestrationEngine.processEnvelope:transaction")(sqlError),
183 ),
154 ),184 ),
155 ),185 );
156 );
157186
158 readModel = committedCommand.nextReadModel;187 readModel = committedCommand.nextReadModel;
159 for (const event of committedCommand.committedEvents) {188 for (const [index, event] of committedCommand.committedEvents.entries()) {
160 yield* PubSub.publish(eventPubSub, event);189 yield* PubSub.publish(eventPubSub, event);
161 }190 if (index === 0) {
162 yield* Deferred.succeed(envelope.result, { sequence: committedCommand.lastSequence });191 yield* Metric.update(
163 }).pipe(192 Metric.withAttributes(
164 Effect.catch((error) =>193 orchestrationCommandAckDuration,
165 Effect.gen(function* () {194 metricAttributes({
166 yield* reconcileReadModelAfterDispatchFailure.pipe(195 ...baseMetricAttributes,
167 Effect.catch(() =>196 ackEventType: event.type,
168 Effect.logWarning(
169 "failed to reconcile orchestration read model after dispatch failure",
170 ).pipe(
171 Effect.annotateLogs({
172 commandId: envelope.command.commandId,
173 snapshotSequence: readModel.snapshotSequence,
174 }),197 }),
175 ),198 ),
199 Duration.millis(Math.max(0, Date.now() - envelope.startedAtMs)),
200 );
201 }
202 }
203 return { sequence: committedCommand.lastSequence };
204 }).pipe(Effect.withSpan(`orchestration.command.${envelope.command.type}`)),
205 ).pipe(
206 Effect.flatMap((exit) =>
207 Effect.gen(function* () {
208 const outcome = Exit.isSuccess(exit)
209 ? "success"
210 : Cause.hasInterruptsOnly(exit.cause)
211 ? "interrupt"
212 : "failure";
213 yield* Metric.update(
214 Metric.withAttributes(
215 orchestrationCommandDuration,
216 metricAttributes(baseMetricAttributes),
217 ),
218 Duration.millis(Math.max(0, Date.now() - processingStartedAtMs)),
219 );
220 yield* Metric.update(
221 Metric.withAttributes(
222 orchestrationCommandsTotal,
223 metricAttributes({
224 ...baseMetricAttributes,
225 outcome,
226 }),
176 ),227 ),
228 1,
177 );229 );
178230
179 if (Schema.is(OrchestrationCommandInvariantError)(error)) {231 if (Exit.isSuccess(exit)) {
180 const aggregateRef = commandToAggregateRef(envelope.command);232 yield* Deferred.succeed(envelope.result, exit.value);
181 yield* commandReceiptRepository233 return;
182 .upsert({
183 commandId: envelope.command.commandId,
184 aggregateKind: aggregateRef.aggregateKind,
185 aggregateId: aggregateRef.aggregateId,
186 acceptedAt: new Date().toISOString(),
187 resultSequence: readModel.snapshotSequence,
188 status: "rejected",
189 error: error.message,
190 })
191 .pipe(Effect.catch(() => Effect.void));
192 }234 }
235
236 const error = Cause.squash(exit.cause) as OrchestrationDispatchError;
237 if (!Schema.is(OrchestrationCommandPreviouslyRejectedError)(error)) {
238 yield* reconcileReadModelAfterDispatchFailure.pipe(
239 Effect.catch(() =>
240 Effect.logWarning(
241 "failed to reconcile orchestration read model after dispatch failure",
242 ).pipe(
243 Effect.annotateLogs({
244 commandId: envelope.command.commandId,
245 snapshotSequence: readModel.snapshotSequence,
246 }),
247 ),
248 ),
249 );
250
251 if (Schema.is(OrchestrationCommandInvariantError)(error)) {
252 yield* commandReceiptRepository
253 .upsert({
254 commandId: envelope.command.commandId,
255 aggregateKind: aggregateRef.aggregateKind,
256 aggregateId: aggregateRef.aggregateId,
257 acceptedAt: new Date().toISOString(),
258 resultSequence: readModel.snapshotSequence,
259 status: "rejected",
260 error: error.message,
261 })
262 .pipe(Effect.catch(() => Effect.void));
263 }
264 }
265
193 yield* Deferred.fail(envelope.result, error);266 yield* Deferred.fail(envelope.result, error);
194 }),267 }),
195 ),268 ),
...@@ -214,7 +287,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {...@@ -214,7 +287,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
214 const dispatch: OrchestrationEngineShape["dispatch"] = (command) =>287 const dispatch: OrchestrationEngineShape["dispatch"] = (command) =>
215 Effect.gen(function* () {288 Effect.gen(function* () {
216 const result = yield* Deferred.make<{ sequence: number }, OrchestrationDispatchError>();289 const result = yield* Deferred.make<{ sequence: number }, OrchestrationDispatchError>();
217 yield* Queue.offer(commandQueue, { command, result });290 yield* Queue.offer(commandQueue, { command, result, startedAtMs: Date.now() });
218 return yield* Deferred.await(result);291 return yield* Deferred.await(result);
219 });292 });
220293
apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+103-88
...@@ -16,6 +16,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";...@@ -16,6 +16,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
1616
17import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";17import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";
18import { GitCore } from "../../git/Services/GitCore.ts";18import { GitCore } from "../../git/Services/GitCore.ts";
19import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts";
19import { ProviderAdapterRequestError, ProviderServiceError } from "../../provider/Errors.ts";20import { ProviderAdapterRequestError, ProviderServiceError } from "../../provider/Errors.ts";
20import { TextGeneration } from "../../git/Services/TextGeneration.ts";21import { TextGeneration } from "../../git/Services/TextGeneration.ts";
21import { ProviderService } from "../../provider/Services/ProviderService.ts";22import { ProviderService } from "../../provider/Services/ProviderService.ts";
...@@ -212,12 +213,12 @@ const make = Effect.gen(function* () {...@@ -212,12 +213,12 @@ const make = Effect.gen(function* () {
212 createdAt: input.createdAt,213 createdAt: input.createdAt,
213 });214 });
214215
215 const resolveThread = Effect.fnUntraced(function* (threadId: ThreadId) {216 const resolveThread = Effect.fn("resolveThread")(function* (threadId: ThreadId) {
216 const readModel = yield* orchestrationEngine.getReadModel();217 const readModel = yield* orchestrationEngine.getReadModel();
217 return readModel.threads.find((entry) => entry.id === threadId);218 return readModel.threads.find((entry) => entry.id === threadId);
218 });219 });
219220
220 const ensureSessionForThread = Effect.fnUntraced(function* (221 const ensureSessionForThread = Effect.fn("ensureSessionForThread")(function* (
221 threadId: ThreadId,222 threadId: ThreadId,
222 createdAt: string,223 createdAt: string,
223 options?: {224 options?: {
...@@ -357,7 +358,7 @@ const make = Effect.gen(function* () {...@@ -357,7 +358,7 @@ const make = Effect.gen(function* () {
357 return startedSession.threadId;358 return startedSession.threadId;
358 });359 });
359360
360 const sendTurnForThread = Effect.fnUntraced(function* (input: {361 const sendTurnForThread = Effect.fn("sendTurnForThread")(function* (input: {
361 readonly threadId: ThreadId;362 readonly threadId: ThreadId;
362 readonly messageText: string;363 readonly messageText: string;
363 readonly attachments?: ReadonlyArray<ChatAttachment>;364 readonly attachments?: ReadonlyArray<ChatAttachment>;
...@@ -409,7 +410,9 @@ const make = Effect.gen(function* () {...@@ -409,7 +410,9 @@ const make = Effect.gen(function* () {
409 });410 });
410 });411 });
411412
412 const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fnUntraced(function* (input: {413 const maybeGenerateAndRenameWorktreeBranchForFirstTurn = Effect.fn(
414 "maybeGenerateAndRenameWorktreeBranchForFirstTurn",
415 )(function* (input: {
413 readonly threadId: ThreadId;416 readonly threadId: ThreadId;
414 readonly branch: string | null;417 readonly branch: string | null;
415 readonly worktreePath: string | null;418 readonly worktreePath: string | null;
...@@ -461,50 +464,52 @@ const make = Effect.gen(function* () {...@@ -461,50 +464,52 @@ const make = Effect.gen(function* () {
461 );464 );
462 });465 });
463466
464 const maybeGenerateThreadTitleForFirstTurn = Effect.fnUntraced(function* (input: {467 const maybeGenerateThreadTitleForFirstTurn = Effect.fn("maybeGenerateThreadTitleForFirstTurn")(
465 readonly threadId: ThreadId;468 function* (input: {
466 readonly cwd: string;469 readonly threadId: ThreadId;
467 readonly messageText: string;470 readonly cwd: string;
468 readonly attachments?: ReadonlyArray<ChatAttachment>;471 readonly messageText: string;
469 readonly titleSeed?: string;472 readonly attachments?: ReadonlyArray<ChatAttachment>;
470 }) {473 readonly titleSeed?: string;
471 const attachments = input.attachments ?? [];474 }) {
472 yield* Effect.gen(function* () {475 const attachments = input.attachments ?? [];
473 const { textGenerationModelSelection: modelSelection } =476 yield* Effect.gen(function* () {
474 yield* serverSettingsService.getSettings;477 const { textGenerationModelSelection: modelSelection } =
475478 yield* serverSettingsService.getSettings;
476 const generated = yield* textGeneration.generateThreadTitle({479
477 cwd: input.cwd,480 const generated = yield* textGeneration.generateThreadTitle({
478 message: input.messageText,481 cwd: input.cwd,
479 ...(attachments.length > 0 ? { attachments } : {}),482 message: input.messageText,
480 modelSelection,483 ...(attachments.length > 0 ? { attachments } : {}),
481 });484 modelSelection,
482 if (!generated) return;485 });
486 if (!generated) return;
483487
484 const thread = yield* resolveThread(input.threadId);488 const thread = yield* resolveThread(input.threadId);
485 if (!thread) return;489 if (!thread) return;
486 if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {490 if (!canReplaceThreadTitle(thread.title, input.titleSeed)) {
487 return;491 return;
488 }492 }
489493
490 yield* orchestrationEngine.dispatch({494 yield* orchestrationEngine.dispatch({
491 type: "thread.meta.update",495 type: "thread.meta.update",
492 commandId: serverCommandId("thread-title-rename"),496 commandId: serverCommandId("thread-title-rename"),
493 threadId: input.threadId,
494 title: generated.title,
495 });
496 }).pipe(
497 Effect.catchCause((cause) =>
498 Effect.logWarning("provider command reactor failed to generate or rename thread title", {
499 threadId: input.threadId,497 threadId: input.threadId,
500 cwd: input.cwd,498 title: generated.title,
501 cause: Cause.pretty(cause),499 });
502 }),500 }).pipe(
503 ),501 Effect.catchCause((cause) =>
504 );502 Effect.logWarning("provider command reactor failed to generate or rename thread title", {
505 });503 threadId: input.threadId,
504 cwd: input.cwd,
505 cause: Cause.pretty(cause),
506 }),
507 ),
508 );
509 },
510 );
506511
507 const processTurnStartRequested = Effect.fnUntraced(function* (512 const processTurnStartRequested = Effect.fn("processTurnStartRequested")(function* (
508 event: Extract<ProviderIntentEvent, { type: "thread.turn-start-requested" }>,513 event: Extract<ProviderIntentEvent, { type: "thread.turn-start-requested" }>,
509 ) {514 ) {
510 const key = turnStartKeyForEvent(event);515 const key = turnStartKeyForEvent(event);
...@@ -583,7 +588,7 @@ const make = Effect.gen(function* () {...@@ -583,7 +588,7 @@ const make = Effect.gen(function* () {
583 );588 );
584 });589 });
585590
586 const processTurnInterruptRequested = Effect.fnUntraced(function* (591 const processTurnInterruptRequested = Effect.fn("processTurnInterruptRequested")(function* (
587 event: Extract<ProviderIntentEvent, { type: "thread.turn-interrupt-requested" }>,592 event: Extract<ProviderIntentEvent, { type: "thread.turn-interrupt-requested" }>,
588 ) {593 ) {
589 const thread = yield* resolveThread(event.payload.threadId);594 const thread = yield* resolveThread(event.payload.threadId);
...@@ -606,7 +611,7 @@ const make = Effect.gen(function* () {...@@ -606,7 +611,7 @@ const make = Effect.gen(function* () {
606 yield* providerService.interruptTurn({ threadId: event.payload.threadId });611 yield* providerService.interruptTurn({ threadId: event.payload.threadId });
607 });612 });
608613
609 const processApprovalResponseRequested = Effect.fnUntraced(function* (614 const processApprovalResponseRequested = Effect.fn("processApprovalResponseRequested")(function* (
610 event: Extract<ProviderIntentEvent, { type: "thread.approval-response-requested" }>,615 event: Extract<ProviderIntentEvent, { type: "thread.approval-response-requested" }>,
611 ) {616 ) {
612 const thread = yield* resolveThread(event.payload.threadId);617 const thread = yield* resolveThread(event.payload.threadId);
...@@ -653,50 +658,52 @@ const make = Effect.gen(function* () {...@@ -653,50 +658,52 @@ const make = Effect.gen(function* () {
653 );658 );
654 });659 });
655660
656 const processUserInputResponseRequested = Effect.fnUntraced(function* (661 const processUserInputResponseRequested = Effect.fn("processUserInputResponseRequested")(
657 event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,662 function* (
658 ) {663 event: Extract<ProviderIntentEvent, { type: "thread.user-input-response-requested" }>,
659 const thread = yield* resolveThread(event.payload.threadId);664 ) {
660 if (!thread) {665 const thread = yield* resolveThread(event.payload.threadId);
661 return;666 if (!thread) {
662 }667 return;
663 const hasSession = thread.session && thread.session.status !== "stopped";668 }
664 if (!hasSession) {669 const hasSession = thread.session && thread.session.status !== "stopped";
665 return yield* appendProviderFailureActivity({670 if (!hasSession) {
666 threadId: event.payload.threadId,671 return yield* appendProviderFailureActivity({
667 kind: "provider.user-input.respond.failed",672 threadId: event.payload.threadId,
668 summary: "Provider user input response failed",673 kind: "provider.user-input.respond.failed",
669 detail: "No active provider session is bound to this thread.",674 summary: "Provider user input response failed",
670 turnId: null,675 detail: "No active provider session is bound to this thread.",
671 createdAt: event.payload.createdAt,676 turnId: null,
672 requestId: event.payload.requestId,677 createdAt: event.payload.createdAt,
673 });678 requestId: event.payload.requestId,
674 }679 });
680 }
675681
676 yield* providerService682 yield* providerService
677 .respondToUserInput({683 .respondToUserInput({
678 threadId: event.payload.threadId,684 threadId: event.payload.threadId,
679 requestId: event.payload.requestId,685 requestId: event.payload.requestId,
680 answers: event.payload.answers,686 answers: event.payload.answers,
681 })687 })
682 .pipe(688 .pipe(
683 Effect.catchCause((cause) =>689 Effect.catchCause((cause) =>
684 appendProviderFailureActivity({690 appendProviderFailureActivity({
685 threadId: event.payload.threadId,691 threadId: event.payload.threadId,
686 kind: "provider.user-input.respond.failed",692 kind: "provider.user-input.respond.failed",
687 summary: "Provider user input response failed",693 summary: "Provider user input response failed",
688 detail: isUnknownPendingUserInputRequestError(cause)694 detail: isUnknownPendingUserInputRequestError(cause)
689 ? stalePendingRequestDetail("user-input", event.payload.requestId)695 ? stalePendingRequestDetail("user-input", event.payload.requestId)
690 : Cause.pretty(cause),696 : Cause.pretty(cause),
691 turnId: null,697 turnId: null,
692 createdAt: event.payload.createdAt,698 createdAt: event.payload.createdAt,
693 requestId: event.payload.requestId,699 requestId: event.payload.requestId,
694 }),700 }),
695 ),701 ),
696 );702 );
697 });703 },
704 );
698705
699 const processSessionStopRequested = Effect.fnUntraced(function* (706 const processSessionStopRequested = Effect.fn("processSessionStopRequested")(function* (
700 event: Extract<ProviderIntentEvent, { type: "thread.session-stop-requested" }>,707 event: Extract<ProviderIntentEvent, { type: "thread.session-stop-requested" }>,
701 ) {708 ) {
702 const thread = yield* resolveThread(event.payload.threadId);709 const thread = yield* resolveThread(event.payload.threadId);
...@@ -727,6 +734,14 @@ const make = Effect.gen(function* () {...@@ -727,6 +734,14 @@ const make = Effect.gen(function* () {
727 const processDomainEvent = Effect.fn("processDomainEvent")(function* (734 const processDomainEvent = Effect.fn("processDomainEvent")(function* (
728 event: ProviderIntentEvent,735 event: ProviderIntentEvent,
729 ) {736 ) {
737 yield* Effect.annotateCurrentSpan({
738 "orchestration.event_type": event.type,
739 "orchestration.thread_id": event.payload.threadId,
740 ...(event.commandId ? { "orchestration.command_id": event.commandId } : {}),
741 });
742 yield* increment(orchestrationEventsProcessedTotal, {
743 eventType: event.type,
744 });
730 switch (event.type) {745 switch (event.type) {
731 case "thread.runtime-mode-set": {746 case "thread.runtime-mode-set": {
732 const thread = yield* resolveThread(event.payload.threadId);747 const thread = yield* resolveThread(event.payload.threadId);
apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+44-41
...@@ -524,7 +524,7 @@ const make = Effect.fn("make")(function* () {...@@ -524,7 +524,7 @@ const make = Effect.fn("make")(function* () {
524 lookup: () => Effect.succeed({ text: "", createdAt: "" }),524 lookup: () => Effect.succeed({ text: "", createdAt: "" }),
525 });525 });
526526
527 const isGitRepoForThread = Effect.fnUntraced(function* (threadId: ThreadId) {527 const isGitRepoForThread = Effect.fn("isGitRepoForThread")(function* (threadId: ThreadId) {
528 const readModel = yield* orchestrationEngine.getReadModel();528 const readModel = yield* orchestrationEngine.getReadModel();
529 const thread = readModel.threads.find((entry) => entry.id === threadId);529 const thread = readModel.threads.find((entry) => entry.id === threadId);
530 if (!thread) {530 if (!thread) {
...@@ -796,9 +796,9 @@ const make = Effect.fn("make")(function* () {...@@ -796,9 +796,9 @@ const make = Effect.fn("make")(function* () {
796 ).pipe(Effect.asVoid);796 ).pipe(Effect.asVoid);
797 });797 });
798798
799 const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fnUntraced(function* (799 const getSourceProposedPlanReferenceForPendingTurnStart = Effect.fn(
800 threadId: ThreadId,800 "getSourceProposedPlanReferenceForPendingTurnStart",
801 ) {801 )(function* (threadId: ThreadId) {
802 const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({802 const pendingTurnStart = yield* projectionTurnRepository.getPendingTurnStartByThreadId({
803 threadId,803 threadId,
804 });804 });
...@@ -818,16 +818,17 @@ const make = Effect.fn("make")(function* () {...@@ -818,16 +818,17 @@ const make = Effect.fn("make")(function* () {
818 } as const;818 } as const;
819 });819 });
820820
821 const getExpectedProviderTurnIdForThread = Effect.fnUntraced(function* (threadId: ThreadId) {821 const getExpectedProviderTurnIdForThread = Effect.fn("getExpectedProviderTurnIdForThread")(
822 const sessions = yield* providerService.listSessions();822 function* (threadId: ThreadId) {
823 const session = sessions.find((entry) => entry.threadId === threadId);823 const sessions = yield* providerService.listSessions();
824 return session?.activeTurnId;824 const session = sessions.find((entry) => entry.threadId === threadId);
825 });825 return session?.activeTurnId;
826826 },
827 const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fnUntraced(function* (827 );
828 threadId: ThreadId,828
829 eventTurnId: TurnId | undefined,829 const getSourceProposedPlanReferenceForAcceptedTurnStart = Effect.fn(
830 ) {830 "getSourceProposedPlanReferenceForAcceptedTurnStart",
831 )(function* (threadId: ThreadId, eventTurnId: TurnId | undefined) {
831 if (eventTurnId === undefined) {832 if (eventTurnId === undefined) {
832 return null;833 return null;
833 }834 }
...@@ -840,34 +841,36 @@ const make = Effect.fn("make")(function* () {...@@ -840,34 +841,36 @@ const make = Effect.fn("make")(function* () {
840 return yield* getSourceProposedPlanReferenceForPendingTurnStart(threadId);841 return yield* getSourceProposedPlanReferenceForPendingTurnStart(threadId);
841 });842 });
842843
843 const markSourceProposedPlanImplemented = Effect.fnUntraced(function* (844 const markSourceProposedPlanImplemented = Effect.fn("markSourceProposedPlanImplemented")(
844 sourceThreadId: ThreadId,845 function* (
845 sourcePlanId: OrchestrationProposedPlanId,846 sourceThreadId: ThreadId,
846 implementationThreadId: ThreadId,847 sourcePlanId: OrchestrationProposedPlanId,
847 implementedAt: string,848 implementationThreadId: ThreadId,
848 ) {849 implementedAt: string,
849 const readModel = yield* orchestrationEngine.getReadModel();850 ) {
850 const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);851 const readModel = yield* orchestrationEngine.getReadModel();
851 const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);852 const sourceThread = readModel.threads.find((entry) => entry.id === sourceThreadId);
852 if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {853 const sourcePlan = sourceThread?.proposedPlans.find((entry) => entry.id === sourcePlanId);
853 return;854 if (!sourceThread || !sourcePlan || sourcePlan.implementedAt !== null) {
854 }855 return;
856 }
855857
856 yield* orchestrationEngine.dispatch({858 yield* orchestrationEngine.dispatch({
857 type: "thread.proposed-plan.upsert",859 type: "thread.proposed-plan.upsert",
858 commandId: CommandId.makeUnsafe(860 commandId: CommandId.makeUnsafe(
859 `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,861 `provider:source-proposed-plan-implemented:${implementationThreadId}:${crypto.randomUUID()}`,
860 ),862 ),
861 threadId: sourceThread.id,863 threadId: sourceThread.id,
862 proposedPlan: {864 proposedPlan: {
863 ...sourcePlan,865 ...sourcePlan,
864 implementedAt,866 implementedAt,
865 implementationThreadId,867 implementationThreadId,
866 updatedAt: implementedAt,868 updatedAt: implementedAt,
867 },869 },
868 createdAt: implementedAt,870 createdAt: implementedAt,
869 });871 });
870 });872 },
873 );
871874
872 const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (875 const processRuntimeEvent = Effect.fn("processRuntimeEvent")(function* (
873 event: ProviderRuntimeEvent,876 event: ProviderRuntimeEvent,
apps/server/src/persistence/Layers/Sqlite.ts+11-1
...@@ -6,6 +6,7 @@ import { ServerConfig } from "../../config.ts";...@@ -6,6 +6,7 @@ import { ServerConfig } from "../../config.ts";
66
7type RuntimeSqliteLayerConfig = {7type RuntimeSqliteLayerConfig = {
8 readonly filename: string;8 readonly filename: string;
9 readonly spanAttributes?: Record<string, unknown>;
9};10};
1011
11type Loader = {12type Loader = {
...@@ -41,7 +42,16 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(...@@ -41,7 +42,16 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(
41 const path = yield* Path.Path;42 const path = yield* Path.Path;
42 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });43 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
4344
44 return Layer.provideMerge(setup, makeRuntimeSqliteLayer({ filename: dbPath }));45 return Layer.provideMerge(
46 setup,
47 makeRuntimeSqliteLayer({
48 filename: dbPath,
49 spanAttributes: {
50 "db.name": path.basename(dbPath),
51 "service.name": "t3-server",
52 },
53 }),
54 );
45}, Layer.unwrap);55}, Layer.unwrap);
4656
47export const SqlitePersistenceMemory = Layer.provideMerge(57export const SqlitePersistenceMemory = Layer.provideMerge(
apps/server/src/provider/Layers/ProviderService.test.ts+126-1
...@@ -20,7 +20,7 @@ import {...@@ -20,7 +20,7 @@ import {
20import { it, assert, vi } from "@effect/vitest";20import { it, assert, vi } from "@effect/vitest";
21import { assertFailure } from "@effect/vitest/utils";21import { assertFailure } from "@effect/vitest/utils";
2222
23import { Effect, Fiber, Layer, Option, PubSub, Ref, Stream } from "effect";23import { Effect, Fiber, Layer, Metric, Option, PubSub, Ref, Stream } from "effect";
24import * as SqlClient from "effect/unstable/sql/SqlClient";24import * as SqlClient from "effect/unstable/sql/SqlClient";
2525
26import {26import {
...@@ -230,6 +230,17 @@ function makeFakeCodexAdapter(provider: ProviderKind = "codex") {...@@ -230,6 +230,17 @@ function makeFakeCodexAdapter(provider: ProviderKind = "codex") {
230const sleep = (ms: number) =>230const sleep = (ms: number) =>
231 Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, ms)));231 Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, ms)));
232232
233const hasMetricSnapshot = (
234 snapshots: ReadonlyArray<Metric.Metric.Snapshot>,
235 id: string,
236 attributes: Readonly<Record<string, string>>,
237) =>
238 snapshots.some(
239 (snapshot) =>
240 snapshot.id === id &&
241 Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
242 );
243
233function makeProviderServiceLayer() {244function makeProviderServiceLayer() {
234 const codex = makeFakeCodexAdapter();245 const codex = makeFakeCodexAdapter();
235 const claude = makeFakeCodexAdapter("claudeAgent");246 const claude = makeFakeCodexAdapter("claudeAgent");
...@@ -1060,6 +1071,120 @@ fanout.layer("ProviderServiceLive fanout", (it) => {...@@ -1060,6 +1071,120 @@ fanout.layer("ProviderServiceLive fanout", (it) => {
1060 );1071 );
1061 }),1072 }),
1062 );1073 );
1074
1075 it.effect("records provider metrics with the routed provider label", () =>
1076 Effect.gen(function* () {
1077 const provider = yield* ProviderService;
1078
1079 const session = yield* provider.startSession(asThreadId("thread-metrics"), {
1080 provider: "claudeAgent",
1081 threadId: asThreadId("thread-metrics"),
1082 cwd: "/tmp/project",
1083 runtimeMode: "full-access",
1084 });
1085
1086 yield* provider.interruptTurn({ threadId: session.threadId });
1087 yield* provider.respondToRequest({
1088 threadId: session.threadId,
1089 requestId: asRequestId("req-metrics-1"),
1090 decision: "accept",
1091 });
1092 yield* provider.respondToUserInput({
1093 threadId: session.threadId,
1094 requestId: asRequestId("req-metrics-2"),
1095 answers: {
1096 sandbox_mode: "workspace-write",
1097 },
1098 });
1099 yield* provider.rollbackConversation({
1100 threadId: session.threadId,
1101 numTurns: 1,
1102 });
1103 yield* provider.stopSession({ threadId: session.threadId });
1104
1105 const snapshots = yield* Metric.snapshot;
1106
1107 assert.equal(
1108 hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
1109 provider: "claudeAgent",
1110 operation: "interrupt",
1111 outcome: "success",
1112 }),
1113 true,
1114 );
1115 assert.equal(
1116 hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
1117 provider: "claudeAgent",
1118 operation: "approval-response",
1119 outcome: "success",
1120 }),
1121 true,
1122 );
1123 assert.equal(
1124 hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
1125 provider: "claudeAgent",
1126 operation: "user-input-response",
1127 outcome: "success",
1128 }),
1129 true,
1130 );
1131 assert.equal(
1132 hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
1133 provider: "claudeAgent",
1134 operation: "rollback",
1135 outcome: "success",
1136 }),
1137 true,
1138 );
1139 assert.equal(
1140 hasMetricSnapshot(snapshots, "t3_provider_sessions_total", {
1141 provider: "claudeAgent",
1142 operation: "stop",
1143 outcome: "success",
1144 }),
1145 true,
1146 );
1147 }),
1148 );
1149
1150 it.effect(
1151 "records sendTurn metrics with the resolved provider when modelSelection is omitted",
1152 () =>
1153 Effect.gen(function* () {
1154 const provider = yield* ProviderService;
1155
1156 const session = yield* provider.startSession(asThreadId("thread-send-metrics"), {
1157 provider: "claudeAgent",
1158 threadId: asThreadId("thread-send-metrics"),
1159 cwd: "/tmp/project-send-metrics",
1160 runtimeMode: "full-access",
1161 });
1162
1163 yield* provider.sendTurn({
1164 threadId: session.threadId,
1165 input: "hello",
1166 attachments: [],
1167 });
1168
1169 const snapshots = yield* Metric.snapshot;
1170
1171 assert.equal(
1172 hasMetricSnapshot(snapshots, "t3_provider_turns_total", {
1173 provider: "claudeAgent",
1174 operation: "send",
1175 outcome: "success",
1176 }),
1177 true,
1178 );
1179 assert.equal(
1180 hasMetricSnapshot(snapshots, "t3_provider_turn_duration", {
1181 provider: "claudeAgent",
1182 operation: "send",
1183 }),
1184 true,
1185 );
1186 }),
1187 );
1063});1188});
10641189
1065const validation = makeProviderServiceLayer();1190const validation = makeProviderServiceLayer();
apps/server/src/provider/Layers/ProviderService.ts+323-161
...@@ -24,6 +24,16 @@ import {...@@ -24,6 +24,16 @@ import {
24} from "@t3tools/contracts";24} from "@t3tools/contracts";
25import { Effect, Layer, Option, PubSub, Queue, Schema, SchemaIssue, Stream } from "effect";25import { Effect, Layer, Option, PubSub, Queue, Schema, SchemaIssue, Stream } from "effect";
2626
27import {
28 increment,
29 providerMetricAttributes,
30 providerRuntimeEventsTotal,
31 providerSessionsTotal,
32 providerTurnDuration,
33 providerTurnsTotal,
34 providerTurnMetricAttributes,
35 withMetrics,
36} from "../../observability/Metrics.ts";
27import { ProviderValidationError } from "../Errors.ts";37import { ProviderValidationError } from "../Errors.ts";
28import { ProviderAdapterRegistry } from "../Services/ProviderAdapterRegistry.ts";38import { ProviderAdapterRegistry } from "../Services/ProviderAdapterRegistry.ts";
29import { ProviderService, type ProviderServiceShape } from "../Services/ProviderService.ts";39import { ProviderService, type ProviderServiceShape } from "../Services/ProviderService.ts";
...@@ -179,7 +189,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -179,7 +189,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
179 const providers = yield* registry.listProviders();189 const providers = yield* registry.listProviders();
180 const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider));190 const adapters = yield* Effect.forEach(providers, (provider) => registry.getByProvider(provider));
181 const processRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect<void> =>191 const processRuntimeEvent = (event: ProviderRuntimeEvent): Effect.Effect<void> =>
182 publishRuntimeEvent(event);192 increment(providerRuntimeEventsTotal, {
193 provider: event.provider,
194 eventType: event.type,
195 }).pipe(Effect.andThen(publishRuntimeEvent(event)));
183196
184 const worker = Effect.forever(197 const worker = Effect.forever(
185 Queue.take(runtimeEventQueue).pipe(Effect.flatMap(processRuntimeEvent)),198 Queue.take(runtimeEventQueue).pipe(Effect.flatMap(processRuntimeEvent)),
...@@ -196,58 +209,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -196,58 +209,72 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
196 readonly binding: ProviderRuntimeBinding;209 readonly binding: ProviderRuntimeBinding;
197 readonly operation: string;210 readonly operation: string;
198 }) {211 }) {
199 const adapter = yield* registry.getByProvider(input.binding.provider);212 yield* Effect.annotateCurrentSpan({
200 const hasResumeCursor =213 "provider.operation": "recover-session",
201 input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;214 "provider.kind": input.binding.provider,
202 const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);215 "provider.thread_id": input.binding.threadId,
203 if (hasActiveSession) {216 });
204 const activeSessions = yield* adapter.listSessions();217 return yield* Effect.gen(function* () {
205 const existing = activeSessions.find(218 const adapter = yield* registry.getByProvider(input.binding.provider);
206 (session) => session.threadId === input.binding.threadId,219 const hasResumeCursor =
207 );220 input.binding.resumeCursor !== null && input.binding.resumeCursor !== undefined;
208 if (existing) {221 const hasActiveSession = yield* adapter.hasSession(input.binding.threadId);
209 yield* upsertSessionBinding(existing, input.binding.threadId);222 if (hasActiveSession) {
210 yield* analytics.record("provider.session.recovered", {223 const activeSessions = yield* adapter.listSessions();
211 provider: existing.provider,224 const existing = activeSessions.find(
212 strategy: "adopt-existing",225 (session) => session.threadId === input.binding.threadId,
213 hasResumeCursor: existing.resumeCursor !== undefined,226 );
214 });227 if (existing) {
215 return { adapter, session: existing } as const;228 yield* upsertSessionBinding(existing, input.binding.threadId);
229 yield* analytics.record("provider.session.recovered", {
230 provider: existing.provider,
231 strategy: "adopt-existing",
232 hasResumeCursor: existing.resumeCursor !== undefined,
233 });
234 return { adapter, session: existing } as const;
235 }
216 }236 }
217 }
218237
219 if (!hasResumeCursor) {238 if (!hasResumeCursor) {
220 return yield* toValidationError(239 return yield* toValidationError(
221 input.operation,240 input.operation,
222 `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,241 `Cannot recover thread '${input.binding.threadId}' because no provider resume state is persisted.`,
223 );242 );
224 }243 }
225244
226 const persistedCwd = readPersistedCwd(input.binding.runtimePayload);245 const persistedCwd = readPersistedCwd(input.binding.runtimePayload);
227 const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);246 const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload);
228247
229 const resumed = yield* adapter.startSession({248 const resumed = yield* adapter.startSession({
230 threadId: input.binding.threadId,249 threadId: input.binding.threadId,
231 provider: input.binding.provider,250 provider: input.binding.provider,
232 ...(persistedCwd ? { cwd: persistedCwd } : {}),251 ...(persistedCwd ? { cwd: persistedCwd } : {}),
233 ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),252 ...(persistedModelSelection ? { modelSelection: persistedModelSelection } : {}),
234 ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),253 ...(hasResumeCursor ? { resumeCursor: input.binding.resumeCursor } : {}),
235 runtimeMode: input.binding.runtimeMode ?? "full-access",254 runtimeMode: input.binding.runtimeMode ?? "full-access",
236 });255 });
237 if (resumed.provider !== adapter.provider) {256 if (resumed.provider !== adapter.provider) {
238 return yield* toValidationError(257 return yield* toValidationError(
239 input.operation,258 input.operation,
240 `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,259 `Adapter/provider mismatch while recovering thread '${input.binding.threadId}'. Expected '${adapter.provider}', received '${resumed.provider}'.`,
241 );260 );
242 }261 }
243262
244 yield* upsertSessionBinding(resumed, input.binding.threadId);263 yield* upsertSessionBinding(resumed, input.binding.threadId);
245 yield* analytics.record("provider.session.recovered", {264 yield* analytics.record("provider.session.recovered", {
246 provider: resumed.provider,265 provider: resumed.provider,
247 strategy: "resume-thread",266 strategy: "resume-thread",
248 hasResumeCursor: resumed.resumeCursor !== undefined,267 hasResumeCursor: resumed.resumeCursor !== undefined,
249 });268 });
250 return { adapter, session: resumed } as const;269 return { adapter, session: resumed } as const;
270 }).pipe(
271 withMetrics({
272 counter: providerSessionsTotal,
273 attributes: providerMetricAttributes(input.binding.provider, {
274 operation: "recover",
275 }),
276 }),
277 );
251 });278 });
252279
253 const resolveRoutableSession = Effect.fn("resolveRoutableSession")(function* (input: {280 const resolveRoutableSession = Effect.fn("resolveRoutableSession")(function* (input: {
...@@ -291,52 +318,69 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -291,52 +318,69 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
291 threadId,318 threadId,
292 provider: parsed.provider ?? "codex",319 provider: parsed.provider ?? "codex",
293 };320 };
294 const settings = yield* serverSettings.getSettings.pipe(321 yield* Effect.annotateCurrentSpan({
295 Effect.mapError((error) =>322 "provider.operation": "start-session",
296 toValidationError(323 "provider.kind": input.provider,
297 "ProviderService.startSession",324 "provider.thread_id": threadId,
298 `Failed to load provider settings: ${error.message}`,325 "provider.runtime_mode": input.runtimeMode,
299 error,326 });
327 return yield* Effect.gen(function* () {
328 const settings = yield* serverSettings.getSettings.pipe(
329 Effect.mapError((error) =>
330 toValidationError(
331 "ProviderService.startSession",
332 `Failed to load provider settings: ${error.message}`,
333 error,
334 ),
300 ),335 ),
301 ),
302 );
303 if (!settings.providers[input.provider].enabled) {
304 return yield* toValidationError(
305 "ProviderService.startSession",
306 `Provider '${input.provider}' is disabled in T3 Code settings.`,
307 );336 );
308 }337 if (!settings.providers[input.provider].enabled) {
309 const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));338 return yield* toValidationError(
310 const effectiveResumeCursor =339 "ProviderService.startSession",
311 input.resumeCursor ??340 `Provider '${input.provider}' is disabled in T3 Code settings.`,
312 (persistedBinding?.provider === input.provider ? persistedBinding.resumeCursor : undefined);341 );
313 const adapter = yield* registry.getByProvider(input.provider);342 }
314 const session = yield* adapter.startSession({343 const persistedBinding = Option.getOrUndefined(yield* directory.getBinding(threadId));
315 ...input,344 const effectiveResumeCursor =
316 ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),345 input.resumeCursor ??
317 });346 (persistedBinding?.provider === input.provider
347 ? persistedBinding.resumeCursor
348 : undefined);
349 const adapter = yield* registry.getByProvider(input.provider);
350 const session = yield* adapter.startSession({
351 ...input,
352 ...(effectiveResumeCursor !== undefined ? { resumeCursor: effectiveResumeCursor } : {}),
353 });
318354
319 if (session.provider !== adapter.provider) {355 if (session.provider !== adapter.provider) {
320 return yield* toValidationError(356 return yield* toValidationError(
321 "ProviderService.startSession",357 "ProviderService.startSession",
322 `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,358 `Adapter/provider mismatch: requested '${adapter.provider}', received '${session.provider}'.`,
323 );359 );
324 }360 }
325361
326 yield* upsertSessionBinding(session, threadId, {362 yield* upsertSessionBinding(session, threadId, {
327 modelSelection: input.modelSelection,363 modelSelection: input.modelSelection,
328 });364 });
329 yield* analytics.record("provider.session.started", {365 yield* analytics.record("provider.session.started", {
330 provider: session.provider,366 provider: session.provider,
331 runtimeMode: input.runtimeMode,367 runtimeMode: input.runtimeMode,
332 hasResumeCursor: session.resumeCursor !== undefined,368 hasResumeCursor: session.resumeCursor !== undefined,
333 hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,369 hasCwd: typeof input.cwd === "string" && input.cwd.trim().length > 0,
334 hasModel:370 hasModel:
335 typeof input.modelSelection?.model === "string" &&371 typeof input.modelSelection?.model === "string" &&
336 input.modelSelection.model.trim().length > 0,372 input.modelSelection.model.trim().length > 0,
337 });373 });
338374
339 return session;375 return session;
376 }).pipe(
377 withMetrics({
378 counter: providerSessionsTotal,
379 attributes: providerMetricAttributes(input.provider, {
380 operation: "start",
381 }),
382 }),
383 );
340 },384 },
341 );385 );
342386
...@@ -357,32 +401,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -357,32 +401,61 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
357 "Either input text or at least one attachment is required",401 "Either input text or at least one attachment is required",
358 );402 );
359 }403 }
360 const routed = yield* resolveRoutableSession({404 yield* Effect.annotateCurrentSpan({
361 threadId: input.threadId,405 "provider.operation": "send-turn",
362 operation: "ProviderService.sendTurn",406 "provider.thread_id": input.threadId,
363 allowRecovery: true,407 "provider.interaction_mode": input.interactionMode,
364 });408 "provider.attachment_count": input.attachments.length,
365 const turn = yield* routed.adapter.sendTurn(input);
366 yield* directory.upsert({
367 threadId: input.threadId,
368 provider: routed.adapter.provider,
369 status: "running",
370 ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
371 runtimePayload: {
372 ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
373 activeTurnId: turn.turnId,
374 lastRuntimeEvent: "provider.sendTurn",
375 lastRuntimeEventAt: new Date().toISOString(),
376 },
377 });
378 yield* analytics.record("provider.turn.sent", {
379 provider: routed.adapter.provider,
380 model: input.modelSelection?.model,
381 interactionMode: input.interactionMode,
382 attachmentCount: input.attachments.length,
383 hasInput: typeof input.input === "string" && input.input.trim().length > 0,
384 });409 });
385 return turn;410 let metricProvider = "unknown";
411 let metricModel = input.modelSelection?.model;
412 return yield* Effect.gen(function* () {
413 const routed = yield* resolveRoutableSession({
414 threadId: input.threadId,
415 operation: "ProviderService.sendTurn",
416 allowRecovery: true,
417 });
418 metricProvider = routed.adapter.provider;
419 metricModel = input.modelSelection?.model;
420 yield* Effect.annotateCurrentSpan({
421 "provider.kind": routed.adapter.provider,
422 ...(input.modelSelection?.model ? { "provider.model": input.modelSelection.model } : {}),
423 });
424 const turn = yield* routed.adapter.sendTurn(input);
425 yield* directory.upsert({
426 threadId: input.threadId,
427 provider: routed.adapter.provider,
428 status: "running",
429 ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}),
430 runtimePayload: {
431 ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}),
432 activeTurnId: turn.turnId,
433 lastRuntimeEvent: "provider.sendTurn",
434 lastRuntimeEventAt: new Date().toISOString(),
435 },
436 });
437 yield* analytics.record("provider.turn.sent", {
438 provider: routed.adapter.provider,
439 model: input.modelSelection?.model,
440 interactionMode: input.interactionMode,
441 attachmentCount: input.attachments.length,
442 hasInput: typeof input.input === "string" && input.input.trim().length > 0,
443 });
444 return turn;
445 }).pipe(
446 withMetrics({
447 counter: providerTurnsTotal,
448 timer: providerTurnDuration,
449 attributes: () =>
450 providerTurnMetricAttributes({
451 provider: metricProvider,
452 model: metricModel,
453 extra: {
454 operation: "send",
455 },
456 }),
457 }),
458 );
386 });459 });
387460
388 const interruptTurn: ProviderServiceShape["interruptTurn"] = Effect.fn("interruptTurn")(461 const interruptTurn: ProviderServiceShape["interruptTurn"] = Effect.fn("interruptTurn")(
...@@ -392,15 +465,33 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -392,15 +465,33 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
392 schema: ProviderInterruptTurnInput,465 schema: ProviderInterruptTurnInput,
393 payload: rawInput,466 payload: rawInput,
394 });467 });
395 const routed = yield* resolveRoutableSession({468 let metricProvider = "unknown";
396 threadId: input.threadId,469 return yield* Effect.gen(function* () {
397 operation: "ProviderService.interruptTurn",470 const routed = yield* resolveRoutableSession({
398 allowRecovery: true,471 threadId: input.threadId,
399 });472 operation: "ProviderService.interruptTurn",
400 yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);473 allowRecovery: true,
401 yield* analytics.record("provider.turn.interrupted", {474 });
402 provider: routed.adapter.provider,475 metricProvider = routed.adapter.provider;
403 });476 yield* Effect.annotateCurrentSpan({
477 "provider.operation": "interrupt-turn",
478 "provider.kind": routed.adapter.provider,
479 "provider.thread_id": input.threadId,
480 "provider.turn_id": input.turnId,
481 });
482 yield* routed.adapter.interruptTurn(routed.threadId, input.turnId);
483 yield* analytics.record("provider.turn.interrupted", {
484 provider: routed.adapter.provider,
485 });
486 }).pipe(
487 withMetrics({
488 counter: providerTurnsTotal,
489 outcomeAttributes: () =>
490 providerMetricAttributes(metricProvider, {
491 operation: "interrupt",
492 }),
493 }),
494 );
404 },495 },
405 );496 );
406497
...@@ -411,16 +502,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -411,16 +502,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
411 schema: ProviderRespondToRequestInput,502 schema: ProviderRespondToRequestInput,
412 payload: rawInput,503 payload: rawInput,
413 });504 });
414 const routed = yield* resolveRoutableSession({505 let metricProvider = "unknown";
415 threadId: input.threadId,506 return yield* Effect.gen(function* () {
416 operation: "ProviderService.respondToRequest",507 const routed = yield* resolveRoutableSession({
417 allowRecovery: true,508 threadId: input.threadId,
418 });509 operation: "ProviderService.respondToRequest",
419 yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);510 allowRecovery: true,
420 yield* analytics.record("provider.request.responded", {511 });
421 provider: routed.adapter.provider,512 metricProvider = routed.adapter.provider;
422 decision: input.decision,513 yield* Effect.annotateCurrentSpan({
423 });514 "provider.operation": "respond-to-request",
515 "provider.kind": routed.adapter.provider,
516 "provider.thread_id": input.threadId,
517 "provider.request_id": input.requestId,
518 });
519 yield* routed.adapter.respondToRequest(routed.threadId, input.requestId, input.decision);
520 yield* analytics.record("provider.request.responded", {
521 provider: routed.adapter.provider,
522 decision: input.decision,
523 });
524 }).pipe(
525 withMetrics({
526 counter: providerTurnsTotal,
527 outcomeAttributes: () =>
528 providerMetricAttributes(metricProvider, {
529 operation: "approval-response",
530 }),
531 }),
532 );
424 },533 },
425 );534 );
426535
...@@ -432,12 +541,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -432,12 +541,30 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
432 schema: ProviderRespondToUserInputInput,541 schema: ProviderRespondToUserInputInput,
433 payload: rawInput,542 payload: rawInput,
434 });543 });
435 const routed = yield* resolveRoutableSession({544 let metricProvider = "unknown";
436 threadId: input.threadId,545 return yield* Effect.gen(function* () {
437 operation: "ProviderService.respondToUserInput",546 const routed = yield* resolveRoutableSession({
438 allowRecovery: true,547 threadId: input.threadId,
439 });548 operation: "ProviderService.respondToUserInput",
440 yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);549 allowRecovery: true,
550 });
551 metricProvider = routed.adapter.provider;
552 yield* Effect.annotateCurrentSpan({
553 "provider.operation": "respond-to-user-input",
554 "provider.kind": routed.adapter.provider,
555 "provider.thread_id": input.threadId,
556 "provider.request_id": input.requestId,
557 });
558 yield* routed.adapter.respondToUserInput(routed.threadId, input.requestId, input.answers);
559 }).pipe(
560 withMetrics({
561 counter: providerTurnsTotal,
562 outcomeAttributes: () =>
563 providerMetricAttributes(metricProvider, {
564 operation: "user-input-response",
565 }),
566 }),
567 );
441 });568 });
442569
443 const stopSession: ProviderServiceShape["stopSession"] = Effect.fn("stopSession")(570 const stopSession: ProviderServiceShape["stopSession"] = Effect.fn("stopSession")(
...@@ -447,18 +574,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -447,18 +574,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
447 schema: ProviderStopSessionInput,574 schema: ProviderStopSessionInput,
448 payload: rawInput,575 payload: rawInput,
449 });576 });
450 const routed = yield* resolveRoutableSession({577 let metricProvider = "unknown";
451 threadId: input.threadId,578 return yield* Effect.gen(function* () {
452 operation: "ProviderService.stopSession",579 const routed = yield* resolveRoutableSession({
453 allowRecovery: false,580 threadId: input.threadId,
454 });581 operation: "ProviderService.stopSession",
455 if (routed.isActive) {582 allowRecovery: false,
456 yield* routed.adapter.stopSession(routed.threadId);583 });
457 }584 metricProvider = routed.adapter.provider;
458 yield* directory.remove(input.threadId);585 yield* Effect.annotateCurrentSpan({
459 yield* analytics.record("provider.session.stopped", {586 "provider.operation": "stop-session",
460 provider: routed.adapter.provider,587 "provider.kind": routed.adapter.provider,
461 });588 "provider.thread_id": input.threadId,
589 });
590 if (routed.isActive) {
591 yield* routed.adapter.stopSession(routed.threadId);
592 }
593 yield* directory.remove(input.threadId);
594 yield* analytics.record("provider.session.stopped", {
595 provider: routed.adapter.provider,
596 });
597 }).pipe(
598 withMetrics({
599 counter: providerSessionsTotal,
600 outcomeAttributes: () =>
601 providerMetricAttributes(metricProvider, {
602 operation: "stop",
603 }),
604 }),
605 );
462 },606 },
463 );607 );
464608
...@@ -524,16 +668,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (...@@ -524,16 +668,34 @@ const makeProviderService = Effect.fn("makeProviderService")(function* (
524 if (input.numTurns === 0) {668 if (input.numTurns === 0) {
525 return;669 return;
526 }670 }
527 const routed = yield* resolveRoutableSession({671 let metricProvider = "unknown";
528 threadId: input.threadId,672 return yield* Effect.gen(function* () {
529 operation: "ProviderService.rollbackConversation",673 const routed = yield* resolveRoutableSession({
530 allowRecovery: true,674 threadId: input.threadId,
531 });675 operation: "ProviderService.rollbackConversation",
532 yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);676 allowRecovery: true,
533 yield* analytics.record("provider.conversation.rolled_back", {677 });
534 provider: routed.adapter.provider,678 metricProvider = routed.adapter.provider;
535 turns: input.numTurns,679 yield* Effect.annotateCurrentSpan({
536 });680 "provider.operation": "rollback-conversation",
681 "provider.kind": routed.adapter.provider,
682 "provider.thread_id": input.threadId,
683 "provider.rollback_turns": input.numTurns,
684 });
685 yield* routed.adapter.rollbackThread(routed.threadId, input.numTurns);
686 yield* analytics.record("provider.conversation.rolled_back", {
687 provider: routed.adapter.provider,
688 turns: input.numTurns,
689 });
690 }).pipe(
691 withMetrics({
692 counter: providerTurnsTotal,
693 outcomeAttributes: () =>
694 providerMetricAttributes(metricProvider, {
695 operation: "rollback",
696 }),
697 }),
698 );
537 });699 });
538700
539 const runStopAll = Effect.fn("runStopAll")(function* () {701 const runStopAll = Effect.fn("runStopAll")(function* () {
apps/server/src/server.test.ts+21-2
...@@ -139,8 +139,17 @@ const buildAppUnderTest = (options?: {...@@ -139,8 +139,17 @@ const buildAppUnderTest = (options?: {
139 const baseDir = options?.config?.baseDir ?? tempBaseDir;139 const baseDir = options?.config?.baseDir ?? tempBaseDir;
140 const devUrl = options?.config?.devUrl;140 const devUrl = options?.config?.devUrl;
141 const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);141 const derivedPaths = yield* deriveServerPaths(baseDir, devUrl);
142 const config = {142 const config: ServerConfigShape = {
143 logLevel: "Info",143 logLevel: "Info",
144 traceMinLevel: "Info",
145 traceTimingEnabled: true,
146 traceBatchWindowMs: 200,
147 traceMaxBytes: 10 * 1024 * 1024,
148 traceMaxFiles: 10,
149 otlpTracesUrl: undefined,
150 otlpMetricsUrl: undefined,
151 otlpExportIntervalMs: 10_000,
152 otlpServiceName: "t3-server",
144 mode: "web",153 mode: "web",
145 port: 0,154 port: 0,
146 host: "127.0.0.1",155 host: "127.0.0.1",
...@@ -154,7 +163,7 @@ const buildAppUnderTest = (options?: {...@@ -154,7 +163,7 @@ const buildAppUnderTest = (options?: {
154 autoBootstrapProjectFromCwd: false,163 autoBootstrapProjectFromCwd: false,
155 logWebSocketEvents: false,164 logWebSocketEvents: false,
156 ...options?.config,165 ...options?.config,
157 } satisfies ServerConfigShape;166 };
158 const layerConfig = Layer.succeed(ServerConfig, config);167 const layerConfig = Layer.succeed(ServerConfig, config);
159168
160 const appLayer = HttpRouter.serve(makeRoutesLayer, {169 const appLayer = HttpRouter.serve(makeRoutesLayer, {
...@@ -533,6 +542,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -533,6 +542,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
533 } as const;542 } as const;
534543
535 yield* buildAppUnderTest({544 yield* buildAppUnderTest({
545 config: {
546 otlpTracesUrl: "http://localhost:4318/v1/traces",
547 otlpMetricsUrl: "http://localhost:4318/v1/metrics",
548 },
536 layers: {549 layers: {
537 keybindings: {550 keybindings: {
538 loadConfigState: Effect.succeed({551 loadConfigState: Effect.succeed({
...@@ -561,6 +574,12 @@ it.layer(NodeServices.layer)("server router seam", (it) => {...@@ -561,6 +574,12 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
561 assert.deepEqual(first.config.keybindings, []);574 assert.deepEqual(first.config.keybindings, []);
562 assert.deepEqual(first.config.issues, []);575 assert.deepEqual(first.config.issues, []);
563 assert.deepEqual(first.config.providers, providers);576 assert.deepEqual(first.config.providers, providers);
577 assert.equal(first.config.observability.logsDirectoryPath.endsWith("/logs"), true);
578 assert.equal(first.config.observability.localTracingEnabled, true);
579 assert.equal(first.config.observability.otlpTracesUrl, "http://localhost:4318/v1/traces");
580 assert.equal(first.config.observability.otlpTracesEnabled, true);
581 assert.equal(first.config.observability.otlpMetricsUrl, "http://localhost:4318/v1/metrics");
582 assert.equal(first.config.observability.otlpMetricsEnabled, true);
564 assert.deepEqual(first.config.settings, DEFAULT_SERVER_SETTINGS);583 assert.deepEqual(first.config.settings, DEFAULT_SERVER_SETTINGS);
565 }584 }
566 assert.deepEqual(second, {585 assert.deepEqual(second, {
apps/server/src/server.ts+2-2
...@@ -29,7 +29,6 @@ import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration";...@@ -29,7 +29,6 @@ import { RoutingTextGenerationLive } from "./git/Layers/RoutingTextGeneration";
29import { TerminalManagerLive } from "./terminal/Layers/Manager";29import { TerminalManagerLive } from "./terminal/Layers/Manager";
30import { GitManagerLive } from "./git/Layers/GitManager";30import { GitManagerLive } from "./git/Layers/GitManager";
31import { KeybindingsLive } from "./keybindings";31import { KeybindingsLive } from "./keybindings";
32import { ServerLoggerLive } from "./serverLogger";
33import { ServerRuntimeStartup, ServerRuntimeStartupLive } from "./serverRuntimeStartup";32import { ServerRuntimeStartup, ServerRuntimeStartupLive } from "./serverRuntimeStartup";
34import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor";33import { OrchestrationReactorLive } from "./orchestration/Layers/OrchestrationReactor";
35import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus";34import { RuntimeReceiptBusLive } from "./orchestration/Layers/RuntimeReceiptBus";
...@@ -42,6 +41,7 @@ import { ProjectFaviconResolverLive } from "./project/Layers/ProjectFaviconResol...@@ -42,6 +41,7 @@ import { ProjectFaviconResolverLive } from "./project/Layers/ProjectFaviconResol
42import { WorkspaceEntriesLive } from "./workspace/Layers/WorkspaceEntries";41import { WorkspaceEntriesLive } from "./workspace/Layers/WorkspaceEntries";
43import { WorkspaceFileSystemLive } from "./workspace/Layers/WorkspaceFileSystem";42import { WorkspaceFileSystemLive } from "./workspace/Layers/WorkspaceFileSystem";
44import { WorkspacePathsLive } from "./workspace/Layers/WorkspacePaths";43import { WorkspacePathsLive } from "./workspace/Layers/WorkspacePaths";
44import { ObservabilityLive } from "./observability/Layers/Observability";
4545
46const PtyAdapterLive = Layer.unwrap(46const PtyAdapterLive = Layer.unwrap(
47 Effect.gen(function* () {47 Effect.gen(function* () {
...@@ -231,7 +231,7 @@ export const makeServerLayer = Layer.unwrap(...@@ -231,7 +231,7 @@ export const makeServerLayer = Layer.unwrap(
231 return serverApplicationLayer.pipe(231 return serverApplicationLayer.pipe(
232 Layer.provideMerge(RuntimeServicesLive),232 Layer.provideMerge(RuntimeServicesLive),
233 Layer.provideMerge(HttpServerLive),233 Layer.provideMerge(HttpServerLive),
234 Layer.provide(ServerLoggerLive),234 Layer.provide(ObservabilityLive),
235 Layer.provideMerge(FetchHttpClient.layer),235 Layer.provideMerge(FetchHttpClient.layer),
236 Layer.provideMerge(PlatformServicesLive),236 Layer.provideMerge(PlatformServicesLive),
237 );237 );
apps/server/src/serverLogger.ts+1-4
...@@ -4,11 +4,8 @@ import { ServerConfig } from "./config";...@@ -4,11 +4,8 @@ import { ServerConfig } from "./config";
44
5export const ServerLoggerLive = Effect.gen(function* () {5export const ServerLoggerLive = Effect.gen(function* () {
6 const config = yield* ServerConfig;6 const config = yield* ServerConfig;
7 const { serverLogPath } = config;
8
9 const fileLogger = Logger.formatSimple.pipe(Logger.toFile(serverLogPath));
10 const minimumLogLevelLayer = Layer.succeed(References.MinimumLogLevel, config.logLevel);7 const minimumLogLevelLayer = Layer.succeed(References.MinimumLogLevel, config.logLevel);
11 const loggerLayer = Logger.layer([Logger.consolePretty(), fileLogger], {8 const loggerLayer = Logger.layer([Logger.consolePretty(), Logger.tracerLogger], {
12 mergeWithExisting: false,9 mergeWithExisting: false,
13 });10 });
1411
apps/server/src/serverRuntimeStartup.ts+62-31
...@@ -141,6 +141,8 @@ export const recordStartupHeartbeat = Effect.gen(function* () {...@@ -141,6 +141,8 @@ export const recordStartupHeartbeat = Effect.gen(function* () {
141});141});
142142
143export const launchStartupHeartbeat = recordStartupHeartbeat.pipe(143export const launchStartupHeartbeat = recordStartupHeartbeat.pipe(
144 Effect.annotateSpans({ "startup.phase": "heartbeat.record" }),
145 Effect.withSpan("server.startup.heartbeat.record"),
144 Effect.ignoreCause({ log: true }),146 Effect.ignoreCause({ log: true }),
145 Effect.forkScoped,147 Effect.forkScoped,
146 Effect.asVoid,148 Effect.asVoid,
...@@ -248,7 +250,14 @@ const maybeOpenBrowser = Effect.gen(function* () {...@@ -248,7 +250,14 @@ const maybeOpenBrowser = Effect.gen(function* () {
248 );250 );
249});251});
250252
253const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>) =>
254 effect.pipe(
255 Effect.annotateSpans({ "startup.phase": phase }),
256 Effect.withSpan(`server.startup.${phase}`),
257 );
258
251const makeServerRuntimeStartup = Effect.gen(function* () {259const makeServerRuntimeStartup = Effect.gen(function* () {
260 const serverConfig = yield* ServerConfig;
252 const keybindings = yield* Keybindings;261 const keybindings = yield* Keybindings;
253 const orchestrationReactor = yield* OrchestrationReactor;262 const orchestrationReactor = yield* OrchestrationReactor;
254 const lifecycleEvents = yield* ServerLifecycleEvents;263 const lifecycleEvents = yield* ServerLifecycleEvents;
...@@ -262,46 +271,65 @@ const makeServerRuntimeStartup = Effect.gen(function* () {...@@ -262,46 +271,65 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
262271
263 const startup = Effect.gen(function* () {272 const startup = Effect.gen(function* () {
264 yield* Effect.logDebug("startup phase: starting keybindings runtime");273 yield* Effect.logDebug("startup phase: starting keybindings runtime");
265 yield* keybindings.start.pipe(274 yield* runStartupPhase(
266 Effect.catch((error) =>275 "keybindings.start",
267 Effect.logWarning("failed to start keybindings runtime", {276 keybindings.start.pipe(
268 path: error.configPath,277 Effect.catch((error) =>
269 detail: error.detail,278 Effect.logWarning("failed to start keybindings runtime", {
270 cause: error.cause,279 path: error.configPath,
271 }),280 detail: error.detail,
281 cause: error.cause,
282 }),
283 ),
284 Effect.forkScoped,
272 ),285 ),
273 Effect.forkScoped,
274 );286 );
275287
276 yield* Effect.logDebug("startup phase: starting server settings runtime");288 yield* Effect.logDebug("startup phase: starting server settings runtime");
277 yield* serverSettings.start.pipe(289 yield* runStartupPhase(
278 Effect.catch((error) =>290 "settings.start",
279 Effect.logWarning("failed to start server settings runtime", {291 serverSettings.start.pipe(
280 path: error.settingsPath,292 Effect.catch((error) =>
281 detail: error.detail,293 Effect.logWarning("failed to start server settings runtime", {
282 cause: error.cause,294 path: error.settingsPath,
283 }),295 detail: error.detail,
296 cause: error.cause,
297 }),
298 ),
299 Effect.forkScoped,
284 ),300 ),
285 Effect.forkScoped,
286 );301 );
287302
288 yield* Effect.logDebug("startup phase: starting orchestration reactors");303 yield* Effect.logDebug("startup phase: starting orchestration reactors");
289 yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope));304 yield* runStartupPhase(
305 "reactors.start",
306 orchestrationReactor.start().pipe(Scope.provide(reactorScope)),
307 );
290308
291 yield* Effect.logDebug("startup phase: preparing welcome payload");309 yield* Effect.logDebug("startup phase: preparing welcome payload");
292 const welcome = yield* autoBootstrapWelcome;310 const welcome = yield* runStartupPhase("welcome.prepare", autoBootstrapWelcome);
293 yield* Effect.logDebug("startup phase: publishing welcome event", {311 yield* Effect.logDebug("startup phase: publishing welcome event", {
294 cwd: welcome.cwd,312 cwd: welcome.cwd,
295 projectName: welcome.projectName,313 projectName: welcome.projectName,
296 bootstrapProjectId: welcome.bootstrapProjectId,314 bootstrapProjectId: welcome.bootstrapProjectId,
297 bootstrapThreadId: welcome.bootstrapThreadId,315 bootstrapThreadId: welcome.bootstrapThreadId,
298 });316 });
299 yield* lifecycleEvents.publish({317 yield* runStartupPhase(
300 version: 1,318 "welcome.publish",
301 type: "welcome",319 lifecycleEvents.publish({
302 payload: welcome,320 version: 1,
303 });321 type: "welcome",
304 });322 payload: welcome,
323 }),
324 );
325 }).pipe(
326 Effect.annotateSpans({
327 "server.mode": serverConfig.mode,
328 "server.port": serverConfig.port,
329 "server.host": serverConfig.host ?? "default",
330 }),
331 Effect.withSpan("server.startup", { kind: "server", root: true }),
332 );
305333
306 yield* Effect.forkScoped(334 yield* Effect.forkScoped(
307 Effect.gen(function* () {335 Effect.gen(function* () {
...@@ -319,18 +347,21 @@ const makeServerRuntimeStartup = Effect.gen(function* () {...@@ -319,18 +347,21 @@ const makeServerRuntimeStartup = Effect.gen(function* () {
319 yield* Effect.logDebug("Accepting commands");347 yield* Effect.logDebug("Accepting commands");
320 yield* commandGate.signalCommandReady;348 yield* commandGate.signalCommandReady;
321 yield* Effect.logDebug("startup phase: waiting for http listener");349 yield* Effect.logDebug("startup phase: waiting for http listener");
322 yield* Deferred.await(httpListening);350 yield* runStartupPhase("http.wait", Deferred.await(httpListening));
323 yield* Effect.logDebug("startup phase: publishing ready event");351 yield* Effect.logDebug("startup phase: publishing ready event");
324 yield* lifecycleEvents.publish({352 yield* runStartupPhase(
325 version: 1,353 "ready.publish",
326 type: "ready",354 lifecycleEvents.publish({
327 payload: { at: new Date().toISOString() },355 version: 1,
328 });356 type: "ready",
357 payload: { at: new Date().toISOString() },
358 }),
359 );
329360
330 yield* Effect.logDebug("startup phase: recording startup heartbeat");361 yield* Effect.logDebug("startup phase: recording startup heartbeat");
331 yield* launchStartupHeartbeat;362 yield* launchStartupHeartbeat;
332 yield* Effect.logDebug("startup phase: browser open check");363 yield* Effect.logDebug("startup phase: browser open check");
333 yield* maybeOpenBrowser;364 yield* runStartupPhase("browser.open", maybeOpenBrowser);
334 yield* Effect.logDebug("startup phase: complete");365 yield* Effect.logDebug("startup phase: complete");
335 }),366 }),
336 );367 );
apps/server/src/telemetry/Layers/AnalyticsService.ts+13-11
...@@ -125,17 +125,19 @@ const makeAnalyticsService = Effect.gen(function* () {...@@ -125,17 +125,19 @@ const makeAnalyticsService = Effect.gen(function* () {
125 }125 }
126 }).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));126 }).pipe(Effect.catch((cause) => Effect.logError("Failed to flush telemetry", { cause })));
127127
128 const record: AnalyticsServiceShape["record"] = Effect.fnUntraced(function* (event, properties) {128 const record: AnalyticsServiceShape["record"] = Effect.fn("record")(
129 if (!telemetryConfig.enabled || !identifier) return;129 function* (event, properties) {
130130 if (!telemetryConfig.enabled || !identifier) return;
131 const enqueueResult = yield* enqueueBufferedEvent(event, properties);131
132 if (enqueueResult.dropped) {132 const enqueueResult = yield* enqueueBufferedEvent(event, properties);
133 yield* Effect.logDebug("analytics buffer full; dropping oldest event", {133 if (enqueueResult.dropped) {
134 size: enqueueResult.size,134 yield* Effect.logDebug("analytics buffer full; dropping oldest event", {
135 event,135 size: enqueueResult.size,
136 });136 event,
137 }137 });
138 });138 }
139 },
140 );
139141
140 yield* Effect.forever(Effect.sleep(1000).pipe(Effect.flatMap(() => flush)), {142 yield* Effect.forever(Effect.sleep(1000).pipe(Effect.flatMap(() => flush)), {
141 disableYield: true,143 disableYield: true,
apps/server/src/terminal/Layers/Manager.ts+55-39
...@@ -23,6 +23,11 @@ import {...@@ -23,6 +23,11 @@ import {
23} from "effect";23} from "effect";
2424
25import { ServerConfig } from "../../config";25import { ServerConfig } from "../../config";
26import {
27 increment,
28 terminalRestartsTotal,
29 terminalSessionsTotal,
30} from "../../observability/Metrics";
26import { runProcess } from "../../processRunner";31import { runProcess } from "../../processRunner";
27import {32import {
28 TerminalCwdError,33 TerminalCwdError,
...@@ -1294,6 +1299,12 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith...@@ -1294,6 +1299,12 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1294 eventType: "started" | "restarted",1299 eventType: "started" | "restarted",
1295 ) {1300 ) {
1296 yield* stopProcess(session);1301 yield* stopProcess(session);
1302 yield* Effect.annotateCurrentSpan({
1303 "terminal.thread_id": session.threadId,
1304 "terminal.id": session.terminalId,
1305 "terminal.event_type": eventType,
1306 "terminal.cwd": input.cwd,
1307 });
12971308
1298 yield* modifyManagerState((state) => {1309 yield* modifyManagerState((state) => {
1299 session.status = "starting";1310 session.status = "starting";
...@@ -1314,45 +1325,49 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith...@@ -1314,45 +1325,49 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1314 let startedShell: string | null = null;1325 let startedShell: string | null = null;
13151326
1316 const startResult = yield* Effect.result(1327 const startResult = yield* Effect.result(
1317 Effect.gen(function* () {1328 increment(terminalSessionsTotal, { lifecycle: eventType }).pipe(
1318 const shellCandidates = resolveShellCandidates(shellResolver);1329 Effect.andThen(
1319 const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);1330 Effect.gen(function* () {
1320 const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);1331 const shellCandidates = resolveShellCandidates(shellResolver);
1321 ptyProcess = spawnResult.process;1332 const terminalEnv = createTerminalSpawnEnv(process.env, session.runtimeEnv);
1322 startedShell = spawnResult.shellLabel;1333 const spawnResult = yield* trySpawn(shellCandidates, terminalEnv, session);
13231334 ptyProcess = spawnResult.process;
1324 const processPid = ptyProcess.pid;1335 startedShell = spawnResult.shellLabel;
1325 const unsubscribeData = ptyProcess.onData((data) => {1336
1326 if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {1337 const processPid = ptyProcess.pid;
1327 return;1338 const unsubscribeData = ptyProcess.onData((data) => {
1328 }1339 if (!enqueueProcessEvent(session, processPid, { type: "output", data })) {
1329 runFork(drainProcessEvents(session, processPid));1340 return;
1330 });1341 }
1331 const unsubscribeExit = ptyProcess.onExit((event) => {1342 runFork(drainProcessEvents(session, processPid));
1332 if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {1343 });
1333 return;1344 const unsubscribeExit = ptyProcess.onExit((event) => {
1334 }1345 if (!enqueueProcessEvent(session, processPid, { type: "exit", event })) {
1335 runFork(drainProcessEvents(session, processPid));1346 return;
1336 });1347 }
13371348 runFork(drainProcessEvents(session, processPid));
1338 yield* modifyManagerState((state) => {1349 });
1339 session.process = ptyProcess;1350
1340 session.pid = processPid;1351 yield* modifyManagerState((state) => {
1341 session.status = "running";1352 session.process = ptyProcess;
1342 session.updatedAt = new Date().toISOString();1353 session.pid = processPid;
1343 session.unsubscribeData = unsubscribeData;1354 session.status = "running";
1344 session.unsubscribeExit = unsubscribeExit;1355 session.updatedAt = new Date().toISOString();
1345 return [undefined, state] as const;1356 session.unsubscribeData = unsubscribeData;
1346 });1357 session.unsubscribeExit = unsubscribeExit;
13471358 return [undefined, state] as const;
1348 yield* publishEvent({1359 });
1349 type: eventType,1360
1350 threadId: session.threadId,1361 yield* publishEvent({
1351 terminalId: session.terminalId,1362 type: eventType,
1352 createdAt: new Date().toISOString(),1363 threadId: session.threadId,
1353 snapshot: snapshot(session),1364 terminalId: session.terminalId,
1354 });1365 createdAt: new Date().toISOString(),
1355 }),1366 snapshot: snapshot(session),
1367 });
1368 }),
1369 ),
1370 ),
1356 );1371 );
13571372
1358 if (startResult._tag === "Success") {1373 if (startResult._tag === "Success") {
...@@ -1722,6 +1737,7 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith...@@ -1722,6 +1737,7 @@ export const makeTerminalManagerWithOptions = Effect.fn("makeTerminalManagerWith
1722 withThreadLock(1737 withThreadLock(
1723 input.threadId,1738 input.threadId,
1724 Effect.gen(function* () {1739 Effect.gen(function* () {
1740 yield* increment(terminalRestartsTotal, { scope: "thread" });
1725 const terminalId = input.terminalId ?? DEFAULT_TERMINAL_ID;1741 const terminalId = input.terminalId ?? DEFAULT_TERMINAL_ID;
1726 yield* assertValidCwd(input.cwd);1742 yield* assertValidCwd(input.cwd);
17271743
apps/server/src/ws.ts+247-121
...@@ -28,6 +28,11 @@ import { Open, resolveAvailableEditors } from "./open";...@@ -28,6 +28,11 @@ import { Open, resolveAvailableEditors } from "./open";
28import { normalizeDispatchCommand } from "./orchestration/Normalizer";28import { normalizeDispatchCommand } from "./orchestration/Normalizer";
29import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine";29import { OrchestrationEngineService } from "./orchestration/Services/OrchestrationEngine";
30import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery";30import { ProjectionSnapshotQuery } from "./orchestration/Services/ProjectionSnapshotQuery";
31import {
32 observeRpcEffect,
33 observeRpcStream,
34 observeRpcStreamEffect,
35} from "./observability/RpcInstrumentation";
31import { ProviderRegistry } from "./provider/Services/ProviderRegistry";36import { ProviderRegistry } from "./provider/Services/ProviderRegistry";
32import { ServerLifecycleEvents } from "./serverLifecycleEvents";37import { ServerLifecycleEvents } from "./serverLifecycleEvents";
33import { ServerRuntimeStartup } from "./serverRuntimeStartup";38import { ServerRuntimeStartup } from "./serverRuntimeStartup";
...@@ -67,85 +72,114 @@ const WsRpcLayer = WsRpcGroup.toLayer(...@@ -67,85 +72,114 @@ const WsRpcLayer = WsRpcGroup.toLayer(
67 issues: keybindingsConfig.issues,72 issues: keybindingsConfig.issues,
68 providers,73 providers,
69 availableEditors: resolveAvailableEditors(),74 availableEditors: resolveAvailableEditors(),
75 observability: {
76 logsDirectoryPath: config.logsDir,
77 localTracingEnabled: true,
78 ...(config.otlpTracesUrl !== undefined ? { otlpTracesUrl: config.otlpTracesUrl } : {}),
79 otlpTracesEnabled: config.otlpTracesUrl !== undefined,
80 ...(config.otlpMetricsUrl !== undefined ? { otlpMetricsUrl: config.otlpMetricsUrl } : {}),
81 otlpMetricsEnabled: config.otlpMetricsUrl !== undefined,
82 },
70 settings,83 settings,
71 };84 };
72 });85 });
7386
74 return WsRpcGroup.of({87 return WsRpcGroup.of({
75 [ORCHESTRATION_WS_METHODS.getSnapshot]: (_input) =>88 [ORCHESTRATION_WS_METHODS.getSnapshot]: (_input) =>
76 projectionSnapshotQuery.getSnapshot().pipe(89 observeRpcEffect(
77 Effect.mapError(90 ORCHESTRATION_WS_METHODS.getSnapshot,
78 (cause) =>91 projectionSnapshotQuery.getSnapshot().pipe(
79 new OrchestrationGetSnapshotError({92 Effect.mapError(
80 message: "Failed to load orchestration snapshot",93 (cause) =>
81 cause,94 new OrchestrationGetSnapshotError({
82 }),95 message: "Failed to load orchestration snapshot",
96 cause,
97 }),
98 ),
83 ),99 ),
100 { "rpc.aggregate": "orchestration" },
84 ),101 ),
85 [ORCHESTRATION_WS_METHODS.dispatchCommand]: (command) =>102 [ORCHESTRATION_WS_METHODS.dispatchCommand]: (command) =>
86 Effect.gen(function* () {103 observeRpcEffect(
87 const normalizedCommand = yield* normalizeDispatchCommand(command);104 ORCHESTRATION_WS_METHODS.dispatchCommand,
88 const result = yield* startup.enqueueCommand(105 Effect.gen(function* () {
89 orchestrationEngine.dispatch(normalizedCommand),106 const normalizedCommand = yield* normalizeDispatchCommand(command);
90 );107 const result = yield* startup.enqueueCommand(
91 if (normalizedCommand.type === "thread.archive") {108 orchestrationEngine.dispatch(normalizedCommand),
92 yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
93 Effect.catch((error) =>
94 Effect.logWarning("failed to close thread terminals after archive", {
95 threadId: normalizedCommand.threadId,
96 error: error.message,
97 }),
98 ),
99 );109 );
100 }110 if (normalizedCommand.type === "thread.archive") {
101 return result;111 yield* terminalManager.close({ threadId: normalizedCommand.threadId }).pipe(
102 }).pipe(112 Effect.catch((error) =>
103 Effect.mapError((cause) =>113 Effect.logWarning("failed to close thread terminals after archive", {
104 Schema.is(OrchestrationDispatchCommandError)(cause)114 threadId: normalizedCommand.threadId,
105 ? cause115 error: error.message,
106 : new OrchestrationDispatchCommandError({116 }),
107 message: "Failed to dispatch orchestration command",117 ),
108 cause,118 );
109 }),119 }
120 return result;
121 }).pipe(
122 Effect.mapError((cause) =>
123 Schema.is(OrchestrationDispatchCommandError)(cause)
124 ? cause
125 : new OrchestrationDispatchCommandError({
126 message: "Failed to dispatch orchestration command",
127 cause,
128 }),
129 ),
110 ),130 ),
131 { "rpc.aggregate": "orchestration" },
111 ),132 ),
112 [ORCHESTRATION_WS_METHODS.getTurnDiff]: (input) =>133 [ORCHESTRATION_WS_METHODS.getTurnDiff]: (input) =>
113 checkpointDiffQuery.getTurnDiff(input).pipe(134 observeRpcEffect(
114 Effect.mapError(135 ORCHESTRATION_WS_METHODS.getTurnDiff,
115 (cause) =>136 checkpointDiffQuery.getTurnDiff(input).pipe(
116 new OrchestrationGetTurnDiffError({137 Effect.mapError(
117 message: "Failed to load turn diff",138 (cause) =>
118 cause,139 new OrchestrationGetTurnDiffError({
119 }),140 message: "Failed to load turn diff",
141 cause,
142 }),
143 ),
120 ),144 ),
145 { "rpc.aggregate": "orchestration" },
121 ),146 ),
122 [ORCHESTRATION_WS_METHODS.getFullThreadDiff]: (input) =>147 [ORCHESTRATION_WS_METHODS.getFullThreadDiff]: (input) =>
123 checkpointDiffQuery.getFullThreadDiff(input).pipe(148 observeRpcEffect(
124 Effect.mapError(149 ORCHESTRATION_WS_METHODS.getFullThreadDiff,
125 (cause) =>150 checkpointDiffQuery.getFullThreadDiff(input).pipe(
126 new OrchestrationGetFullThreadDiffError({151 Effect.mapError(
127 message: "Failed to load full thread diff",152 (cause) =>
128 cause,153 new OrchestrationGetFullThreadDiffError({
129 }),154 message: "Failed to load full thread diff",
155 cause,
156 }),
157 ),
130 ),158 ),
159 { "rpc.aggregate": "orchestration" },
131 ),160 ),
132 [ORCHESTRATION_WS_METHODS.replayEvents]: (input) =>161 [ORCHESTRATION_WS_METHODS.replayEvents]: (input) =>
133 Stream.runCollect(162 observeRpcEffect(
134 orchestrationEngine.readEvents(163 ORCHESTRATION_WS_METHODS.replayEvents,
135 clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),164 Stream.runCollect(
136 ),165 orchestrationEngine.readEvents(
137 ).pipe(166 clamp(input.fromSequenceExclusive, { maximum: Number.MAX_SAFE_INTEGER, minimum: 0 }),
138 Effect.map((events) => Array.from(events)),167 ),
139 Effect.mapError(168 ).pipe(
140 (cause) =>169 Effect.map((events) => Array.from(events)),
141 new OrchestrationReplayEventsError({170 Effect.mapError(
142 message: "Failed to replay orchestration events",171 (cause) =>
143 cause,172 new OrchestrationReplayEventsError({
144 }),173 message: "Failed to replay orchestration events",
174 cause,
175 }),
176 ),
145 ),177 ),
178 { "rpc.aggregate": "orchestration" },
146 ),179 ),
147 [WS_METHODS.subscribeOrchestrationDomainEvents]: (_input) =>180 [WS_METHODS.subscribeOrchestrationDomainEvents]: (_input) =>
148 Stream.unwrap(181 observeRpcStreamEffect(
182 WS_METHODS.subscribeOrchestrationDomainEvents,
149 Effect.gen(function* () {183 Effect.gen(function* () {
150 const snapshot = yield* orchestrationEngine.getReadModel();184 const snapshot = yield* orchestrationEngine.getReadModel();
151 const fromSequenceExclusive = snapshot.snapshotSequence;185 const fromSequenceExclusive = snapshot.snapshotSequence;
...@@ -200,82 +234,167 @@ const WsRpcLayer = WsRpcGroup.toLayer(...@@ -200,82 +234,167 @@ const WsRpcLayer = WsRpcGroup.toLayer(
200 Stream.flatMap((events) => Stream.fromIterable(events)),234 Stream.flatMap((events) => Stream.fromIterable(events)),
201 );235 );
202 }),236 }),
237 { "rpc.aggregate": "orchestration" },
203 ),238 ),
204 [WS_METHODS.serverGetConfig]: (_input) => loadServerConfig,239 [WS_METHODS.serverGetConfig]: (_input) =>
240 observeRpcEffect(WS_METHODS.serverGetConfig, loadServerConfig, {
241 "rpc.aggregate": "server",
242 }),
205 [WS_METHODS.serverRefreshProviders]: (_input) =>243 [WS_METHODS.serverRefreshProviders]: (_input) =>
206 providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),244 observeRpcEffect(
245 WS_METHODS.serverRefreshProviders,
246 providerRegistry.refresh().pipe(Effect.map((providers) => ({ providers }))),
247 { "rpc.aggregate": "server" },
248 ),
207 [WS_METHODS.serverUpsertKeybinding]: (rule) =>249 [WS_METHODS.serverUpsertKeybinding]: (rule) =>
208 Effect.gen(function* () {250 observeRpcEffect(
209 const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);251 WS_METHODS.serverUpsertKeybinding,
210 return { keybindings: keybindingsConfig, issues: [] };252 Effect.gen(function* () {
253 const keybindingsConfig = yield* keybindings.upsertKeybindingRule(rule);
254 return { keybindings: keybindingsConfig, issues: [] };
255 }),
256 { "rpc.aggregate": "server" },
257 ),
258 [WS_METHODS.serverGetSettings]: (_input) =>
259 observeRpcEffect(WS_METHODS.serverGetSettings, serverSettings.getSettings, {
260 "rpc.aggregate": "server",
261 }),
262 [WS_METHODS.serverUpdateSettings]: ({ patch }) =>
263 observeRpcEffect(WS_METHODS.serverUpdateSettings, serverSettings.updateSettings(patch), {
264 "rpc.aggregate": "server",
211 }),265 }),
212 [WS_METHODS.serverGetSettings]: (_input) => serverSettings.getSettings,
213 [WS_METHODS.serverUpdateSettings]: ({ patch }) => serverSettings.updateSettings(patch),
214 [WS_METHODS.projectsSearchEntries]: (input) =>266 [WS_METHODS.projectsSearchEntries]: (input) =>
215 workspaceEntries.search(input).pipe(267 observeRpcEffect(
216 Effect.mapError(268 WS_METHODS.projectsSearchEntries,
217 (cause) =>269 workspaceEntries.search(input).pipe(
218 new ProjectSearchEntriesError({270 Effect.mapError(
219 message: `Failed to search workspace entries: ${cause.detail}`,271 (cause) =>
220 cause,272 new ProjectSearchEntriesError({
221 }),273 message: `Failed to search workspace entries: ${cause.detail}`,
274 cause,
275 }),
276 ),
222 ),277 ),
278 { "rpc.aggregate": "workspace" },
223 ),279 ),
224 [WS_METHODS.projectsWriteFile]: (input) =>280 [WS_METHODS.projectsWriteFile]: (input) =>
225 workspaceFileSystem.writeFile(input).pipe(281 observeRpcEffect(
226 Effect.mapError((cause) => {282 WS_METHODS.projectsWriteFile,
227 const message = Schema.is(WorkspacePathOutsideRootError)(cause)283 workspaceFileSystem.writeFile(input).pipe(
228 ? "Workspace file path must stay within the project root."284 Effect.mapError((cause) => {
229 : "Failed to write workspace file";285 const message = Schema.is(WorkspacePathOutsideRootError)(cause)
230 return new ProjectWriteFileError({286 ? "Workspace file path must stay within the project root."
231 message,287 : "Failed to write workspace file";
232 cause,288 return new ProjectWriteFileError({
233 });289 message,
234 }),290 cause,
291 });
292 }),
293 ),
294 { "rpc.aggregate": "workspace" },
235 ),295 ),
236 [WS_METHODS.shellOpenInEditor]: (input) => open.openInEditor(input),296 [WS_METHODS.shellOpenInEditor]: (input) =>
237 [WS_METHODS.gitStatus]: (input) => gitManager.status(input),297 observeRpcEffect(WS_METHODS.shellOpenInEditor, open.openInEditor(input), {
238 [WS_METHODS.gitPull]: (input) => git.pullCurrentBranch(input.cwd),298 "rpc.aggregate": "workspace",
299 }),
300 [WS_METHODS.gitStatus]: (input) =>
301 observeRpcEffect(WS_METHODS.gitStatus, gitManager.status(input), {
302 "rpc.aggregate": "git",
303 }),
304 [WS_METHODS.gitPull]: (input) =>
305 observeRpcEffect(WS_METHODS.gitPull, git.pullCurrentBranch(input.cwd), {
306 "rpc.aggregate": "git",
307 }),
239 [WS_METHODS.gitRunStackedAction]: (input) =>308 [WS_METHODS.gitRunStackedAction]: (input) =>
240 Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>309 observeRpcStream(
241 gitManager310 WS_METHODS.gitRunStackedAction,
242 .runStackedAction(input, {311 Stream.callback<GitActionProgressEvent, GitManagerServiceError>((queue) =>
243 actionId: input.actionId,312 gitManager
244 progressReporter: {313 .runStackedAction(input, {
245 publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),314 actionId: input.actionId,
246 },315 progressReporter: {
247 })316 publish: (event) => Queue.offer(queue, event).pipe(Effect.asVoid),
248 .pipe(317 },
249 Effect.matchCauseEffect({318 })
250 onFailure: (cause) => Queue.failCause(queue, cause),319 .pipe(
251 onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),320 Effect.matchCauseEffect({
252 }),321 onFailure: (cause) => Queue.failCause(queue, cause),
253 ),322 onSuccess: () => Queue.end(queue).pipe(Effect.asVoid),
323 }),
324 ),
325 ),
326 { "rpc.aggregate": "git" },
254 ),327 ),
255 [WS_METHODS.gitResolvePullRequest]: (input) => gitManager.resolvePullRequest(input),328 [WS_METHODS.gitResolvePullRequest]: (input) =>
329 observeRpcEffect(WS_METHODS.gitResolvePullRequest, gitManager.resolvePullRequest(input), {
330 "rpc.aggregate": "git",
331 }),
256 [WS_METHODS.gitPreparePullRequestThread]: (input) =>332 [WS_METHODS.gitPreparePullRequestThread]: (input) =>
257 gitManager.preparePullRequestThread(input),333 observeRpcEffect(
258 [WS_METHODS.gitListBranches]: (input) => git.listBranches(input),334 WS_METHODS.gitPreparePullRequestThread,
259 [WS_METHODS.gitCreateWorktree]: (input) => git.createWorktree(input),335 gitManager.preparePullRequestThread(input),
260 [WS_METHODS.gitRemoveWorktree]: (input) => git.removeWorktree(input),336 { "rpc.aggregate": "git" },
261 [WS_METHODS.gitCreateBranch]: (input) => git.createBranch(input),337 ),
262 [WS_METHODS.gitCheckout]: (input) => Effect.scoped(git.checkoutBranch(input)),338 [WS_METHODS.gitListBranches]: (input) =>
263 [WS_METHODS.gitInit]: (input) => git.initRepo(input),339 observeRpcEffect(WS_METHODS.gitListBranches, git.listBranches(input), {
264 [WS_METHODS.terminalOpen]: (input) => terminalManager.open(input),340 "rpc.aggregate": "git",
265 [WS_METHODS.terminalWrite]: (input) => terminalManager.write(input),341 }),
266 [WS_METHODS.terminalResize]: (input) => terminalManager.resize(input),342 [WS_METHODS.gitCreateWorktree]: (input) =>
267 [WS_METHODS.terminalClear]: (input) => terminalManager.clear(input),343 observeRpcEffect(WS_METHODS.gitCreateWorktree, git.createWorktree(input), {
268 [WS_METHODS.terminalRestart]: (input) => terminalManager.restart(input),344 "rpc.aggregate": "git",
269 [WS_METHODS.terminalClose]: (input) => terminalManager.close(input),345 }),
346 [WS_METHODS.gitRemoveWorktree]: (input) =>
347 observeRpcEffect(WS_METHODS.gitRemoveWorktree, git.removeWorktree(input), {
348 "rpc.aggregate": "git",
349 }),
350 [WS_METHODS.gitCreateBranch]: (input) =>
351 observeRpcEffect(WS_METHODS.gitCreateBranch, git.createBranch(input), {
352 "rpc.aggregate": "git",
353 }),
354 [WS_METHODS.gitCheckout]: (input) =>
355 observeRpcEffect(WS_METHODS.gitCheckout, Effect.scoped(git.checkoutBranch(input)), {
356 "rpc.aggregate": "git",
357 }),
358 [WS_METHODS.gitInit]: (input) =>
359 observeRpcEffect(WS_METHODS.gitInit, git.initRepo(input), { "rpc.aggregate": "git" }),
360 [WS_METHODS.terminalOpen]: (input) =>
361 observeRpcEffect(WS_METHODS.terminalOpen, terminalManager.open(input), {
362 "rpc.aggregate": "terminal",
363 }),
364 [WS_METHODS.terminalWrite]: (input) =>
365 observeRpcEffect(WS_METHODS.terminalWrite, terminalManager.write(input), {
366 "rpc.aggregate": "terminal",
367 }),
368 [WS_METHODS.terminalResize]: (input) =>
369 observeRpcEffect(WS_METHODS.terminalResize, terminalManager.resize(input), {
370 "rpc.aggregate": "terminal",
371 }),
372 [WS_METHODS.terminalClear]: (input) =>
373 observeRpcEffect(WS_METHODS.terminalClear, terminalManager.clear(input), {
374 "rpc.aggregate": "terminal",
375 }),
376 [WS_METHODS.terminalRestart]: (input) =>
377 observeRpcEffect(WS_METHODS.terminalRestart, terminalManager.restart(input), {
378 "rpc.aggregate": "terminal",
379 }),
380 [WS_METHODS.terminalClose]: (input) =>
381 observeRpcEffect(WS_METHODS.terminalClose, terminalManager.close(input), {
382 "rpc.aggregate": "terminal",
383 }),
270 [WS_METHODS.subscribeTerminalEvents]: (_input) =>384 [WS_METHODS.subscribeTerminalEvents]: (_input) =>
271 Stream.callback<TerminalEvent>((queue) =>385 observeRpcStream(
272 Effect.acquireRelease(386 WS_METHODS.subscribeTerminalEvents,
273 terminalManager.subscribe((event) => Queue.offer(queue, event)),387 Stream.callback<TerminalEvent>((queue) =>
274 (unsubscribe) => Effect.sync(unsubscribe),388 Effect.acquireRelease(
389 terminalManager.subscribe((event) => Queue.offer(queue, event)),
390 (unsubscribe) => Effect.sync(unsubscribe),
391 ),
275 ),392 ),
393 { "rpc.aggregate": "terminal" },
276 ),394 ),
277 [WS_METHODS.subscribeServerConfig]: (_input) =>395 [WS_METHODS.subscribeServerConfig]: (_input) =>
278 Stream.unwrap(396 observeRpcStreamEffect(
397 WS_METHODS.subscribeServerConfig,
279 Effect.gen(function* () {398 Effect.gen(function* () {
280 const keybindingsUpdates = keybindings.streamChanges.pipe(399 const keybindingsUpdates = keybindings.streamChanges.pipe(
281 Stream.map((event) => ({400 Stream.map((event) => ({
...@@ -310,9 +429,11 @@ const WsRpcLayer = WsRpcGroup.toLayer(...@@ -310,9 +429,11 @@ const WsRpcLayer = WsRpcGroup.toLayer(
310 Stream.merge(keybindingsUpdates, Stream.merge(providerStatuses, settingsUpdates)),429 Stream.merge(keybindingsUpdates, Stream.merge(providerStatuses, settingsUpdates)),
311 );430 );
312 }),431 }),
432 { "rpc.aggregate": "server" },
313 ),433 ),
314 [WS_METHODS.subscribeServerLifecycle]: (_input) =>434 [WS_METHODS.subscribeServerLifecycle]: (_input) =>
315 Stream.unwrap(435 observeRpcStreamEffect(
436 WS_METHODS.subscribeServerLifecycle,
316 Effect.gen(function* () {437 Effect.gen(function* () {
317 const snapshot = yield* lifecycleEvents.snapshot;438 const snapshot = yield* lifecycleEvents.snapshot;
318 const snapshotEvents = Array.from(snapshot.events).toSorted(439 const snapshotEvents = Array.from(snapshot.events).toSorted(
...@@ -323,6 +444,7 @@ const WsRpcLayer = WsRpcGroup.toLayer(...@@ -323,6 +444,7 @@ const WsRpcLayer = WsRpcGroup.toLayer(
323 );444 );
324 return Stream.concat(Stream.fromIterable(snapshotEvents), liveEvents);445 return Stream.concat(Stream.fromIterable(snapshotEvents), liveEvents);
325 }),446 }),
447 { "rpc.aggregate": "server" },
326 ),448 ),
327 });449 });
328 }),450 }),
...@@ -330,9 +452,13 @@ const WsRpcLayer = WsRpcGroup.toLayer(...@@ -330,9 +452,13 @@ const WsRpcLayer = WsRpcGroup.toLayer(
330452
331export const websocketRpcRouteLayer = Layer.unwrap(453export const websocketRpcRouteLayer = Layer.unwrap(
332 Effect.gen(function* () {454 Effect.gen(function* () {
333 const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup).pipe(455 const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, {
334 Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)),456 spanPrefix: "ws.rpc",
335 );457 spanAttributes: {
458 "rpc.transport": "websocket",
459 "rpc.system": "effect-rpc",
460 },
461 }).pipe(Effect.provide(Layer.mergeAll(WsRpcLayer, RpcSerialization.layerJson)));
336 return HttpRouter.add(462 return HttpRouter.add(
337 "GET",463 "GET",
338 "/ws",464 "/ws",
apps/web/src/components/ChatView.browser.tsx+6
...@@ -135,6 +135,12 @@ function createBaseServerConfig(): ServerConfig {...@@ -135,6 +135,12 @@ function createBaseServerConfig(): ServerConfig {
135 },135 },
136 ],136 ],
137 availableEditors: [],137 availableEditors: [],
138 observability: {
139 logsDirectoryPath: "/repo/project/.t3/logs",
140 localTracingEnabled: true,
141 otlpTracesEnabled: false,
142 otlpMetricsEnabled: false,
143 },
138 settings: {144 settings: {
139 ...DEFAULT_SERVER_SETTINGS,145 ...DEFAULT_SERVER_SETTINGS,
140 ...DEFAULT_CLIENT_SETTINGS,146 ...DEFAULT_CLIENT_SETTINGS,
apps/web/src/components/KeybindingsToast.browser.tsx+6
...@@ -56,6 +56,12 @@ function createBaseServerConfig(): ServerConfig {...@@ -56,6 +56,12 @@ function createBaseServerConfig(): ServerConfig {
56 },56 },
57 ],57 ],
58 availableEditors: [],58 availableEditors: [],
59 observability: {
60 logsDirectoryPath: "/repo/project/.t3/logs",
61 localTracingEnabled: true,
62 otlpTracesEnabled: false,
63 otlpMetricsEnabled: false,
64 },
59 settings: {65 settings: {
60 enableAssistantStreaming: false,66 enableAssistantStreaming: false,
61 defaultThreadEnvMode: "local" as const,67 defaultThreadEnvMode: "local" as const,
apps/web/src/components/settings/SettingsPanels.browser.tsx created+91
...@@ -0,0 +1,91 @@
1import "../../index.css";
2
3import { DEFAULT_SERVER_SETTINGS, type NativeApi, type ServerConfig } from "@t3tools/contracts";
4import { page } from "vitest/browser";
5import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
6import { render } from "vitest-browser-react";
7
8import { __resetNativeApiForTests } from "../../nativeApi";
9import { AppAtomRegistryProvider } from "../../rpc/atomRegistry";
10import { resetServerStateForTests, setServerConfigSnapshot } from "../../rpc/serverState";
11import { GeneralSettingsPanel } from "./SettingsPanels";
12
13function createBaseServerConfig(): ServerConfig {
14 return {
15 cwd: "/repo/project",
16 keybindingsConfigPath: "/repo/project/.t3code-keybindings.json",
17 keybindings: [],
18 issues: [],
19 providers: [],
20 availableEditors: ["cursor"],
21 observability: {
22 logsDirectoryPath: "/repo/project/.t3/logs",
23 localTracingEnabled: true,
24 otlpTracesUrl: "http://localhost:4318/v1/traces",
25 otlpTracesEnabled: true,
26 otlpMetricsEnabled: false,
27 },
28 settings: DEFAULT_SERVER_SETTINGS,
29 };
30}
31
32describe("GeneralSettingsPanel observability", () => {
33 beforeEach(() => {
34 resetServerStateForTests();
35 __resetNativeApiForTests();
36 localStorage.clear();
37 document.body.innerHTML = "";
38 });
39
40 afterEach(() => {
41 resetServerStateForTests();
42 __resetNativeApiForTests();
43 document.body.innerHTML = "";
44 });
45
46 it("shows diagnostics inside About with a single logs-folder action", async () => {
47 setServerConfigSnapshot(createBaseServerConfig());
48
49 await render(
50 <AppAtomRegistryProvider>
51 <GeneralSettingsPanel />
52 </AppAtomRegistryProvider>,
53 );
54
55 await expect.element(page.getByText("About")).toBeInTheDocument();
56 await expect.element(page.getByText("Diagnostics")).toBeInTheDocument();
57 await expect.element(page.getByText("Open logs folder")).toBeInTheDocument();
58 await expect
59 .element(page.getByText("/repo/project/.t3/logs", { exact: true }))
60 .toBeInTheDocument();
61 await expect
62 .element(
63 page.getByText(
64 "Local trace file. OTLP exporting traces to http://localhost:4318/v1/traces.",
65 ),
66 )
67 .toBeInTheDocument();
68 });
69
70 it("opens the logs folder in the preferred editor", async () => {
71 const openInEditor = vi.fn<NativeApi["shell"]["openInEditor"]>().mockResolvedValue(undefined);
72 window.nativeApi = {
73 shell: {
74 openInEditor,
75 },
76 } as unknown as NativeApi;
77
78 setServerConfigSnapshot(createBaseServerConfig());
79
80 await render(
81 <AppAtomRegistryProvider>
82 <GeneralSettingsPanel />
83 </AppAtomRegistryProvider>,
84 );
85
86 const openLogsButton = page.getByText("Open logs folder");
87 await openLogsButton.click();
88
89 expect(openInEditor).toHaveBeenCalledWith("/repo/project/.t3/logs", "cursor");
90 });
91});
apps/web/src/components/settings/SettingsPanels.tsx+87-22
...@@ -61,6 +61,7 @@ import { ProjectFavicon } from "../ProjectFavicon";...@@ -61,6 +61,7 @@ import { ProjectFavicon } from "../ProjectFavicon";
61import {61import {
62 useServerAvailableEditors,62 useServerAvailableEditors,
63 useServerKeybindingsConfigPath,63 useServerKeybindingsConfigPath,
64 useServerObservability,
64 useServerProviders,65 useServerProviders,
65} from "../../rpc/serverState";66} from "../../rpc/serverState";
6667
...@@ -518,8 +519,13 @@ export function GeneralSettingsPanel() {...@@ -518,8 +519,13 @@ export function GeneralSettingsPanel() {
518 const { theme, setTheme } = useTheme();519 const { theme, setTheme } = useTheme();
519 const settings = useSettings();520 const settings = useSettings();
520 const { updateSettings } = useUpdateSettings();521 const { updateSettings } = useUpdateSettings();
521 const [isOpeningKeybindings, setIsOpeningKeybindings] = useState(false);522 const [openingPathByTarget, setOpeningPathByTarget] = useState({
522 const [openKeybindingsError, setOpenKeybindingsError] = useState<string | null>(null);523 keybindings: false,
524 logsDirectory: false,
525 });
526 const [openPathErrorByTarget, setOpenPathErrorByTarget] = useState<
527 Partial<Record<"keybindings" | "logsDirectory", string | null>>
528 >({});
523 const [openProviderDetails, setOpenProviderDetails] = useState<Record<ProviderKind, boolean>>({529 const [openProviderDetails, setOpenProviderDetails] = useState<Record<ProviderKind, boolean>>({
524 codex: Boolean(530 codex: Boolean(
525 settings.providers.codex.binaryPath !== DEFAULT_UNIFIED_SETTINGS.providers.codex.binaryPath ||531 settings.providers.codex.binaryPath !== DEFAULT_UNIFIED_SETTINGS.providers.codex.binaryPath ||
...@@ -561,8 +567,21 @@ export function GeneralSettingsPanel() {...@@ -561,8 +567,21 @@ export function GeneralSettingsPanel() {
561567
562 const keybindingsConfigPath = useServerKeybindingsConfigPath();568 const keybindingsConfigPath = useServerKeybindingsConfigPath();
563 const availableEditors = useServerAvailableEditors();569 const availableEditors = useServerAvailableEditors();
570 const observability = useServerObservability();
564 const serverProviders = useServerProviders();571 const serverProviders = useServerProviders();
565 const codexHomePath = settings.providers.codex.homePath;572 const codexHomePath = settings.providers.codex.homePath;
573 const logsDirectoryPath = observability?.logsDirectoryPath ?? null;
574 const diagnosticsDescription = (() => {
575 const exports: string[] = [];
576 if (observability?.otlpTracesEnabled && observability.otlpTracesUrl) {
577 exports.push(`traces to ${observability.otlpTracesUrl}`);
578 }
579 if (observability?.otlpMetricsEnabled && observability.otlpMetricsUrl) {
580 exports.push(`metrics to ${observability.otlpMetricsUrl}`);
581 }
582 const mode = observability?.localTracingEnabled ? "Local trace file" : "Terminal logs only";
583 return exports.length > 0 ? `${mode}. OTLP exporting ${exports.join(" and ")}.` : `${mode}.`;
584 })();
566585
567 const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);586 const textGenerationModelSelection = resolveAppModelSelectionState(settings, serverProviders);
568 const textGenProvider = textGenerationModelSelection.provider;587 const textGenProvider = textGenerationModelSelection.provider;
...@@ -579,27 +598,49 @@ export function GeneralSettingsPanel() {...@@ -579,27 +598,49 @@ export function GeneralSettingsPanel() {
579 DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,598 DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection ?? null,
580 );599 );
581600
601 const openInPreferredEditor = useCallback(
602 (target: "keybindings" | "logsDirectory", path: string | null, failureMessage: string) => {
603 if (!path) return;
604 setOpenPathErrorByTarget((existing) => ({ ...existing, [target]: null }));
605 setOpeningPathByTarget((existing) => ({ ...existing, [target]: true }));
606
607 const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);
608 if (!editor) {
609 setOpenPathErrorByTarget((existing) => ({
610 ...existing,
611 [target]: "No available editors found.",
612 }));
613 setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
614 return;
615 }
616
617 void ensureNativeApi()
618 .shell.openInEditor(path, editor)
619 .catch((error) => {
620 setOpenPathErrorByTarget((existing) => ({
621 ...existing,
622 [target]: error instanceof Error ? error.message : failureMessage,
623 }));
624 })
625 .finally(() => {
626 setOpeningPathByTarget((existing) => ({ ...existing, [target]: false }));
627 });
628 },
629 [availableEditors],
630 );
631
582 const openKeybindingsFile = useCallback(() => {632 const openKeybindingsFile = useCallback(() => {
583 if (!keybindingsConfigPath) return;633 openInPreferredEditor("keybindings", keybindingsConfigPath, "Unable to open keybindings file.");
584 setOpenKeybindingsError(null);634 }, [keybindingsConfigPath, openInPreferredEditor]);
585 setIsOpeningKeybindings(true);635
586 const editor = resolveAndPersistPreferredEditor(availableEditors ?? []);636 const openLogsDirectory = useCallback(() => {
587 if (!editor) {637 openInPreferredEditor("logsDirectory", logsDirectoryPath, "Unable to open logs folder.");
588 setOpenKeybindingsError("No available editors found.");638 }, [logsDirectoryPath, openInPreferredEditor]);
589 setIsOpeningKeybindings(false);639
590 return;640 const openKeybindingsError = openPathErrorByTarget.keybindings ?? null;
591 }641 const openDiagnosticsError = openPathErrorByTarget.logsDirectory ?? null;
592 void ensureNativeApi()642 const isOpeningKeybindings = openingPathByTarget.keybindings;
593 .shell.openInEditor(keybindingsConfigPath, editor)643 const isOpeningLogsDirectory = openingPathByTarget.logsDirectory;
594 .catch((error) => {
595 setOpenKeybindingsError(
596 error instanceof Error ? error.message : "Unable to open keybindings file.",
597 );
598 })
599 .finally(() => {
600 setIsOpeningKeybindings(false);
601 });
602 }, [availableEditors, keybindingsConfigPath]);
603644
604 const addCustomModel = useCallback(645 const addCustomModel = useCallback(
605 (provider: ProviderKind) => {646 (provider: ProviderKind) => {
...@@ -1408,6 +1449,30 @@ export function GeneralSettingsPanel() {...@@ -1408,6 +1449,30 @@ export function GeneralSettingsPanel() {
1408 description="Current version of the application."1449 description="Current version of the application."
1409 />1450 />
1410 )}1451 )}
1452 <SettingsRow
1453 title="Diagnostics"
1454 description={diagnosticsDescription}
1455 status={
1456 <>
1457 <span className="block break-all font-mono text-[11px] text-foreground">
1458 {logsDirectoryPath ?? "Resolving logs directory..."}
1459 </span>
1460 {openDiagnosticsError ? (
1461 <span className="mt-1 block text-destructive">{openDiagnosticsError}</span>
1462 ) : null}
1463 </>
1464 }
1465 control={
1466 <Button
1467 size="xs"
1468 variant="outline"
1469 disabled={!logsDirectoryPath || isOpeningLogsDirectory}
1470 onClick={openLogsDirectory}
1471 >
1472 {isOpeningLogsDirectory ? "Opening..." : "Open logs folder"}
1473 </Button>
1474 }
1475 />
1411 </SettingsSection>1476 </SettingsSection>
1412 </SettingsPageContainer>1477 </SettingsPageContainer>
1413 );1478 );
apps/web/src/rpc/serverState.test.ts+6
...@@ -57,6 +57,12 @@ const baseServerConfig: ServerConfig = {...@@ -57,6 +57,12 @@ const baseServerConfig: ServerConfig = {
57 issues: [],57 issues: [],
58 providers: defaultProviders,58 providers: defaultProviders,
59 availableEditors: ["cursor"],59 availableEditors: ["cursor"],
60 observability: {
61 logsDirectoryPath: "/tmp/workspace/.config/logs",
62 localTracingEnabled: true,
63 otlpTracesEnabled: false,
64 otlpMetricsEnabled: false,
65 },
60 settings: DEFAULT_SERVER_SETTINGS,66 settings: DEFAULT_SERVER_SETTINGS,
61};67};
6268
apps/web/src/rpc/serverState.ts+5
...@@ -50,6 +50,7 @@ const selectAvailableEditors = (config: ServerConfig | null): ReadonlyArray<Edit...@@ -50,6 +50,7 @@ const selectAvailableEditors = (config: ServerConfig | null): ReadonlyArray<Edit
50const selectKeybindings = (config: ServerConfig | null) => config?.keybindings ?? EMPTY_KEYBINDINGS;50const selectKeybindings = (config: ServerConfig | null) => config?.keybindings ?? EMPTY_KEYBINDINGS;
51const selectKeybindingsConfigPath = (config: ServerConfig | null) =>51const selectKeybindingsConfigPath = (config: ServerConfig | null) =>
52 config?.keybindingsConfigPath ?? null;52 config?.keybindingsConfigPath ?? null;
53const selectObservability = (config: ServerConfig | null) => config?.observability ?? null;
53const selectProviders = (config: ServerConfig | null) =>54const selectProviders = (config: ServerConfig | null) =>
54 config?.providers ?? EMPTY_SERVER_PROVIDERS;55 config?.providers ?? EMPTY_SERVER_PROVIDERS;
55const selectSettings = (config: ServerConfig | null): ServerSettings =>56const selectSettings = (config: ServerConfig | null): ServerSettings =>
...@@ -281,6 +282,10 @@ export function useServerKeybindingsConfigPath(): string | null {...@@ -281,6 +282,10 @@ export function useServerKeybindingsConfigPath(): string | null {
281 return useAtomValue(serverConfigAtom, selectKeybindingsConfigPath);282 return useAtomValue(serverConfigAtom, selectKeybindingsConfigPath);
282}283}
283284
285export function useServerObservability(): ServerConfig["observability"] | null {
286 return useAtomValue(serverConfigAtom, selectObservability);
287}
288
284export function useServerWelcomeSubscription(289export function useServerWelcomeSubscription(
285 listener: (payload: ServerLifecycleWelcomePayload) => void,290 listener: (payload: ServerLifecycleWelcomePayload) => void,
286): void {291): void {
apps/web/src/wsNativeApi.test.ts+6
...@@ -159,6 +159,12 @@ const baseServerConfig: ServerConfig = {...@@ -159,6 +159,12 @@ const baseServerConfig: ServerConfig = {
159 issues: [],159 issues: [],
160 providers: defaultProviders,160 providers: defaultProviders,
161 availableEditors: ["cursor"],161 availableEditors: ["cursor"],
162 observability: {
163 logsDirectoryPath: "/tmp/workspace/.config/logs",
164 localTracingEnabled: true,
165 otlpTracesEnabled: false,
166 otlpMetricsEnabled: false,
167 },
162 settings: DEFAULT_SERVER_SETTINGS,168 settings: DEFAULT_SERVER_SETTINGS,
163};169};
164170
docs/observability.md created+519
...@@ -0,0 +1,519 @@
1# Observability
2
3T3 Code has one server-side observability model:
4
5- pretty logs go to stdout for humans
6- completed spans go to a local NDJSON trace file
7- traces and metrics can also be exported over OTLP to a real backend like Grafana LGTM
8
9The local trace file is the persisted source of truth. There is no separate persisted server log file anymore.
10
11## Where To Find Things
12
13### Logs
14
15Logs are human-facing only:
16
17- destination: stdout
18- format: `Logger.consolePretty()`
19- persistence: none
20
21If you want a log message to show up in the trace file, emit it inside an active span with `Effect.log...`. `Logger.tracerLogger` will attach it as a span event.
22
23### Traces
24
25Completed spans are written as NDJSON records to `serverTracePath` (by default, `~/.t3/userdata/logs/server.trace.ndjson`).
26
27Important fields in each record:
28
29- `name`: span name
30- `traceId`, `spanId`, `parentSpanId`: correlation
31- `durationMs`: elapsed time
32- `attributes`: structured context
33- `events`: embedded logs and custom events
34- `exit`: `Success`, `Failure`, or `Interrupted`
35
36The schema lives in `apps/server/src/observability/TraceRecord.ts`.
37
38### Metrics
39
40Metrics are not written to a local file.
41
42- local persistence: none
43- remote export: OTLP only, when configured
44- current definitions: `apps/server/src/observability/Metrics.ts`
45
46If OTLP is not configured, metrics still exist in-process, but you will not have a local artifact to inspect.
47
48### Related Artifacts
49
50Provider event NDJSON files still exist for provider runtime streams. Those are separate from the main server trace file.
51
52## Run The Server In Instrumented Mode
53
54There are two useful modes:
55
56- local-only: stdout + local `server.trace.ndjson`
57- full local observability: stdout + local trace file + OTLP export to Grafana/Tempo/Prometheus
58
59The local trace file is always on. OTLP export is opt-in.
60
61### Option 1: Local Traces Only
62
63You do not need any extra env vars. Just run the app normally and inspect `server.trace.ndjson`.
64
65Examples:
66
67```bash
68npx t3
69```
70
71```bash
72bun dev
73```
74
75```bash
76bun dev:desktop
77```
78
79### Option 2: Run With A Local LGTM Stack
80
81#### 1. Start Grafana LGTM
82
83```bash
84docker run --name lgtm \
85 -p 3000:3000 \
86 -p 4317:4317 \
87 -p 4318:4318 \
88 --rm -ti \
89 grafana/otel-lgtm
90```
91
92Then open `http://localhost:3000`.
93
94Default Grafana login:
95
96- username: `admin`
97- password: `admin`
98
99#### 2. Export OTLP env vars
100
101```bash
102export T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces
103export T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics
104export T3CODE_OTLP_SERVICE_NAME=t3-local
105```
106
107Optional:
108
109```bash
110export T3CODE_TRACE_MIN_LEVEL=Info
111export T3CODE_TRACE_TIMING_ENABLED=true
112```
113
114#### 3. Launch the app from that same shell
115
116CLI:
117
118```bash
119npx t3
120```
121
122Monorepo web/server dev:
123
124```bash
125bun dev
126```
127
128Monorepo desktop dev:
129
130```bash
131bun dev:desktop
132```
133
134Packaged desktop app:
135
136Launch the actual app executable from the same shell so the desktop app and embedded backend inherit `T3CODE_OTLP_*`.
137
138macOS app bundle example:
139
140```bash
141T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
142T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
143T3CODE_OTLP_SERVICE_NAME=t3-desktop \
144"/Applications/T3 Code.app/Contents/MacOS/T3 Code"
145```
146
147Direct binary example:
148
149```bash
150T3CODE_OTLP_TRACES_URL=http://localhost:4318/v1/traces \
151T3CODE_OTLP_METRICS_URL=http://localhost:4318/v1/metrics \
152T3CODE_OTLP_SERVICE_NAME=t3-desktop \
153./path/to/your/desktop-app-binary
154```
155
156Do not rely on launching from Finder, Spotlight, the dock, or the Start menu after setting shell env vars. Those launches usually will not pick them up.
157
158#### 4. Fully restart after changing env
159
160The backend reads observability config at process start. If you change OTLP env vars, stop the app completely and start it again.
161
162## How To Use Traces And Metrics To Debug The Server
163
164### Start With The Local Trace File
165
166The trace file is the fastest way to inspect raw span data.
167
168Tail it:
169
170```bash
171tail -f "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
172```
173
174In monorepo dev, use:
175
176```bash
177tail -f ./dev/logs/server.trace.ndjson
178```
179
180Show failed spans:
181
182```bash
183jq -c 'select(.exit._tag != "Success") | {
184 name,
185 durationMs,
186 exit,
187 attributes
188}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
189```
190
191Show slow spans:
192
193```bash
194jq -c 'select(.durationMs > 1000) | {
195 name,
196 durationMs,
197 traceId,
198 spanId
199}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
200```
201
202Inspect embedded log events:
203
204```bash
205jq -c 'select(any(.events[]?; .attributes["effect.logLevel"] != null)) | {
206 name,
207 durationMs,
208 events: [
209 .events[]
210 | select(.attributes["effect.logLevel"] != null)
211 | {
212 message: .name,
213 level: .attributes["effect.logLevel"]
214 }
215 ]
216}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
217```
218
219Follow one trace:
220
221```bash
222jq -r 'select(.traceId == "TRACE_ID_HERE") | [
223 .name,
224 .spanId,
225 (.parentSpanId // "-"),
226 .durationMs
227] | @tsv' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
228```
229
230Filter orchestration commands:
231
232```bash
233jq -c 'select(.attributes["orchestration.command_type"] != null) | {
234 name,
235 durationMs,
236 commandType: .attributes["orchestration.command_type"],
237 aggregateKind: .attributes["orchestration.aggregate_kind"]
238}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
239```
240
241Filter git activity:
242
243```bash
244jq -c 'select(.attributes["git.operation"] != null) | {
245 name,
246 durationMs,
247 operation: .attributes["git.operation"],
248 cwd: .attributes["git.cwd"],
249 hookEvents: [
250 .events[]
251 | select(.name == "git.hook.started" or .name == "git.hook.finished")
252 ]
253}' "$T3CODE_HOME/userdata/logs/server.trace.ndjson"
254```
255
256### Use Tempo When You Need A Real Trace Viewer
257
258Tempo is better than raw NDJSON when you want to:
259
260- search across many traces
261- inspect parent/child relationships visually
262- compare many slow traces
263- drill into one failing request without hand-joining by `traceId`
264
265Recommended flow in Grafana:
266
2671. Open `Explore`.
2682. Pick the `Tempo` data source.
2693. Set the time range to something recent like `Last 15 minutes`.
2704. Start broad. Do not begin with a very narrow query.
2715. Look for spans from your configured service name, then narrow by span name or attributes.
272
273Good first searches:
274
275- service name such as `t3-local`, `t3-dev`, or `t3-desktop`
276- span names like `sql.execute`, `git.runCommand`, `provider.sendTurn`
277- orchestration spans with attributes like `orchestration.command_type`
278
279Once you know traces are arriving, narrower TraceQL queries like `name = "sql.execute"` become useful.
280
281### Use Metrics To See Systemic Problems
282
283Traces are best for one request. Metrics are best for trends.
284
285Good metric families to watch:
286
287- `t3_rpc_request_duration`
288- `t3_orchestration_command_duration`
289- `t3_orchestration_command_ack_duration`
290- `t3_provider_turn_duration`
291- `t3_git_command_duration`
292- `t3_db_query_duration`
293
294Counters tell you volume and failure rate:
295
296- `t3_rpc_requests_total`
297- `t3_orchestration_commands_total`
298- `t3_provider_turns_total`
299- `t3_git_commands_total`
300- `t3_db_queries_total`
301
302Use metrics when the question is:
303
304- "is this always slow?"
305- "did this get worse after a change?"
306- "which command type is failing most often?"
307
308Use traces when the question is:
309
310- "what happened in this specific request?"
311- "which child span caused this one slow interaction?"
312- "what logs were emitted inside the failing flow?"
313
314### What The New Ack Metric Means
315
316`t3_orchestration_command_ack_duration` measures:
317
318- start: command dispatch enters the orchestration engine
319- end: the first committed domain event for that command is published by the server
320
321That is a server-side acknowledgment metric. It does not measure:
322
323- websocket transit to the browser
324- client receipt
325- React render time
326
327If you need those later, add client-side instrumentation or a dedicated server fanout metric.
328
329## Common Workflows
330
331### "Why did this request fail?"
332
3331. Start with the local NDJSON file.
3342. Find spans where `exit._tag != "Success"`.
3353. Group by `traceId`.
3364. Inspect sibling spans and span events.
3375. If needed, move to Tempo for the full trace tree.
338
339### "Why is the UI feeling slow?"
340
3411. Search for slow top-level spans in the trace file or Tempo.
3422. Check child spans for sqlite, git, provider, or terminal work.
3433. Look at the matching duration metrics to see whether the slowness is systemic.
344
345### "Did this command take too long to acknowledge?"
346
3471. Check `t3_orchestration_command_ack_duration` by `commandType`.
3482. If it is high, inspect the corresponding orchestration trace.
3493. Look at child spans for projection, sqlite, provider, or git work.
350
351### "Are git hooks causing latency?"
352
3531. Filter `git.operation` spans.
3542. Inspect `git.hook.started` and `git.hook.finished` events.
3553. Compare hook timing to the enclosing git span duration.
356
357### "Why do I have spans locally but nothing in Grafana?"
358
359Usually one of these is true:
360
361- `T3CODE_OTLP_TRACES_URL` was not set
362- the app was launched from a different environment than the one where you exported the vars
363- the app was not fully restarted after changing env
364- Grafana is looking at the wrong time range or service name
365
366If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.
367
368## How To Think About Adding Tracing To Future Code
369
370### Prefer Boundaries Over Tiny Helpers
371
372Good span boundaries:
373
374- RPC methods
375- orchestration command handling
376- provider adapter calls
377- external process calls
378- persistence writes
379- queue handoffs
380
381Avoid tracing every tiny helper. Most helpers should inherit the active span rather than create a new one.
382
383### Reuse `Effect.fn(...)` Where It Already Exists
384
385The codebase already uses `Effect.fn("name")` heavily. That should usually be your first tracing boundary.
386
387For ad hoc work:
388
389```ts
390import { Effect } from "effect";
391
392const runThing = Effect.gen(function* () {
393 yield* Effect.annotateCurrentSpan({
394 "thing.id": "abc123",
395 "thing.kind": "example",
396 });
397
398 yield* Effect.logInfo("starting thing");
399 return yield* doWork();
400}).pipe(Effect.withSpan("thing.run"));
401```
402
403### Put High-Cardinality Detail On Spans
404
405Use span annotations for IDs, paths, and other detailed context:
406
407```ts
408yield *
409 Effect.annotateCurrentSpan({
410 "provider.thread_id": input.threadId,
411 "provider.request_id": input.requestId,
412 "git.cwd": input.cwd,
413 });
414```
415
416### Keep Metric Labels Low Cardinality
417
418Good metric labels:
419
420- operation kind
421- method name
422- provider kind
423- aggregate kind
424- outcome
425
426Bad metric labels:
427
428- raw thread IDs
429- command IDs
430- file paths
431- cwd
432- full prompts
433- full model strings when a normalized family label would do
434
435Detailed context belongs on spans, not metrics.
436
437### Use Logs As Span Events
438
439Logs inside a span become part of the trace story:
440
441```ts
442yield * Effect.logInfo("starting provider turn");
443yield * Effect.logDebug("waiting for approval response");
444```
445
446Those messages show up as span events because `Logger.tracerLogger` is installed.
447
448### Use The Pipeable Metrics API
449
450`withMetrics(...)` is the default way to attach a counter and timer to an effect:
451
452```ts
453import { someCounter, someDuration, withMetrics } from "../observability/Metrics.ts";
454
455const program = doWork().pipe(
456 withMetrics({
457 counter: someCounter,
458 timer: someDuration,
459 attributes: {
460 operation: "work",
461 },
462 }),
463);
464```
465
466## Detailed API Reference
467
468### Runtime Wiring
469
470The server observability layer is assembled in `apps/server/src/observability/Layers/Observability.ts`.
471
472It provides:
473
474- pretty stdout logger
475- `Logger.tracerLogger`
476- local NDJSON tracer
477- optional OTLP trace exporter
478- optional OTLP metrics exporter
479- Effect trace-level and timing refs
480
481### Env Vars
482
483Local trace file:
484
485- `T3CODE_TRACE_FILE`: override trace file path
486- `T3CODE_TRACE_MAX_BYTES`: per-file rotation size, default `10485760`
487- `T3CODE_TRACE_MAX_FILES`: rotated file count, default `10`
488- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `200`
489- `T3CODE_TRACE_MIN_LEVEL`: minimum trace level, default `Info`
490- `T3CODE_TRACE_TIMING_ENABLED`: enable timing metadata, default `true`
491
492OTLP export:
493
494- `T3CODE_OTLP_TRACES_URL`: OTLP trace endpoint
495- `T3CODE_OTLP_METRICS_URL`: OTLP metric endpoint
496- `T3CODE_OTLP_EXPORT_INTERVAL_MS`: export interval, default `10000`
497- `T3CODE_OTLP_SERVICE_NAME`: service name, default `t3-server`
498
499If the OTLP URLs are unset, local tracing still works and metrics stay in-process only.
500
501### What Is Instrumented Today
502
503Current high-value span and metric boundaries include:
504
505- Effect RPC websocket request spans from `effect/rpc`
506- RPC request metrics in `apps/server/src/observability/RpcInstrumentation.ts`
507- startup phases
508- orchestration command processing
509- orchestration command acknowledgment latency
510- provider session and turn operations
511- git command execution and git hook events
512- terminal session lifecycle
513- sqlite query execution
514
515### Current Constraints
516
517- logs outside spans are not persisted
518- metrics are not snapshotted locally
519- the old `serverLogPath` still exists in config for compatibility, but the trace file is the persisted artifact that matters
packages/contracts/src/server.ts+11
...@@ -72,6 +72,16 @@ export type ServerProvider = typeof ServerProvider.Type;...@@ -72,6 +72,16 @@ export type ServerProvider = typeof ServerProvider.Type;
72export const ServerProviders = Schema.Array(ServerProvider);72export const ServerProviders = Schema.Array(ServerProvider);
73export type ServerProviders = typeof ServerProviders.Type;73export type ServerProviders = typeof ServerProviders.Type;
7474
75export const ServerObservability = Schema.Struct({
76 logsDirectoryPath: TrimmedNonEmptyString,
77 localTracingEnabled: Schema.Boolean,
78 otlpTracesUrl: Schema.optional(TrimmedNonEmptyString),
79 otlpTracesEnabled: Schema.Boolean,
80 otlpMetricsUrl: Schema.optional(TrimmedNonEmptyString),
81 otlpMetricsEnabled: Schema.Boolean,
82});
83export type ServerObservability = typeof ServerObservability.Type;
84
75export const ServerConfig = Schema.Struct({85export const ServerConfig = Schema.Struct({
76 cwd: TrimmedNonEmptyString,86 cwd: TrimmedNonEmptyString,
77 keybindingsConfigPath: TrimmedNonEmptyString,87 keybindingsConfigPath: TrimmedNonEmptyString,
...@@ -79,6 +89,7 @@ export const ServerConfig = Schema.Struct({...@@ -79,6 +89,7 @@ export const ServerConfig = Schema.Struct({
79 issues: ServerConfigIssues,89 issues: ServerConfigIssues,
80 providers: ServerProviders,90 providers: ServerProviders,
81 availableEditors: Schema.Array(EditorId),91 availableEditors: Schema.Array(EditorId),
92 observability: ServerObservability,
82 settings: ServerSettings,93 settings: ServerSettings,
83});94});
84export type ServerConfig = typeof ServerConfig.Type;95export type ServerConfig = typeof ServerConfig.Type;
turbo.json+11-1
...@@ -11,7 +11,17 @@...@@ -11,7 +11,17 @@
11 "T3CODE_NO_BROWSER",11 "T3CODE_NO_BROWSER",
12 "T3CODE_HOME",12 "T3CODE_HOME",
13 "T3CODE_AUTH_TOKEN",13 "T3CODE_AUTH_TOKEN",
14 "T3CODE_DESKTOP_WS_URL"14 "T3CODE_DESKTOP_WS_URL",
15 "T3CODE_TRACE_MIN_LEVEL",
16 "T3CODE_TRACE_TIMING_ENABLED",
17 "T3CODE_TRACE_FILE",
18 "T3CODE_TRACE_MAX_BYTES",
19 "T3CODE_TRACE_MAX_FILES",
20 "T3CODE_TRACE_BATCH_WINDOW_MS",
21 "T3CODE_OTLP_TRACES_URL",
22 "T3CODE_OTLP_METRICS_URL",
23 "T3CODE_OTLP_EXPORT_INTERVAL_MS",
24 "T3CODE_OTLP_SERVICE_NAME"
15 ],25 ],
16 "tasks": {26 "tasks": {
17 "build": {27 "build": {