| 1 | import { PRIMARY_LOCAL_ENVIRONMENT_ID } from "@t3tools/contracts"; |
| 2 | import { |
| 3 | makeLocalFileTracer, |
| 4 | makeTraceSink, |
| 5 | type SignalExport, |
| 6 | } from "@t3tools/shared/observability"; |
| 7 | import * as SharedObservability from "@t3tools/shared/observability"; |
| 8 | import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; |
| 9 | import { |
| 10 | parsePersistedServerObservabilitySettings, |
| 11 | type PersistedServerObservabilitySettings, |
| 12 | } from "@t3tools/shared/serverSettings"; |
| 13 | import * as Context from "effect/Context"; |
| 14 | import * as DateTime from "effect/DateTime"; |
| 15 | import * as Effect from "effect/Effect"; |
| 16 | import * as FileSystem from "effect/FileSystem"; |
| 17 | import * as Layer from "effect/Layer"; |
| 18 | import * as Logger from "effect/Logger"; |
| 19 | import * as Option from "effect/Option"; |
| 20 | import * as Path from "effect/Path"; |
| 21 | import * as PlatformError from "effect/PlatformError"; |
| 22 | import * as References from "effect/References"; |
| 23 | import * as Ref from "effect/Ref"; |
| 24 | import * as Schema from "effect/Schema"; |
| 25 | import * as Scope from "effect/Scope"; |
| 26 | import * as Semaphore from "effect/Semaphore"; |
| 27 | import * as SynchronizedRef from "effect/SynchronizedRef"; |
| 28 | import * as Tracer from "effect/Tracer"; |
| 29 | import { OtlpExporter, OtlpLogger, OtlpTracer } from "effect/observability"; |
| 30 | |
| 31 | import * as DesktopEnvironment from "./DesktopEnvironment.ts"; |
| 32 | |
| 33 | const DESKTOP_LOG_FILE_MAX_BYTES = 10 * 1024 * 1024; |
| 34 | const DESKTOP_LOG_FILE_MAX_FILES = 10; |
| 35 | const DESKTOP_BACKEND_CHILD_LOG_FIBER_ID = "#backend-child"; |
| 36 | const DESKTOP_TRACE_BATCH_WINDOW_MS = 1_000; |
| 37 | const DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_BYTES = 1024 * 1024; |
| 38 | const DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_CHUNKS = 256; |
| 39 | |
| 40 | export interface RotatingLogFileWriter { |
| 41 | readonly writeBytes: (chunk: Uint8Array) => Effect.Effect<void>; |
| 42 | readonly writeText: (chunk: string) => Effect.Effect<void>; |
| 43 | } |
| 44 | |
| 45 | export interface DesktopBackendOutputLogShape { |
| 46 | readonly beginSession: (input: { readonly details: string }) => Effect.Effect<void>; |
| 47 | readonly writeOutputChunk: ( |
| 48 | streamName: "stdout" | "stderr", |
| 49 | chunk: Uint8Array, |
| 50 | ) => Effect.Effect<void>; |
| 51 | readonly persistFailureSnapshot: (input: { readonly details: string }) => Effect.Effect<void>; |
| 52 | readonly persistFailure: (input: { readonly details: string }) => Effect.Effect<void>; |
| 53 | readonly discardSession: Effect.Effect<void>; |
| 54 | } |
| 55 | |
| 56 | // Factory for per-instance backend output logs. `forInstance(id)` returns |
| 57 | // a writer that targets a distinct rotating log file — the primary |
| 58 | // instance keeps `server-child.log` so the historical path stays stable |
| 59 | // for ops; other instances get `server-child-<sanitized-id>.log`. |
| 60 | // |
| 61 | // Writers are cached per id within a single factory instance so repeated |
| 62 | // `forInstance` calls (e.g. during a backend restart that re-resolves |
| 63 | // services) reuse the same rotating writer rather than racing each other |
| 64 | // on the same file. |
| 65 | export class DesktopBackendOutputLogFactory extends Context.Service< |
| 66 | DesktopBackendOutputLogFactory, |
| 67 | { |
| 68 | readonly forInstance: (id: string) => Effect.Effect<DesktopBackendOutputLogShape>; |
| 69 | } |
| 70 | >()("@t3tools/desktop/app/DesktopObservability/DesktopBackendOutputLogFactory") {} |
| 71 | |
| 72 | const textEncoder = new TextEncoder(); |
| 73 | const textDecoder = new TextDecoder(); |
| 74 | |
| 75 | export type DesktopLogAnnotations = Record<string, unknown>; |
| 76 | |
| 77 | export interface DesktopComponentLogger { |
| 78 | readonly annotate: <A, E, R>( |
| 79 | effect: Effect.Effect<A, E, R>, |
| 80 | annotations?: DesktopLogAnnotations, |
| 81 | ) => Effect.Effect<A, E, R>; |
| 82 | readonly logDebug: (message: string, annotations?: DesktopLogAnnotations) => Effect.Effect<void>; |
| 83 | readonly logInfo: (message: string, annotations?: DesktopLogAnnotations) => Effect.Effect<void>; |
| 84 | readonly logWarning: ( |
| 85 | message: string, |
| 86 | annotations?: DesktopLogAnnotations, |
| 87 | ) => Effect.Effect<void>; |
| 88 | readonly logError: (message: string, annotations?: DesktopLogAnnotations) => Effect.Effect<void>; |
| 89 | } |
| 90 | |
| 91 | export function makeComponentLogger(component: string): DesktopComponentLogger { |
| 92 | const annotate: DesktopComponentLogger["annotate"] = (effect, annotations) => |
| 93 | effect.pipe( |
| 94 | Effect.annotateLogs({ |
| 95 | component, |
| 96 | ...annotations, |
| 97 | }), |
| 98 | ); |
| 99 | |
| 100 | return { |
| 101 | annotate, |
| 102 | logDebug: (message, annotations) => annotate(Effect.logDebug(message), annotations), |
| 103 | logInfo: (message, annotations) => annotate(Effect.logInfo(message), annotations), |
| 104 | logWarning: (message, annotations) => annotate(Effect.logWarning(message), annotations), |
| 105 | logError: (message, annotations) => annotate(Effect.logError(message), annotations), |
| 106 | }; |
| 107 | } |
| 108 | |
| 109 | class DesktopLogFileWriterConfigurationError extends Schema.TaggedError<DesktopLogFileWriterConfigurationError>()( |
| 110 | "DesktopLogFileWriterConfigurationError", |
| 111 | { |
| 112 | option: Schema.Literals(["maxBytes", "maxFiles"]), |
| 113 | value: Schema.Number, |
| 114 | }, |
| 115 | ) { |
| 116 | override get message() { |
| 117 | return `${this.option} must be >= 1 (received ${this.value})`; |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | type DesktopLogFileWriterError = |
| 122 | | DesktopLogFileWriterConfigurationError |
| 123 | | PlatformError.PlatformError; |
| 124 | |
| 125 | const sanitizeLogValue = (value: string): string => value.replace(/\s+/g, " ").trim(); |
| 126 | |
| 127 | const DesktopBackendChildLogRecord = Schema.Struct({ |
| 128 | message: Schema.String, |
| 129 | level: Schema.Literals(["INFO", "ERROR"]), |
| 130 | timestamp: Schema.String, |
| 131 | annotations: Schema.Record(Schema.String, Schema.Unknown), |
| 132 | spans: Schema.Record(Schema.String, Schema.Unknown), |
| 133 | fiberId: Schema.String, |
| 134 | }); |
| 135 | |
| 136 | const encodeDesktopBackendChildLogRecord = Schema.encodeEffect( |
| 137 | Schema.fromJsonString(DesktopBackendChildLogRecord), |
| 138 | ); |
| 139 | |
| 140 | const DesktopBackendOutputLogNoop: DesktopBackendOutputLogShape = { |
| 141 | beginSession: () => Effect.void, |
| 142 | writeOutputChunk: () => Effect.void, |
| 143 | persistFailureSnapshot: () => Effect.void, |
| 144 | persistFailure: () => Effect.void, |
| 145 | discardSession: Effect.void, |
| 146 | }; |
| 147 | |
| 148 | interface BufferedBackendOutputChunk { |
| 149 | readonly streamName: "stdout" | "stderr"; |
| 150 | readonly chunk: Uint8Array; |
| 151 | readonly offset: number; |
| 152 | } |
| 153 | |
| 154 | interface BackendOutputSession { |
| 155 | readonly runId: string; |
| 156 | readonly startDetails: string; |
| 157 | readonly chunks: ReadonlyArray<BufferedBackendOutputChunk>; |
| 158 | readonly byteLength: number; |
| 159 | } |
| 160 | |
| 161 | export function appendBoundedOutputChunk( |
| 162 | session: BackendOutputSession, |
| 163 | streamName: "stdout" | "stderr", |
| 164 | chunk: Uint8Array, |
| 165 | ): BackendOutputSession { |
| 166 | if (chunk.byteLength === 0) { |
| 167 | return session; |
| 168 | } |
| 169 | |
| 170 | const retainedChunk = |
| 171 | chunk.byteLength > DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_BYTES |
| 172 | ? chunk.slice(chunk.byteLength - DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_BYTES) |
| 173 | : chunk.slice(); |
| 174 | const chunks = [...session.chunks, { streamName, chunk: retainedChunk, offset: 0 }]; |
| 175 | let byteLength = session.byteLength + retainedChunk.byteLength; |
| 176 | let overflow = Math.max(0, byteLength - DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_BYTES); |
| 177 | let firstRetainedIndex = 0; |
| 178 | |
| 179 | while (overflow > 0) { |
| 180 | const first = chunks[firstRetainedIndex]; |
| 181 | if (!first) break; |
| 182 | const retainedByteLength = first.chunk.byteLength - first.offset; |
| 183 | if (retainedByteLength <= overflow) { |
| 184 | overflow -= retainedByteLength; |
| 185 | byteLength -= retainedByteLength; |
| 186 | firstRetainedIndex += 1; |
| 187 | continue; |
| 188 | } |
| 189 | |
| 190 | chunks[firstRetainedIndex] = { |
| 191 | ...first, |
| 192 | offset: first.offset + overflow, |
| 193 | }; |
| 194 | byteLength -= overflow; |
| 195 | overflow = 0; |
| 196 | } |
| 197 | |
| 198 | const excessChunks = Math.max( |
| 199 | 0, |
| 200 | chunks.length - firstRetainedIndex - DESKTOP_BACKEND_OUTPUT_BUFFER_MAX_CHUNKS, |
| 201 | ); |
| 202 | for (let index = firstRetainedIndex; index < firstRetainedIndex + excessChunks; index += 1) { |
| 203 | const chunk = chunks[index]; |
| 204 | byteLength -= chunk ? chunk.chunk.byteLength - chunk.offset : 0; |
| 205 | } |
| 206 | firstRetainedIndex += excessChunks; |
| 207 | |
| 208 | return { |
| 209 | ...session, |
| 210 | chunks: chunks.slice(firstRetainedIndex), |
| 211 | byteLength, |
| 212 | }; |
| 213 | } |
| 214 | |
| 215 | const currentDesktopRunId = Effect.gen(function* () { |
| 216 | const annotations = yield* References.CurrentLogAnnotations; |
| 217 | const runId = annotations.runId; |
| 218 | return typeof runId === "string" && runId.length > 0 ? runId : "unknown"; |
| 219 | }); |
| 220 | |
| 221 | const refreshFileSize = ( |
| 222 | fileSystem: FileSystem.FileSystem, |
| 223 | filePath: string, |
| 224 | ): Effect.Effect<number, never> => |
| 225 | fileSystem.stat(filePath).pipe( |
| 226 | Effect.map((stat) => Number(stat.size)), |
| 227 | Effect.orElseSucceed(() => 0), |
| 228 | ); |
| 229 | |
| 230 | export const makeRotatingLogFileWriter = Effect.fn("makeRotatingLogFileWriter")(function* (input: { |
| 231 | readonly filePath: string; |
| 232 | readonly maxBytes?: number; |
| 233 | readonly maxFiles?: number; |
| 234 | }): Effect.fn.Return< |
| 235 | RotatingLogFileWriter, |
| 236 | DesktopLogFileWriterError, |
| 237 | FileSystem.FileSystem | Path.Path |
| 238 | > { |
| 239 | const fileSystem = yield* FileSystem.FileSystem; |
| 240 | const path = yield* Path.Path; |
| 241 | const maxBytes = input.maxBytes ?? DESKTOP_LOG_FILE_MAX_BYTES; |
| 242 | const maxFiles = input.maxFiles ?? DESKTOP_LOG_FILE_MAX_FILES; |
| 243 | const directory = path.dirname(input.filePath); |
| 244 | const baseName = path.basename(input.filePath); |
| 245 | |
| 246 | if (maxBytes < 1) { |
| 247 | return yield* new DesktopLogFileWriterConfigurationError({ |
| 248 | option: "maxBytes", |
| 249 | value: maxBytes, |
| 250 | }); |
| 251 | } |
| 252 | if (maxFiles < 1) { |
| 253 | return yield* new DesktopLogFileWriterConfigurationError({ |
| 254 | option: "maxFiles", |
| 255 | value: maxFiles, |
| 256 | }); |
| 257 | } |
| 258 | |
| 259 | yield* fileSystem.makeDirectory(directory, { recursive: true }); |
| 260 | |
| 261 | const withSuffix = (index: number) => `${input.filePath}.${index}`; |
| 262 | const currentSize = yield* Ref.make(yield* refreshFileSize(fileSystem, input.filePath)); |
| 263 | const mutex = yield* Semaphore.make(1); |
| 264 | |
| 265 | const pruneOverflowBackups = Effect.gen(function* () { |
| 266 | const entries = yield* fileSystem.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); |
| 267 | for (const entry of entries) { |
| 268 | if (!entry.startsWith(`${baseName}.`)) continue; |
| 269 | const suffix = Number(entry.slice(baseName.length + 1)); |
| 270 | if (!Number.isInteger(suffix) || suffix <= maxFiles) continue; |
| 271 | yield* fileSystem.remove(path.join(directory, entry), { force: true }).pipe(Effect.ignore); |
| 272 | } |
| 273 | }); |
| 274 | |
| 275 | const rotate = Effect.gen(function* () { |
| 276 | yield* fileSystem.remove(withSuffix(maxFiles), { force: true }).pipe(Effect.ignore); |
| 277 | for (let index = maxFiles - 1; index >= 1; index -= 1) { |
| 278 | const source = withSuffix(index); |
| 279 | const sourceExists = yield* fileSystem.exists(source).pipe(Effect.orElseSucceed(() => false)); |
| 280 | if (sourceExists) { |
| 281 | yield* fileSystem.rename(source, withSuffix(index + 1)); |
| 282 | } |
| 283 | } |
| 284 | const currentExists = yield* fileSystem |
| 285 | .exists(input.filePath) |
| 286 | .pipe(Effect.orElseSucceed(() => false)); |
| 287 | if (currentExists) { |
| 288 | yield* fileSystem.rename(input.filePath, withSuffix(1)); |
| 289 | } |
| 290 | yield* Ref.set(currentSize, 0); |
| 291 | }).pipe( |
| 292 | Effect.catch(() => |
| 293 | refreshFileSize(fileSystem, input.filePath).pipe( |
| 294 | Effect.flatMap((size) => Ref.set(currentSize, size)), |
| 295 | ), |
| 296 | ), |
| 297 | ); |
| 298 | |
| 299 | const writeBytes = (chunk: Uint8Array): Effect.Effect<void> => { |
| 300 | if (chunk.byteLength === 0) return Effect.void; |
| 301 | |
| 302 | return mutex.withPermits(1)( |
| 303 | Effect.gen(function* () { |
| 304 | const beforeSize = yield* Ref.get(currentSize); |
| 305 | if (beforeSize > 0 && beforeSize + chunk.byteLength > maxBytes) { |
| 306 | yield* rotate; |
| 307 | } |
| 308 | |
| 309 | yield* fileSystem.writeFile(input.filePath, chunk, { flag: "a" }); |
| 310 | const afterSize = (yield* Ref.get(currentSize)) + chunk.byteLength; |
| 311 | yield* Ref.set(currentSize, afterSize); |
| 312 | |
| 313 | if (afterSize > maxBytes) { |
| 314 | yield* rotate; |
| 315 | } |
| 316 | }).pipe( |
| 317 | Effect.catch(() => |
| 318 | refreshFileSize(fileSystem, input.filePath).pipe( |
| 319 | Effect.flatMap((size) => Ref.set(currentSize, size)), |
| 320 | ), |
| 321 | ), |
| 322 | ), |
| 323 | ); |
| 324 | }; |
| 325 | |
| 326 | yield* pruneOverflowBackups; |
| 327 | |
| 328 | return { |
| 329 | writeBytes, |
| 330 | writeText: (chunk) => writeBytes(textEncoder.encode(chunk)), |
| 331 | } satisfies RotatingLogFileWriter; |
| 332 | }); |
| 333 | |
| 334 | const noPersistedObservabilitySettings: PersistedServerObservabilitySettings = { |
| 335 | otlpTracesUrl: undefined, |
| 336 | otlpMetricsUrl: undefined, |
| 337 | otlpLogsUrl: undefined, |
| 338 | }; |
| 339 | |
| 340 | const readPersistedObservabilitySettings: Effect.Effect< |
| 341 | PersistedServerObservabilitySettings, |
| 342 | never, |
| 343 | FileSystem.FileSystem | DesktopEnvironment.DesktopEnvironment |
| 344 | > = Effect.gen(function* () { |
| 345 | const fileSystem = yield* FileSystem.FileSystem; |
| 346 | const environment = yield* DesktopEnvironment.DesktopEnvironment; |
| 347 | const raw = yield* fileSystem.readFileString(environment.serverSettingsPath).pipe(Effect.option); |
| 348 | return Option.isNone(raw) |
| 349 | ? noPersistedObservabilitySettings |
| 350 | : parsePersistedServerObservabilitySettings(raw.value); |
| 351 | }); |
| 352 | |
| 353 | /** |
| 354 | * Resolved as the server resolves them, with persisted Settings as the |
| 355 | * fallback. Settings is read once for every signal, so the main process |
| 356 | * cannot resolve traces against one revision of the file and logs against |
| 357 | * another. |
| 358 | */ |
| 359 | const resolveOtlpEndpoints = Effect.gen(function* () { |
| 360 | const otel = yield* OtelEnvironment.load; |
| 361 | if (otel.disabled) { |
| 362 | return { |
| 363 | traces: undefined, |
| 364 | metrics: undefined, |
| 365 | logs: undefined, |
| 366 | warnings: otel.warnings, |
| 367 | resourceAttributes: otel.resourceAttributes, |
| 368 | }; |
| 369 | } |
| 370 | |
| 371 | const environment = yield* DesktopEnvironment.DesktopEnvironment; |
| 372 | const persisted = yield* readPersistedObservabilitySettings; |
| 373 | const signalExport: SignalExport = { |
| 374 | protocol: environment.otlpProtocol, |
| 375 | headers: Option.getOrUndefined(environment.otlpHeaders), |
| 376 | exportIntervalMs: environment.otlpExportIntervalMs, |
| 377 | }; |
| 378 | return { |
| 379 | traces: OtelEnvironment.resolveSignalEndpoint( |
| 380 | otel, |
| 381 | "traces", |
| 382 | { url: Option.getOrUndefined(environment.otlpTracesUrl), export: signalExport }, |
| 383 | persisted.otlpTracesUrl, |
| 384 | ), |
| 385 | metrics: OtelEnvironment.resolveSignalEndpoint( |
| 386 | otel, |
| 387 | "metrics", |
| 388 | { url: Option.getOrUndefined(environment.otlpMetricsUrl), export: signalExport }, |
| 389 | persisted.otlpMetricsUrl, |
| 390 | ), |
| 391 | logs: OtelEnvironment.resolveSignalEndpoint( |
| 392 | otel, |
| 393 | "logs", |
| 394 | { url: Option.getOrUndefined(environment.otlpLogsUrl), export: signalExport }, |
| 395 | persisted.otlpLogsUrl, |
| 396 | ), |
| 397 | warnings: otel.warnings, |
| 398 | resourceAttributes: otel.resourceAttributes, |
| 399 | }; |
| 400 | }); |
| 401 | |
| 402 | const writeDevelopmentConsoleOutput = ( |
| 403 | streamName: "stdout" | "stderr", |
| 404 | chunk: Uint8Array, |
| 405 | ): Effect.Effect<void> => |
| 406 | Effect.sync(() => { |
| 407 | const output = streamName === "stderr" ? process.stderr : process.stdout; |
| 408 | output.write(chunk); |
| 409 | }).pipe(Effect.ignore); |
| 410 | |
| 411 | const writeBackendChildLogRecord = Effect.fn("desktop.observability.writeBackendChildLogRecord")( |
| 412 | function* ( |
| 413 | logFile: RotatingLogFileWriter, |
| 414 | input: { |
| 415 | readonly message: string; |
| 416 | readonly level: "INFO" | "ERROR"; |
| 417 | readonly annotations: Record<string, unknown>; |
| 418 | }, |
| 419 | ): Effect.fn.Return<void> { |
| 420 | return yield* Effect.gen(function* () { |
| 421 | const timestamp = DateTime.formatIso(yield* DateTime.now); |
| 422 | const encoded = yield* encodeDesktopBackendChildLogRecord({ |
| 423 | message: input.message, |
| 424 | level: input.level, |
| 425 | timestamp, |
| 426 | annotations: input.annotations, |
| 427 | spans: {}, |
| 428 | fiberId: DESKTOP_BACKEND_CHILD_LOG_FIBER_ID, |
| 429 | }); |
| 430 | yield* logFile.writeText(`${encoded}\n`); |
| 431 | }).pipe(Effect.ignore({ log: true })); |
| 432 | }, |
| 433 | ); |
| 434 | |
| 435 | const PRIMARY_BACKEND_LOG_INSTANCE_ID = PRIMARY_LOCAL_ENVIRONMENT_ID; |
| 436 | |
| 437 | const sanitizeInstanceIdForFileName = (id: string): string => id.replace(/[^a-zA-Z0-9._-]+/g, "_"); |
| 438 | |
| 439 | const backendLogFilePathForInstance = ( |
| 440 | environment: DesktopEnvironment.DesktopEnvironment["Service"], |
| 441 | id: string, |
| 442 | ): string => { |
| 443 | // Primary keeps the historical "server-child.log" path so ops scripts |
| 444 | // and packaged-build log inspection still find it where it always lived. |
| 445 | if (id === PRIMARY_BACKEND_LOG_INSTANCE_ID) { |
| 446 | return environment.path.join(environment.logDir, "server-child.log"); |
| 447 | } |
| 448 | const sanitized = sanitizeInstanceIdForFileName(id); |
| 449 | return environment.path.join(environment.logDir, `server-child-${sanitized}.log`); |
| 450 | }; |
| 451 | |
| 452 | // Just the IO sink. Cacheable by resolved file path so two ids that |
| 453 | // sanitize to the same filename share a single RotatingLogFileWriter |
| 454 | // (no race on currentSize tracking). Splitting the sink off from the |
| 455 | // per-call shape lets the shape annotate writes with the *caller's* |
| 456 | // id rather than whatever id created the cached writer first. |
| 457 | const makeBackendOutputSinkForInstance = ( |
| 458 | environment: DesktopEnvironment.DesktopEnvironment["Service"], |
| 459 | id: string, |
| 460 | ): Effect.Effect< |
| 461 | Option.Option<RotatingLogFileWriter>, |
| 462 | never, |
| 463 | FileSystem.FileSystem | Path.Path | Scope.Scope |
| 464 | > => |
| 465 | makeRotatingLogFileWriter({ |
| 466 | filePath: backendLogFilePathForInstance(environment, id), |
| 467 | }).pipe(Effect.option); |
| 468 | |
| 469 | const makeBackendOutputLogShape = ( |
| 470 | environment: DesktopEnvironment.DesktopEnvironment["Service"], |
| 471 | id: string, |
| 472 | sink: Option.Option<RotatingLogFileWriter>, |
| 473 | ): Effect.Effect<DesktopBackendOutputLogShape> => |
| 474 | Option.match(sink, { |
| 475 | onNone: () => Effect.succeed(DesktopBackendOutputLogNoop), |
| 476 | onSome: (logFile) => |
| 477 | Effect.gen(function* () { |
| 478 | const sessionRef = yield* Ref.make(Option.none<BackendOutputSession>()); |
| 479 | const writeFailure = Effect.fn("desktop.observability.backendOutput.writeFailure")( |
| 480 | function* (session: BackendOutputSession, details: string) { |
| 481 | yield* writeBackendChildLogRecord(logFile, { |
| 482 | message: "backend child process failure output start", |
| 483 | level: "ERROR", |
| 484 | annotations: { |
| 485 | component: "desktop-backend-child", |
| 486 | runId: session.runId, |
| 487 | instanceId: id, |
| 488 | phase: "START", |
| 489 | details: session.startDetails, |
| 490 | }, |
| 491 | }); |
| 492 | for (const output of session.chunks) { |
| 493 | yield* writeBackendChildLogRecord(logFile, { |
| 494 | message: "backend child process output", |
| 495 | level: output.streamName === "stderr" ? "ERROR" : "INFO", |
| 496 | annotations: { |
| 497 | component: "desktop-backend-child", |
| 498 | runId: session.runId, |
| 499 | instanceId: id, |
| 500 | stream: output.streamName, |
| 501 | text: textDecoder.decode(output.chunk.subarray(output.offset)), |
| 502 | }, |
| 503 | }); |
| 504 | } |
| 505 | yield* writeBackendChildLogRecord(logFile, { |
| 506 | message: "backend child process failure output end", |
| 507 | level: "ERROR", |
| 508 | annotations: { |
| 509 | component: "desktop-backend-child", |
| 510 | runId: session.runId, |
| 511 | instanceId: id, |
| 512 | phase: "END", |
| 513 | details: sanitizeLogValue(details), |
| 514 | }, |
| 515 | }); |
| 516 | }, |
| 517 | ); |
| 518 | return { |
| 519 | beginSession: Effect.fn("desktop.observability.backendOutput.beginSession")(function* ({ |
| 520 | details, |
| 521 | }) { |
| 522 | const runId = yield* currentDesktopRunId; |
| 523 | yield* Ref.set( |
| 524 | sessionRef, |
| 525 | Option.some({ |
| 526 | runId, |
| 527 | startDetails: sanitizeLogValue(details), |
| 528 | chunks: [], |
| 529 | byteLength: 0, |
| 530 | }), |
| 531 | ); |
| 532 | }), |
| 533 | writeOutputChunk: Effect.fnUntraced(function* (streamName, chunk) { |
| 534 | if (environment.isDevelopment) { |
| 535 | yield* writeDevelopmentConsoleOutput(streamName, chunk); |
| 536 | } |
| 537 | yield* Ref.update( |
| 538 | sessionRef, |
| 539 | Option.map((session) => appendBoundedOutputChunk(session, streamName, chunk)), |
| 540 | ); |
| 541 | }), |
| 542 | persistFailureSnapshot: Effect.fn( |
| 543 | "desktop.observability.backendOutput.persistFailureSnapshot", |
| 544 | )(function* ({ details }) { |
| 545 | const session = yield* Ref.get(sessionRef); |
| 546 | if (Option.isSome(session)) { |
| 547 | yield* writeFailure(session.value, details); |
| 548 | } |
| 549 | }), |
| 550 | persistFailure: Effect.fn("desktop.observability.backendOutput.persistFailure")( |
| 551 | function* ({ details }) { |
| 552 | const session = yield* Ref.modify(sessionRef, (current) => [current, Option.none()]); |
| 553 | if (Option.isNone(session)) return; |
| 554 | yield* writeFailure(session.value, details); |
| 555 | }, |
| 556 | ), |
| 557 | discardSession: Ref.set(sessionRef, Option.none()), |
| 558 | } satisfies DesktopBackendOutputLogShape; |
| 559 | }), |
| 560 | }); |
| 561 | |
| 562 | const layerBackendOutputLogFactory = Layer.effect( |
| 563 | DesktopBackendOutputLogFactory, |
| 564 | Effect.gen(function* () { |
| 565 | const environment = yield* DesktopEnvironment.DesktopEnvironment; |
| 566 | const fileSystem = yield* FileSystem.FileSystem; |
| 567 | const path = yield* Path.Path; |
| 568 | const factoryScope = yield* Scope.Scope; |
| 569 | // Per-file-path cache of the IO sink only. The per-call shape |
| 570 | // wraps the sink with the caller's instance id so a cache hit on |
| 571 | // a path collision (e.g. "wsl:default" and "wsl_default" both |
| 572 | // resolve to server-child-wsl_default.log) doesn't attribute the |
| 573 | // second caller's writes to the first caller's id. Each sink pins |
| 574 | // itself to the factory's scope so all log resources tear down |
| 575 | // together at app exit. Mutex serializes concurrent first-time |
| 576 | // lookups for the same file path. |
| 577 | const cacheRef = yield* SynchronizedRef.make< |
| 578 | ReadonlyMap<string, Option.Option<RotatingLogFileWriter>> |
| 579 | >(new Map()); |
| 580 | |
| 581 | const makeForId = (id: string): Effect.Effect<DesktopBackendOutputLogShape> => |
| 582 | SynchronizedRef.modifyEffect(cacheRef, (cache) => { |
| 583 | const cacheKey = backendLogFilePathForInstance(environment, id); |
| 584 | const cached = cache.get(cacheKey); |
| 585 | if (cached !== undefined) { |
| 586 | return makeBackendOutputLogShape(environment, id, cached).pipe( |
| 587 | Effect.map((outputLog) => [outputLog, cache] as const), |
| 588 | ); |
| 589 | } |
| 590 | return makeBackendOutputSinkForInstance(environment, id).pipe( |
| 591 | Effect.provideService(FileSystem.FileSystem, fileSystem), |
| 592 | Effect.provideService(Path.Path, path), |
| 593 | Scope.provide(factoryScope), |
| 594 | Effect.map((sink) => { |
| 595 | const next = new Map(cache); |
| 596 | next.set(cacheKey, sink); |
| 597 | return { sink, next }; |
| 598 | }), |
| 599 | Effect.flatMap(({ sink, next }) => |
| 600 | makeBackendOutputLogShape(environment, id, sink).pipe( |
| 601 | Effect.map( |
| 602 | (outputLog) => |
| 603 | [ |
| 604 | outputLog, |
| 605 | next as ReadonlyMap<string, Option.Option<RotatingLogFileWriter>>, |
| 606 | ] as const, |
| 607 | ), |
| 608 | ), |
| 609 | ), |
| 610 | ); |
| 611 | }); |
| 612 | |
| 613 | return DesktopBackendOutputLogFactory.of({ |
| 614 | forInstance: (id) => makeForId(id), |
| 615 | }); |
| 616 | }), |
| 617 | ); |
| 618 | |
| 619 | /** |
| 620 | * Logs and traces for the main process, assembled together because they share |
| 621 | * one read of the environment and Settings, and because a process gets exactly |
| 622 | * one logger set. |
| 623 | */ |
| 624 | const layerTelemetry = Layer.unwrap( |
| 625 | Effect.gen(function* () { |
| 626 | const environment = yield* DesktopEnvironment.DesktopEnvironment; |
| 627 | const endpoints = yield* resolveOtlpEndpoints; |
| 628 | const resource = { |
| 629 | serviceName: "t3code-desktop", |
| 630 | attributes: { |
| 631 | "service.namespace": "t3code", |
| 632 | "service.runtime": "desktop", |
| 633 | "service.mode": environment.isDevelopment ? "development" : "packaged", |
| 634 | }, |
| 635 | }; |
| 636 | |
| 637 | // `Logger.layer` writes the whole logger set rather than adding to it, so |
| 638 | // every logger the main process wants has to be named in this one call. |
| 639 | // Splitting the OTLP logger back out into a layer of its own silently |
| 640 | // drops either it or the console output. |
| 641 | // |
| 642 | // Swapping `Logger.tracerLogger` out for the OTLP logger matches the |
| 643 | // server: both reach a collector, but the tracer logger covers only |
| 644 | // messages logged inside a recorded span and files them under traces, |
| 645 | // while the OTLP logger carries every message as a log record stamped |
| 646 | // with its trace and span ids. Keeping both would export every in-span |
| 647 | // message twice. |
| 648 | const layerLogger = Logger.layer( |
| 649 | endpoints.logs === undefined |
| 650 | ? [Logger.consolePretty(), Logger.tracerLogger] |
| 651 | : [ |
| 652 | Logger.consolePretty(), |
| 653 | OtlpLogger.make({ |
| 654 | url: endpoints.logs.url, |
| 655 | exportInterval: `${endpoints.logs.export.exportIntervalMs} millis`, |
| 656 | headers: endpoints.logs.export.headers, |
| 657 | resource, |
| 658 | }), |
| 659 | ], |
| 660 | { mergeWithExisting: false }, |
| 661 | ).pipe( |
| 662 | Layer.provide(OtlpExporter.layerFlusher), |
| 663 | Layer.provide( |
| 664 | SharedObservability.layerOtlpSerialization( |
| 665 | endpoints.logs?.export.protocol ?? environment.otlpProtocol, |
| 666 | ), |
| 667 | ), |
| 668 | ); |
| 669 | |
| 670 | const layerTracer = Layer.unwrap( |
| 671 | Effect.gen(function* () { |
| 672 | const tracePath = environment.path.join(environment.logDir, "desktop.trace.ndjson"); |
| 673 | const sink = yield* makeTraceSink({ |
| 674 | filePath: tracePath, |
| 675 | maxBytes: DESKTOP_LOG_FILE_MAX_BYTES, |
| 676 | maxFiles: DESKTOP_LOG_FILE_MAX_FILES, |
| 677 | batchWindowMs: DESKTOP_TRACE_BATCH_WINDOW_MS, |
| 678 | }); |
| 679 | const delegate = |
| 680 | endpoints.traces === undefined |
| 681 | ? undefined |
| 682 | : yield* OtlpTracer.make({ |
| 683 | url: endpoints.traces.url, |
| 684 | exportInterval: `${endpoints.traces.export.exportIntervalMs} millis`, |
| 685 | headers: endpoints.traces.export.headers, |
| 686 | resource, |
| 687 | }).pipe( |
| 688 | Effect.provide( |
| 689 | SharedObservability.layerOtlpSerialization(endpoints.traces.export.protocol), |
| 690 | ), |
| 691 | ); |
| 692 | const tracer = yield* makeLocalFileTracer({ |
| 693 | filePath: tracePath, |
| 694 | maxBytes: DESKTOP_LOG_FILE_MAX_BYTES, |
| 695 | maxFiles: DESKTOP_LOG_FILE_MAX_FILES, |
| 696 | batchWindowMs: DESKTOP_TRACE_BATCH_WINDOW_MS, |
| 697 | sink, |
| 698 | ...(delegate ? { delegate } : {}), |
| 699 | }); |
| 700 | |
| 701 | return Layer.succeed(Tracer.Tracer, tracer); |
| 702 | }), |
| 703 | ).pipe(Layer.provide(OtlpExporter.layerFlusher)); |
| 704 | |
| 705 | // Metrics stay off until the main process records one. `OtlpMetrics` |
| 706 | // exports on every interval even when the registry is empty, so wiring |
| 707 | // it up today would post an empty payload every ten seconds to any |
| 708 | // collector configured for the backend. Restore this when a desktop |
| 709 | // metric exists, and add it to the `Layer.mergeAll` below. |
| 710 | // |
| 711 | // const metricsLayer = |
| 712 | // endpoints.metrics === undefined |
| 713 | // ? Layer.empty |
| 714 | // : OtlpMetrics.layer({ |
| 715 | // url: endpoints.metrics.url, |
| 716 | // exportInterval: `${endpoints.metrics.export.exportIntervalMs} millis`, |
| 717 | // headers: endpoints.metrics.export.headers, |
| 718 | // resource, |
| 719 | // }).pipe(Layer.provide(SharedObservability.layerOtlpSerialization(endpoints.metrics.export.protocol))); |
| 720 | |
| 721 | // Logged once the loggers above are installed, so the warnings use them. |
| 722 | const layerOtelWarnings = Layer.effectDiscard( |
| 723 | Effect.forEach(endpoints.warnings, (warning) => Effect.logWarning(warning)), |
| 724 | ); |
| 725 | |
| 726 | return layerOtelWarnings.pipe( |
| 727 | Layer.provideMerge(Layer.mergeAll(layerLogger, layerTracer)), |
| 728 | Layer.provide(OtelEnvironment.layerResourceAttributes(endpoints.resourceAttributes)), |
| 729 | ); |
| 730 | }), |
| 731 | ); |
| 732 | |
| 733 | export const layer = Layer.mergeAll( |
| 734 | layerBackendOutputLogFactory, |
| 735 | layerTelemetry, |
| 736 | Layer.succeed(References.MinimumLogLevel, "Info"), |
| 737 | Layer.succeed(Tracer.MinimumTraceLevel, "Info"), |
| 738 | Layer.succeed(References.TracerTimingEnabled, true), |
| 739 | ); |