From 4292720fd927f4c81c21da4e596bdfcedbde32ce Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Oct 2026 10:42:47 -0700 Subject: [PATCH] fix(devices): report agent tool startup failures (#18502) --- apps/server/src/device/DeviceHost.ts | 35 ++- apps/server/src/device/DeviceService.test.ts | 38 ++- apps/server/src/device/DeviceService.ts | 8 +- .../server/src/device/LocalDeviceHost.test.ts | 224 +++++++++++++++++- apps/server/src/device/LocalDeviceHost.ts | 62 ++++- 5 files changed, 357 insertions(+), 10 deletions(-) diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index 14537985b9b852d12499ecc698ade49574e8e055..f8bb80b5b4e8228b1630bb52e45141d18308bab3 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -39,6 +39,39 @@ export class DeviceHostTimeoutError extends Schema.TaggedError()( + "AgentDeviceStartError", + { + hostId: Schema.String, + stateDir: Schema.String, + exitCode: Schema.NullOr(Schema.Int), + reason: Schema.Literals([ + "daemon_lock_not_directory", + "daemon_registration_unproven", + "daemon_retirement_unconfirmed", + "command_failed", + ]), + cause: Schema.Defect(), + }, +) { + override get message(): string { + const exit = this.exitCode === null ? "" : ` (exit code ${this.exitCode})`; + const detail = + this.reason === "daemon_lock_not_directory" + ? "Daemon registration ownership could not be verified: daemon.lock is not a directory." + : this.reason === "daemon_registration_unproven" + ? "Daemon registration ownership could not be verified." + : this.reason === "daemon_retirement_unconfirmed" + ? "Daemon replacement could not be confirmed." + : "The AgentDevice startup command failed."; + const recovery = + this.reason === "command_failed" + ? `Inspect daemon diagnostics in ${this.stateDir}.` + : `Inspect daemon.lock and daemon diagnostics in ${this.stateDir}. Stop older clients and daemons with their original CLI before manually repairing retained state.`; + return `Agent tools failed to start${exit}. ${detail} ${recovery}`; + } +} + export interface DeviceHubEndpoint { /** Loopback origin of expo-device-hub, e.g. `http://127.0.0.1:3400`. */ readonly origin: string; @@ -96,7 +129,7 @@ export class DeviceHost extends Context.Service< onPhase: (phase: "installing" | "starting", detail?: string) => Effect.Effect, ) => Effect.Effect< DeviceHostAgentReady, - DeviceHostError | DeviceHostTimeoutError | NodeRuntimeUnavailableError + DeviceHostError | DeviceHostTimeoutError | AgentDeviceStartError | NodeRuntimeUnavailableError >; /** Current endpoints when already running, without starting anything. */ readonly current: Effect.Effect; diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 55a7ac28a67cf5b4d284883fba73233353d17740..82ff8cec897825e3fae1a5e85ba3daab5df26e4d 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -66,7 +66,10 @@ const fixture = Effect.fn("fixture")(function* ( onBoot: Effect.Effect = Effect.void, bootError?: string, failListAfterShutdown = false, - runtimeFailure?: NodeRuntimeUnavailableError | DeviceHost.DeviceHostError, + runtimeFailure?: + | NodeRuntimeUnavailableError + | DeviceHost.DeviceHostError + | DeviceHost.AgentDeviceStartError, inspectError = false, installTool?: Parameters[3], ) { @@ -107,7 +110,8 @@ const fixture = Effect.fn("fixture")(function* ( platformAvailability: (platform) => Effect.succeed({ platform, available: true }), ensureReady: (onPhase) => Effect.gen(function* () { - if (runtimeFailure) return yield* runtimeFailure; + if (runtimeFailure && runtimeFailure._tag !== "AgentDeviceStartError") + return yield* runtimeFailure; starts.push("start"); yield* onPhase("installing", "Updating device hub from 0.9.0 to 0.10.1…"); return ready; @@ -217,6 +221,36 @@ const fixture = Effect.fn("fixture")(function* ( }); describe("device setup consent", () => { + it.effect("exposes daemon startup diagnostics through agent readiness and host status", () => + Effect.gen(function* () { + const startupError = new DeviceHost.AgentDeviceStartError({ + hostId: LOCAL_DEVICE_HOST_ID, + stateDir: "/test/device/agent-device", + exitCode: 1, + reason: "daemon_lock_not_directory", + cause: { stdout: "private startup diagnostics", code: 1 }, + }); + const { service, settings } = yield* fixture(Effect.void, undefined, false, startupError); + yield* Ref.update(settings, (current) => ({ + ...current, + enableDeviceSupport: true, + enableAgentDeviceAccess: true, + })); + const error = yield* service.agentReadinessIfSupported().pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "DeviceHostUnavailableError", + reason: startupError.message, + cause: startupError, + }); + expect(error.message).toContain("daemon.lock is not a directory"); + expect(error.message).not.toContain("private startup diagnostics"); + expect((yield* service.state).hostStatuses[LOCAL_DEVICE_HOST_ID]).toEqual({ + status: "failed", + detail: startupError.message, + }); + }).pipe(Effect.scoped), + ); + it.effect( "preserves missing-runtime guidance and causes through manual and agent readiness", () => diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 4226755d48eb1160e4eb99f38c83fa712f0f8e1f..3d1da7baa49acf70b99263259caddbb67cd7f33b 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -337,9 +337,11 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ? nodeRuntimeUnavailableMessage("Local device support") : error._tag === "DeviceHostTimeoutError" ? `Agent tools did not start within ${error.timeoutMs} ms.` - : error.step === "probe" - ? "Could not connect to this host over SSH." - : `Device support failed during ${error.step}.`, + : error._tag === "AgentDeviceStartError" + ? error.message + : error.step === "probe" + ? "Could not connect to this host over SSH." + : `Device support failed during ${error.step}.`, cause: error, }), ), diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts index 288b70763a9a0072bd4656a598caf2590d675890..eef93e4bdaec50494f6ba2b1349fc14aa3b1682b 100644 --- a/apps/server/src/device/LocalDeviceHost.test.ts +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -6,14 +6,236 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as FileSystem from "effect/FileSystem"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import * as NetService from "@t3tools/shared/Net"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; +import { + AGENT_DEVICE_VERSION, + DEVICE_HUB_VERSION, + agentDeviceStateDir, +} from "./DeviceToolchain.ts"; + +const processOutput = (code: number, stdout = "", stderr = "", timedOut = false) => ({ + code: ChildProcessSpawner.ExitCode(code), + stdout, + stderr, + timedOut, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, +}); + +const agentFixture = Effect.fn("agentFixture")(function* ( + startup: ( + stateDir: string, + ) => Effect.Effect, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-agent-start-" }); + const stateDir = agentDeviceStateDir(path, path.join(baseDir, "userdata")); + for (const [name, version, entry] of [ + ["expo-device-hub", DEVICE_HUB_VERSION, "dist/server/cli.mjs"], + ["agent-device", AGENT_DEVICE_VERSION, "bin/agent-device.mjs"], + ]) { + const installDir = path.join(baseDir, "tools", name!, version!); + const entryPath = path.join(installDir, "node_modules", name!, entry!); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, ""); + yield* fs.writeFileString(path.join(installDir, ".install-complete"), version!); + } + const host = yield* LocalDeviceHost.make().pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(baseDir, baseDir), NetService.layer)), + Effect.provideService(HostProcess.Environment, { HOME: baseDir, PATH: "" }), + Effect.provideService(HostProcess.IsExecutable, false), + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ), + ), + ), + Effect.provideService(ProcessRunner.ProcessRunner, { + run: (input) => + input.args.includes("devices") ? startup(stateDir) : Effect.succeed(processOutput(0)), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok"))), + ), + ), + ); + yield* host.ensureReady(() => Effect.void); + return { host, fs, stateDir }; +}); + +describe("AgentDevice startup", () => { + it.effect("reports an incompatible lock immediately and retains it for manual repair", () => + Effect.gen(function* () { + const result = processOutput( + 1, + JSON.stringify({ + success: false, + error: { + message: "private command output", + details: { + reason: "daemon_registration_unproven", + inspection: { state: "unproven", reason: "non-directory" }, + }, + }, + }), + ); + const { host, fs, stateDir } = yield* agentFixture(() => Effect.succeed(result)); + const lock = `${stateDir}/daemon.lock`; + const legacyOwner = JSON.stringify({ pid: 85669, processStartTime: "old start time" }); + yield* fs.writeFileString(lock, legacyOwner); + const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "AgentDeviceStartError", + reason: "daemon_lock_not_directory", + exitCode: 1, + cause: result, + }); + expect(error.message).toContain("Daemon registration ownership could not be verified"); + expect(error.message).toContain("daemon.lock is not a directory"); + expect(error.message).toContain(stateDir); + expect(error.message).toContain("original CLI"); + expect(error.message).not.toContain("private command output"); + expect(yield* fs.readFileString(lock)).toBe(legacyOwner); + expect(yield* host.current).not.toBeNull(); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("reports unknown nonzero exits without exposing arbitrary command output", () => + Effect.gen(function* () { + const result = processOutput(2, "invalid JSON", "private credentials"); + const { host, stateDir } = yield* agentFixture(() => Effect.succeed(result)); + const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "AgentDeviceStartError", + reason: "command_failed", + exitCode: 2, + cause: result, + }); + expect(error.message).toContain("exit code 2"); + expect(error.message).toContain(stateDir); + expect(error.message).not.toContain("private credentials"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("preserves process execution failures instead of waiting for a daemon file", () => + Effect.gen(function* () { + const cause = new ProcessRunner.ProcessSpawnError({ + command: "/missing/node", + argumentCount: 3, + cause: new Error("missing executable"), + }); + const { host } = yield* agentFixture(() => Effect.fail(cause)); + const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "DeviceHostError", + step: "starting agent tools", + cause, + }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("returns a command timeout without starting another readiness wait", () => + Effect.gen(function* () { + const { host } = yield* agentFixture(() => Effect.succeed(processOutput(124, "", "", true))); + const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "DeviceHostTimeoutError", timeoutMs: 30_000 }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("uses a registered daemon when reading startup command output fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cause = new ProcessRunner.ProcessReadError({ + command: process.execPath, + argumentCount: 3, + stream: "stdout", + cause: new Error("output read failed after registration"), + }); + let attempts = 0; + const { host } = yield* agentFixture((stateDir) => + Effect.gen(function* () { + attempts++; + yield* fs + .writeFileString( + `${stateDir}/daemon.json`, + JSON.stringify({ httpPort: 12345, token: "test-token" }), + ) + .pipe(Effect.orDie); + return yield* cause; + }), + ); + const ready = yield* host.ensureAgentReady(() => Effect.void); + expect(ready.agentDevice).toMatchObject({ + baseUrl: "http://127.0.0.1:12345", + token: "test-token", + }); + expect((yield* host.ensureAgentReady(() => Effect.void)).agentDevice).toEqual( + ready.agentDevice, + ); + expect(attempts).toBe(1); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("allows retry and uses a registered daemon even when discovery exits nonzero", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let attempts = 0; + const { host } = yield* agentFixture((stateDir) => + Effect.gen(function* () { + attempts++; + if (attempts > 1) { + yield* fs + .writeFileString( + `${stateDir}/daemon.json`, + JSON.stringify({ httpPort: 12345, token: "test-token" }), + ) + .pipe(Effect.orDie); + } + return processOutput(1); + }), + ); + yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip); + const ready = yield* host.ensureAgentReady(() => Effect.void); + expect(ready.agentDevice).toMatchObject({ + baseUrl: "http://127.0.0.1:12345", + token: "test-token", + }); + expect((yield* host.ensureAgentReady(() => Effect.void)).agentDevice).toEqual( + ready.agentDevice, + ); + expect(attempts).toBe(2); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); const diagnose = ( files: ReadonlyArray, diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 779221ab93cf7ca9b804588c6f4e03956a855a3b..63245da93d2d8a70506f137f7af7017f13d64af9 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -86,6 +86,19 @@ const AgentDeviceDaemonFile = Schema.Struct({ }); const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); +const decodeAgentDeviceFailure = Schema.decodeUnknownOption( + Schema.fromJsonString( + Schema.Struct({ + error: Schema.Struct({ + details: Schema.Struct({ + reason: Schema.String, + inspection: Schema.optional(Schema.Struct({ reason: Schema.String })), + }), + }), + }), + ), +); + interface HubProcess { readonly nodePath: string; readonly child: ChildProcessSpawner.ChildProcessHandle; @@ -498,7 +511,12 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( agentTool: DeviceToolPaths, nodePath: string, - ): Effect.fn.Return { + ): Effect.fn.Return< + DeviceHost.AgentDeviceEndpoint, + | DeviceHost.DeviceHostError + | DeviceHost.DeviceHostTimeoutError + | DeviceHost.AgentDeviceStartError + > { const stateDir = agentDeviceStateDir(path, config.stateDir); yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); const existing = yield* readDaemonFile().pipe(Effect.option); @@ -536,7 +554,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { } // There is no `daemon start`; the first command in a state dir spawns the // daemon and blocks until it answers. `devices` is the cheapest one. - yield* runner + const runResult = yield* runner .run({ command: nodePath, args: [agentTool.entryPath, "devices", "--json"], @@ -544,7 +562,44 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), timeoutBehavior: "timedOutResult", }) - .pipe(Effect.ignore); + .pipe(Effect.result); + // Discovery can fail after the daemon starts. A published endpoint still + // lets agents connect; absent one, preserve the CLI failure instead of polling. + const started = yield* readDaemonFile().pipe(Effect.option); + if (started._tag === "Some") return toEndpoint(started.value); + if (runResult._tag === "Failure") { + return yield* new DeviceHost.DeviceHostError({ + hostId, + step: "starting agent tools", + cause: runResult.failure, + }); + } + const result = runResult.success; + if (result.timedOut) { + return yield* new DeviceHost.DeviceHostTimeoutError({ + hostId, + timeoutMs: DAEMON_READY_TIMEOUT_MS, + }); + } + if (result.code !== 0) { + const failure = decodeAgentDeviceFailure(result.stdout); + const details = Option.isSome(failure) ? failure.value.error.details : undefined; + const reason = + details?.reason === "daemon_registration_unproven" + ? details.inspection?.reason === "non-directory" + ? "daemon_lock_not_directory" + : "daemon_registration_unproven" + : details?.reason === "daemon_retirement_unconfirmed" + ? "daemon_retirement_unconfirmed" + : "command_failed"; + return yield* new DeviceHost.AgentDeviceStartError({ + hostId, + stateDir, + exitCode: result.code === null ? null : Number(result.code), + reason, + cause: result, + }); + } const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; while (true) { const file = yield* readDaemonFile().pipe(Effect.option); @@ -658,6 +713,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { unknown, | DeviceHost.DeviceHostError | DeviceHost.DeviceHostTimeoutError + | DeviceHost.AgentDeviceStartError | NodeRuntimeUnavailableError >, DeviceHost.DeviceHostAgentReady -- 2.54.0