authorgravatar for me@pixp.ccAlex <me@pixp.cc> 2026-10-13 10:42:47-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-13 12:42:47-05:00
log4292720fd927f4c81c21da4e596bdfcedbde32ce
tree915565debd4933913f27474c3c007b665bb05eec
parent4d7683557ad4a2411d6de405e500e4a56b43d0e1
signature Signed by PGP key B5690EEEBB952194

fix(devices): report agent tool startup failures (#18502)

5 files changed, 357 insertions(+), 10 deletions(-)

apps/server/src/device/DeviceHost.ts+34-1
......@@ -39,6 +39,39 @@ export class DeviceHostTimeoutError extends Schema.TaggedError<DeviceHostTimeout
3939 }
4040}
4141
42export class AgentDeviceStartError extends Schema.TaggedError<AgentDeviceStartError>()(
43 "AgentDeviceStartError",
44 {
45 hostId: Schema.String,
46 stateDir: Schema.String,
47 exitCode: Schema.NullOr(Schema.Int),
48 reason: Schema.Literals([
49 "daemon_lock_not_directory",
50 "daemon_registration_unproven",
51 "daemon_retirement_unconfirmed",
52 "command_failed",
53 ]),
54 cause: Schema.Defect(),
55 },
56) {
57 override get message(): string {
58 const exit = this.exitCode === null ? "" : ` (exit code ${this.exitCode})`;
59 const detail =
60 this.reason === "daemon_lock_not_directory"
61 ? "Daemon registration ownership could not be verified: daemon.lock is not a directory."
62 : this.reason === "daemon_registration_unproven"
63 ? "Daemon registration ownership could not be verified."
64 : this.reason === "daemon_retirement_unconfirmed"
65 ? "Daemon replacement could not be confirmed."
66 : "The AgentDevice startup command failed.";
67 const recovery =
68 this.reason === "command_failed"
69 ? `Inspect daemon diagnostics in ${this.stateDir}.`
70 : `Inspect daemon.lock and daemon diagnostics in ${this.stateDir}. Stop older clients and daemons with their original CLI before manually repairing retained state.`;
71 return `Agent tools failed to start${exit}. ${detail} ${recovery}`;
72 }
73}
74
4275export interface DeviceHubEndpoint {
4376 /** Loopback origin of expo-device-hub, e.g. `http://127.0.0.1:3400`. */
4477 readonly origin: string;
......@@ -96,7 +129,7 @@ export class DeviceHost extends Context.Service<
96129 onPhase: (phase: "installing" | "starting", detail?: string) => Effect.Effect<void>,
97130 ) => Effect.Effect<
98131 DeviceHostAgentReady,
99 DeviceHostError | DeviceHostTimeoutError | NodeRuntimeUnavailableError
132 DeviceHostError | DeviceHostTimeoutError | AgentDeviceStartError | NodeRuntimeUnavailableError
100133 >;
101134 /** Current endpoints when already running, without starting anything. */
102135 readonly current: Effect.Effect<DeviceHostReady | null>;
apps/server/src/device/DeviceService.test.ts+36-2
......@@ -66,7 +66,10 @@ const fixture = Effect.fn("fixture")(function* (
6666 onBoot: Effect.Effect<void> = Effect.void,
6767 bootError?: string,
6868 failListAfterShutdown = false,
69 runtimeFailure?: NodeRuntimeUnavailableError | DeviceHost.DeviceHostError,
69 runtimeFailure?:
70 | NodeRuntimeUnavailableError
71 | DeviceHost.DeviceHostError
72 | DeviceHost.AgentDeviceStartError,
7073 inspectError = false,
7174 installTool?: Parameters<typeof DeviceService.makeWithHosts>[3],
7275) {
......@@ -107,7 +110,8 @@ const fixture = Effect.fn("fixture")(function* (
107110 platformAvailability: (platform) => Effect.succeed({ platform, available: true }),
108111 ensureReady: (onPhase) =>
109112 Effect.gen(function* () {
110 if (runtimeFailure) return yield* runtimeFailure;
113 if (runtimeFailure && runtimeFailure._tag !== "AgentDeviceStartError")
114 return yield* runtimeFailure;
111115 starts.push("start");
112116 yield* onPhase("installing", "Updating device hub from 0.9.0 to 0.10.1…");
113117 return ready;
......@@ -217,6 +221,36 @@ const fixture = Effect.fn("fixture")(function* (
217221});
218222
219223describe("device setup consent", () => {
224 it.effect("exposes daemon startup diagnostics through agent readiness and host status", () =>
225 Effect.gen(function* () {
226 const startupError = new DeviceHost.AgentDeviceStartError({
227 hostId: LOCAL_DEVICE_HOST_ID,
228 stateDir: "/test/device/agent-device",
229 exitCode: 1,
230 reason: "daemon_lock_not_directory",
231 cause: { stdout: "private startup diagnostics", code: 1 },
232 });
233 const { service, settings } = yield* fixture(Effect.void, undefined, false, startupError);
234 yield* Ref.update(settings, (current) => ({
235 ...current,
236 enableDeviceSupport: true,
237 enableAgentDeviceAccess: true,
238 }));
239 const error = yield* service.agentReadinessIfSupported().pipe(Effect.flip);
240 expect(error).toMatchObject({
241 _tag: "DeviceHostUnavailableError",
242 reason: startupError.message,
243 cause: startupError,
244 });
245 expect(error.message).toContain("daemon.lock is not a directory");
246 expect(error.message).not.toContain("private startup diagnostics");
247 expect((yield* service.state).hostStatuses[LOCAL_DEVICE_HOST_ID]).toEqual({
248 status: "failed",
249 detail: startupError.message,
250 });
251 }).pipe(Effect.scoped),
252 );
253
220254 it.effect(
221255 "preserves missing-runtime guidance and causes through manual and agent readiness",
222256 () =>
apps/server/src/device/DeviceService.ts+5-3
......@@ -337,9 +337,11 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function*
337337 ? nodeRuntimeUnavailableMessage("Local device support")
338338 : error._tag === "DeviceHostTimeoutError"
339339 ? `Agent tools did not start within ${error.timeoutMs} ms.`
340 : error.step === "probe"
341 ? "Could not connect to this host over SSH."
342 : `Device support failed during ${error.step}.`,
340 : error._tag === "AgentDeviceStartError"
341 ? error.message
342 : error.step === "probe"
343 ? "Could not connect to this host over SSH."
344 : `Device support failed during ${error.step}.`,
343345 cause: error,
344346 }),
345347 ),
apps/server/src/device/LocalDeviceHost.test.ts+223-1
......@@ -6,14 +6,236 @@ import * as Layer from "effect/Layer";
66import * as Option from "effect/Option";
77import * as Path from "effect/Path";
88import * as FileSystem from "effect/FileSystem";
9import * as Sink from "effect/Sink";
10import * as Stream from "effect/Stream";
911
1012import * as LocalDeviceHost from "./LocalDeviceHost.ts";
1113import * as NodeServices from "@effect/platform-node/NodeServices";
1214import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
13import { HttpClient } from "effect/http";
15import { HttpClient, HttpClientResponse } from "effect/http";
1416import * as NetService from "@t3tools/shared/Net";
1517import * as ServerConfig from "../config.ts";
1618import * as ProcessRunner from "../processRunner.ts";
19import {
20 AGENT_DEVICE_VERSION,
21 DEVICE_HUB_VERSION,
22 agentDeviceStateDir,
23} from "./DeviceToolchain.ts";
24
25const processOutput = (code: number, stdout = "", stderr = "", timedOut = false) => ({
26 code: ChildProcessSpawner.ExitCode(code),
27 stdout,
28 stderr,
29 timedOut,
30 stdoutTruncated: false,
31 stderrTruncated: false,
32 stdoutInvalidUtf8: false,
33 stderrInvalidUtf8: false,
34});
35
36const agentFixture = Effect.fn("agentFixture")(function* (
37 startup: (
38 stateDir: string,
39 ) => Effect.Effect<ProcessRunner.ProcessRunOutput, ProcessRunner.ProcessRunError>,
40) {
41 const fs = yield* FileSystem.FileSystem;
42 const path = yield* Path.Path;
43 const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-agent-start-" });
44 const stateDir = agentDeviceStateDir(path, path.join(baseDir, "userdata"));
45 for (const [name, version, entry] of [
46 ["expo-device-hub", DEVICE_HUB_VERSION, "dist/server/cli.mjs"],
47 ["agent-device", AGENT_DEVICE_VERSION, "bin/agent-device.mjs"],
48 ]) {
49 const installDir = path.join(baseDir, "tools", name!, version!);
50 const entryPath = path.join(installDir, "node_modules", name!, entry!);
51 yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true });
52 yield* fs.writeFileString(entryPath, "");
53 yield* fs.writeFileString(path.join(installDir, ".install-complete"), version!);
54 }
55 const host = yield* LocalDeviceHost.make().pipe(
56 Effect.provide(Layer.mergeAll(ServerConfig.layerTest(baseDir, baseDir), NetService.layer)),
57 Effect.provideService(HostProcess.Environment, { HOME: baseDir, PATH: "" }),
58 Effect.provideService(HostProcess.IsExecutable, false),
59 Effect.provideService(HostProcess.Platform, "darwin"),
60 Effect.provideService(
61 ChildProcessSpawner.ChildProcessSpawner,
62 ChildProcessSpawner.make(() =>
63 Effect.succeed(
64 ChildProcessSpawner.makeHandle({
65 pid: ChildProcessSpawner.ProcessId(123),
66 stdout: Stream.empty,
67 stderr: Stream.empty,
68 all: Stream.empty,
69 exitCode: Effect.never,
70 isRunning: Effect.succeed(true),
71 kill: () => Effect.void,
72 stdin: Sink.drain,
73 getInputFd: () => Sink.drain,
74 getOutputFd: () => Stream.empty,
75 unref: Effect.succeed(Effect.void),
76 }),
77 ),
78 ),
79 ),
80 Effect.provideService(ProcessRunner.ProcessRunner, {
81 run: (input) =>
82 input.args.includes("devices") ? startup(stateDir) : Effect.succeed(processOutput(0)),
83 }),
84 Effect.provideService(
85 HttpClient.HttpClient,
86 HttpClient.make((request) =>
87 Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok"))),
88 ),
89 ),
90 );
91 yield* host.ensureReady(() => Effect.void);
92 return { host, fs, stateDir };
93});
94
95describe("AgentDevice startup", () => {
96 it.effect("reports an incompatible lock immediately and retains it for manual repair", () =>
97 Effect.gen(function* () {
98 const result = processOutput(
99 1,
100 JSON.stringify({
101 success: false,
102 error: {
103 message: "private command output",
104 details: {
105 reason: "daemon_registration_unproven",
106 inspection: { state: "unproven", reason: "non-directory" },
107 },
108 },
109 }),
110 );
111 const { host, fs, stateDir } = yield* agentFixture(() => Effect.succeed(result));
112 const lock = `${stateDir}/daemon.lock`;
113 const legacyOwner = JSON.stringify({ pid: 85669, processStartTime: "old start time" });
114 yield* fs.writeFileString(lock, legacyOwner);
115 const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip);
116 expect(error).toMatchObject({
117 _tag: "AgentDeviceStartError",
118 reason: "daemon_lock_not_directory",
119 exitCode: 1,
120 cause: result,
121 });
122 expect(error.message).toContain("Daemon registration ownership could not be verified");
123 expect(error.message).toContain("daemon.lock is not a directory");
124 expect(error.message).toContain(stateDir);
125 expect(error.message).toContain("original CLI");
126 expect(error.message).not.toContain("private command output");
127 expect(yield* fs.readFileString(lock)).toBe(legacyOwner);
128 expect(yield* host.current).not.toBeNull();
129 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
130 );
131
132 it.effect("reports unknown nonzero exits without exposing arbitrary command output", () =>
133 Effect.gen(function* () {
134 const result = processOutput(2, "invalid JSON", "private credentials");
135 const { host, stateDir } = yield* agentFixture(() => Effect.succeed(result));
136 const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip);
137 expect(error).toMatchObject({
138 _tag: "AgentDeviceStartError",
139 reason: "command_failed",
140 exitCode: 2,
141 cause: result,
142 });
143 expect(error.message).toContain("exit code 2");
144 expect(error.message).toContain(stateDir);
145 expect(error.message).not.toContain("private credentials");
146 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
147 );
148
149 it.effect("preserves process execution failures instead of waiting for a daemon file", () =>
150 Effect.gen(function* () {
151 const cause = new ProcessRunner.ProcessSpawnError({
152 command: "/missing/node",
153 argumentCount: 3,
154 cause: new Error("missing executable"),
155 });
156 const { host } = yield* agentFixture(() => Effect.fail(cause));
157 const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip);
158 expect(error).toMatchObject({
159 _tag: "DeviceHostError",
160 step: "starting agent tools",
161 cause,
162 });
163 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
164 );
165
166 it.effect("returns a command timeout without starting another readiness wait", () =>
167 Effect.gen(function* () {
168 const { host } = yield* agentFixture(() => Effect.succeed(processOutput(124, "", "", true)));
169 const error = yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip);
170 expect(error).toMatchObject({ _tag: "DeviceHostTimeoutError", timeoutMs: 30_000 });
171 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
172 );
173
174 it.effect("uses a registered daemon when reading startup command output fails", () =>
175 Effect.gen(function* () {
176 const fs = yield* FileSystem.FileSystem;
177 const cause = new ProcessRunner.ProcessReadError({
178 command: process.execPath,
179 argumentCount: 3,
180 stream: "stdout",
181 cause: new Error("output read failed after registration"),
182 });
183 let attempts = 0;
184 const { host } = yield* agentFixture((stateDir) =>
185 Effect.gen(function* () {
186 attempts++;
187 yield* fs
188 .writeFileString(
189 `${stateDir}/daemon.json`,
190 JSON.stringify({ httpPort: 12345, token: "test-token" }),
191 )
192 .pipe(Effect.orDie);
193 return yield* cause;
194 }),
195 );
196 const ready = yield* host.ensureAgentReady(() => Effect.void);
197 expect(ready.agentDevice).toMatchObject({
198 baseUrl: "http://127.0.0.1:12345",
199 token: "test-token",
200 });
201 expect((yield* host.ensureAgentReady(() => Effect.void)).agentDevice).toEqual(
202 ready.agentDevice,
203 );
204 expect(attempts).toBe(1);
205 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
206 );
207
208 it.effect("allows retry and uses a registered daemon even when discovery exits nonzero", () =>
209 Effect.gen(function* () {
210 const fs = yield* FileSystem.FileSystem;
211 let attempts = 0;
212 const { host } = yield* agentFixture((stateDir) =>
213 Effect.gen(function* () {
214 attempts++;
215 if (attempts > 1) {
216 yield* fs
217 .writeFileString(
218 `${stateDir}/daemon.json`,
219 JSON.stringify({ httpPort: 12345, token: "test-token" }),
220 )
221 .pipe(Effect.orDie);
222 }
223 return processOutput(1);
224 }),
225 );
226 yield* host.ensureAgentReady(() => Effect.void).pipe(Effect.flip);
227 const ready = yield* host.ensureAgentReady(() => Effect.void);
228 expect(ready.agentDevice).toMatchObject({
229 baseUrl: "http://127.0.0.1:12345",
230 token: "test-token",
231 });
232 expect((yield* host.ensureAgentReady(() => Effect.void)).agentDevice).toEqual(
233 ready.agentDevice,
234 );
235 expect(attempts).toBe(2);
236 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
237 );
238});
17239
18240const diagnose = (
19241 files: ReadonlyArray<string>,
apps/server/src/device/LocalDeviceHost.ts+59-3
......@@ -86,6 +86,19 @@ const AgentDeviceDaemonFile = Schema.Struct({
8686});
8787const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile));
8888
89const decodeAgentDeviceFailure = Schema.decodeUnknownOption(
90 Schema.fromJsonString(
91 Schema.Struct({
92 error: Schema.Struct({
93 details: Schema.Struct({
94 reason: Schema.String,
95 inspection: Schema.optional(Schema.Struct({ reason: Schema.String })),
96 }),
97 }),
98 }),
99 ),
100);
101
89102interface HubProcess {
90103 readonly nodePath: string;
91104 readonly child: ChildProcessSpawner.ChildProcessHandle;
......@@ -498,7 +511,12 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
498511 const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* (
499512 agentTool: DeviceToolPaths,
500513 nodePath: string,
501 ): Effect.fn.Return<DeviceHost.AgentDeviceEndpoint, DeviceHost.DeviceHostTimeoutError> {
514 ): Effect.fn.Return<
515 DeviceHost.AgentDeviceEndpoint,
516 | DeviceHost.DeviceHostError
517 | DeviceHost.DeviceHostTimeoutError
518 | DeviceHost.AgentDeviceStartError
519 > {
502520 const stateDir = agentDeviceStateDir(path, config.stateDir);
503521 yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore);
504522 const existing = yield* readDaemonFile().pipe(Effect.option);
......@@ -536,7 +554,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
536554 }
537555 // There is no `daemon start`; the first command in a state dir spawns the
538556 // daemon and blocks until it answers. `devices` is the cheapest one.
539 yield* runner
557 const runResult = yield* runner
540558 .run({
541559 command: nodePath,
542560 args: [agentTool.entryPath, "devices", "--json"],
......@@ -544,7 +562,44 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
544562 timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS),
545563 timeoutBehavior: "timedOutResult",
546564 })
547 .pipe(Effect.ignore);
565 .pipe(Effect.result);
566 // Discovery can fail after the daemon starts. A published endpoint still
567 // lets agents connect; absent one, preserve the CLI failure instead of polling.
568 const started = yield* readDaemonFile().pipe(Effect.option);
569 if (started._tag === "Some") return toEndpoint(started.value);
570 if (runResult._tag === "Failure") {
571 return yield* new DeviceHost.DeviceHostError({
572 hostId,
573 step: "starting agent tools",
574 cause: runResult.failure,
575 });
576 }
577 const result = runResult.success;
578 if (result.timedOut) {
579 return yield* new DeviceHost.DeviceHostTimeoutError({
580 hostId,
581 timeoutMs: DAEMON_READY_TIMEOUT_MS,
582 });
583 }
584 if (result.code !== 0) {
585 const failure = decodeAgentDeviceFailure(result.stdout);
586 const details = Option.isSome(failure) ? failure.value.error.details : undefined;
587 const reason =
588 details?.reason === "daemon_registration_unproven"
589 ? details.inspection?.reason === "non-directory"
590 ? "daemon_lock_not_directory"
591 : "daemon_registration_unproven"
592 : details?.reason === "daemon_retirement_unconfirmed"
593 ? "daemon_retirement_unconfirmed"
594 : "command_failed";
595 return yield* new DeviceHost.AgentDeviceStartError({
596 hostId,
597 stateDir,
598 exitCode: result.code === null ? null : Number(result.code),
599 reason,
600 cause: result,
601 });
602 }
548603 const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS;
549604 while (true) {
550605 const file = yield* readDaemonFile().pipe(Effect.option);
......@@ -658,6 +713,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
658713 unknown,
659714 | DeviceHost.DeviceHostError
660715 | DeviceHost.DeviceHostTimeoutError
716 | DeviceHost.AgentDeviceStartError
661717 | NodeRuntimeUnavailableError
662718 >,
663719 DeviceHost.DeviceHostAgentReady