authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-20 22:08:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-20 22:08:51-07:00
log790be6d755026d61bd13a970dd77176c12ca0425
tree2e7e4e07378fa3769ccf2a419189e33502e97bb3
parent61b25b88d7922a480790066341867f3f8ae42fcb
signature Signed by PGP key B5690EEEBB952194

fix(devices): safely reclaim obsolete managed tool versions (#12819)

6 files changed, 325 insertions(+), 1 deletions(-)

apps/server/src/device/DeviceService.ts+1-1
...@@ -1111,7 +1111,7 @@ export const make = Effect.gen(function* () {...@@ -1111,7 +1111,7 @@ export const make = Effect.gen(function* () {
1111 return {1111 return {
1112 ...service,1112 ...service,
1113 agentCli: resolveNodeExecutable("Device automation").pipe(1113 agentCli: resolveNodeExecutable("Device automation").pipe(
1114 Effect.flatMap(() => ensureAgentDevice(config.baseDir)),1114 Effect.andThen(ensureAgentDevice(config.baseDir)),
1115 Effect.provideService(FileSystem.FileSystem, fs),1115 Effect.provideService(FileSystem.FileSystem, fs),
1116 Effect.provideService(Path.Path, path),1116 Effect.provideService(Path.Path, path),
1117 Effect.provideService(ProcessRunner.ProcessRunner, runner),1117 Effect.provideService(ProcessRunner.ProcessRunner, runner),
apps/server/src/device/LocalDeviceHost.ts+11
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import { pruneLocalDeviceTools } from "./deviceToolMaintenance.ts";
1import { deviceToolInstallMessage } from "@t3tools/contracts";2import { deviceToolInstallMessage } from "@t3tools/contracts";
2/**3/**
3 * The device host that is this machine.4 * The device host that is this machine.
...@@ -593,6 +594,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {...@@ -593,6 +594,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
593 );594 );
594 yield* onPhase("starting");595 yield* onPhase("starting");
595 const hub = yield* spawnHub(hubTool, nodePath);596 const hub = yield* spawnHub(hubTool, nodePath);
597 yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe(
598 Effect.provideService(Path.Path, path),
599 Effect.provideService(ProcessRunner.ProcessRunner, runner),
600 Effect.ignore,
601 );
596 const candidate = helperPaths(hubTool);602 const candidate = helperPaths(hubTool);
597 const [axExists, cliExists] = yield* Effect.all([603 const [axExists, cliExists] = yield* Effect.all([
598 fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)),604 fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)),
...@@ -655,6 +661,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {...@@ -655,6 +661,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
655 agentToolRef = { entryPath: agentTool.entryPath, nodePath: running.hub.nodePath };661 agentToolRef = { entryPath: agentTool.entryPath, nodePath: running.hub.nodePath };
656 yield* onPhase("starting");662 yield* onPhase("starting");
657 const agentDevice = yield* startAgentDeviceDaemon(agentTool, running.hub.nodePath);663 const agentDevice = yield* startAgentDeviceDaemon(agentTool, running.hub.nodePath);
664 yield* pruneLocalDeviceTools(config.baseDir, running.hub.nodePath, "agent").pipe(
665 Effect.provideService(Path.Path, path),
666 Effect.provideService(ProcessRunner.ProcessRunner, runner),
667 Effect.ignore,
668 );
658 const next = { ...running, agentDevice };669 const next = { ...running, agentDevice };
659 yield* Ref.set(runningRef, next);670 yield* Ref.set(runningRef, next);
660 return { ...toReady(next), agentDevice };671 return { ...toReady(next), agentDevice };
apps/server/src/device/deviceToolMaintenance.test.ts created+159
...@@ -0,0 +1,159 @@
1import * as Effect from "effect/Effect";
2import * as NodePathLayer from "@effect/platform-node/NodePath";
3import * as ProcessRunner from "../processRunner.ts";
4import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
5// @effect-diagnostics nodeBuiltinImport:off - tests the same standalone script used by local and SSH hosts.
6import { describe, expect, it } from "@effect/vitest";
7import * as NodeFSP from "node:fs/promises";
8import * as NodeOS from "node:os";
9import * as NodePath from "node:path";
10import * as NodeChildProcess from "node:child_process";
11import * as NodeUtil from "node:util";
12import { pruneLocalDeviceTools, deviceToolMaintenanceScript } from "./deviceToolMaintenance.ts";
13
14const exec = NodeUtil.promisify(NodeChildProcess.execFile);
15
16describe.each([false, true])("device tool cleanup, flat=%s", (flat) => {
17 it("keeps current, previous, active and incomplete installs, pruning unused completed versions", async () => {
18 const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-cleanup-"));
19 const name = "expo-device-hub";
20 const directory = (version: string) =>
21 flat ? NodePath.join(root, `${name}@${version}`) : NodePath.join(root, name, version);
22 try {
23 for (const version of ["0.1.0", "0.2.0", "0.3.0", "0.4.0", "0.5.0", "0.6.0"]) {
24 await NodeFSP.mkdir(directory(version), { recursive: true });
25 if (version === "0.5.0") continue;
26 const sentinel = NodePath.join(directory(version), ".install-complete");
27 await NodeFSP.writeFile(sentinel, version);
28 await NodeFSP.utimes(
29 sentinel,
30 Number(version.split(".")[1]),
31 Number(version.split(".")[1]),
32 );
33 }
34 const script =
35 deviceToolMaintenanceScript +
36 `
37(async () => {
38 const root = ${JSON.stringify(root)};
39 await pruneTools(root, [['${name}', '0.6.0']], ${flat});
40})().catch(error => { console.error(error); process.exitCode = 1; });`;
41 await exec(process.execPath, [
42 "-e",
43 script,
44 NodePath.join(directory("0.2.0"), "active-helper.cjs"),
45 ]);
46 await expect(NodeFSP.stat(directory("0.1.0"))).rejects.toThrow();
47 await expect(NodeFSP.stat(directory("0.3.0"))).rejects.toThrow();
48 for (const version of ["0.2.0", "0.4.0", "0.5.0", "0.6.0"])
49 expect((await NodeFSP.stat(directory(version))).isDirectory()).toBe(true);
50 } finally {
51 await NodeFSP.rm(root, { recursive: true, force: true });
52 }
53 });
54
55 it("keeps every install when the process scan fails", async () => {
56 const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-scan-"));
57 try {
58 for (const version of ["0.1.0", "0.2.0", "0.3.0"]) {
59 const dir = flat
60 ? NodePath.join(root, `expo-device-hub@${version}`)
61 : NodePath.join(root, "expo-device-hub", version);
62 await NodeFSP.mkdir(dir, { recursive: true });
63 await NodeFSP.writeFile(NodePath.join(dir, ".install-complete"), version);
64 }
65 await exec(process.execPath, [
66 "-e",
67 deviceToolMaintenanceScript +
68 `
69 require('node:child_process').spawnSync = () => ({ status: 1, stdout: '' });
70 pruneTools(${JSON.stringify(root)}, [['expo-device-hub', '0.3.0']], ${flat}).catch(() => process.exitCode = 1);
71 `,
72 ]);
73 const parent = flat ? root : NodePath.join(root, "expo-device-hub");
74 expect((await NodeFSP.readdir(parent)).length).toBe(3);
75 } finally {
76 await NodeFSP.rm(root, { recursive: true, force: true });
77 }
78 });
79
80 it("does not prune before the required version has completed installation", async () => {
81 const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-cleanup-"));
82 try {
83 const dir = flat
84 ? NodePath.join(root, "expo-device-hub@0.1.0")
85 : NodePath.join(root, "expo-device-hub/0.1.0");
86 await NodeFSP.mkdir(dir, { recursive: true });
87 await NodeFSP.writeFile(NodePath.join(dir, ".install-complete"), "0.1.0");
88 await exec(process.execPath, [
89 "-e",
90 deviceToolMaintenanceScript +
91 `pruneTools(${JSON.stringify(root)}, [['expo-device-hub','0.6.0']], ${flat}).catch(() => process.exitCode = 1);`,
92 ]);
93 expect((await NodeFSP.stat(dir)).isDirectory()).toBe(true);
94 } finally {
95 await NodeFSP.rm(root, { recursive: true, force: true });
96 }
97 });
98});
99
100it.effect("maintenance failures retain safe context and the original process result", () =>
101 Effect.gen(function* () {
102 const output = {
103 code: ChildProcessSpawner.ExitCode(1),
104 stdout: "",
105 stderr: "private child diagnostics",
106 timedOut: false,
107 stdoutTruncated: false,
108 stderrTruncated: false,
109 stdoutInvalidUtf8: false,
110 stderrInvalidUtf8: false,
111 };
112 for (const [operation, run] of [["prune", pruneLocalDeviceTools]] as const) {
113 const error = yield* run("/tools", process.execPath, "hub").pipe(
114 Effect.provideService(ProcessRunner.ProcessRunner, { run: () => Effect.succeed(output) }),
115 Effect.flip,
116 );
117 expect(error).toMatchObject({
118 _tag: "DeviceToolMaintenanceError",
119 operation,
120 tool: "hub",
121 exitCode: 1,
122 cause: output,
123 });
124 expect(error.message).toBe(`Device tool ${operation} failed for hub (exit code 1).`);
125 expect(error.message).not.toContain(output.stderr);
126 }
127 }).pipe(Effect.provide(NodePathLayer.layer)),
128);
129
130it("serializes competing maintenance processes after reclaiming a stale lock", async () => {
131 const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-contention-"));
132 try {
133 const lock = NodePath.join(root, ".maintenance-lock");
134 await NodeFSP.mkdir(lock);
135 await NodeFSP.writeFile(
136 NodePath.join(lock, "stale-owner.json"),
137 JSON.stringify({ pid: 2147483647, identity: "dead" }),
138 );
139 const script =
140 deviceToolMaintenanceScript +
141 `
142(async () => {
143 const root = ${JSON.stringify(root)};
144 const marker = maintenancePath.join(root, 'critical-section');
145 for (let attempt = 0; attempt < 8; attempt++) await withToolMaintenance(root, () => {
146 maintenanceFs.writeFileSync(marker, String(process.pid), { flag: 'wx' });
147 for (let check = 0; check < 100; check++) {
148 if (maintenanceFs.readFileSync(marker, 'utf8') !== String(process.pid)) throw Error('Overlapping maintenance');
149 }
150 maintenanceFs.unlinkSync(marker);
151 });
152})().catch(error => { console.error(error); process.exitCode = 1; });`;
153 await Promise.all(Array.from({ length: 6 }, () => exec(process.execPath, ["-e", script])));
154 await expect(NodeFSP.stat(lock)).rejects.toThrow();
155 expect(await NodeFSP.readdir(root)).toEqual([]);
156 } finally {
157 await NodeFSP.rm(root, { recursive: true, force: true });
158 }
159});
apps/server/src/device/deviceToolMaintenance.ts created+148
...@@ -0,0 +1,148 @@
1// @effect-diagnostics preferSchemaOverJson:off - JSON string literals safely embed paths and arguments in generated JavaScript.
2import * as Schema from "effect/Schema";
3import * as Effect from "effect/Effect";
4import * as Path from "effect/Path";
5import * as ProcessRunner from "../processRunner.ts";
6import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts";
7
8/** Shared with the SSH bootstrap. Cleanup runs only after successful startup. */
9export const deviceToolMaintenanceScript = String.raw`
10const maintenanceFs = require('node:fs');
11const maintenancePath = require('node:path');
12const maintenanceAlive = pid => {
13 try { process.kill(pid, 0); return true; }
14 catch (error) { return error.code !== 'ESRCH'; }
15};
16async function withToolMaintenance(root, operation) {
17 maintenanceFs.mkdirSync(root, { recursive: true });
18 const lock = maintenancePath.join(root, '.maintenance-lock');
19 const nonce = require('node:crypto').randomUUID();
20 const ownerFile = process.pid + '.' + nonce + '.json';
21 const candidate = lock + '.' + nonce;
22 const holder = { pid: process.pid };
23 const deadline = Date.now() + 30000;
24 const removeEmptyLock = () => {
25 try { maintenanceFs.rmdirSync(lock); }
26 catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST', 'EPERM'].includes(error.code)) throw error; }
27 };
28 maintenanceFs.mkdirSync(candidate);
29 try {
30 maintenanceFs.writeFileSync(maintenancePath.join(candidate, ownerFile), JSON.stringify(holder));
31 while (true) {
32 try {
33 // Publish a populated directory atomically; rename cannot replace another populated lock.
34 maintenanceFs.renameSync(candidate, lock);
35 break;
36 }
37 catch (error) {
38 if (!['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES'].includes(error.code)) throw error;
39 let files = [];
40 try { files = maintenanceFs.readdirSync(lock); } catch (error) { if (error.code !== 'ENOENT') throw error; }
41 if (files.length === 1) {
42 const previousFile = maintenancePath.join(lock, files[0]);
43 let previous;
44 try { previous = JSON.parse(maintenanceFs.readFileSync(previousFile, 'utf8')); } catch {}
45 if (Number.isSafeInteger(previous?.pid) && previous.pid > 0 && !maintenanceAlive(previous.pid)) {
46 // The unique filename belongs only to that owner. Never unlink a replacement owner's file.
47 try { maintenanceFs.unlinkSync(previousFile); } catch (error) { if (error.code !== 'ENOENT') throw error; }
48 }
49 }
50 // A concurrent acquirer publishes its owner file with the directory, so this cannot remove it.
51 removeEmptyLock();
52 if (Date.now() >= deadline) throw Error('Device tool maintenance is locked. Retry when the other operation finishes.');
53 await new Promise(resolve => setTimeout(resolve, 50));
54 }
55 }
56 try { return operation(); }
57 finally {
58 maintenanceFs.unlinkSync(maintenancePath.join(lock, ownerFile));
59 removeEmptyLock();
60 }
61 } finally {
62 maintenanceFs.rmSync(candidate, { recursive: true, force: true });
63 }
64}
65function pruneTools(root, specs, flat) {
66 return withToolMaintenance(root, () => {
67 // Keep installs used by any running helper, including older T3 releases.
68 const scan = process.platform === 'win32'
69 ? require('node:child_process').spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', 'Get-CimInstance Win32_Process | Select-Object -ExpandProperty CommandLine'], { encoding: 'utf8', timeout: 10000 })
70 : require('node:child_process').spawnSync('ps', ['-ax', '-o', 'command='], { encoding: 'utf8', timeout: 10000 });
71 if (scan.status !== 0 || !scan.stdout) return;
72 for (const [name, required] of specs) {
73 const parent = flat ? root : maintenancePath.join(root, name);
74 let names;
75 try { names = maintenanceFs.readdirSync(parent); } catch { continue; }
76 const completed = [];
77 for (const item of names) {
78 const version = flat ? (item.startsWith(name + '@') ? item.slice(name.length + 1) : '') : item;
79 if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?$/.test(version)) continue;
80 const directory = maintenancePath.join(parent, item);
81 try {
82 if (!maintenanceFs.lstatSync(directory).isDirectory()) continue;
83 if (maintenanceFs.readFileSync(maintenancePath.join(directory, '.install-complete'), 'utf8').trim() !== version) continue;
84 completed.push({ version, directory, modified: maintenanceFs.statSync(maintenancePath.join(directory, '.install-complete')).mtimeMs });
85 } catch {}
86 }
87 // Never prune until the required install has completed. Retain the last other successful install.
88 if (!completed.some(value => value.version === required)) continue;
89 const previous = completed.filter(value => value.version !== required).sort((a, b) => b.modified - a.modified || b.version.localeCompare(a.version, 'en', { numeric: true }))[0]?.version;
90 for (const { version, directory } of completed) {
91 if (version === required || version === previous || scan.stdout.includes(directory + maintenancePath.sep)) continue;
92 maintenanceFs.rmSync(directory, { recursive: true, force: true });
93 }
94 }
95 });
96}
97`;
98
99class DeviceToolMaintenanceError extends Schema.TaggedError<DeviceToolMaintenanceError>()(
100 "DeviceToolMaintenanceError",
101 {
102 operation: Schema.Literal("prune"),
103 tool: Schema.Literals(["hub", "agent"]),
104 exitCode: Schema.NullOr(Schema.Int),
105 cause: Schema.Defect(),
106 },
107) {
108 override get message() {
109 return `Device tool ${this.operation} failed for ${this.tool} (exit code ${this.exitCode ?? "unknown"}).`;
110 }
111}
112
113const runMaintenance = Effect.fn("DeviceToolchain.maintenance")(function* (
114 nodePath: string,
115 script: string,
116 operation: "prune",
117 tool: "hub" | "agent",
118) {
119 const runner = yield* ProcessRunner.ProcessRunner;
120 const result = yield* runner.run({
121 command: nodePath,
122 args: [
123 "-e",
124 deviceToolMaintenanceScript +
125 "\n" +
126 script +
127 ".catch(error => { console.error(error.message); process.exitCode = 1; });",
128 ],
129 });
130 if (result.code !== 0)
131 return yield* Effect.fail(
132 new DeviceToolMaintenanceError({ operation, tool, exitCode: result.code, cause: result }),
133 );
134});
135
136export const pruneLocalDeviceTools = Effect.fn("DeviceToolchain.prune")(function* (
137 baseDir: string,
138 nodePath: string,
139 tool: "hub" | "agent",
140) {
141 const path = yield* Path.Path;
142 yield* runMaintenance(
143 nodePath,
144 `pruneTools(${JSON.stringify(path.join(baseDir, "tools"))}, ${JSON.stringify(tool === "hub" ? [["expo-device-hub", DEVICE_HUB_VERSION]] : [["agent-device", AGENT_DEVICE_VERSION]])}, false)`,
145 "prune",
146 tool,
147 );
148});
apps/server/src/device/sshDeviceScript.test.ts+3
...@@ -125,6 +125,9 @@ else { const child=spawn(process.execPath,[path.join(path.dirname(process.argv[1...@@ -125,6 +125,9 @@ else { const child=spawn(process.execPath,[path.join(path.dirname(process.argv[1
125 );125 );
126 await NodeFSP.mkdir(NodePath.join(root, "hosts/one"), { recursive: true });126 await NodeFSP.mkdir(NodePath.join(root, "hosts/one"), { recursive: true });
127 await NodeFSP.writeFile(NodePath.join(root, "hosts/one/fail-start-once"), "");127 await NodeFSP.writeFile(NodePath.join(root, "hosts/one/fail-start-once"), "");
128 // Unavailable advisory bookkeeping must not prevent either helper from starting.
129 await NodeFSP.writeFile(NodePath.join(root, "tools/.maintenance-lock"), "blocked");
130 await NodeFSP.writeFile(NodePath.join(root, "tools/.users"), "unwritable lease directory");
128 try {131 try {
129 const [manual, concurrent] = await Promise.all([132 const [manual, concurrent] = await Promise.all([
130 invoke("one", "start"),133 invoke("one", "start"),
apps/server/src/device/sshDeviceScript.ts+3
...@@ -1,3 +1,4 @@...@@ -1,3 +1,4 @@
1import { deviceToolMaintenanceScript } from "./deviceToolMaintenance.ts";
1import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts";2import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts";
23
3export const quoteRemoteArg = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`;4export const quoteRemoteArg = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`;
...@@ -28,6 +29,7 @@ const mode = ${JSON.stringify(mode)};...@@ -28,6 +29,7 @@ const mode = ${JSON.stringify(mode)};
28const hubVersion = ${JSON.stringify(DEVICE_HUB_VERSION)};29const hubVersion = ${JSON.stringify(DEVICE_HUB_VERSION)};
29const agentVersion = ${JSON.stringify(AGENT_DEVICE_VERSION)};30const agentVersion = ${JSON.stringify(AGENT_DEVICE_VERSION)};
30` +31` +
32 deviceToolMaintenanceScript +
31 String.raw`33 String.raw`
32const fs = require('node:fs');34const fs = require('node:fs');
33const path = require('node:path');35const path = require('node:path');
...@@ -214,6 +216,7 @@ async function install(name, version, entry) {...@@ -214,6 +216,7 @@ async function install(name, version, entry) {
214 }216 }
215 const vendor = path.resolve(path.dirname(hubEntry), '../../vendor/serve-sim/dist');217 const vendor = path.resolve(path.dirname(hubEntry), '../../vendor/serve-sim/dist');
216 const optional = file => fs.existsSync(file) ? file : null;218 const optional = file => fs.existsSync(file) ? file : null;
219 await pruneTools(path.join(root, 'tools'), [['expo-device-hub', hubVersion], ...(mode === 'agent-start' ? [['agent-device', agentVersion]] : [])], true).catch(() => {});
217 console.log(JSON.stringify({ nodePath: process.execPath, platforms, tools: versions(), hubPort: hub.port, ...agentResult,220 console.log(JSON.stringify({ nodePath: process.execPath, platforms, tools: versions(), hubPort: hub.port, ...agentResult,
218 helpers: { serveSimAxSettings: optional(path.join(vendor, 'simax/serve-sim-ax-settings')), serveSimCli: optional(path.join(vendor, 'serve-sim.js')) } }));221 helpers: { serveSimAxSettings: optional(path.join(vendor, 'simax/serve-sim-ax-settings')), serveSimCli: optional(path.join(vendor, 'serve-sim.js')) } }));
219 } finally { releaseHost(); }222 } finally { releaseHost(); }