authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 05:51:48-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 05:51:48-07:00
log36f91c45687f97fc8e12ff89eaa8cff6f23abe8b
tree01809de508a614446def1fb2b6d1b6ccf43f7357
parent737192594f47d87f28b5f5a3fda3ef792a0d7972
signature Signed by PGP key B5690EEEBB952194

fix(ssh): reconnecting keeps the remote server and its agents running (#19432)

Co-authored-by: puccez <emanuele.puccetti@icloud.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: luke2x <97411838+luke2x@users.noreply.github.com>

2 files changed, 488 insertions(+), 46 deletions(-)

packages/ssh/src/tunnel.test.ts+455-33
...@@ -10,6 +10,7 @@ import * as Effect from "effect/Effect";...@@ -10,6 +10,7 @@ import * as Effect from "effect/Effect";
10import * as Fiber from "effect/Fiber";10import * as Fiber from "effect/Fiber";
11import * as FileSystem from "effect/FileSystem";11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";12import * as Layer from "effect/Layer";
13import * as Option from "effect/Option";
13import * as Result from "effect/Result";14import * as Result from "effect/Result";
14import * as Sink from "effect/Sink";15import * as Sink from "effect/Sink";
15import * as Stream from "effect/Stream";16import * as Stream from "effect/Stream";
...@@ -250,7 +251,7 @@ describe("ssh tunnel scripts", () => {...@@ -250,7 +251,7 @@ describe("ssh tunnel scripts", () => {
250251
251 assert.include(252 assert.include(
252 launch,253 launch,
253 '[ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && kill -0 "$REMOTE_PID" 2>/dev/null',254 '[ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && server_running "$REMOTE_PID"',
254 );255 );
255 assert.include(launch, "RUNNER_CHANGED=1");256 assert.include(launch, "RUNNER_CHANGED=1");
256 assert.include(launch, "ensure_remote_node_path()");257 assert.include(launch, "ensure_remote_node_path()");
...@@ -299,7 +300,10 @@ describe("ssh tunnel scripts", () => {...@@ -299,7 +300,10 @@ describe("ssh tunnel scripts", () => {
299 assert.include(launch, "resolve_default_runtime_port()");300 assert.include(launch, "resolve_default_runtime_port()");
300 assert.include(launch, 'DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port');301 assert.include(launch, 'DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port');
301 assert.include(launch, "if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(port))");302 assert.include(launch, "if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(port))");
302 assert.include(launch, 'PID_TO_STOP="${REMOTE_PID:-$DEFAULT_RUNTIME_PID}"');303 assert.include(
304 launch,
305 'if [ -n "$DEFAULT_REMOTE_PORT" ] && [ "$REMOTE_MANAGED" = "managed" ] && [ "$DEFAULT_RUNTIME_PID" = "$REMOTE_PID" ]; then',
306 );
303 assert.include(launch, 'REMOTE_PORT="$DEFAULT_REMOTE_PORT"');307 assert.include(launch, 'REMOTE_PORT="$DEFAULT_REMOTE_PORT"');
304 assert.include(launch, 'rm -f "$PID_FILE"');308 assert.include(launch, 'rm -f "$PID_FILE"');
305 assert.include(launch, "printf 'external\\n' >\"$MANAGED_FILE\"");309 assert.include(launch, "printf 'external\\n' >\"$MANAGED_FILE\"");
...@@ -492,6 +496,59 @@ describe("ssh tunnel scripts", () => {...@@ -492,6 +496,59 @@ describe("ssh tunnel scripts", () => {
492 }).pipe(Effect.provide(layerProcess));496 }).pipe(Effect.provide(layerProcess));
493 });497 });
494498
499 it.effect("replaces a stale SSH forward without stopping the remote server", () =>
500 Effect.gen(function* () {
501 const probeStarted = yield* Deferred.make<void>();
502 let stale = false;
503 let forwards = 0;
504 let stops = 0;
505 let kills = 0;
506 const spawner = ChildProcessSpawner.make((command) =>
507 Effect.sync(() => {
508 const args = commandArgs(command);
509 if (args.includes("-N")) {
510 forwards += 1;
511 stale = false;
512 return makeRunningProcess(() => {
513 kills += 1;
514 });
515 }
516 if (args.includes("--")) return makeSuccessfulProcess('{"remotePort":3773}\n');
517 if (args.includes("sh")) stops += 1;
518 return makeSuccessfulProcess("");
519 }),
520 );
521 const httpClient = HttpClient.make((request) =>
522 stale
523 ? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never))
524 : Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status: 200 }))),
525 );
526 const layer = Layer.mergeAll(
527 NodeServices.layer,
528 Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
529 Layer.succeed(HttpClient.HttpClient, httpClient),
530 Layer.succeed(NetService.NetService, testNetService),
531 SshAuth.SshPasswordPrompt.disabledLayer,
532 SshTunnel.SshEnvironmentManager.layer({ resolveCliRunner: Effect.succeed(ARCHIVE) }),
533 );
534 yield* Effect.gen(function* () {
535 const manager = yield* SshTunnel.SshEnvironmentManager;
536 const target = { alias: "devbox", hostname: "devbox", username: null, port: null };
537 yield* manager.ensureEnvironment(target);
538 stale = true;
539 const reconnect = yield* Effect.forkChild(manager.ensureEnvironment(target));
540 yield* Deferred.await(probeStarted);
541 yield* TestClock.adjust("2 seconds");
542 yield* Fiber.join(reconnect);
543 assert.equal(forwards, 2);
544 assert.equal(kills, 1);
545 assert.equal(stops, 0);
546 yield* manager.disconnectEnvironment(target);
547 assert.equal(stops, 1);
548 }).pipe(Effect.provide(layer), Effect.scoped);
549 }),
550 );
551
495 it.effect.each(["successful stop", "failed stop"] as const)(552 it.effect.each(["successful stop", "failed stop"] as const)(
496 "closes the tunnel scope and starts fresh after a %s",553 "closes the tunnel scope and starts fresh after a %s",
497 (mode) => {554 (mode) => {
...@@ -767,6 +824,39 @@ describe("remote port picker", () => {...@@ -767,6 +824,39 @@ describe("remote port picker", () => {
767 );824 );
768});825});
769826
827// Runs a generated script with an isolated HOME, as an SSH remote would.
828const runScript = (home: string, args: ReadonlyArray<string>) =>
829 Effect.gen(function* () {
830 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
831 const child = yield* spawner.spawn(
832 ChildProcess.make("sh", args, {
833 env: { PATH: process.env.PATH ?? "", HOME: home },
834 extendEnv: false,
835 }),
836 );
837 const [stdout, stderr, exitCode] = yield* Effect.all(
838 [
839 child.stdout.pipe(
840 Stream.decodeText(),
841 Stream.runFold(
842 () => "",
843 (acc, chunk) => acc + chunk,
844 ),
845 ),
846 child.stderr.pipe(
847 Stream.decodeText(),
848 Stream.runFold(
849 () => "",
850 (acc, chunk) => acc + chunk,
851 ),
852 ),
853 child.exitCode.pipe(Effect.map(Number)),
854 ],
855 { concurrency: "unbounded" },
856 );
857 return { stdout, stderr, exitCode };
858 });
859
770// The archive runner is generated shell; string assertions cannot prove the860// The archive runner is generated shell; string assertions cannot prove the
771// lock excludes concurrent installers. Run the real script against a tiny861// lock excludes concurrent installers. Run the real script against a tiny
772// fake archive served from a file:// mirror.862// fake archive served from a file:// mirror.
...@@ -776,37 +866,7 @@ describe("archive runner script", () => {...@@ -776,37 +866,7 @@ describe("archive runner script", () => {
776 const windowsHost = hostPlatform === "win32";866 const windowsHost = hostPlatform === "win32";
777 const archiveVersion = "1.2.3-preview.20260911.4";867 const archiveVersion = "1.2.3-preview.20260911.4";
778868
779 const runRunner = (home: string, runner: string) =>869 const runRunner = (home: string, runner: string) => runScript(home, [runner, "--version"]);
780 Effect.gen(function* () {
781 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
782 const child = yield* spawner.spawn(
783 ChildProcess.make("sh", [runner, "--version"], {
784 env: { PATH: process.env.PATH ?? "", HOME: home },
785 extendEnv: false,
786 }),
787 );
788 const [stdout, stderr, exitCode] = yield* Effect.all(
789 [
790 child.stdout.pipe(
791 Stream.decodeText(),
792 Stream.runFold(
793 () => "",
794 (acc, chunk) => acc + chunk,
795 ),
796 ),
797 child.stderr.pipe(
798 Stream.decodeText(),
799 Stream.runFold(
800 () => "",
801 (acc, chunk) => acc + chunk,
802 ),
803 ),
804 child.exitCode.pipe(Effect.map(Number)),
805 ],
806 { concurrency: "unbounded" },
807 );
808 return { stdout, stderr, exitCode };
809 });
810870
811 // A fake "executable" that answers --version, packed the way the release871 // A fake "executable" that answers --version, packed the way the release
812 // workflow packs the real archive: one top-level directory named after the872 // workflow packs the real archive: one top-level directory named after the
...@@ -879,3 +939,365 @@ describe("archive runner script", () => {...@@ -879,3 +939,365 @@ describe("archive runner script", () => {
879 60_000,939 60_000,
880 );940 );
881});941});
942
943// Reuse decisions also live in generated shell. Drive the real launch script
944// against a fake server that answers readiness probes and writes the default
945// runtime file the way `t3 serve` does.
946describe("launch script", () => {
947 const windowsHost = HostProcess.Platform.defaultValue() === "win32";
948 const FAKE_SERVER = `const fs = require("node:fs");
949const http = require("node:http");
950const path = require("node:path");
951const args = process.argv.slice(2);
952const flag = (name) => args[args.indexOf(name) + 1];
953// --answer-once stalls every request after the first, like a busy server;
954// --exit-after-answer exits right after the first answer.
955const answerOnce = args.includes("--answer-once");
956const exitAfterAnswer = args.includes("--exit-after-answer");
957let answered = false;
958const server = http.createServer((_request, response) => {
959 if (answerOnce && answered) return;
960 answered = true;
961 response.end("ok", () => {
962 if (exitAfterAnswer) process.exit(0);
963 });
964});
965server.listen(Number(flag("--port")), "127.0.0.1", () => {
966 const port = server.address().port;
967 const userdata = path.join(flag("--base-dir"), "userdata");
968 fs.mkdirSync(userdata, { recursive: true });
969 fs.writeFileSync(
970 path.join(userdata, "server-runtime.json"),
971 JSON.stringify({ version: 1, pid: process.pid, port, origin: "http://127.0.0.1:" + port }),
972 );
973 console.log(port);
974});
975`;
976
977 const isAlive = (pid: number) => {
978 try {
979 process.kill(pid, 0);
980 return true;
981 } catch {
982 return false;
983 }
984 };
985
986 const freeLoopbackPort = Effect.callback<number>((resume) => {
987 const server = NodeNet.createServer();
988 server.listen(0, "127.0.0.1", () => {
989 const address = server.address();
990 server.close(() =>
991 resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)),
992 );
993 });
994 });
995
996 // Whether anything still accepts connections on the port. Unlike a PID
997 // check, this cannot mistake an unreaped, exited server for a running one.
998 const isListening = (port: number) =>
999 Effect.callback<boolean>((resume) => {
1000 const socket = NodeNet.connect(port, "127.0.0.1");
1001 socket.once("connect", () => {
1002 socket.destroy();
1003 resume(Effect.succeed(true));
1004 });
1005 socket.once("error", () => resume(Effect.succeed(false)));
1006 });
1007
1008 const launchResult = (stdout: string) => JSON.parse(stdout.trim().split("\n").at(-1) ?? "");
1009
1010 // Writes the launch script for a fake server into a temporary HOME. The
1011 // seeded remembered port keeps the test off the default 3773, which a live
1012 // T3 on this machine may own.
1013 const prepareLaunch = Effect.gen(function* () {
1014 const fs = yield* FileSystem.FileSystem;
1015 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launch-script-" });
1016 const home = `${root}/home`;
1017 const stateDir = `${home}/.t3/ssh-launch/test-state`;
1018 const fakeServer = `${root}/fake-t3.cjs`;
1019 const launch = `${root}/launch.sh`;
1020 yield* fs.makeDirectory(stateDir, { recursive: true });
1021 yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
1022 yield* fs.writeFileString(fakeServer, FAKE_SERVER);
1023 yield* fs.writeFileString(
1024 launch,
1025 SshTunnel.buildRemoteLaunchScript({ nodeScriptPath: fakeServer }),
1026 );
1027 const readManagedPid = fs
1028 .readFileString(`${stateDir}/pid`)
1029 .pipe(Effect.map((pid) => Number(pid.trim())));
1030 // Launched servers outlive the script, so stop each PID the script
1031 // recorded, including one it recorded just before a failed assertion.
1032 // SIGKILL also ends a server a test left stopped.
1033 const launchedPids: Array<number> = [];
1034 yield* Effect.addFinalizer(() =>
1035 readManagedPid.pipe(
1036 Effect.map((pid) => [...launchedPids, pid]),
1037 Effect.orElseSucceed(() => launchedPids),
1038 Effect.map((pids) => {
1039 for (const pid of pids) {
1040 if (isAlive(pid)) process.kill(pid, "SIGKILL");
1041 }
1042 }),
1043 ),
1044 );
1045 const runLaunchScript = runScript(home, [launch, "test-state"]);
1046 const runLaunch = runLaunchScript.pipe(
1047 Effect.map((result) => {
1048 assert.equal(result.exitCode, 0, result.stderr);
1049 return launchResult(result.stdout);
1050 }),
1051 );
1052 const launchManaged = Effect.gen(function* () {
1053 const launched = yield* runLaunch;
1054 const pid = yield* readManagedPid;
1055 launchedPids.push(pid);
1056 assert.equal(launched.serverKind, "managed");
1057 return { launched, pid };
1058 });
1059 // Starts a separate server on the default home, as a service would, and
1060 // returns the port it reports. An unreaped server runs under a parent that
1061 // never waits on it, so once it exits it lingers as a zombie.
1062 const startDefaultHomeServer = (
1063 flags: ReadonlyArray<string>,
1064 options?: { readonly unreaped?: boolean },
1065 ) =>
1066 Effect.gen(function* () {
1067 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1068 const args = [fakeServer, "serve", "--port", "0", "--base-dir", `${home}/.t3`, ...flags];
1069 const server = yield* spawner.spawn(
1070 options?.unreaped
1071 ? ChildProcess.make("sh", ["-c", 'node "$@" & exec sleep 600', "sh", ...args])
1072 : ChildProcess.make("node", args),
1073 );
1074 const port = yield* server.stdout.pipe(
1075 Stream.decodeText(),
1076 Stream.splitLines,
1077 Stream.runHead,
1078 );
1079 return Number(Option.getOrThrow(port));
1080 });
1081 return {
1082 home,
1083 stateDir,
1084 launch,
1085 readManagedPid,
1086 runLaunchScript,
1087 runLaunch,
1088 launchManaged,
1089 startDefaultHomeServer,
1090 };
1091 });
1092
1093 it.effect.skipIf(windowsHost)(
1094 "reuses its own managed server and yields only to a different default-home server",
1095 () =>
1096 Effect.gen(function* () {
1097 const fs = yield* FileSystem.FileSystem;
1098 const { stateDir, readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
1099 yield* prepareLaunch;
1100 const { launched, pid: managedPid } = yield* launchManaged;
1101
1102 // The managed server wrote the default runtime file itself, so a
1103 // reconnect must keep it rather than stop it as an external server.
1104 assert.deepEqual(yield* runLaunch, launched);
1105 assert.equal(yield* readManagedPid, managedPid);
1106 assert.isTrue(isAlive(managedPid));
1107
1108 // A remembered port that is stale must not hide our own server: the
1109 // runtime file names its port.
1110 yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
1111 assert.deepEqual(yield* runLaunch, launched);
1112 assert.equal(yield* readManagedPid, managedPid);
1113 assert.equal(
1114 (yield* fs.readFileString(`${stateDir}/port`)).trim(),
1115 `${launched.remotePort}`,
1116 );
1117
1118 // A different server on the default home, such as a service, still
1119 // takes over from the managed one.
1120 const servicePort = yield* startDefaultHomeServer([]);
1121 assert.deepEqual(yield* runLaunch, {
1122 remotePort: servicePort,
1123 serverKind: "external",
1124 });
1125 assert.isFalse(yield* isListening(launched.remotePort));
1126 assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
1127 }).pipe(Effect.provide(NodeServices.layer)),
1128 60_000,
1129 );
1130
1131 // A stopped process keeps its listener but answers nothing, like a server
1132 // stuck in a long SQLite stall.
1133 it.effect.skipIf(windowsHost)(
1134 "waits for its busy managed server instead of replacing it",
1135 () =>
1136 Effect.gen(function* () {
1137 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1138 const { readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1139 const { launched, pid } = yield* launchManaged;
1140
1141 // It resumes after the two 2-second probes that used to replace it.
1142 process.kill(pid, "SIGSTOP");
1143 yield* spawner.spawn(ChildProcess.make("sh", ["-c", `sleep 5; kill -CONT ${pid}`]));
1144 assert.deepEqual(yield* runLaunch, launched);
1145 assert.equal(yield* readManagedPid, pid);
1146 assert.isTrue(isAlive(pid));
1147 }).pipe(Effect.provide(NodeServices.layer)),
1148 60_000,
1149 );
1150
1151 it.effect.skipIf(windowsHost)(
1152 "fails without stopping a managed server that never becomes ready",
1153 () =>
1154 Effect.gen(function* () {
1155 const fs = yield* FileSystem.FileSystem;
1156 const { stateDir, launch, readManagedPid, runLaunchScript, launchManaged } =
1157 yield* prepareLaunch;
1158 const { launched, pid } = yield* launchManaged;
1159
1160 // Shorten the one-minute readiness budget so the test stays fast.
1161 const script = yield* fs.readFileString(launch);
1162 assert.include(script, 'wait_ready "60000"');
1163 yield* fs.writeFileString(
1164 launch,
1165 script.replaceAll('wait_ready "60000"', 'wait_ready "1500"'),
1166 );
1167 process.kill(pid, "SIGSTOP");
1168 const result = yield* runLaunchScript;
1169 assert.equal(result.exitCode, 1, result.stdout);
1170 assert.include(result.stderr, "the server was left running");
1171 assert.isTrue(isAlive(pid));
1172 assert.equal(yield* readManagedPid, pid);
1173 assert.equal(
1174 (yield* fs.readFileString(`${stateDir}/port`)).trim(),
1175 `${launched.remotePort}`,
1176 );
1177 assert.equal((yield* fs.readFileString(`${stateDir}/managed`)).trim(), "managed");
1178 }).pipe(Effect.provide(NodeServices.layer)),
1179 60_000,
1180 );
1181
1182 it.effect.skipIf(windowsHost)(
1183 "still replaces its managed server when the runner changes",
1184 () =>
1185 Effect.gen(function* () {
1186 const fs = yield* FileSystem.FileSystem;
1187 const { stateDir, readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1188 const { pid: previousPid } = yield* launchManaged;
1189
1190 // A different runner on disk means the client updated: replace the server.
1191 yield* fs.writeFileString(`${stateDir}/run-t3.sh`, "# previous runner\n");
1192 assert.equal((yield* runLaunch).serverKind, "managed");
1193 const replacementPid = yield* readManagedPid;
1194 assert.notEqual(replacementPid, previousPid);
1195 assert.isFalse(isAlive(previousPid));
1196 assert.isTrue(isAlive(replacementPid));
1197 }).pipe(Effect.provide(NodeServices.layer)),
1198 60_000,
1199 );
1200
1201 it.effect.skipIf(windowsHost)(
1202 "replaces a managed server that exited but was not reaped",
1203 () =>
1204 Effect.gen(function* () {
1205 const fs = yield* FileSystem.FileSystem;
1206 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1207 const { home, stateDir, readManagedPid, runLaunch, launchManaged } = yield* prepareLaunch;
1208 const { launched, pid } = yield* launchManaged;
1209 process.kill(pid, "SIGKILL");
1210
1211 // A crashed server whose parent never reaps it stays a zombie, which
1212 // kill -0 still reports. Its pid file and runtime record keep naming it.
1213 const parent = yield* spawner.spawn(
1214 ChildProcess.make("sh", ["-c", "sleep 600 & echo $!; exec sleep 600"]),
1215 );
1216 const zombie = Number(
1217 Option.getOrThrow(
1218 yield* parent.stdout.pipe(Stream.decodeText(), Stream.splitLines, Stream.runHead),
1219 ),
1220 );
1221 process.kill(zombie, "SIGKILL");
1222 assert.include((yield* runScript(home, ["-c", `ps -o stat= -p ${zombie}`])).stdout, "Z");
1223 yield* fs.writeFileString(`${stateDir}/pid`, `${zombie}\n`);
1224 yield* fs.writeFileString(
1225 `${home}/.t3/userdata/server-runtime.json`,
1226 JSON.stringify({
1227 version: 1,
1228 pid: zombie,
1229 port: launched.remotePort,
1230 origin: `http://127.0.0.1:${launched.remotePort}`,
1231 }),
1232 );
1233
1234 assert.equal((yield* runLaunch).serverKind, "managed");
1235 const replacementPid = yield* readManagedPid;
1236 assert.notEqual(replacementPid, zombie);
1237 assert.isTrue(isAlive(replacementPid));
1238 }).pipe(Effect.provide(NodeServices.layer)),
1239 60_000,
1240 );
1241
1242 it.effect.skipIf(windowsHost)(
1243 "adopts a server it has no managed record for instead of restarting it",
1244 () =>
1245 Effect.gen(function* () {
1246 const fs = yield* FileSystem.FileSystem;
1247 const { stateDir, launchManaged, runLaunch } = yield* prepareLaunch;
1248 const { launched, pid } = yield* launchManaged;
1249
1250 // An interrupted launch, or a recycled PID, can leave a PID that matches
1251 // the runtime file without the managed marker. A changed runner must not
1252 // turn that into a restart of a server this launcher cannot claim.
1253 yield* fs.remove(`${stateDir}/managed`);
1254 yield* fs.writeFileString(`${stateDir}/run-t3.sh`, "# superseded runner\n");
1255 assert.deepEqual(yield* runLaunch, {
1256 remotePort: launched.remotePort,
1257 serverKind: "external",
1258 });
1259 assert.isTrue(isAlive(pid));
1260 }).pipe(Effect.provide(NodeServices.layer)),
1261 60_000,
1262 );
1263
1264 it.effect.skipIf(windowsHost)(
1265 "keeps an adopted default-home server without launching another beside it",
1266 () =>
1267 Effect.gen(function* () {
1268 const fs = yield* FileSystem.FileSystem;
1269 const { stateDir, runLaunch, startDefaultHomeServer } = yield* prepareLaunch;
1270 // Ownership files from an earlier managed launch that lost its PID.
1271 yield* fs.writeFileString(`${stateDir}/managed`, "managed\n");
1272 const servicePort = yield* startDefaultHomeServer(["--answer-once"]);
1273
1274 // The server answers adoption and then stalls. A second readiness
1275 // probe would launch another server on the same home.
1276 assert.deepEqual(yield* runLaunch, { remotePort: servicePort, serverKind: "external" });
1277 assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
1278 }).pipe(Effect.provide(NodeServices.layer)),
1279 60_000,
1280 );
1281
1282 it.effect.skipIf(windowsHost)(
1283 "launches a replacement when the adopted server exits during the handover",
1284 () =>
1285 Effect.gen(function* () {
1286 const { readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
1287 yield* prepareLaunch;
1288 const { pid: previousPid } = yield* launchManaged;
1289 yield* startDefaultHomeServer(["--exit-after-answer"], { unreaped: true });
1290
1291 // The service answers adoption and then exits while the managed
1292 // server is being stopped, staying a zombie, so the script must launch
1293 // a new server rather than return the dead one. The freed port may be
1294 // reused.
1295 const result = yield* runLaunch;
1296 assert.equal(result.serverKind, "managed");
1297 const replacementPid = yield* readManagedPid;
1298 assert.notEqual(replacementPid, previousPid);
1299 assert.isTrue(isAlive(replacementPid));
1300 }).pipe(Effect.provide(NodeServices.layer)),
1301 60_000,
1302 );
1303});
packages/ssh/src/tunnel.ts+33-13
...@@ -620,6 +620,14 @@ wait_for_pid_exit() {...@@ -620,6 +620,14 @@ wait_for_pid_exit() {
620 sleep 0.1620 sleep 0.1
621 done621 done
622}622}
623# kill -0 also succeeds for a server that exited but was not reaped yet, so
624# a zombie does not count as running. Without ps to tell, trust kill -0.
625server_running() {
626 [ -n "$1" ] && kill -0 "$1" 2>/dev/null || return 1
627 case "$(ps -o stat= -p "$1" 2>/dev/null || true)" in
628 *Z*) return 1 ;;
629 esac
630}
623resolve_default_runtime_port() {631resolve_default_runtime_port() {
624 if [ "$T3_ARCHIVE_MODE" = "1" ]; then632 if [ "$T3_ARCHIVE_MODE" = "1" ]; then
625 "$RUNNER_FILE" __ssh-helper runtime-port "$DEFAULT_RUNTIME_FILE"633 "$RUNNER_FILE" __ssh-helper runtime-port "$DEFAULT_RUNTIME_FILE"
...@@ -652,18 +660,25 @@ REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)"...@@ -652,18 +660,25 @@ REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)"
652DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"660DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
653DEFAULT_RUNTIME_PID=""661DEFAULT_RUNTIME_PID=""
654DEFAULT_REMOTE_PORT=""662DEFAULT_REMOTE_PORT=""
663ADOPTED_PID=""
655if [ -n "$DEFAULT_RUNTIME_INFO" ]; then664if [ -n "$DEFAULT_RUNTIME_INFO" ]; then
656 DEFAULT_RUNTIME_PID="\${DEFAULT_RUNTIME_INFO%% *}"665 DEFAULT_RUNTIME_PID="\${DEFAULT_RUNTIME_INFO%% *}"
657 DEFAULT_REMOTE_PORT="\${DEFAULT_RUNTIME_INFO#* }"666 DEFAULT_REMOTE_PORT="\${DEFAULT_RUNTIME_INFO#* }"
658fi667fi
659if [ -n "$DEFAULT_REMOTE_PORT" ]; then668# The managed server runs on the default home, so it writes the default
669# runtime file too. When that file names the PID this launcher recorded as
670# managed, it describes our own server and its real port, not one to adopt.
671if [ -n "$DEFAULT_REMOTE_PORT" ] && [ "$REMOTE_MANAGED" = "managed" ] && [ "$DEFAULT_RUNTIME_PID" = "$REMOTE_PID" ]; then
672 REMOTE_PORT="$DEFAULT_REMOTE_PORT"
673 printf '%s\\n' "$REMOTE_PORT" >"$PORT_FILE"
674elif [ -n "$DEFAULT_REMOTE_PORT" ]; then
660 REMOTE_PORT="$DEFAULT_REMOTE_PORT"675 REMOTE_PORT="$DEFAULT_REMOTE_PORT"
661 if wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then676 if wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
677 ADOPTED_PID="$DEFAULT_RUNTIME_PID"
662 if [ "$REMOTE_MANAGED" = "managed" ]; then678 if [ "$REMOTE_MANAGED" = "managed" ]; then
663 PID_TO_STOP="\${REMOTE_PID:-$DEFAULT_RUNTIME_PID}"679 if [ -n "$REMOTE_PID" ] && kill -0 "$REMOTE_PID" 2>/dev/null; then
664 if [ -n "$PID_TO_STOP" ] && kill -0 "$PID_TO_STOP" 2>/dev/null; then680 kill "$REMOTE_PID" 2>/dev/null || true
665 kill "$PID_TO_STOP" 2>/dev/null || true681 wait_for_pid_exit "$REMOTE_PID"
666 wait_for_pid_exit "$PID_TO_STOP"
667 fi682 fi
668 REMOTE_PID=""683 REMOTE_PID=""
669 REMOTE_PORT="$DEFAULT_REMOTE_PORT"684 REMOTE_PORT="$DEFAULT_REMOTE_PORT"
...@@ -684,24 +699,27 @@ if [ -n "$DEFAULT_REMOTE_PORT" ]; then...@@ -684,24 +699,27 @@ if [ -n "$DEFAULT_REMOTE_PORT" ]; then
684 fi699 fi
685fi700fi
686if [ "$REMOTE_MANAGED" = "external" ]; then701if [ "$REMOTE_MANAGED" = "external" ]; then
687 if [ -z "$REMOTE_PORT" ] || ! wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then702 # A server adopted above has just answered. While it is still alive, probing
703 # it again could catch a momentary stall and launch a second server beside it.
704 if server_running "$ADOPTED_PID"; then
705 :
706 elif [ -z "$REMOTE_PORT" ] || ! wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then
688 REMOTE_PID=""707 REMOTE_PID=""
689 REMOTE_PORT=""708 REMOTE_PORT=""
690 REMOTE_MANAGED=""709 REMOTE_MANAGED=""
691 fi710 fi
692elif [ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && kill -0 "$REMOTE_PID" 2>/dev/null; then711elif [ -n "$REMOTE_PID" ] && [ -n "$REMOTE_PORT" ] && server_running "$REMOTE_PID"; then
712 # Reconnect owns the local tunnel, not the lifetime of an already running
713 # server. A changed runner is the update path and still replaces it.
693 if [ "$RUNNER_CHANGED" -eq 1 ]; then714 if [ "$RUNNER_CHANGED" -eq 1 ]; then
694 kill "$REMOTE_PID" 2>/dev/null || true715 kill "$REMOTE_PID" 2>/dev/null || true
695 wait_for_pid_exit "$REMOTE_PID"716 wait_for_pid_exit "$REMOTE_PID"
696 REMOTE_PID=""717 REMOTE_PID=""
697 REMOTE_PORT=""718 REMOTE_PORT=""
698 REMOTE_MANAGED=""719 REMOTE_MANAGED=""
699 elif ! wait_ready "@@T3_REUSE_READY_TIMEOUT_MS@@"; then720 elif ! wait_ready "@@T3_READY_TIMEOUT_MS@@"; then
700 kill "$REMOTE_PID" 2>/dev/null || true721 printf 'Remote T3 server is still running but did not become ready. Retry the connection; the server was left running.\\n' >&2
701 wait_for_pid_exit "$REMOTE_PID"722 exit 1
702 REMOTE_PID=""
703 REMOTE_PORT=""
704 REMOTE_MANAGED=""
705 fi723 fi
706else724else
707 REMOTE_PID=""725 REMOTE_PID=""
...@@ -1654,6 +1672,8 @@ const makeSshEnvironmentManager = Effect.fn("ssh/tunnel.SshEnvironmentManager.ma...@@ -1654,6 +1672,8 @@ const makeSshEnvironmentManager = Effect.fn("ssh/tunnel.SshEnvironmentManager.ma
1654 remotePort: entry.remotePort,1672 remotePort: entry.remotePort,
1655 cause: readinessExit.cause,1673 cause: readinessExit.cause,
1656 });1674 });
1675 // Closing a stale forward must not run the manager-owned remote stop.
1676 tunnels.delete(key);
1657 yield* closeTunnelEntry(entry);1677 yield* closeTunnelEntry(entry);
1658 }1678 }
16591679