1/**
2 * Scaffolding for the `alchemy dev` stress suite.
3 *
4 * Everything here exists to make one thing possible: run the REAL CLI
5 * against a throwaway copy of this project and then rewrite that copy's
6 * files underneath it, while keeping every assertion independent of what
7 * the CLI happens to be printing at the time.
8 */
9import { spawn, spawnSync, type ChildProcess } from "node:child_process";
10import * as fs from "node:fs";
11import * as path from "node:path";
12
13/** The checked-in example project (the copy source). */
14export const EXAMPLE_ROOT = path.resolve(import.meta.dirname, "..");
15
16/**
17 * Spawn the CLI entry directly rather than through `bun run` / `cli.js`, so
18 * signals reach the actual CLI process whose scope teardown kills the exec
19 * child and the provider sidecars.
20 */
21export const ALCHEMY_BIN = path.join(
22 EXAMPLE_ROOT,
23 "node_modules",
24 "alchemy",
25 "bin",
26 "alchemy.js",
27);
28
29/** Entries never copied into a scratch project. */
30const SKIP_COPY = new Set([
31 "node_modules",
32 "test",
33 ".stress",
34 ".alchemy",
35 "dist",
36 "tsconfig.json",
37 "bun.lock",
38]);
39
40/**
41 * Copy the checked-in project into `.stress/<name>/`.
42 *
43 * The scratch dir lives INSIDE the example so Node/Bun resolution still
44 * walks up into `examples/dev-stress/node_modules` and finds the workspace
45 * `alchemy` link — while every mutation the suite makes stays out of the
46 * source-controlled tree, and survives a mid-suite failure without leaving
47 * the repo dirty.
48 */
49/**
50 * Reset the floci emulator to a clean slate before a run. A KILLED previous
51 * run (bun's timeout, Ctrl-C on the test process) never reaches the CLI's
52 * graceful shutdown, so its emulated ECS services stay registered in the
53 * long-lived `alchemy-floci` container — their reconcilers re-spawn task
54 * containers forever and squat the suite's pinned host ports, deadlocking
55 * every subsequent run. The emulator is disposable by design (in-memory
56 * state, local image), so a fresh one per run makes the suite self-healing
57 * regardless of how the previous run died.
58 */
59export const resetFlociEmulator = (): void => {
60 const names = spawnSync(
61 "docker",
62 ["ps", "-aq", "--filter", "name=^(alchemy-floci$|floci-ec2|floci-ecs|floci-microvm)"],
63 { encoding: "utf8", timeout: 30_000 },
64 )
65 .stdout?.trim()
66 .split("\n")
67 .filter(Boolean);
68 if (names && names.length > 0) {
69 spawnSync("docker", ["rm", "-f", ...names], {
70 stdio: "ignore",
71 timeout: 120_000,
72 });
73 }
74};
75
76/**
77 * Free the suite's pinned ports before a run. A KILLED previous run leaves
78 * orphaned workerd / dev-child processes listening on them (they are not
79 * children of the test process, so bun's cleanup never reaches them), and
80 * every `strictPort` worker then fails its first apply. Scoped strictly to
81 * the ports this suite owns.
82 */
83export const freePinnedPorts = (ports: readonly number[]): void => {
84 const pids = new Set<string>();
85 for (const port of ports) {
86 const out = spawnSync("lsof", ["-ti", `tcp:${port}`, "-sTCP:LISTEN"], {
87 encoding: "utf8",
88 timeout: 15_000,
89 }).stdout;
90 for (const pid of (out ?? "").trim().split("\n")) {
91 if (pid) pids.add(pid);
92 }
93 }
94 if (pids.size === 0) return;
95 spawnSync("kill", [...pids], { stdio: "ignore", timeout: 15_000 });
96 // Grace, then force anything still holding a port.
97 spawnSync("sleep", ["2"], { stdio: "ignore", timeout: 5_000 });
98 spawnSync("kill", ["-9", ...pids], { stdio: "ignore", timeout: 15_000 });
99};
100
101export const makeScratchProject = (name: string): string => {
102 const dir = path.join(EXAMPLE_ROOT, ".stress", name);
103 fs.rmSync(dir, { recursive: true, force: true });
104 fs.mkdirSync(dir, { recursive: true });
105 for (const entry of fs.readdirSync(EXAMPLE_ROOT)) {
106 if (SKIP_COPY.has(entry) || entry.startsWith(".git")) continue;
107 fs.cpSync(path.join(EXAMPLE_ROOT, entry), path.join(dir, entry), {
108 recursive: true,
109 });
110 }
111 return dir;
112};
113
114export interface DevServerOptions {
115 /** Scratch project directory the CLI runs in. */
116 readonly cwd: string;
117 /** Stage name — isolates local state from any other run. */
118 readonly stage: string;
119 /** Extra environment for the CLI process. */
120 readonly env?: Record<string, string>;
121}
122
123/**
124 * Hermetic environment for the CLI: an alchemy profile that exists nowhere
125 * on disk plus stripped AWS credentials, so the run can only resolve to the
126 * local emulator, and placeholder Cloudflare credentials so the local
127 * Cloudflare providers' env lookups are satisfied without any cloud call
128 * being possible.
129 *
130 * `CI=1` is what makes the placeholders usable: without it a profile with
131 * no configured Cloudflare section refuses to fall back to environment
132 * credentials in a non-interactive process.
133 */
134export const hermeticEnv = (extra?: Record<string, string>) => {
135 const env: NodeJS.ProcessEnv = {
136 ...process.env,
137 CI: "1",
138 ALCHEMY_PROFILE: "dev-stress-cli-test",
139 CLOUDFLARE_ACCOUNT_ID: "0123456789abcdef0123456789abcdef",
140 CLOUDFLARE_API_TOKEN: "dev-stress-placeholder",
141 ...extra,
142 };
143 delete env.AWS_ACCESS_KEY_ID;
144 delete env.AWS_SECRET_ACCESS_KEY;
145 delete env.AWS_SESSION_TOKEN;
146 delete env.AWS_PROFILE;
147 return env;
148};
149
150export class DevServer {
151 readonly proc: ChildProcess;
152 readonly cwd: string;
153 readonly stage: string;
154 /** Everything the CLI has written to stdout+stderr, in order. */
155 output = "";
156 private readonly logPath: string;
157 private readonly logStream: fs.WriteStream;
158
159 constructor(options: DevServerOptions) {
160 this.cwd = options.cwd;
161 this.stage = options.stage;
162 this.logPath = path.join(options.cwd, "dev-stress.log");
163 this.logStream = fs.createWriteStream(this.logPath, { flags: "a" });
164 this.proc = spawn(
165 "bun",
166 [ALCHEMY_BIN, "dev", "--stage", options.stage],
167 {
168 cwd: options.cwd,
169 // Own process group so teardown can deliver Ctrl-C to the whole
170 // tree (CLI + `--watch` exec child + provider sidecars) the way a
171 // terminal would.
172 detached: true,
173 stdio: ["ignore", "pipe", "pipe"],
174 env: hermeticEnv(options.env),
175 },
176 );
177 const pump = (stream: NodeJS.ReadableStream | null) =>
178 stream?.on("data", (chunk: Buffer) => {
179 const text = chunk.toString();
180 this.output += text;
181 this.logStream.write(text);
182 if (process.env.DEBUG) process.stderr.write(text);
183 });
184 pump(this.proc.stdout);
185 pump(this.proc.stderr);
186 }
187
188 /** `true` while the CLI process itself is still running. */
189 get alive(): boolean {
190 return this.proc.exitCode === null && this.proc.signalCode === null;
191 }
192
193 /** A cursor into {@link output}, for "did anything new happen since?". */
194 mark(): number {
195 return this.output.length;
196 }
197
198 /** Output written since `cursor`. */
199 since(cursor: number): string {
200 return this.output.slice(cursor);
201 }
202
203 /** Last `n` characters of output — for failure messages. */
204 tail(n = 6_000): string {
205 return this.output.slice(-n);
206 }
207
208 /**
209 * Number of plan renders. `alchemy dev` prints one `Plan:` line per run
210 * of the stack, so this counts EXEC-CHILD RESTARTS.
211 *
212 * It is the discriminator between the two dev reload paths. Both paths
213 * put new code in front of a request within a couple of seconds — the
214 * local Worker provider's bundler watch loop always wins the race — but
215 * only a file inside the STACK's import graph also makes `bun --watch`
216 * re-run the stack, which shows up here a beat later. Measured on this
217 * fixture: a bundler-only edit swaps at ~2s and never re-plans; a
218 * stack-graph edit swaps at ~2s and re-plans at ~4s.
219 */
220 get planCount(): number {
221 return this.output.match(/^Plan:/gm)?.length ?? 0;
222 }
223
224 /** Number of applies that ran to completion (`Done: N succeeded`). */
225 get doneCount(): number {
226 return this.output.match(/^Done:/gm)?.length ?? 0;
227 }
228
229 /**
230 * Resolve once the CLI has rendered more than `count` plans — i.e. the
231 * stack has re-run. The re-plan trails the bundler's hot swap, so this
232 * must be WAITED for, never sampled right after new code starts serving.
233 */
234 waitForPlanAfter(count: number, options?: { tries?: number }): Promise<true> {
235 return pollUntil(
236 `the stack to re-run (plan #${count + 1})`,
237 () => (this.planCount > count ? (true as const) : undefined),
238 { tries: options?.tries ?? 120, delayMs: 500, server: this },
239 );
240 }
241
242 /**
243 * Resolve once no new plan has appeared for `quietMs`. Phases that edit
244 * a file inside the STACK's import graph leave a re-plan trailing the
245 * hot swap by a few seconds; a later phase that baselines `planCount`
246 * (to assert "this path never re-plans") must absorb that trail first
247 * or the trailing plan lands inside its window and reads as a spurious
248 * re-run.
249 */
250 async settlePlans(quietMs = 8_000, capMs = 60_000): Promise<void> {
251 const deadline = Date.now() + capMs;
252 let last = this.planCount;
253 let quietSince = Date.now();
254 while (Date.now() < deadline) {
255 await Bun.sleep(500);
256 const current = this.planCount;
257 if (current !== last) {
258 last = current;
259 quietSince = Date.now();
260 } else if (Date.now() - quietSince >= quietMs) {
261 return;
262 }
263 }
264 }
265
266 /**
267 * Throws (with an output tail) unless the CLI process is still running.
268 * The single most important invariant of the whole suite: nothing the
269 * suite does to the project may kill the dev server.
270 */
271 assertAlive(context: string): void {
272 if (this.alive) return;
273 throw new Error(
274 `alchemy dev died during: ${context}\n` +
275 `exit=${this.proc.exitCode} signal=${this.proc.signalCode}\n` +
276 `--- output tail ---\n${this.tail()}`,
277 );
278 }
279
280 /**
281 * What the CLI has printed since `cursor`, with ANSI colour stripped —
282 * `bun test` output is not a TTY, but the exec child still colours its
283 * plan lines when stdout is a pipe it believes supports colour.
284 */
285 plain(cursor = 0): string {
286 // eslint-disable-next-line no-control-regex
287 return this.output.slice(cursor).replaceAll(/\x1b\[[0-9;]*m/g, "");
288 }
289
290 /** Extract a plain value for `key` from the stack outputs the CLI printed. */
291 outputValue(key: string): string | undefined {
292 const matches = this.output.match(
293 new RegExp(`${key}:\\s*['\"]?([^\\s'\",]+)`, "g"),
294 );
295 // The newest print wins: outputs are re-printed on every re-apply.
296 return matches
297 ?.at(-1)
298 ?.match(new RegExp(`${key}:\\s*['\"]?([^\\s'\",]+)`))?.[1];
299 }
300
301 /** Extract a URL for `key` from the stack outputs the CLI printed. */
302 outputUrl(key: string): string | undefined {
303 const matches = this.output.match(
304 new RegExp(`${key}:\\s*['"]?(http[^\\s'",]+)`, "g"),
305 );
306 // The newest print wins: outputs are re-printed on every re-apply.
307 return matches?.at(-1)?.match(/(http[^\s'",]+)/)?.[1];
308 }
309
310 // ── file mutation ────────────────────────────────────────────────────
311
312 abs(relative: string): string {
313 return path.join(this.cwd, relative);
314 }
315
316 read(relative: string): string {
317 return fs.readFileSync(this.abs(relative), "utf8");
318 }
319
320 write(relative: string, contents: string): void {
321 const file = this.abs(relative);
322 fs.mkdirSync(path.dirname(file), { recursive: true });
323 fs.writeFileSync(file, contents);
324 }
325
326 remove(relative: string): void {
327 fs.rmSync(this.abs(relative), { force: true, recursive: true });
328 }
329
330 move(from: string, to: string): void {
331 const target = this.abs(to);
332 fs.mkdirSync(path.dirname(target), { recursive: true });
333 fs.renameSync(this.abs(from), target);
334 }
335
336 exists(relative: string): boolean {
337 return fs.existsSync(this.abs(relative));
338 }
339
340 /**
341 * Replace the body of a `// <<NAME>> … // <</NAME>>` region. Surgical
342 * region edits keep the rest of the stack byte-identical, so a diff the
343 * suite makes is exactly the change under test — no regex drift.
344 */
345 patchRegion(relative: string, name: string, body: string): void {
346 const source = this.read(relative);
347 const pattern = new RegExp(
348 `([ \\t]*// <<${name}>>\\n)[\\s\\S]*?([ \\t]*// <<\\/${name}>>)`,
349 );
350 if (!pattern.test(source)) {
351 throw new Error(`region <<${name}>> not found in ${relative}`);
352 }
353 this.write(relative, source.replace(pattern, `$1${body}$2`));
354 }
355
356 // ── teardown ─────────────────────────────────────────────────────────
357
358 /**
359 * Ctrl-C the whole process GROUP, then escalate. Signalling only the CLI
360 * orphans the exec child, which keeps holding the stack's state lock.
361 */
362 async shutdown(timeoutMs = 30_000): Promise<void> {
363 if (!this.proc.pid) return;
364 const killGroup = (signal: NodeJS.Signals) => {
365 try {
366 process.kill(-this.proc.pid!, signal);
367 } catch {
368 // group already gone
369 }
370 };
371 const exited = new Promise((resolve) => this.proc.once("exit", resolve));
372 killGroup("SIGINT");
373 await Promise.race([exited, Bun.sleep(timeoutMs)]);
374 if (this.alive) {
375 killGroup("SIGKILL");
376 await Promise.race([exited, Bun.sleep(5_000)]);
377 }
378 this.logStream.end();
379 }
380
381 /** Tear the stack's local resources down out of band. */
382 destroyStack(timeoutMs = 180_000): void {
383 spawnSync(
384 "bun",
385 [ALCHEMY_BIN, "destroy", "--stage", this.stage, "--yes"],
386 {
387 cwd: this.cwd,
388 stdio: process.env.DEBUG ? "inherit" : "ignore",
389 timeout: timeoutMs,
390 env: hermeticEnv(),
391 },
392 );
393 }
394}
395
396// ── polling helpers ─────────────────────────────────────────────────────
397
398export class PollTimeout extends Error {}
399
400/**
401 * Every request the suite makes carries a deadline. A local Worker that is
402 * hot-swapped while a request is in flight can drop that request on the
403 * floor — workerd cancels it and the dev proxy never answers the client —
404 * and a bare `fetch` would then wait forever, wedging the whole suite
405 * (observed: a `/roundtrip` in flight across a MicrovmWorker bundle swap).
406 * The deadline is generous enough for a MicroVM boot-to-terminate
407 * roundtrip, which is the slowest single request here.
408 */
409export const REQUEST_TIMEOUT_MS = 120_000;
410
411export const fetchWithDeadline = (
412 url: string | URL,
413 init?: RequestInit,
414 timeoutMs = REQUEST_TIMEOUT_MS,
415): Promise<Response> =>
416 fetch(url, { ...init, signal: init?.signal ?? AbortSignal.timeout(timeoutMs) });
417
418/** Bounded poll for a (possibly async) producer to yield a value. */
419export const pollUntil = async <T>(
420 what: string,
421 f: () => T | undefined | Promise<T | undefined>,
422 {
423 tries = 60,
424 delayMs = 500,
425 server,
426 }: { tries?: number; delayMs?: number; server?: DevServer } = {},
427): Promise<T> => {
428 for (let i = 0; i < tries; i++) {
429 // A dead CLI can never satisfy the predicate — fail immediately with
430 // the crash context instead of burning the whole budget.
431 server?.assertAlive(what);
432 const value = await f();
433 if (value !== undefined) return value;
434 await Bun.sleep(delayMs);
435 }
436 throw new PollTimeout(
437 `Timed out after ${(tries * delayMs) / 1000}s waiting for ${what}.` +
438 (server ? `\n--- alchemy dev output (tail) ---\n${server.tail()}` : ""),
439 );
440};
441
442/** Fetch with retries — a freshly (re)started worker takes a moment. */
443export const fetchOk = async (
444 url: string | URL,
445 init?: RequestInit,
446 { tries = 40, delayMs = 500 }: { tries?: number; delayMs?: number } = {},
447): Promise<Response> => {
448 let last: Response | undefined;
449 let error: unknown;
450 for (let i = 0; i < tries; i++) {
451 try {
452 last = await fetchWithDeadline(url, init);
453 if (last.ok) return last;
454 } catch (cause) {
455 error = cause; // not listening yet, mid-swap, or dropped
456 }
457 await Bun.sleep(delayMs);
458 }
459 throw new Error(
460 `${init?.method ?? "GET"} ${url} never returned 2xx ` +
461 `(last status: ${last?.status}, last error: ${String(error)})`,
462 );
463};
464
465export const fetchJson = async <T>(
466 url: string | URL,
467 init?: RequestInit,
468 options?: { tries?: number; delayMs?: number },
469): Promise<T> => (await fetchOk(url, init, options)).json() as Promise<T>;
470
471/**
472 * Poll a URL until its JSON body satisfies `predicate`. This is how the
473 * suite observes a reload landing: not by reading the CLI's logs, but by
474 * asking the running worker what code it is currently executing.
475 */
476export const waitForJson = async <T>(
477 what: string,
478 url: string | URL,
479 predicate: (body: T) => boolean,
480 options?: { tries?: number; delayMs?: number; server?: DevServer },
481): Promise<T> => {
482 // What the endpoint last said, so a timeout explains itself: "served the
483 // OLD marker for 240s" and "answered 502 the whole time" are different
484 // bugs, and a bare timeout hides which one it was.
485 let last = "no response yet";
486 try {
487 return await pollUntil(
488 what,
489 async () => {
490 try {
491 const res = await fetchWithDeadline(url);
492 const text = await res.text();
493 last = `${res.status} ${text.slice(0, 300)}`;
494 if (!res.ok) return undefined;
495 const body = JSON.parse(text) as T;
496 return predicate(body) ? body : undefined;
497 } catch (cause) {
498 last = `error: ${String(cause).slice(0, 300)}`;
499 return undefined; // mid-restart, or dropped
500 }
501 },
502 { tries: 120, delayMs: 500, ...options },
503 );
504 } catch (cause) {
505 if (cause instanceof PollTimeout) {
506 cause.message = `${cause.message.split("\n")[0]}\n--- last response from ${url} ---\n${last}\n${cause.message.split("\n").slice(1).join("\n")}`;
507 }
508 throw cause;
509 }
510};
511
512/** `http://localhost:<port>` */
513export const at = (port: number, path = "/"): URL =>
514 new URL(path, `http://localhost:${port}`);
515
516/** Docker gate — floci (the AWS emulator) and Containers both need it. */
517export const dockerAvailable =
518 spawnSync("docker", ["info"], { stdio: "ignore", timeout: 30_000 })
519 .status === 0;