1/**
2 * `alchemy dev` stress suite.
3 *
4 * Every other dev test in this repo asserts that a stack CONVERGES. This
5 * one asserts that the dev server SURVIVES — that no sequence of edits a
6 * developer can perform while `alchemy dev` is running kills it, wedges it,
7 * or leaves it serving stale code.
8 *
9 * Topology under test (one cross-cloud stack, see ../alchemy.run.ts):
10 *
11 * AWS (floci emulator) Cloudflare (workerd + docker)
12 * ──────────────────── ─────────────────────────────
13 * Lambda ApiFunction ◀──────── Worker ApiWorker (path `main`)
14 * ├ S3 StressBucket cross- Worker EchoWorker (Effect-native `main`)
15 * ├ DynamoDB Table cloud ├ KV + R2 + Durable Object
16 * └ SQS + consumer hop └ Container SandboxContainer
17 * Website.StaticSite Website.StaticSite (build mode)
18 * (dev-command child)
19 *
20 * The suite runs ONE dev server for its whole lifetime and accumulates
21 * churn on it — that accumulation IS the stress. Phases run in order and
22 * each one leaves the stack healthy for the next.
23 *
24 * Two distinct reload paths are told apart throughout by
25 * `server.planCount`, which counts `Plan:` renders (i.e. exec-child
26 * restarts). Both paths put new code in front of a request within ~2s —
27 * the local Worker provider's bundler watch loop always wins that race —
28 * so the discriminator is what happens AFTER the swap:
29 *
30 * - a file only the BUNDLER sees → hot swap, and `planCount` never
31 * (`src/api/marker.ts`) moves, however long you wait.
32 * - a file the STACK imports → hot swap, and then `bun --watch`
33 * (`src/echo/marker.ts`) re-runs the stack (~4s), replan,
34 * re-apply. `planCount` goes UP.
35 *
36 * A failed rebuild takes its own Worker off the air until the next good
37 * save; what must never happen is the CLI dying or an UNRELATED resource
38 * going with it. That is what the resilience phases assert.
39 *
40 * Requires Docker: the AWS half runs in floci and the Cloudflare Container
41 * runs as a real container. No cloud credentials are used or needed.
42 */
43import { afterAll, beforeAll, expect, test } from "bun:test";
44import * as fs from "node:fs";
45import { PORTS } from "../src/ports.ts";
46import {
47 at,
48 DevServer,
49 dockerAvailable,
50 fetchJson,
51 fetchOk,
52 freePinnedPorts,
53 makeScratchProject,
54 PollTimeout,
55 resetFlociEmulator,
56 pollUntil,
57 waitForJson,
58} from "./harness.ts";
59import {
60 echoInboxClass,
61 echoQueueBindings,
62 echoQueueLayers,
63 echoQueueRoutes,
64 echoStreamImport,
65 extraWorkerDeclaration,
66 extraWorkerOutput,
67 extraWorkerSource,
68 lambdaArchiveBindings,
69 lambdaArchiveRoutes,
70 portSquatterDeclaration,
71 portSquatterSource,
72 reportFunctionDeclaration,
73 reportFunctionImport,
74 reportFunctionOutput,
75 reportFunctionSource,
76 secondImageBindings,
77 secondImageImport,
78 secondImageLayer,
79 secondImageRoutes,
80 secondImageSource,
81 secondImageWorkerImport,
82} from "./mutations.ts";
83
84const STAGE = "dev-stress";
85
86// Long budgets: the first apply builds a Lambda bundle and a MicroVM
87// image, boots floci, starts four workerd instances, runs a site build,
88// and pulls a container image. Later phases are fast because only the
89// delta re-applies.
90//
91// These are deliberately LARGER than the sum of every bounded poll inside
92// a phase. `bun test` kills every child process the file spawned the
93// moment a test hits its timeout (`killed N dangling processes`) — which
94// would take the dev server down and turn one slow assertion into a
95// cascade of meaningless failures in every later phase. A phase must
96// therefore always fail through a `pollUntil` timeout (server still
97// alive, output tail attached), never through bun's.
98const BOOT_TIMEOUT = 1_800_000;
99const PHASE_TIMEOUT = 1_800_000;
100
101let server: DevServer;
102let cfSiteUrl: string;
103/** Cursor into the CLI output, advanced past phases that expect errors. */
104let cleanCursor = 0;
105
106/** URL on a floci host-routed address (`*.localhost.floci.io`). */
107const at2 = (host: string, port: number, path: string) =>
108 new URL(path, `http://${host}:${port}`);
109
110const echo = (path: string) => at(PORTS.echo, path);
111const api = (path: string) => at(PORTS.api, path);
112const extra = (path: string) => at(PORTS.extra, path);
113const awsSite = (path: string) => at(PORTS.awsSite, path);
114const extraAlt = (path: string) => at(PORTS.extraAlt, path);
115const microvm = (path: string) => at(PORTS.microvm, path);
116const ecs = (path: string) => at(PORTS.ecs, path);
117const ecsInline = (path: string) => at(PORTS.ecsInline, path);
118
119/**
120 * Poll an ECS-served URL until its TEXT body matches. The busybox httpd
121 * containers serve plain files; a container mid-roll refuses connections.
122 */
123const waitForText = async (
124 what: string,
125 url: URL,
126 expected: string,
127 options?: { tries?: number; delayMs?: number },
128): Promise<void> => {
129 let last = "no response yet";
130 try {
131 await pollUntil(
132 what,
133 async () => {
134 try {
135 const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
136 const text = (await res.text()).trim();
137 last = `${res.status} ${text.slice(0, 200)}`;
138 return res.ok && text === expected ? true : undefined;
139 } catch (cause) {
140 last = `error: ${String(cause).slice(0, 200)}`;
141 return undefined;
142 }
143 },
144 { tries: 240, delayMs: 1_000, server, ...options },
145 );
146 } catch (cause) {
147 if (cause instanceof PollTimeout) {
148 cause.message = `${cause.message.split("\n")[0]}\n--- last response from ${url} ---\n${last}\n${cause.message.split("\n").slice(1).join("\n")}`;
149 }
150 throw cause;
151 }
152};
153
154/** Source text for a marker module — the suite's unit of observable change. */
155const markerModule = (name: string, value: string) =>
156 `export const ${name} = ${JSON.stringify(value)};\n`;
157
158let lambdaMarkerSeq = 0;
159/**
160 * Bump the Lambda's marker and wait for the emulator to serve it, observed
161 * THROUGH the cross-cloud hop. Called after every phase that makes the
162 * engine re-reconcile the Function, because that is exactly where hot
163 * swap used to die: an engine update re-points the function's code at a
164 * content-addressed S3 key, detaching it from the watch loop's stable dev
165 * key, and the next marker edit was silently ignored until the next engine
166 * update (fixed in FlociFunctionProvider — this pins it).
167 */
168const expectLambdaHotSwapStillWorks = async (context: string) => {
169 const marker = `lambda-after-${context}-${++lambdaMarkerSeq}`;
170 server.write("src/lambda/marker.ts", markerModule("LAMBDA_MARKER", marker));
171 await waitForJson<{ marker: string }>(
172 `the Lambda to hot-swap to ${marker} (${context})`,
173 api("/aws/"),
174 (body) => body.marker === marker,
175 { tries: 120, delayMs: 1_000, server },
176 );
177};
178
179const messageModule = (value: string) =>
180 `export const message = () => ${JSON.stringify(value)};\n`;
181
182/** Is a port refusing connections (i.e. nothing is serving there)? */
183const isClosed = async (url: URL) => {
184 try {
185 await fetch(url, { signal: AbortSignal.timeout(2_000) });
186 return false;
187 } catch {
188 return true;
189 }
190};
191
192/**
193 * Has `url` stopped serving the worker that used to answer with `marker`?
194 * A deleted local Worker normally closes its port, but a lingering dev
195 * proxy answering 5xx counts as "gone" too — what matters is that the
196 * removed code is no longer reachable.
197 */
198const stoppedServing = async (url: URL, marker: string) => {
199 try {
200 const res = await fetch(url, { signal: AbortSignal.timeout(2_000) });
201 if (!res.ok) return true;
202 const body = (await res.json()) as { marker?: string };
203 return body.marker !== marker;
204 } catch {
205 return true;
206 }
207};
208
209beforeAll(async () => {
210 // Docker is a hard requirement, not a reason to skip: floci hosts the
211 // whole AWS half and the Container and MicroVM are real containers. A
212 // machine without it must fail this suite, not quietly pass it.
213 if (!dockerAvailable) {
214 throw new Error(
215 "dev-stress requires Docker (floci, Cloudflare Containers, MicroVMs). " +
216 "Start the Docker daemon and re-run.",
217 );
218 }
219 resetFlociEmulator();
220 freePinnedPorts(Object.values(PORTS));
221 const cwd = makeScratchProject(STAGE);
222 server = new DevServer({ cwd, stage: STAGE });
223
224 // The first apply is done when the CLI prints its `Done:` summary.
225 await pollUntil(
226 "the first dev apply to complete",
227 () => (server.doneCount >= 1 ? true : undefined),
228 { tries: 1_700, delayMs: 1_000, server },
229 );
230}, BOOT_TIMEOUT);
231
232afterAll(async () => {
233 if (!server) return;
234 await server.shutdown();
235 if (!process.env.NO_DESTROY) {
236 server.destroyStack();
237 if (!process.env.DEBUG) {
238 fs.rmSync(server.cwd, { recursive: true, force: true });
239 }
240 }
241}, 300_000);
242
243// ───────────────────────────────────────────────────────────────────────
244// Phase 1 — the stack converges, locally, across both clouds.
245// ───────────────────────────────────────────────────────────────────────
246
247test(
248 "boots: every local resource serves, and the cross-cloud hop works",
249 async () => {
250 server.assertAlive("boot");
251
252 // ── Cloudflare, Effect-native worker ──
253 expect(
254 await fetchJson<{ marker: string }>(echo("/marker")),
255 ).toEqual({ marker: "echo-v1" });
256
257 expect(await fetchJson<{ value: string }>(echo("/kv?key=boot"))).toEqual({
258 value: "kv:boot",
259 });
260
261 const r2 = await fetchJson<{ text: string; keys: string[] }>(echo("/r2"));
262 expect(r2.text).toBe("hello from r2");
263 expect(r2.keys).toContain("hello.txt");
264
265 // Durable Object state is real state, not a stub.
266 const first = await fetchJson<{ count: number }>(echo("/counter"));
267 const second = await fetchJson<{ count: number }>(echo("/counter"));
268 expect(second.count).toBe(first.count + 1);
269
270 // ── Cloudflare Container (docker), reached through the DO ──
271 const sandbox = await fetchJson<{ greeting: string; marker: string }>(
272 echo("/sandbox"),
273 undefined,
274 // The image may still be pulling/starting on the first request.
275 { tries: 180, delayMs: 1_000 },
276 );
277 expect(sandbox.greeting).toBe("hello-from-container");
278 expect(sandbox.marker).toBe("sandbox-v1");
279
280 // ── #1334: the container reaches a service on the HOST through an env
281 // var written as `http://localhost:…` (the dev runtime rewrites the
282 // loopback host to `host.docker.localhost`; Linux unix-socket-tunnels
283 // it into the sidecar netns instead of SYNing the docker bridge) ──
284 const hostFetch = await fetchJson<{ target: string; body: string }>(
285 echo("/sandbox/host-fetch"),
286 undefined,
287 { tries: 60, delayMs: 1_000 },
288 );
289 expect(hostFetch.target).toBe("http://host.docker.localhost:8793");
290 expect(hostFetch.body).toContain("aws-site-env-v1");
291
292 // ── Cloudflare, path-`main` worker ──
293 expect(
294 await fetchJson<{ marker: string; message: string }>(api("/marker")),
295 ).toEqual({ marker: "api-v1", message: "message-v1" });
296
297 const env = await fetchJson<{
298 API_VARIABLE: string;
299 AWS_LAMBDA_URL: string;
300 }>(api("/env"));
301 expect(env.API_VARIABLE).toBe("api-variable-v1");
302 // Local identity: the Lambda URL is the emulator's, not real AWS.
303 expect(env.AWS_LAMBDA_URL).toContain("localhost:4566");
304
305 // ── CROSS-CLOUD: local workerd → the floci-hosted Lambda URL ──
306 const lambda = await fetchJson<{ marker: string; variable: string }>(
307 api("/aws/"),
308 undefined,
309 { tries: 120, delayMs: 1_000 },
310 );
311 expect(lambda).toEqual({
312 marker: "lambda-v1",
313 variable: "lambda-variable-v1",
314 });
315
316 // ── AWS data plane, driven through the cross-cloud hop ──
317 expect(await fetchJson<{ text: string }>(api("/aws/s3"))).toEqual({
318 text: "hello from s3",
319 });
320 expect(await fetchJson<{ text: string }>(api("/aws/dynamo"))).toEqual({
321 text: "hello from dynamo",
322 });
323
324 // SQS produce → floci poller → the same Lambda's consumer → DynamoDB.
325 const message = { id: crypto.randomUUID() };
326 await fetchOk(api("/aws/queue/send"), {
327 method: "POST",
328 body: JSON.stringify(message),
329 });
330 const delivered = await waitForJson<{ body: string | null }>(
331 "the queue message to be consumed",
332 api(`/aws/queue/messages?id=${message.id}`),
333 (body) => body.body !== null,
334 { tries: 120, delayMs: 500, server },
335 );
336 expect(JSON.parse(delivered.body!)).toEqual(message);
337
338 // ── AWS ECS: floci runs the services' tasks as real containers on
339 // the host daemon; bridge networking publishes the baked ports. First
340 // contact can wait out the image build + scheduler launch.
341 await waitForText(
342 "the context-built ECS service to serve",
343 ecs("/"),
344 "ecs-site-v1",
345 { tries: 300 },
346 );
347 await waitForText(
348 "the ECS service's baked Dockerfile marker",
349 ecs("/baked.txt"),
350 "dockerfile-v1",
351 );
352 await waitForText(
353 "the ECS service's task-definition env",
354 ecs("/env.txt"),
355 "ecs-env-v1",
356 );
357 await waitForText(
358 "the inline-Dockerfile ECS service to serve",
359 ecsInline("/"),
360 "ecs-inline-v1",
361 { tries: 300 },
362 );
363
364 // ── AWS Website: the dev-command child, on its pinned port ──
365 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
366 const siteEnv = await fetchJson<{ marker: string; pid: number }>(
367 awsSite("/__dev-env"),
368 );
369 expect(siteEnv.marker).toBe("aws-site-env-v1");
370
371 // ── Cloudflare Website: build mode, served by a local Worker ──
372 cfSiteUrl = await pollUntil(
373 "cfSiteUrl in the stack outputs",
374 () => server.outputUrl("cfSiteUrl"),
375 { tries: 60, delayMs: 1_000, server },
376 );
377 expect(cfSiteUrl).toMatch(/^http:\/\/localhost:\d+/);
378 expect(await (await fetchOk(new URL("/", cfSiteUrl))).text()).toContain(
379 "cf-site-v1",
380 );
381
382 // Nothing in a clean boot may look like a failure.
383 expect(server.output).not.toContain("alchemy dev: run failed");
384 expect(server.output).not.toContain("alchemy dev: apply failed");
385 cleanCursor = server.mark();
386 },
387 PHASE_TIMEOUT,
388);
389
390test(
391 "cross-cloud: a Cloudflare Worker boots an AWS Lambda MicroVM and drives both its protocols",
392 async () => {
393 // Boot → wait for RUNNING → auth token → typed RPC + raw HTTPS →
394 // terminate, all from local workerd against the floci emulator.
395 const roundtrip = await fetchJson<{
396 microvmId: string;
397 reply: string;
398 marker: string;
399 echo: string;
400 }>(microvm("/roundtrip?message=stress"), undefined, {
401 tries: 240,
402 delayMs: 1_000,
403 });
404 expect(roundtrip.microvmId).toBeTruthy();
405 expect(roundtrip.reply).toBe("hello, stress!");
406 expect(roundtrip.marker).toBe("vm-v1");
407 expect(roundtrip.echo).toBe("stress");
408 server.assertAlive("microvm roundtrip");
409 },
410 PHASE_TIMEOUT,
411);
412
413// ───────────────────────────────────────────────────────────────────────
414// Phase 2 — the two reload paths, told apart.
415// ───────────────────────────────────────────────────────────────────────
416
417test(
418 "hot reload (bundler path): editing a file the stack never imports swaps the script without re-running the stack",
419 async () => {
420 const plansBefore = server.planCount;
421 const counterBefore = (
422 await fetchJson<{ count: number }>(echo("/counter"))
423 ).count;
424
425 server.write("src/api/marker.ts", markerModule("API_MARKER", "api-v2"));
426
427 await waitForJson<{ marker: string }>(
428 "ApiWorker to serve api-v2",
429 api("/marker"),
430 (body) => body.marker === "api-v2",
431 { server },
432 );
433
434 // The whole point of this path: the stack process never re-ran. A
435 // re-plan would trail the swap by a couple of seconds, so settle well
436 // past that before claiming it never came.
437 await Bun.sleep(20_000);
438 expect(server.planCount).toBe(plansBefore);
439 server.assertAlive("bundler hot swap");
440
441 // And nothing else moved: the sibling worker's DO kept its state.
442 const counterAfter = (await fetchJson<{ count: number }>(echo("/counter")))
443 .count;
444 expect(counterAfter).toBe(counterBefore + 1);
445 },
446 PHASE_TIMEOUT,
447);
448
449test(
450 "hot reload (watch path): editing a file the stack imports re-runs the stack and keeps sidecar children alive",
451 async () => {
452 const plansBefore = server.planCount;
453 // The dev-command child lives in the provider sidecar, which is meant
454 // to survive user-code restarts. Its pid is the proof.
455 const sitePidBefore = (
456 await fetchJson<{ pid: number }>(awsSite("/__dev-env"))
457 ).pid;
458
459 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v2"));
460
461 await waitForJson<{ marker: string }>(
462 "EchoWorker to serve echo-v2",
463 echo("/marker"),
464 (body) => body.marker === "echo-v2",
465 { server },
466 );
467
468 // The re-plan trails the hot swap — wait for it rather than sampling.
469 await server.waitForPlanAfter(plansBefore);
470 server.assertAlive("watch-path reload");
471
472 // The sidecar-hosted dev server was NOT bounced by a user-code reload.
473 const sitePidAfter = (await fetchJson<{ pid: number }>(awsSite("/__dev-env")))
474 .pid;
475 expect(sitePidAfter).toBe(sitePidBefore);
476
477 // The bundler-path worker was not disturbed either.
478 expect(
479 (await fetchJson<{ marker: string }>(api("/marker"))).marker,
480 ).toBe("api-v2");
481 },
482 PHASE_TIMEOUT,
483);
484
485test(
486 "hot reload (AWS): editing the Lambda's source hot-swaps its code in the emulator",
487 async () => {
488 server.write(
489 "src/lambda/marker.ts",
490 markerModule("LAMBDA_MARKER", "lambda-v2"),
491 );
492
493 // Observed THROUGH the cross-cloud hop, so this also re-proves that the
494 // Worker → Lambda edge still resolves after both sides reloaded.
495 await waitForJson<{ marker: string }>(
496 "the Lambda to serve lambda-v2",
497 api("/aws/"),
498 (body) => body.marker === "lambda-v2",
499 { tries: 240, delayMs: 1_000, server },
500 );
501
502 server.assertAlive("lambda hot swap");
503 // Bindings survived the swap.
504 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe(
505 "hello from s3",
506 );
507 },
508 PHASE_TIMEOUT,
509);
510
511// ───────────────────────────────────────────────────────────────────────
512// Phase 3 — files that move out from under the bundler.
513// ───────────────────────────────────────────────────────────────────────
514
515test(
516 "surviving a moved module: the importer points at a missing path, then the file arrives",
517 async () => {
518 const importer = server.read("src/api-worker.ts");
519
520 // Deliberately out of order: rewrite the import FIRST, so the bundler
521 // is asked to resolve a module that does not exist yet.
522 server.write(
523 "src/api-worker.ts",
524 importer.replace('"./api/message.ts"', '"./api/nested/message.ts"'),
525 );
526 await Bun.sleep(8_000);
527 // A failed rebuild takes ITS worker off the air — that part is fine.
528 // What must hold is that the dev server lives and every unrelated
529 // resource is untouched.
530 server.assertAlive("broken import window");
531 expect(
532 (await fetchJson<{ marker: string }>(echo("/marker"))).marker,
533 ).toBe("echo-v2");
534 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
535
536 // Complete the move, with a NEW value so "the rebuild happened" cannot
537 // be confused with "the old bundle is still serving".
538 server.remove("src/api/message.ts");
539 server.write("src/api/nested/message.ts", messageModule("message-moved"));
540 await waitForJson<{ message: string }>(
541 "ApiWorker to rebuild against the moved module",
542 api("/marker"),
543 (body) => body.message === "message-moved",
544 { tries: 180, delayMs: 500, server },
545 );
546
547 // The NEW path is watched too, not just resolved once.
548 server.write("src/api/nested/message.ts", messageModule("message-v2"));
549 await waitForJson<{ message: string }>(
550 "the moved module's edit to land",
551 api("/marker"),
552 (body) => body.message === "message-v2",
553 { server },
554 );
555 server.assertAlive("moved module reload");
556 },
557 PHASE_TIMEOUT,
558);
559
560// ───────────────────────────────────────────────────────────────────────
561// Phase 4 — broken states. The dev server must log and wait, never exit.
562// ───────────────────────────────────────────────────────────────────────
563
564test(
565 "surviving a syntax error in a stack-imported module",
566 async () => {
567 const plansBefore = server.planCount;
568 const cursor = server.mark();
569 // Unterminated string literal: the module cannot even be parsed, so
570 // importing the stack throws before any Alchemy code runs.
571 server.write("src/echo/marker.ts", 'export const ECHO_MARKER = "oops\n');
572
573 await pollUntil(
574 "the CLI to report the failed run",
575 () =>
576 /alchemy dev: run failed|SyntaxError|Unterminated|Unexpected end/.test(
577 server.since(cursor),
578 ) || undefined,
579 { tries: 120, delayMs: 500, server },
580 );
581 server.assertAlive("stack syntax error");
582
583 // EchoWorker is down (its bundle no longer builds), but every resource
584 // that does not depend on the broken module keeps serving: they live in
585 // the provider sidecar, which the exec child's crash does not touch.
586 expect(
587 (await fetchJson<{ marker: string }>(api("/marker"))).marker,
588 ).toBe("api-v2");
589 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
590
591 // Recovery on the next save.
592 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v3"));
593 await waitForJson<{ marker: string }>(
594 "EchoWorker to recover on echo-v3",
595 echo("/marker"),
596 (body) => body.marker === "echo-v3",
597 { tries: 240, delayMs: 500, server },
598 );
599 await server.waitForPlanAfter(plansBefore);
600 cleanCursor = server.mark();
601 },
602 PHASE_TIMEOUT,
603);
604
605test(
606 "surviving a throw at module scope",
607 async () => {
608 const config = server.read("src/stack-config.ts");
609 server.write(
610 "src/stack-config.ts",
611 `${config}\nthrow new Error("dev-stress: deliberate module-scope failure");\n`,
612 );
613
614 await pollUntil(
615 "the CLI to report the failed run",
616 () =>
617 server.since(cleanCursor).includes("deliberate module-scope failure")
618 ? true
619 : undefined,
620 { tries: 120, delayMs: 500, server },
621 );
622 server.assertAlive("module-scope throw");
623 expect(server.since(cleanCursor)).toContain("alchemy dev: run failed");
624
625 // Restore and converge.
626 server.write("src/stack-config.ts", config);
627 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v4"));
628 await waitForJson<{ marker: string }>(
629 "EchoWorker to recover on echo-v4",
630 echo("/marker"),
631 (body) => body.marker === "echo-v4",
632 { tries: 240, delayMs: 500, server },
633 );
634 cleanCursor = server.mark();
635 },
636 PHASE_TIMEOUT,
637);
638
639test(
640 "surviving an apply failure: healthy resources keep serving while one resource cannot reconcile",
641 async () => {
642 // A second Worker that demands a port EchoWorker already holds. The
643 // stack imports and plans fine; the reconcile is what fails — which is
644 // the case that has to leave the rest of the stack serving. (A Worker
645 // with a missing `main` is NOT such a case: the local provider creates
646 // it happily.)
647 server.write("src/extra/squatter.ts", portSquatterSource);
648 server.patchRegion("alchemy.run.ts", "EXTRA", portSquatterDeclaration);
649
650 await pollUntil(
651 "the CLI to report the failed apply",
652 () =>
653 /alchemy dev: apply failed/.test(server.since(cleanCursor)) ||
654 undefined,
655 { tries: 240, delayMs: 500, server },
656 );
657 server.assertAlive("apply failure");
658 expect(server.since(cleanCursor)).toContain("already in use");
659
660 // Everything healthy is still healthy.
661 expect(
662 (await fetchJson<{ marker: string }>(echo("/marker"))).marker,
663 ).toBe("echo-v4");
664 expect(
665 (await fetchJson<{ marker: string }>(api("/marker"))).marker,
666 ).toBe("api-v2");
667 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe(
668 "hello from s3",
669 );
670
671 server.patchRegion("alchemy.run.ts", "EXTRA", "");
672 server.remove("src/extra/squatter.ts");
673 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-v5"));
674 await waitForJson<{ marker: string }>(
675 "the stack to converge again",
676 echo("/marker"),
677 (body) => body.marker === "echo-v5",
678 { tries: 240, delayMs: 500, server },
679 );
680 cleanCursor = server.mark();
681 },
682 PHASE_TIMEOUT,
683);
684
685// ───────────────────────────────────────────────────────────────────────
686// Phase 5 — the resource GRAPH changes shape. This is the substance of the
687// suite: resources appear, get renamed, get replaced and disappear while
688// the dev server keeps running, across both clouds and every resource kind
689// the stack has (Workers, a Lambda, buckets, a queue + its event source, a
690// Durable Object class, a MicroVM image).
691// ───────────────────────────────────────────────────────────────────────
692
693test(
694 "graph churn: a Cloudflare Worker can be added, renamed onto a new port, and removed",
695 async () => {
696 // ── add ──
697 server.write("src/extra/extra-worker.ts", extraWorkerSource("extra-v1"));
698 server.patchRegion(
699 "alchemy.run.ts",
700 "EXTRA",
701 extraWorkerDeclaration("ExtraWorker", "extra"),
702 );
703 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", extraWorkerOutput);
704 await waitForJson<{ marker: string }>(
705 "the added ExtraWorker to serve",
706 extra("/"),
707 (body) => body.marker === "extra-v1",
708 { tries: 240, delayMs: 500, server },
709 );
710 // The worker serves before the stack's re-apply prints its outputs —
711 // the output line is the proof the ENGINE saw the new resource.
712 await pollUntil(
713 "extraUrl in the stack outputs",
714 () => server.outputUrl("extraUrl"),
715 { tries: 120, delayMs: 500, server },
716 );
717 server.assertAlive("worker added");
718
719 // ── rename (a new logical id is a create + a delete) ──
720 // The new generation lands on its own port: the engine may create
721 // before it deletes, and two `strictPort` workers cannot share one.
722 server.patchRegion(
723 "alchemy.run.ts",
724 "EXTRA",
725 extraWorkerDeclaration("RenamedWorker", "extraAlt"),
726 );
727 server.write("src/extra/extra-worker.ts", extraWorkerSource("extra-v2"));
728 await waitForJson<{ marker: string }>(
729 "the renamed worker to serve on its new port",
730 extraAlt("/"),
731 (body) => body.marker === "extra-v2",
732 { tries: 240, delayMs: 500, server },
733 );
734 // …and the old generation is gone.
735 await pollUntil(
736 "the old worker's port to stop serving",
737 async () => ((await isClosed(extra("/"))) ? true : undefined),
738 { tries: 120, delayMs: 500, server },
739 );
740 server.assertAlive("worker renamed");
741
742 // ── remove (output reference first, so the stack never names a
743 // binding that no longer exists) ──
744 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", "");
745 server.patchRegion("alchemy.run.ts", "EXTRA", "");
746 server.remove("src/extra/extra-worker.ts");
747 await pollUntil(
748 "the removed worker to stop serving",
749 async () =>
750 (await stoppedServing(extraAlt("/"), "extra-v2")) ? true : undefined,
751 { tries: 240, delayMs: 500, server },
752 );
753 server.assertAlive("worker removed");
754
755 // The survivors are untouched.
756 expect(
757 (await fetchJson<{ marker: string }>(echo("/marker"))).marker,
758 ).toBe("echo-v5");
759 cleanCursor = server.mark();
760 },
761 PHASE_TIMEOUT,
762);
763
764test(
765 "graph churn: a whole AWS Lambda (with its own S3 bucket) can be added and removed",
766 async () => {
767 // ── add: a new module, a new top-level import in the stack, a new
768 // Function resource, a new Bucket resource, and two new bindings ──
769 server.write(
770 "src/extra/ReportFunction.ts",
771 reportFunctionSource("report-v1"),
772 );
773 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", reportFunctionImport);
774 server.patchRegion("alchemy.run.ts", "EXTRA", reportFunctionDeclaration);
775 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", reportFunctionOutput);
776
777 const reportUrl = await pollUntil(
778 "the new Lambda's function URL in the stack outputs",
779 () => server.outputUrl("reportUrl"),
780 { tries: 480, delayMs: 500, server },
781 );
782 // Local identity: a floci URL, never real AWS.
783 expect(reportUrl).toContain("localhost:4566");
784
785 const report = await fetchJson<{ marker: string; text: string }>(
786 new URL("/report", reportUrl),
787 undefined,
788 { tries: 240, delayMs: 500 },
789 );
790 expect(report).toEqual({ marker: "report-v1", text: "report-v1" });
791 server.assertAlive("lambda added");
792
793 // ── change it: a new marker means a new bundle for a Function that
794 // only came into existence a moment ago ──
795 server.write(
796 "src/extra/ReportFunction.ts",
797 reportFunctionSource("report-v2"),
798 );
799 await waitForJson<{ marker: string }>(
800 "the added Lambda to hot-swap to report-v2",
801 new URL("/report", reportUrl),
802 (body) => body.marker === "report-v2",
803 { tries: 480, delayMs: 500, server },
804 );
805
806 // ── remove: the Function, its Bucket and its URL all go away ──
807 server.patchRegion("alchemy.run.ts", "EXTRA_OUTPUTS", "");
808 server.patchRegion("alchemy.run.ts", "EXTRA", "");
809 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", "");
810 server.remove("src/extra/ReportFunction.ts");
811 await pollUntil(
812 "the removed Lambda to stop answering",
813 async () => {
814 try {
815 const res = await fetch(new URL("/report", reportUrl), {
816 signal: AbortSignal.timeout(5_000),
817 });
818 return res.ok ? undefined : true;
819 } catch {
820 return true;
821 }
822 },
823 { tries: 240, delayMs: 500, server },
824 );
825 server.assertAlive("lambda removed");
826
827 // The original Lambda is untouched by its sibling's whole lifecycle.
828 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe(
829 "hello from s3",
830 );
831 cleanCursor = server.mark();
832 },
833 PHASE_TIMEOUT,
834);
835
836test(
837 "graph churn: a Queue, its consumer event source, and a new Durable Object class are grafted onto a LIVE Worker",
838 async () => {
839 const plansBefore = server.planCount;
840
841 // Three kinds of change at once, all on a Worker that is already
842 // serving: a new resource (the Queue), a new event source (its
843 // consumer), and a new Durable Object class — which means a class
844 // migration on a running script.
845 server.patchRegion("src/EchoWorker.ts", "ECHO_IMPORTS", echoStreamImport);
846 server.patchRegion("src/EchoWorker.ts", "ECHO_BINDINGS", echoQueueBindings);
847 server.patchRegion("src/EchoWorker.ts", "ECHO_ROUTES", echoQueueRoutes);
848 server.patchRegion("src/EchoWorker.ts", "ECHO_LAYERS", echoQueueLayers);
849 server.patchRegion("src/EchoWorker.ts", "ECHO_CLASSES", echoInboxClass);
850
851 // The bundler hot-swaps the new script BEFORE the stack re-apply has
852 // registered the `Inbox` class and the queue consumer, so for a beat
853 // the Worker serves a bundle whose DO binding workerd rejects
854 // (`DurableObject 'Inbox' not found`). A message sent in that window
855 // has no consumer and is lost. Wait for the re-apply to land first;
856 // the self-healing is what's under test, not the window.
857 await server.waitForPlanAfter(plansBefore, { tries: 480 });
858 await pollUntil(
859 "the re-apply that registers the Queue, consumer and DO to finish",
860 () =>
861 /\[EchoQueueConsumer\] created/.test(server.since(cleanCursor))
862 ? true
863 : undefined,
864 { tries: 480, delayMs: 500, server },
865 );
866
867 const id = crypto.randomUUID();
868 await pollUntil(
869 "the new queue route to accept a message",
870 async () => {
871 try {
872 const res = await fetch(echo(`/queue/send?id=${id}`), {
873 signal: AbortSignal.timeout(5_000),
874 });
875 return res.ok ? true : undefined;
876 } catch {
877 return undefined;
878 }
879 },
880 { tries: 480, delayMs: 500, server },
881 );
882
883 // Produce → the local broker delivers → the new DO records it.
884 const received = await waitForJson<{ ids: string[] }>(
885 "the queue message to reach the new Durable Object",
886 echo("/queue/received"),
887 (body) => body.ids.includes(id),
888 { tries: 240, delayMs: 500, server },
889 );
890 expect(received.ids).toContain(id);
891 server.assertAlive("queue + consumer + DO added");
892
893 // The Worker's pre-existing bindings still work after the graft.
894 expect((await fetchJson<{ value: string }>(echo("/kv?key=graft"))).value)
895 .toBe("kv:graft");
896 expect(
897 (await fetchJson<{ count: number }>(echo("/counter"))).count,
898 ).toBeGreaterThan(0);
899
900 // ── and back out again: the routes, the consumer, the Queue and the
901 // DO class all disappear from a running Worker ──
902 server.patchRegion("src/EchoWorker.ts", "ECHO_ROUTES", "");
903 server.patchRegion("src/EchoWorker.ts", "ECHO_BINDINGS", "");
904 server.patchRegion("src/EchoWorker.ts", "ECHO_LAYERS", "");
905 server.patchRegion("src/EchoWorker.ts", "ECHO_CLASSES", "");
906 server.patchRegion("src/EchoWorker.ts", "ECHO_IMPORTS", "");
907 await pollUntil(
908 "the queue route to disappear",
909 async () => {
910 try {
911 const res = await fetch(echo(`/queue/send?id=${id}`), {
912 signal: AbortSignal.timeout(5_000),
913 });
914 // The fall-through route answers with the marker instead.
915 if (!res.ok) return true;
916 const body = (await res.json()) as { sent?: string };
917 return body.sent === undefined ? true : undefined;
918 } catch {
919 return undefined;
920 }
921 },
922 { tries: 480, delayMs: 500, server },
923 );
924 server.assertAlive("queue + consumer + DO removed");
925 expect(
926 (await fetchJson<{ marker: string }>(echo("/marker"))).marker,
927 ).toBe("echo-v5");
928 cleanCursor = server.mark();
929 },
930 PHASE_TIMEOUT,
931);
932
933test(
934 "graph churn: a second S3 bucket and its bindings are added to the LIVE Lambda, then removed",
935 async () => {
936 server.patchRegion(
937 "src/ApiFunction.ts",
938 "LAMBDA_BINDINGS",
939 lambdaArchiveBindings,
940 );
941 server.patchRegion(
942 "src/ApiFunction.ts",
943 "LAMBDA_ROUTES",
944 lambdaArchiveRoutes,
945 );
946
947 // Driven through the cross-cloud hop, so this also re-proves the
948 // Worker → Lambda edge after the Lambda's binding set changed.
949 const archived = await waitForJson<{ text: string }>(
950 "the new bucket binding to answer on the Lambda",
951 api("/aws/archive"),
952 (body) => body.text === "archived",
953 { tries: 480, delayMs: 500, server },
954 );
955 expect(archived.text).toBe("archived");
956 server.assertAlive("lambda bucket added");
957
958 // Pre-existing bindings survived the change.
959 expect((await fetchJson<{ text: string }>(api("/aws/dynamo"))).text).toBe(
960 "hello from dynamo",
961 );
962 // The engine just UPDATED the Function — hot swap must still work.
963 await expectLambdaHotSwapStillWorks("archive-added");
964
965 server.patchRegion("src/ApiFunction.ts", "LAMBDA_ROUTES", "");
966 server.patchRegion("src/ApiFunction.ts", "LAMBDA_BINDINGS", "");
967 await pollUntil(
968 "the archive route to disappear",
969 async () => {
970 try {
971 const res = await fetch(api("/aws/archive"), {
972 signal: AbortSignal.timeout(10_000),
973 });
974 return res.status === 404 ? true : undefined;
975 } catch {
976 return undefined;
977 }
978 },
979 { tries: 480, delayMs: 500, server },
980 );
981 server.assertAlive("lambda bucket removed");
982 await expectLambdaHotSwapStillWorks("archive-removed");
983 cleanCursor = server.mark();
984 },
985 PHASE_TIMEOUT,
986);
987
988test(
989 "ECS hot reload: context content, the Dockerfile itself, an env prop, and an inline-Dockerfile prop all roll the running containers",
990 async () => {
991 // ── 1. a file in the build context (the classic watch path) ──
992 server.write("site/ecs/index.html", "ecs-site-v2\n");
993 await waitForText(
994 "the ECS service to serve the edited context file",
995 ecs("/"),
996 "ecs-site-v2",
997 );
998 server.assertAlive("ecs context reload");
999
1000 // ── 2. the Dockerfile ITSELF (also a file in the context, but the
1001 // change only lands if the image actually rebuilds) ──
1002 server.write(
1003 "site/ecs/Dockerfile",
1004 server
1005 .read("site/ecs/Dockerfile")
1006 .replace("BAKED_MARKER=dockerfile-v1", "BAKED_MARKER=dockerfile-v2"),
1007 );
1008 await waitForText(
1009 "the ECS service to serve the rebuilt Dockerfile marker",
1010 ecs("/baked.txt"),
1011 "dockerfile-v2",
1012 );
1013 // The context file survived the rebuild.
1014 await waitForText(
1015 "the context file to still serve after the Dockerfile rebuild",
1016 ecs("/"),
1017 "ecs-site-v2",
1018 );
1019 server.assertAlive("ecs dockerfile reload");
1020
1021 // ── 3. an env PROP (no file event at all — the engine registers a new
1022 // task-definition revision, and the running task must roll onto it;
1023 // regression: only file-watch triggers restarted tasks, so prop-driven
1024 // updates left containers serving the old revision forever) ──
1025 server.patchRegion(
1026 "alchemy.run.ts",
1027 "ECS_ENV",
1028 ' STRESS_ENV: "ecs-env-v2",\n',
1029 );
1030 await waitForText(
1031 "the ECS task to roll onto the new env",
1032 ecs("/env.txt"),
1033 "ecs-env-v2",
1034 );
1035 server.assertAlive("ecs env-prop reload");
1036
1037 // ── 4. an INLINE Dockerfile (a pure prop change that is nonetheless a
1038 // Dockerfile edit — same engine path as 3, but the image itself must
1039 // rebuild) ──
1040 server.patchRegion(
1041 "alchemy.run.ts",
1042 "ECS_INLINE_MARKER",
1043 ` "RUN mkdir -p /www && echo -n ecs-inline-v2 > /www/index.html",\n`,
1044 );
1045 await waitForText(
1046 "the inline-Dockerfile service to serve the rebuilt image",
1047 ecsInline("/"),
1048 "ecs-inline-v2",
1049 );
1050 server.assertAlive("ecs inline-dockerfile reload");
1051
1052 // The sibling service was untouched by the inline rebuild.
1053 await waitForText(
1054 "the context service to still serve",
1055 ecs("/"),
1056 "ecs-site-v2",
1057 { tries: 30 },
1058 );
1059 cleanCursor = server.mark();
1060 },
1061 PHASE_TIMEOUT,
1062);
1063
1064test(
1065 "EC2 hot reload: the hosted instance serves, and editing its program updates it in place",
1066 async () => {
1067 // The box's address comes from floci: `i-….localhost.floci.io`
1068 // resolves to 127.0.0.1 and the mux publishes the SG app port.
1069 const dns = await pollUntil(
1070 "ec2Dns in the stack outputs",
1071 () => server.outputValue("ec2Dns"),
1072 { tries: 240, delayMs: 500, server },
1073 );
1074 expect(dns).toMatch(/\.localhost\.floci\.io$/);
1075 const marker = at2(dns, PORTS.ec2, "/marker");
1076
1077 // First boot: container start + userData (bundle sync from emulated
1078 // S3, runtime install) + the Bun HTTP server binding the app port.
1079 await waitForJson<{ marker: string }>(
1080 "the EC2 box to serve its hosted program",
1081 marker,
1082 (body) => body.marker === "ec2-v1",
1083 { tries: 90, delayMs: 1_000, server },
1084 );
1085
1086 // The reload path is the ENGINE's: a content edit re-plans, the
1087 // provider re-uploads the bundle in place and reboots the instance —
1088 // same instance id, same address, new code.
1089 const reloadStartedAt = Date.now();
1090 server.write("src/ec2/marker.ts", markerModule("EC2_MARKER", "ec2-v2"));
1091 await waitForJson<{ marker: string }>(
1092 "the EC2 box to serve ec2-v2 after the in-place update",
1093 marker,
1094 (body) => body.marker === "ec2-v2",
1095 { tries: 150, delayMs: 1_000, server },
1096 );
1097 console.log(
1098 `ec2 hosted-program reload -> serving ec2-v2 in ${Date.now() - reloadStartedAt}ms`,
1099 );
1100 // Same box, same address — the update was in place, not a replacement.
1101 expect(server.outputValue("ec2Dns")).toBe(dns);
1102 server.assertAlive("ec2 hot reload");
1103 },
1104 PHASE_TIMEOUT,
1105);
1106
1107test(
1108 "Cloudflare Container hot reload: editing the container's program rebuilds the image and restarts it",
1109 async () => {
1110 // The container module is imported by the stack, so this edit replans;
1111 // the reload rides the image content hash in the worker's restart
1112 // config (regression: the sidecar-lifetime artifact memo made the diff
1113 // compare the first run's hash forever, and the config only carried
1114 // the image's stable paths — the running container served stale code
1115 // until a full dev-session restart).
1116 server.patchRegion(
1117 "src/SandboxContainer.ts",
1118 "SANDBOX_MARKER",
1119 ' marker: "sandbox-v2",\n',
1120 );
1121 await waitForJson<{ marker: string }>(
1122 "the sandbox container to serve the rebuilt program",
1123 echo("/sandbox"),
1124 (body) => body.marker === "sandbox-v2",
1125 { tries: 300, delayMs: 1_000, server },
1126 );
1127 server.assertAlive("container hot reload");
1128
1129 // The #1334 loopback rewrite still holds on the rebuilt container.
1130 const hostFetch = await fetchJson<{ body: string }>(
1131 echo("/sandbox/host-fetch"),
1132 undefined,
1133 { tries: 60, delayMs: 1_000 },
1134 );
1135 expect(hostFetch.body).toContain("aws-site-env-v1");
1136 cleanCursor = server.mark();
1137 },
1138 PHASE_TIMEOUT,
1139);
1140
1141test(
1142 "graph churn: a replacement-forcing prop change swaps a DynamoDB table under a running Lambda",
1143 async () => {
1144 const source = server.read("src/ApiFunction.ts");
1145 // The partition key is immutable on DynamoDB — changing it is a
1146 // REPLACEMENT, not an update: a new table is created, the Lambda's
1147 // binding is repointed, and the old table is deleted.
1148 server.write(
1149 "src/ApiFunction.ts",
1150 source
1151 .replace('partitionKey: "id",', 'partitionKey: "pk",')
1152 .replace('attributes: { id: "S" },', 'attributes: { pk: "S" },')
1153 // every item key in the handler, in one sweep
1154 .replaceAll("id: { S:", "pk: { S:"),
1155 );
1156
1157 // The old table answers `/dynamo` until the replacement lands, so the
1158 // engine's own plan line is the signal — then the route must work
1159 // against the NEW table (a fresh write lands in it).
1160 const replacedCursor = server.mark();
1161 await pollUntil(
1162 "the engine to plan the table replacement",
1163 () =>
1164 /\[StressTable\] replace/.test(server.plain(replacedCursor)) ||
1165 undefined,
1166 { tries: 240, delayMs: 500, server },
1167 );
1168 // A replacement prints `creating replacement` → `created` and then
1169 // cleans up the old generation; the cleanup line is the end of it.
1170 await pollUntil(
1171 "the replaced table to finish",
1172 () =>
1173 /\[StressTable\] Replaced resource cleanup complete/.test(
1174 server.plain(replacedCursor),
1175 ) || undefined,
1176 { tries: 480, delayMs: 500, server },
1177 );
1178 await waitForJson<{ text: string }>(
1179 "the replaced table to serve through the Lambda",
1180 api("/aws/dynamo"),
1181 (body) => body.text === "hello from dynamo",
1182 { tries: 480, delayMs: 500, server },
1183 );
1184 server.assertAlive("table replaced");
1185 await expectLambdaHotSwapStillWorks("table-replaced");
1186
1187 // Put it back — the swap back is a second replacement.
1188 const restoredCursor = server.mark();
1189 server.write("src/ApiFunction.ts", source);
1190 await pollUntil(
1191 "the table's second replacement to finish",
1192 () =>
1193 /\[StressTable\] Replaced resource cleanup complete/.test(
1194 server.plain(restoredCursor),
1195 ) || undefined,
1196 { tries: 480, delayMs: 500, server },
1197 );
1198 await waitForJson<{ text: string }>(
1199 "the restored table to serve through the Lambda",
1200 api("/aws/dynamo"),
1201 (body) => body.text === "hello from dynamo",
1202 { tries: 480, delayMs: 500, server },
1203 );
1204 server.assertAlive("table restored");
1205 await expectLambdaHotSwapStillWorks("table-restored");
1206 cleanCursor = server.mark();
1207 },
1208 PHASE_TIMEOUT,
1209);
1210
1211test(
1212 "graph churn: a SECOND AWS MicroVM image is added, driven from the Worker, and removed",
1213 async () => {
1214 // A new image resource (floci builds it), a new top-level import in
1215 // both the stack and the Worker, two new MicroVM bindings on a running
1216 // Worker, and a route that boots and terminates an instance of it.
1217 server.write("src/extra/WorkerImage.ts", secondImageSource("worker-vm-v1"));
1218 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", secondImageImport);
1219 server.patchRegion("alchemy.run.ts", "EXTRA_LAYERS", secondImageLayer);
1220 server.patchRegion(
1221 "src/MicrovmWorker.ts",
1222 "VM_IMPORTS",
1223 secondImageWorkerImport,
1224 );
1225 server.patchRegion(
1226 "src/MicrovmWorker.ts",
1227 "VM_BINDINGS",
1228 secondImageBindings,
1229 );
1230 server.patchRegion("src/MicrovmWorker.ts", "VM_ROUTES", secondImageRoutes);
1231
1232 const booted = await waitForJson<{ microvmId: string }>(
1233 "the second MicroVM image to build and boot an instance",
1234 microvm("/second"),
1235 (body) => typeof body.microvmId === "string" && body.microvmId.length > 0,
1236 { tries: 900, delayMs: 1_000, server },
1237 );
1238 expect(booted.microvmId).toBeTruthy();
1239 server.assertAlive("second microvm added");
1240
1241 // The FIRST image still works — adding a sibling did not disturb it.
1242 expect(
1243 (
1244 await fetchJson<{ marker: string }>(
1245 microvm("/roundtrip?message=sibling"),
1246 undefined,
1247 { tries: 300, delayMs: 1_000 },
1248 )
1249 ).marker,
1250 ).toBe("vm-v1");
1251
1252 // ── remove the second image and everything that referenced it ──
1253 server.patchRegion("src/MicrovmWorker.ts", "VM_ROUTES", "");
1254 server.patchRegion("src/MicrovmWorker.ts", "VM_BINDINGS", "");
1255 server.patchRegion("src/MicrovmWorker.ts", "VM_IMPORTS", "");
1256 server.patchRegion("alchemy.run.ts", "EXTRA_LAYERS", "");
1257 server.patchRegion("alchemy.run.ts", "EXTRA_IMPORTS", "");
1258 server.remove("src/extra/WorkerImage.ts");
1259 await pollUntil(
1260 "the second MicroVM's route to disappear",
1261 async () => {
1262 try {
1263 const res = await fetch(microvm("/second"), {
1264 signal: AbortSignal.timeout(10_000),
1265 });
1266 const text = await res.text();
1267 return text === "ok" ? true : undefined;
1268 } catch {
1269 return undefined;
1270 }
1271 },
1272 { tries: 600, delayMs: 500, server },
1273 );
1274 server.assertAlive("second microvm removed");
1275 cleanCursor = server.mark();
1276 },
1277 PHASE_TIMEOUT,
1278);
1279
1280test(
1281 "graph churn: the entire AWS Lambda subsystem is deleted and restored while Cloudflare keeps serving",
1282 async () => {
1283 const awsHalf = server.read("alchemy.run.ts");
1284
1285 // Delete the Lambda, its bucket, its table, its queue and its event
1286 // source in one edit — and the cross-cloud binding that referenced it.
1287 server.patchRegion("alchemy.run.ts", "AWS_OUTPUTS", "");
1288 server.patchRegion("alchemy.run.ts", "AWS_LAMBDA_URL", "");
1289 server.patchRegion("alchemy.run.ts", "AWS_HALF", "");
1290
1291 await pollUntil(
1292 "the AWS half to be gone from ApiWorker's bindings",
1293 async () => {
1294 try {
1295 const body = (await (
1296 await fetch(api("/env"), { signal: AbortSignal.timeout(5_000) })
1297 ).json()) as { AWS_LAMBDA_URL: string | null };
1298 return body.AWS_LAMBDA_URL === null ? true : undefined;
1299 } catch {
1300 return undefined;
1301 }
1302 },
1303 { tries: 600, delayMs: 500, server },
1304 );
1305 server.assertAlive("aws half deleted");
1306
1307 // Cloudflare is completely unaffected by the AWS half vanishing.
1308 expect(
1309 (await fetchJson<{ marker: string }>(echo("/marker"))).marker,
1310 ).toBe("echo-v5");
1311 expect((await fetchJson<{ text: string }>(echo("/r2"))).text).toBe(
1312 "hello from r2",
1313 );
1314 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
1315
1316 // ── restore the whole subsystem ──
1317 server.write("alchemy.run.ts", awsHalf);
1318 await waitForJson<{ marker: string }>(
1319 "the restored Lambda to serve through the cross-cloud hop",
1320 api("/aws/"),
1321 (body) => body.marker.startsWith("lambda-"),
1322 { tries: 900, delayMs: 500, server },
1323 );
1324 expect((await fetchJson<{ text: string }>(api("/aws/s3"))).text).toBe(
1325 "hello from s3",
1326 );
1327 server.assertAlive("aws half restored");
1328 // A brand-new Function generation must get a working watch loop.
1329 await expectLambdaHotSwapStillWorks("aws-half-restored");
1330 cleanCursor = server.mark();
1331 },
1332 PHASE_TIMEOUT,
1333);
1334
1335test(
1336 "binding churn: changing a binding value re-applies the consumer; changing a dev child's env restarts it",
1337 async () => {
1338 // A plain var binding on the path-`main` Worker.
1339 server.patchRegion(
1340 "alchemy.run.ts",
1341 "API_VARIABLE",
1342 ' API_VARIABLE: "api-variable-v2",\n',
1343 );
1344 await waitForJson<{ API_VARIABLE: string }>(
1345 "ApiWorker to pick up the new binding value",
1346 api("/env"),
1347 (body) => body.API_VARIABLE === "api-variable-v2",
1348 { tries: 240, delayMs: 500, server },
1349 );
1350
1351 // A brand-new binding on a Worker that is already running.
1352 server.patchRegion(
1353 "alchemy.run.ts",
1354 "API_EXTRA_ENV",
1355 ' API_EXTRA: "api-extra-v1",\n',
1356 );
1357 await waitForJson<{ API_EXTRA: string | null }>(
1358 "ApiWorker to gain a binding it never had",
1359 api("/env"),
1360 (body) => body.API_EXTRA === "api-extra-v1",
1361 { tries: 240, delayMs: 500, server },
1362 );
1363
1364 // …and lose it again.
1365 server.patchRegion("alchemy.run.ts", "API_EXTRA_ENV", "");
1366 await waitForJson<{ API_EXTRA: string | null }>(
1367 "ApiWorker to lose the binding again",
1368 api("/env"),
1369 (body) => body.API_EXTRA === null,
1370 { tries: 240, delayMs: 500, server },
1371 );
1372
1373 // `Command.Dev`'s restart surface: a change to the resolved config must
1374 // restart the child (new pid), not silently leave the old one running.
1375 const pidBefore = (await fetchJson<{ pid: number }>(awsSite("/__dev-env")))
1376 .pid;
1377 server.patchRegion(
1378 "alchemy.run.ts",
1379 "SITE_MARKER",
1380 ' SITE_MARKER: "aws-site-env-v2",\n',
1381 );
1382 const restarted = await waitForJson<{ marker: string; pid: number }>(
1383 "the dev-command child to restart with the new env",
1384 awsSite("/__dev-env"),
1385 (body) => body.marker === "aws-site-env-v2",
1386 { tries: 240, delayMs: 500, server },
1387 );
1388 expect(restarted.pid).not.toBe(pidBefore);
1389 server.assertAlive("binding churn");
1390 await expectLambdaHotSwapStillWorks("binding-churn");
1391 cleanCursor = server.mark();
1392 },
1393 PHASE_TIMEOUT,
1394);
1395
1396// ───────────────────────────────────────────────────────────────────────
1397// Phase 6 — rapid-fire edits. Convergence to the LAST write, and evidence
1398// that the watchers coalesce instead of replaying every keystroke.
1399// ───────────────────────────────────────────────────────────────────────
1400
1401test(
1402 "rapid-fire bundler edits converge on the last write without re-running the stack",
1403 async () => {
1404 // The previous phase edited stack-graph files (alchemy.run.ts, the
1405 // Lambda source); absorb any trailing re-plan before baselining.
1406 await server.settlePlans();
1407 const plansBefore = server.planCount;
1408 const BURST = 25;
1409 for (let i = 1; i <= BURST; i++) {
1410 server.write(
1411 "src/api/marker.ts",
1412 markerModule("API_MARKER", `api-storm-${i}`),
1413 );
1414 await Bun.sleep(40);
1415 }
1416
1417 await waitForJson<{ marker: string }>(
1418 `ApiWorker to settle on api-storm-${BURST}`,
1419 api("/marker"),
1420 (body) => body.marker === `api-storm-${BURST}`,
1421 { tries: 240, delayMs: 500, server },
1422 );
1423 // A burst on the bundler path must never escalate into stack re-runs.
1424 expect(server.planCount).toBe(plansBefore);
1425 server.assertAlive("bundler burst");
1426 },
1427 PHASE_TIMEOUT,
1428);
1429
1430test(
1431 "rapid-fire stack edits converge on the last write and are coalesced",
1432 async () => {
1433 const plansBefore = server.planCount;
1434 const BURST = 15;
1435 for (let i = 1; i <= BURST; i++) {
1436 server.write(
1437 "src/echo/marker.ts",
1438 markerModule("ECHO_MARKER", `echo-storm-${i}`),
1439 );
1440 await Bun.sleep(60);
1441 }
1442
1443 await waitForJson<{ marker: string }>(
1444 `EchoWorker to settle on echo-storm-${BURST}`,
1445 echo("/marker"),
1446 (body) => body.marker === `echo-storm-${BURST}`,
1447 { tries: 360, delayMs: 500, server },
1448 );
1449
1450 // The hot swap lands before the stack re-runs; wait for the re-plan
1451 // and let any coalesced follow-up runs drain before counting.
1452 await server.waitForPlanAfter(plansBefore);
1453 await Bun.sleep(15_000);
1454 const restarts = server.planCount - plansBefore;
1455 expect(restarts).toBeGreaterThanOrEqual(1);
1456 // Debouncing: a sub-second burst of 15 saves must not produce 15
1457 // complete plan/apply cycles.
1458 expect(restarts).toBeLessThan(BURST);
1459 server.assertAlive("stack burst");
1460 },
1461 PHASE_TIMEOUT,
1462);
1463
1464test(
1465 "simultaneous cross-cloud edits all land",
1466 async () => {
1467 // One tick, four surfaces: the bundler path, the watch path, the AWS
1468 // half, and the stack graph itself.
1469 server.write("src/api/marker.ts", markerModule("API_MARKER", "api-final"));
1470 server.write("src/echo/marker.ts", markerModule("ECHO_MARKER", "echo-final"));
1471 server.write(
1472 "src/lambda/marker.ts",
1473 markerModule("LAMBDA_MARKER", "lambda-final"),
1474 );
1475 server.patchRegion(
1476 "alchemy.run.ts",
1477 "API_VARIABLE",
1478 ' API_VARIABLE: "api-variable-final",\n',
1479 );
1480
1481 await waitForJson<{ marker: string }>(
1482 "EchoWorker to serve echo-final",
1483 echo("/marker"),
1484 (body) => body.marker === "echo-final",
1485 { tries: 360, delayMs: 500, server },
1486 );
1487 await waitForJson<{ marker: string }>(
1488 "ApiWorker to serve api-final",
1489 api("/marker"),
1490 (body) => body.marker === "api-final",
1491 { tries: 360, delayMs: 500, server },
1492 );
1493 await waitForJson<{ API_VARIABLE: string }>(
1494 "ApiWorker to carry api-variable-final",
1495 api("/env"),
1496 (body) => body.API_VARIABLE === "api-variable-final",
1497 { tries: 360, delayMs: 500, server },
1498 );
1499 await waitForJson<{ marker: string }>(
1500 "the Lambda to serve lambda-final",
1501 api("/aws/"),
1502 (body) => body.marker === "lambda-final",
1503 { tries: 360, delayMs: 1_000, server },
1504 );
1505 server.assertAlive("simultaneous edits");
1506 },
1507 PHASE_TIMEOUT,
1508);
1509
1510// ───────────────────────────────────────────────────────────────────────
1511// Phase 7 — after all that churn, the whole stack is still the stack.
1512// ───────────────────────────────────────────────────────────────────────
1513
1514test(
1515 "after the full churn every resource still serves and the CLI never restarted",
1516 async () => {
1517 server.assertAlive("final health check");
1518
1519 // Cloudflare
1520 expect((await fetchJson<{ value: string }>(echo("/kv?key=final"))).value)
1521 .toBe("kv:final");
1522 expect((await fetchJson<{ text: string }>(echo("/r2"))).text).toBe(
1523 "hello from r2",
1524 );
1525 expect(
1526 (await fetchJson<{ count: number }>(echo("/counter"))).count,
1527 ).toBeGreaterThan(0);
1528 const finalSandbox = await fetchJson<{ greeting: string; marker: string }>(
1529 echo("/sandbox"),
1530 undefined,
1531 { tries: 60, delayMs: 1_000 },
1532 );
1533 expect(finalSandbox.greeting).toBe("hello-from-container");
1534 expect(finalSandbox.marker).toBe("sandbox-v2");
1535
1536 // AWS, through the cross-cloud hop
1537 expect((await fetchJson<{ text: string }>(api("/aws/dynamo"))).text).toBe(
1538 "hello from dynamo",
1539 );
1540 const message = { id: crypto.randomUUID() };
1541 await fetchOk(api("/aws/queue/send"), {
1542 method: "POST",
1543 body: JSON.stringify(message),
1544 });
1545 const delivered = await waitForJson<{ body: string | null }>(
1546 "the post-churn queue message to be consumed",
1547 api(`/aws/queue/messages?id=${message.id}`),
1548 (body) => body.body !== null,
1549 { tries: 180, delayMs: 500, server },
1550 );
1551 expect(JSON.parse(delivered.body!)).toEqual(message);
1552
1553 // ECS: both services still serve after every reload they went through.
1554 await waitForText("the ECS service post-churn", ecs("/"), "ecs-site-v2", {
1555 tries: 60,
1556 });
1557 await waitForText(
1558 "the inline ECS service post-churn",
1559 ecsInline("/"),
1560 "ecs-inline-v2",
1561 { tries: 60 },
1562 );
1563
1564 // Websites. The Cloudflare site's port is not pinned (a
1565 // `Website.StaticSite` owns its Worker's dev options), so re-read the
1566 // newest value the CLI printed rather than trusting the boot one.
1567 expect(await (await fetchOk(awsSite("/"))).text()).toContain("aws-site-v1");
1568 cfSiteUrl = server.outputUrl("cfSiteUrl") ?? cfSiteUrl;
1569 expect(await (await fetchOk(new URL("/", cfSiteUrl))).text()).toContain(
1570 "cf-site-v1",
1571 );
1572
1573 // No failure was logged since the last phase that expected one.
1574 expect(server.since(cleanCursor)).not.toContain("alchemy dev: run failed");
1575 expect(server.since(cleanCursor)).not.toContain(
1576 "alchemy dev: apply failed",
1577 );
1578 },
1579 PHASE_TIMEOUT,
1580);
1581
1582test(
1583 "the MicroVM image survives every unrelated edit, and rebuilds when its own source changes",
1584 async () => {
1585 // Nothing the suite did to the other resources touched the image.
1586 expect(server.output).not.toContain("[StressMicrovm] replace");
1587 expect(
1588 (
1589 await fetchJson<{ marker: string }>(
1590 microvm("/roundtrip?message=post-churn"),
1591 undefined,
1592 { tries: 240, delayMs: 1_000 },
1593 )
1594 ).marker,
1595 ).toBe("vm-v1");
1596
1597 // Now change the program that runs INSIDE the VM. The dev provider
1598 // builds the new image with a HOST-side cached `docker build` and hands
1599 // floci a pre-built `docker://` reference, so the rebuild itself is
1600 // seconds — the wait below is dominated by the roundtrip's own
1601 // boot-a-VM-per-request cost.
1602 const rebuildStartedAt = Date.now();
1603 server.write("src/vm/marker.ts", markerModule("VM_MARKER", "vm-v2"));
1604 await waitForJson<{ marker: string }>(
1605 "the rebuilt MicroVM image to serve vm-v2",
1606 microvm("/roundtrip?message=rebuild"),
1607 (body) => body.marker === "vm-v2",
1608 { tries: 600, delayMs: 1_000, server },
1609 );
1610 const rebuildMs = Date.now() - rebuildStartedAt;
1611 console.log(`microvm image rebuild -> serving vm-v2 in ${rebuildMs}ms`);
1612 // Pre-docker:// this took minutes (zip upload + cold server-side
1613 // build); the cached host build must keep the whole edit-to-serving
1614 // path within a couple of VM boots.
1615 expect(rebuildMs).toBeLessThan(120_000);
1616 server.assertAlive("microvm image rebuild");
1617 },
1618 PHASE_TIMEOUT,
1619);
1620
1621test(
1622 "shuts down cleanly on Ctrl-C and stops serving",
1623 async () => {
1624 await server.shutdown();
1625 expect(server.alive).toBe(false);
1626
1627 for (const url of [echo("/marker"), api("/marker"), awsSite("/")]) {
1628 await pollUntil(
1629 `${url} to stop serving after shutdown`,
1630 async () => ((await isClosed(url)) ? true : undefined),
1631 { tries: 60, delayMs: 500 },
1632 );
1633 }
1634 },
1635 PHASE_TIMEOUT,
1636);