clover's git

the-knife

1	import * as machines from "@distilled.cloud/fly-io/machines";
2	import { type Machine } from "@distilled.cloud/fly-io/machines";
3	import * as Fly from "@/Fly";
4	import {
5	waitHealthy,
6	autostopMode,
7	checksPassing,
8	ensureStarted,
9	observeReplicaSet,
10	predecessorShutdown,
11	deleteReplicaSet,
12	ReplicaNotCreated,
13	ReplicaRetirementIncomplete,
14	retireMachines,
15	retireMachine,
16	} from "@/Fly/replicas";
17	import * as Test from "@/Test/Alchemy";
18	import { expect, assert, it, describe } from "alchemy-test";
19	import * as Effect from "effect/Effect";
20	import * as Result from "effect/Result";
21	import {
22	assertAppGone,
23	census,
24	checks,
25	deployWorker,
26	assertCommitted,
27	} from "./fixtures/bluegreen.ts";
28	import * as Retry from "@distilled.cloud/fly-io/Retry";
29	import { DestroyError } from "@/Apply";
30	import { AppDeletionAmbiguous } from "@/Fly/App";
31	import * as Cause from "effect/Cause";
32	import * as Fiber from "effect/Fiber";
33	import * as FetchHttpClient from "effect/http/FetchHttpClient";
34	import { engineActor } from "./fixtures/actors.ts";
35	import {
36	transportProxy,
37	throughProxy,
38	type TransportEvent,
39	} from "./fixtures/transport.ts";
40	import { readinessRoles, DeploymentRecoveryAmbiguous } from "@/Fly/bluegreen";
41	import * as Schedule from "effect/Schedule";
42	import * as HttpClient from "effect/http/HttpClient";
43	import * as Exit from "effect/Exit";
44	import {
45	makeReadinessControl,
46	repairReadiness,
47	} from "./fixtures/http-readiness-control.ts";
48	import { makeMachineLeases } from "@/Fly/leases";
49	import * as Clock from "effect/Clock";
50	import * as HttpClientRequest from "effect/http/HttpClientRequest";
51	import { sanitizeExecFailure } from "./fixtures/exec-lease.ts";
52	import { type MachineContainer, type MachineProps } from "@/Fly/Machine";
53	import { ReplicaChecksNotPassing } from "@/Fly/replicas";
54	import type { ScratchStack } from "@/Test/Alchemy";
55	import * as Path from "effect/Path";
56	import * as Deferred from "effect/Deferred";
57	import {
58	assertReadinessCommit,
59	readinessActor,
60	readinessChecksPassing,
61	readinessProxy,
62	retires,
63	type ReadinessEvent,
64	} from "./fixtures/idle-cadence-readiness.ts";
65	import * as Docker from "@/Docker";
66	import * as TestCore from "@/Test/Core";
67	import { scratchStack, withProviders } from "@/Test/Core";
68	import * as Redacted from "effect/Redacted";
69	import {
70	GatewayTimeout,
71	HTTP_STATUS_MAP,
72	RETRYABLE_HTTP_STATUSES,
73	} from "@distilled.cloud/core/errors";
74	import { Credentials } from "@distilled.cloud/fly-io/Credentials";
75	import * as Ref from "effect/Ref";
76	import * as Schema from "effect/Schema";
77	import { dropCompletedCreate } from "./fixtures/bluegreen-create-proxy.ts";
78	import {
79	observeStops,
80	type StopRequest,
81	writeLegacyProtocol,
82	} from "./fixtures/legacy-protocol-writer.ts";
83	import * as FileSystem from "effect/FileSystem";
84	import {
85	assertBarrierInventory,
86	assertClean,
87	assertConverged,
88	assertSingleRunner,
89	census as censusProcessDeath,
90	deploy,
91	evidencePaths,
92	heldLeases,
93	identity,
94	machine,
95	matchesBarrier,
96	nowSeconds,
97	observeLeaseExpiry,
98	persistedRow,
99	phases,
100	processDeathFile,
101	readWitness,
102	writeEvidence,
103	type Phase,
104	type Witness,
105	} from "./fixtures/process-death.ts";
106	import { GatewayTimeout as GatewayTimeoutErrors } from "@distilled.cloud/fly-io/Errors";
107	import { alchemyMetadataKeys as keys } from "@/Fly/Metadata";
108	import {
109	appName,
110	candidateId,
111	metadata,
112	protocolClient,
113	reconcile,
114	reply,
115	withControlledClient,
116	} from "./fixtures/protocol-branches.ts";
117	import * as Data from "effect/Data";
118	import { randomBytes, createHash } from "node:crypto";
119	import {
120	Site,
121	Token,
122	TRIGGER_SECRET,
123	Writer,
124	writerLayer,
125	} from "./fixtures/bluegreen-runtime-secrets/writer.ts";
126	import * as ConfigProvider from "effect/ConfigProvider";
127	import {
128	BoundSecrets,
129	CacheOne,
130	CacheTwo,
131	Site as SiteBluegreenSecrets,
132	} from "./fixtures/bluegreen-secrets.ts";
133	import * as Stream from "effect/Stream";
134	import {
135	assertOrder,
136	assertReplacement,
137	assertStopped,
138	makeScenario,
139	requireValue,
140	} from "./fixtures/bluegreen-worker-test.ts";
141	import {
142	Finalized,
143	RunnerInterrupted,
144	Witness as WitnessSignalOverlap,
145	assertBoundary,
146	assertConverged as assertConvergedSignalOverlap,
147	assertInventory,
148	assertReleased,
149	assertSingleRunner as assertSingleRunnerSignalOverlap,
150	boundary,
151	cases,
152	firstReturnedUncordon,
153	matches,
154	observeExpiry,
155	observeLeases,
156	pathsFor,
157	readEvidence,
158	signalOverlapFile,
159	successful,
160	type SignalCase,
161	} from "./fixtures/signal-overlap.ts";
162	import * as Alchemy from "@/index";
163	import { localState, makeLocalState } from "@/State/LocalState";
164	import {
165	delayedResourceWrite,
166	ResourceRow,
167	} from "./fixtures/state-persistence.ts";
168	import MountedBlueGreen from "./fixtures/mounted-bluegreen.ts";
169
170	describe.sequential(
171	"deployment",
172	{
173	tags: [
174	"provider:fly",
175	"provider:fly:app",
176	"provider:fly:machine",
177	"provider:fly:service",
178	"live",
179	],
180	},
181	() => {
182	const { test } = Test.make({ providers: Fly.providers() });
183
184	test.provider(
185	"S01 bluegreen worker checks, promotion, replacement, and graceful teardown",
186	(stack) =>
187	Effect.gen(function* () {
188	yield* stack.destroy();
189	const deploy = (version: string, path = "/") =>
190	stack.deploy(
191	Effect.gen(function* () {
192	const app = yield* Fly.App("Site");
193	return yield* Fly.Machine("Worker", {
194	app,
195	name: "bluegreen-worker-long-name-base",
196	image: "nginx:alpine",
197	env: { VERSION: version },
198	deploy: {
199	strategy: "bluegreen",
200	healthTimeout: "30 seconds",
201	},
202	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
203	checks: {
204	ready: {
205	type: "http",
206	port: 80,
207	path,
208	interval: "2s",
209	timeout: "1s",
210	},
211	},
212	});
213	}),
214	);
215	const first = yield* deploy("one");
216	const committed = yield* machines.getMachine({
217	app_name: first.appName,
218	machine_id: first.machineId,
219	});
220	// Active commit metadata can reset reports after the provider's readiness validation.
221	const machine = yield* waitHealthy(first.appName, committed, 30_000);
222	expect(machine.instance_id).toBe(committed.instance_id);
223	yield* Effect.logInfo("Observed promoted worker", {
224	checks: machine.checks?.map(({ name, status }) => ({
225	name,
226	status,
227	})),
228	configured: machine.config?.checks,
229	metadata: machine.config?.metadata,
230	});
231	expect(machine.cordoned).toBe(false);
232	expect(
233	machine.checks?.some(
234	(check) => check.name === "ready" && check.status === "passing",
235	),
236	).toBe(true);
237	expect(machine.config?.metadata?.["alchemy.phase"]).toBe("active");
238	expect(first.name.length).toBeLessThanOrEqual(30);
239	const second = yield* deploy("two");
240	expect(second.machineId).not.toBe(first.machineId);
241	expect(second.baseName).toBe(first.baseName);
242	const listed = yield* machines.listMachines({
243	app_name: first.appName,
244	});
245	expect(
246	listed
247	.filter((machine) => machine.state !== "destroyed")
248	.map((machine) => machine.id),
249	).toEqual([second.machineId]);
250	yield* stack.destroy();
251	yield* assertAppGone(first.appName);
252	}),
253	{ timeout: 180_000 },
254	);
255
256	test.provider(
257	"S04 unhealthy replacement preserves the old routed ID and cleans unpromoted candidates",
258	(stack) =>
259	Effect.gen(function* () {
260	yield* stack.destroy();
261	const initial = yield* deployWorker(stack, "one");
262	const failed = yield* deployWorker(stack, "broken", {
263	checks: { ready: { ...checks.ready, path: "/missing" } },
264	deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
265	}).pipe(Effect.result);
266	expect(Result.isFailure(failed)).toBe(true);
267	if (Result.isFailure(failed))
268	expect(failed.failure).toMatchObject({
269	_tag: "Fly.ReplicaChecksNotPassing",
270	});
271	const live = yield* census(initial.appName);
272	expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
273	expect(live[0]!.state).toBe("started");
274	expect(live[0]!.cordoned).toBe(false);
275	yield* stack.destroy();
276	yield* assertAppGone(initial.appName);
277	}),
278	{ timeout: 180_000 },
279	);
280	},
281	);
282
283	describe.sequential(
284	"App deletion",
285	{
286	tags: [
287	"provider:fly",
288	"provider:fly:app",
289	"provider:fly:machine",
290	"provider:fly:service",
291	"live",
292	],
293	},
294	() => {
295	const { test } = Test.make({ providers: Fly.providers() });
296
297	test.provider(
298	"F11 ambiguous name-addressed App deletion does not blindly retry",
299	(stack) =>
300	Effect.gen(function* () {
301	yield* stack.destroy();
302	const app = yield* stack.deploy(Fly.App("Site"));
303	const proxy = yield* transportProxy();
304	const actor = yield* engineActor(
305	stack,
306	"F11 ambiguous name-addressed App deletion does not blindly retry",
307	"test/Fly/BlueGreen.test.ts",
308	proxy.url,
309	);
310	yield* Effect.sync(() => {
311	proxy.arm({
312	match: (event) =>
313	event.method === "DELETE" &&
314	event.path === `/v1/apps/${app.appName}`,
315	action: "drop-response",
316	remaining: Infinity,
317	});
318	});
319	const result = yield* actor
320	.destroy()
321	.pipe(Effect.timeout("45 seconds"), Effect.result);
322	assert(Result.isFailure(result));
323	assert(result.failure instanceof DestroyError);
324	expect(result.failure.blocked).toEqual([]);
325	expect(result.failure.failures).toHaveLength(1);
326	const failure = result.failure.failures[0]!;
327	expect(failure.logicalId).toBe("Site");
328	expect(failure.resourceType).toBe("Fly.App");
329	const cause = Cause.findError(failure.cause);
330	assert(Result.isSuccess(cause));
331	assert(cause.success instanceof AppDeletionAmbiguous);
332	expect(cause.success).toMatchObject({
333	_tag: "Fly.AppDeletionAmbiguous",
334	appName: app.appName,
335	evidence: "HttpClientError",
336	});
337	const attempts = proxy.events.filter(
338	(event) =>
339	event.stage === "request" &&
340	event.method === "DELETE" &&
341	event.path === `/v1/apps/${app.appName}`,
342	);
343	expect(attempts).toHaveLength(1);
344	expect(
345	proxy.events.some(
346	(event) =>
347	event.stage === "dropped" &&
348	event.status! >= 200 &&
349	event.status! < 300,
350	),
351	).toBe(true);
352	yield* Effect.sync(proxy.clear);
353	yield* assertAppGone(app.appName);
354	// An explicit cleanup after observing absence is not an automatic transport retry.
355	yield* stack.destroy();
356	}).pipe(Effect.scoped),
357	{ timeout: 180_000 },
358	);
359
360	test.provider(
361	"F11 ambiguous App deletion cannot retry into an independently recreated same-name successor",
362	(stack) =>
363	Effect.gen(function* () {
364	yield* stack.destroy();
365	const app = yield* stack.deploy(Fly.App("Site"));
366	expect(app.orgSlug).toBeDefined();
367	const proxy = yield* transportProxy();
368	yield* Effect.sync(() =>
369	proxy.arm({
370	match: (event) =>
371	event.method === "DELETE" &&
372	event.path === `/v1/apps/${app.appName}`,
373	action: "hold-response",
374	remaining: 1,
375	}),
376	);
377	yield* Effect.gen(function* () {
378	const actor = yield* engineActor(
379	stack,
380	"F11 ambiguous App deletion cannot retry into an independently recreated same-name successor",
381	"test/Fly/BlueGreen.test.ts",
382	proxy.url,
383	);
384	const deletion = yield* actor
385	.destroy()
386	.pipe(Effect.scoped, Effect.result, Effect.forkScoped);
387	yield* proxy.wait(
388	(event) =>
389	event.stage === "held" &&
390	event.status! >= 200 &&
391	event.status! < 300,
392	);
393	yield* assertAppGone(app.appName);
394	// This independent API actor demonstrates the missing cross-process tombstone, not safe automatic recreation.
395	yield* machines
396	.createApp({ name: app.appName, org_slug: app.orgSlug! })
397	.pipe(Retry.none, Effect.provide(FetchHttpClient.layer));
398	yield* Effect.gen(function* () {
399	const successor = yield* machines
400	.createMachine({
401	app_name: app.appName,
402	name: "successor-sentinel",
403	region: "iad",
404	config: { image: "nginx:alpine" },
405	})
406	.pipe(Effect.provide(FetchHttpClient.layer));
407	yield* Effect.sync(proxy.dropHeld);
408	const result = yield* Fiber.join(deletion).pipe(
409	Effect.timeout("45 seconds"),
410	);
411	expect(
412	proxy.events.filter(
413	(event) =>
414	event.stage === "request" &&
415	event.method === "DELETE" &&
416	event.path === `/v1/apps/${app.appName}`,
417	),
418	).toHaveLength(1);
419	assert(Result.isFailure(result));
420	assert(result.failure instanceof DestroyError);
421	expect(result.failure.blocked).toEqual([]);
422	expect(result.failure.failures).toHaveLength(1);
423	const failure = result.failure.failures[0]!;
424	expect(failure.logicalId).toBe("Site");
425	expect(failure.resourceType).toBe("Fly.App");
426	const cause = Cause.findError(failure.cause);
427	assert(Result.isSuccess(cause));
428	assert(cause.success instanceof AppDeletionAmbiguous);
429	expect(cause.success).toMatchObject({
430	_tag: "Fly.AppDeletionAmbiguous",
431	appName: app.appName,
432	evidence: "HttpClientError",
433	});
434	const surviving = yield* machines
435	.getMachine({
436	app_name: app.appName,
437	machine_id: successor.id!,
438	})
439	.pipe(Effect.provide(FetchHttpClient.layer));
440	expect(surviving.id).toBe(successor.id);
441	expect(surviving.name).toBe("successor-sentinel");
442	}).pipe(
443	Effect.ensuring(
444	// Only the independent fixture owner removes its successor; the uncertain engine is not retried against it.
445	machines.deleteApp({ app_name: app.appName }).pipe(
446	Retry.none,
447	Effect.catchTag("NotFound", () => Effect.void),
448	Effect.provide(FetchHttpClient.layer),
449	Effect.orDie,
450	),
451	),
452	);
453	}).pipe(
454	Effect.ensuring(
455	Effect.sync(() => {
456	proxy.clear();
457	proxy.dropHeld();
458	}),
459	),
460	Effect.scoped,
461	);
462	yield* assertAppGone(app.appName);
463	yield* stack.destroy();
464	}).pipe(Effect.scoped),
465	{ timeout: 300_000 },
466	);
467	},
468	);
469
470	describe.sequential(
471	"App deletion verification",
472	{
473	tags: [
474	"provider:fly",
475	"provider:fly:app",
476	"provider:fly:machine",
477	"provider:fly:service",
478	"live",
479	],
480	},
481	() => {
482	const { test } = Test.make({ providers: Fly.providers() });
483
484	for (const fault of [
485	{ action: "cut-request", stage: "cut", error: "HttpClientError" },
486	{ action: "hold-response", stage: "held", error: "TimeoutError" },
487	] as const) {
488	test.provider(
489	`F11 accepted App deletion reports bounded ambiguity for ${fault.action} verification GET`,
490	(stack) =>
491	Effect.gen(function* () {
492	yield* stack.destroy();
493	const app = yield* stack.deploy(Fly.App("Site"));
494	const proxy = yield* transportProxy();
495	const actor = yield* engineActor(
496	stack,
497	`F11 accepted App deletion reports bounded ambiguity for ${fault.action} verification GET`,
498	"test/Fly/BlueGreen.test.ts",
499	proxy.url,
500	);
501	const path = `/v1/apps/${app.appName}`;
502	yield* Effect.gen(function* () {
503	yield* Effect.sync(() =>
504	proxy.arm({
505	match: (event) =>
506	event.method === "GET" &&
507	event.path === path &&
508	proxy.events.some(
509	(accepted) =>
510	accepted.stage === "forwarded" &&
511	accepted.method === "DELETE" &&
512	accepted.path === path &&
513	accepted.status! >= 200 &&
514	accepted.status! < 300,
515	),
516	action: fault.action,
517	remaining: Infinity,
518	}),
519	);
520	const deletion = yield* actor
521	.destroy()
522	.pipe(Effect.result, Effect.forkScoped);
523	const accepted = yield* proxy.wait(
524	(event) =>
525	event.stage === "forwarded" &&
526	event.method === "DELETE" &&
527	event.path === path &&
528	event.status! >= 200 &&
529	event.status! < 300,
530	);
531	const verification = yield* proxy.wait(
532	(event) =>
533	event.stage === fault.stage &&
534	event.method === "GET" &&
535	event.path === path,
536	);
537	expect(verification.sequence).toBeGreaterThan(accepted.sequence);
538	const result = yield* Fiber.join(deletion).pipe(
539	Effect.timeout("45 seconds"),
540	);
541	assert(Result.isFailure(result));
542	assert(result.failure instanceof DestroyError);
543	expect(result.failure.blocked).toEqual([]);
544	expect(result.failure.failures).toHaveLength(1);
545	const failure = result.failure.failures[0]!;
546	expect(failure.logicalId).toBe("Site");
547	expect(failure.resourceType).toBe("Fly.App");
548	const cause = Cause.findError(failure.cause);
549	assert(Result.isSuccess(cause));
550	assert(cause.success instanceof AppDeletionAmbiguous);
551	expect(cause.success).toMatchObject({
552	_tag: "Fly.AppDeletionAmbiguous",
553	appName: app.appName,
554	evidence: `delete accepted but absence verification failed: ${fault.error}`,
555	});
556	expect(cause.success.message).toContain(
557	"reconcile the App before retrying or recreating its name",
558	);
559	expect(
560	proxy.events.filter(
561	(event) =>
562	event.stage === "request" &&
563	event.method === "DELETE" &&
564	event.path === path,
565	),
566	).toHaveLength(1);
567	}).pipe(
568	Effect.scoped,
569	Effect.ensuring(
570	Effect.sync(() => {
571	proxy.clear();
572	proxy.dropHeld();
573	}),
574	),
575	);
576	yield* assertAppGone(app.appName);
577	// Explicit cleanup follows out-of-band absence, not a blind DELETE retry.
578	yield* stack.destroy();
579	yield* assertAppGone(app.appName);
580	}).pipe(Effect.scoped),
581	{ timeout: 300_000 },
582	);
583	}
584	},
585	);
586
587	describe.sequential(
588	"autostop",
589	{ tags: ["provider:fly", "provider:fly:service"] },
590	() => {
591	const { test } = Test.make({ providers: Fly.providers() });
592
593	it.effect(
594	"S03 check identity rejects duplicate, missing, warning and unknown reports",
595	() =>
596	Effect.sync(() => {
597	const config = {
598	checks: { ready: { type: "http", port: 80 } },
599	services: [{ internal_port: 80, checks: [{ type: "http" }] }],
600	};
601	const checks = [
602	{ name: "ready", status: "passing" },
603	{ name: "servicecheck-00-http-80", status: "passing" },
604	];
605	expect(checksPassing({ state: "started", checks }, config)).toBe(
606	true,
607	);
608	expect(checksPassing({ state: "stopped", checks }, config)).toBe(
609	false,
610	);
611	for (const invalid of [
612	[],
613	checks.slice(0, 1),
614	[checks[0]!, checks[0]!],
615	[...checks, { name: "unknown", status: "passing" }],
616	checks.map((check) => ({ ...check, status: "warning" })),
617	]) {
618	expect(
619	checksPassing({ state: "started", checks: invalid }, config),
620	).toBe(false);
621	}
622	}),
623	{ tags: ["unit", "local"] },
624	);
625
626	it.effect(
627	"S05 deterministic representatives, floors, scale-up and mixed-service roles",
628	() =>
629	Effect.sync(() => {
630	const services = [{ autostop: "stop", min_machines_running: 0 }];
631	expect(readinessRoles({ services }, 3, [])).toEqual([
632	"run",
633	"idle",
634	"idle",
635	]);
636	const running = [
637	{
638	state: "started",
639	config: { metadata: { "alchemy.replica": "1" } },
640	},
641	];
642	expect(readinessRoles({ services }, 3, running)).toEqual([
643	"idle",
644	"run",
645	"idle",
646	]);
647	expect(
648	readinessRoles(
649	{ services: [{ autostop: "suspend", min_machines_running: 2 }] },
650	3,
651	running,
652	),
653	).toEqual(["run", "run", "idle"]);
654	expect(
655	readinessRoles(
656	{ services: [...services, { autostop: "off" }] },
657	3,
658	[],
659	),
660	).toEqual(["run", "run", "run"]);
661	}),
662	{ tags: ["unit", "local"] },
663	);
664
665	for (const autostop of ["stop", "suspend"] as const) {
666	test.provider(
667	`S05 S11 ${autostop} all-idle replacement preserves nonrepresentatives`,
668	(stack) =>
669	Effect.gen(function* () {
670	yield* stack.destroy();
671	const deploy = (version: string, healthTimeout = 20_000) =>
672	stack.deploy(
673	Effect.gen(function* () {
674	const app = yield* Fly.App("Site");
675	return yield* Fly.Machine("Worker", {
676	app,
677	image: "nginx:alpine",
678	count: 2,
679	env: { VERSION: version },
680	deploy: { strategy: "bluegreen", healthTimeout },
681	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
682	services: [
683	{
684	protocol: "tcp",
685	internalPort: 80,
686	autostop,
687	autostart: true,
688	minMachinesRunning: 0,
689	checks: [
690	{
691	type: "http",
692	port: 80,
693	path: "/",
694	interval: "2s",
695	timeout: "1s",
696	},
697	],
698	},
699	],
700	});
701	}),
702	);
703	const first = yield* deploy("one");
704	const firstIdle = yield* machines.getMachine({
705	app_name: first.appName,
706	machine_id: first.machineIds[1]!,
707	});
708	expect(firstIdle.config?.metadata?.["alchemy.readiness-role"]).toBe(
709	"idle",
710	);
711	expect(firstIdle.state).not.toBe("started");
712	const primary = {
713	app_name: first.appName,
714	machine_id: first.machineId,
715	};
716	if (autostop === "suspend") yield* machines.suspendMachine(primary);
717	else
718	yield* machines.stopMachine({
719	...primary,
720	signal: "SIGQUIT",
721	timeout: "10s",
722	});
723	yield* machines.waitMachine({
724	...primary,
725	state: autostop === "suspend" ? "suspended" : "stopped",
726	timeout: 8,
727	});
728	const second = yield* deploy("two");
729	expect(
730	second.machineIds.every((id) => !first.machineIds.includes(id)),
731	).toBe(true);
732	const live = (yield* machines.listMachines({
733	app_name: second.appName,
734	})).filter((machine) => machine.state !== "destroyed");
735	expect(live.map((machine) => machine.id).sort()).toEqual(
736	[...second.machineIds].sort(),
737	);
738	const nonrepresentative = live.find(
739	(machine) =>
740	machine.config?.metadata?.["alchemy.replica"] === "1",
741	);
742	expect(nonrepresentative?.state).not.toBe("started");
743	expect(nonrepresentative?.config?.metadata?.["alchemy.phase"]).toBe(
744	"active",
745	);
746	expect(
747	autostopMode(nonrepresentative?.config?.services?.[0]?.autostop),
748	).toBe(autostop);
749	const committed = {
750	app_name: second.appName,
751	machine_id: second.machineId,
752	};
753	if (autostop === "suspend")
754	yield* machines.suspendMachine(committed);
755	else
756	yield* machines.stopMachine({
757	...committed,
758	signal: "SIGQUIT",
759	timeout: "10s",
760	});
761	yield* machines.waitMachine({
762	...committed,
763	state: autostop === "suspend" ? "suspended" : "stopped",
764	timeout: 8,
765	});
766	const unchanged = yield* deploy("two", 22_000);
767	expect(unchanged.machineIds).toEqual(second.machineIds);
768	expect((yield* machines.getMachine(committed)).state).not.toBe(
769	"started",
770	);
771	yield* stack.destroy();
772	expect(
773	yield* machines.listMachines({ app_name: second.appName }).pipe(
774	Effect.map((machines) =>
775	machines.filter((machine) => machine.state !== "destroyed"),
776	),
777	Effect.catchTag("NotFound", () => Effect.succeed([])),
778	),
779	).toEqual([]);
780	}),
781	{
782	tags: ["provider:fly:app", "provider:fly:machine", "live"],
783	timeout: 120_000,
784	},
785	);
786	}
787
788	for (const autostop of ["stop", "suspend"] as const) {
789	test.provider(
790	`P3 ${autostop} cordoned readiness and restored instance probe`,
791	(stack) =>
792	Effect.gen(function* () {
793	yield* stack.destroy();
794	const output = yield* stack.deploy(
795	Effect.gen(function* () {
796	const app = yield* Fly.App("Site");
797	return yield* Fly.Machine("Worker", {
798	app,
799	image: "nginx:alpine",
800	skipLaunch: true,
801	services: [
802	{
803	protocol: "tcp",
804	internalPort: 80,
805	autostop,
806	autostart: true,
807	minMachinesRunning: 0,
808	ports: [{ port: 80, handlers: ["http"] }],
809	checks: [
810	{
811	type: "http",
812	port: 80,
813	path: "/",
814	interval: "2s",
815	timeout: "1s",
816	},
817	{
818	type: "tcp",
819	port: 80,
820	interval: "2s",
821	timeout: "1s",
822	},
823	],
824	},
825	{
826	protocol: "tcp",
827	internalPort: 81,
828	autostop,
829	autostart: true,
830	checks: [
831	{
832	type: "http",
833	port: 80,
834	path: "/",
835	interval: "2s",
836	timeout: "1s",
837	},
838	],
839	},
840	],
841	});
842	}),
843	);
844	const request = {
845	app_name: output.appName,
846	machine_id: output.machineId,
847	};
848	const original = yield* machines.getMachine(request);
849	yield* machines.cordonMachine(request);
850	const prepared = yield* machines.updateMachine({
851	...request,
852	config: {
853	...original.config,
854	services: original.config?.services?.map((service) => ({
855	...service,
856	autostop: "off",
857	})),
858	},
859	skip_launch: true,
860	skip_service_registration: true,
861	});
862	const ready = yield* ensureStarted(
863	output.appName,
864	prepared,
865	false,
866	30_000,
867	);
868	expect(ready.cordoned).toBe(true);
869	yield* machines.uncordonMachine(request);
870	const restored = yield* machines.updateMachine({
871	...request,
872	config: original.config,
873	});
874	const fresh = yield* ensureStarted(
875	output.appName,
876	restored,
877	false,
878	30_000,
879	);
880	yield* waitHealthy(output.appName, fresh, 30_000);
881	yield* Effect.logInfo("P3 restored idle policy", {
882	autostop,
883	beforeInstance: ready.instance_id,
884	afterInstance: fresh.instance_id,
885	state: fresh.state,
886	checks: fresh.checks?.map((check) => ({
887	name: check.name,
888	status: check.status,
889	})),
890	});
891	expect(autostopMode(fresh.config?.services?.[0]?.autostop)).toBe(
892	autostop,
893	);
894	if (autostop === "suspend") yield* machines.suspendMachine(request);
895	else
896	yield* machines.stopMachine({
897	...request,
898	signal: "SIGQUIT",
899	timeout: "10s",
900	});
901	yield* machines.waitMachine({
902	...request,
903	state: autostop === "suspend" ? "suspended" : "stopped",
904	timeout: 8,
905	});
906	const idle = yield* machines.getMachine(request);
907	expect(idle.state).toBe(
908	autostop === "suspend" ? "suspended" : "stopped",
909	);
910	yield* stack.destroy();
911	expect(
912	yield* machines.getMachine(request).pipe(
913	Effect.map((machine) => machine.state === "destroyed"),
914	Effect.catchTag("NotFound", () => Effect.succeed(true)),
915	),
916	).toBe(true);
917	}),
918	{
919	tags: ["provider:fly:app", "provider:fly:machine", "live"],
920	timeout: 120_000,
921	},
922	);
923	}
924
925	for (const autostop of ["stop", "suspend"] as const) {
926	test.provider(
927	`P3 public ${autostop} restored policy autostarts on real traffic`,
928	(stack) =>
929	Effect.gen(function* () {
930	yield* stack.destroy();
931	const output = yield* stack.deploy(
932	Effect.gen(function* () {
933	const app = yield* Fly.App("Site");
934	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
935	return yield* Fly.Machine("Worker", {
936	app,
937	image: "nginx:alpine",
938	deploy: {
939	strategy: "bluegreen",
940	healthTimeout: "20 seconds",
941	},
942	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
943	services: [
944	{
945	protocol: "tcp",
946	internalPort: 80,
947	autostop,
948	autostart: true,
949	minMachinesRunning: 0,
950	ports: [{ port: 80, handlers: ["http"] }],
951	checks: [
952	{
953	type: "http",
954	port: 80,
955	path: "/",
956	interval: "2s",
957	timeout: "1s",
958	},
959	],
960	},
961	],
962	});
963	}),
964	);
965	const request = {
966	app_name: output.appName,
967	machine_id: output.machineId,
968	};
969	if (autostop === "suspend") yield* machines.suspendMachine(request);
970	else
971	yield* machines.stopMachine({
972	...request,
973	signal: "SIGQUIT",
974	timeout: "10s",
975	});
976	yield* machines.waitMachine({
977	...request,
978	state: autostop === "suspend" ? "suspended" : "stopped",
979	timeout: 8,
980	});
981	const client = yield* HttpClient.HttpClient;
982	const response = yield* client
983	.get(`http://${output.appName}.fly.dev`)
984	.pipe(
985	Effect.retry({
986	schedule: Schedule.spaced("2 seconds"),
987	times: 8,
988	}),
989	);
990	expect(response.status).toBe(200);
991	expect(yield* response.text).toContain("Welcome to nginx");
992	const awakened = yield* machines.getMachine(request);
993	expect(awakened.state).toBe("started");
994	expect(autostopMode(awakened.config?.services?.[0]?.autostop)).toBe(
995	autostop,
996	);
997	yield* stack.destroy();
998	expect(
999	yield* machines.getMachine(request).pipe(
1000	Effect.map((machine) => machine.state === "destroyed"),
1001	Effect.catchTag("NotFound", () => Effect.succeed(true)),
1002	),
1003	).toBe(true);
1004	}),
1005	{
1006	tags: [
1007	"provider:fly:app",
1008	"provider:fly:ipassignment",
1009	"provider:fly:machine",
1010	"live",
1011	],
1012	timeout: 120_000,
1013	},
1014	);
1015	}
1016	},
1017	);
1018
1019	describe.sequential(
1020	"commit recovery",
1021	{
1022	tags: [
1023	"provider:fly",
1024	"provider:fly:app",
1025	"provider:fly:ipassignment",
1026	"provider:fly:machine",
1027	"provider:fly:secret",
1028	"provider:fly:service",
1029	"live",
1030	],
1031	},
1032	() => {
1033	const file = "test/Fly/BlueGreen.test.ts";
1034	const { test } = Test.make({ providers: Fly.providers() });
1035
1036	for (const interrupted of [false, true]) {
1037	test.provider(
1038	`FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
1039	(stack) =>
1040	Effect.gen(function* () {
1041	yield* stack.destroy();
1042	const readiness = yield* makeReadinessControl();
1043	const site = yield* readiness.deployApp(stack);
1044	const proxy = yield* transportProxy();
1045	yield* Effect.sync(() =>
1046	proxy.arm({
1047	match: (event) =>
1048	event.path.endsWith("/metadata") &&
1049	event.phase === "validating",
1050	action: "hold-response",
1051	remaining: 1,
1052	}),
1053	);
1054	const actor = yield* engineActor(
1055	stack,
1056	`FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
1057	file,
1058	proxy.url,
1059	);
1060	const attempt = yield* readiness
1061	.deployWorker(actor, "one")
1062	.pipe(Effect.scoped, Effect.result, Effect.forkScoped);
1063	const barrier = yield* proxy
1064	.wait(
1065	(event) =>
1066	event.stage === "held" &&
1067	event.phase === "validating" &&
1068	event.status! >= 200 &&
1069	event.status! < 300,
1070	)
1071	.pipe(
1072	Effect.raceFirst(
1073	Effect.gen(function* () {
1074	const result = yield* Fiber.join(attempt);
1075	if (Result.isFailure(result))
1076	return yield* Effect.fail(result.failure);
1077	return yield* Effect.fail(
1078	new Error(
1079	"Deployment completed without reaching the validating barrier",
1080	),
1081	);
1082	}),
1083	),
1084	);
1085	const machineId = barrier.machineId!;
1086	expect(machineId).toBeDefined();
1087
1088	const firstRouting = proxy.events.findIndex(
1089	(event) =>
1090	event.stage === "request" && event.path.endsWith("/uncordon"),
1091	);
1092	expect(firstRouting).toBeGreaterThan(0);
1093	const preparation = proxy.events.slice(0, firstRouting);
1094	const promoting = preparation.findLastIndex(
1095	(event) =>
1096	event.stage === "completed" &&
1097	event.phase === "promoting" &&
1098	event.path.endsWith("/metadata") &&
1099	event.status! < 300,
1100	);
1101	const ready = preparation.findLastIndex(
1102	(event) =>
1103	event.stage === "completed" &&
1104	event.method === "GET" &&
1105	event.path.endsWith(`/machines/${machineId}`) &&
1106	event.state === "started" &&
1107	event.cordoned === true &&
1108	event.checks?.find((check) => check.name === "ready")
1109	?.status === "passing",
1110	);
1111	expect(promoting).toBeGreaterThanOrEqual(0);
1112	expect(ready).toBeGreaterThan(promoting);
1113
1114	yield* readiness.turnOff(site.appName, machineId);
1115	if (interrupted) {
1116	const interruption = yield* Fiber.interrupt(attempt).pipe(
1117	Effect.forkScoped,
1118	);
1119	yield* Effect.yieldNow;
1120	yield* Effect.sync(proxy.release);
1121	yield* Fiber.join(interruption).pipe(
1122	Effect.timeout("180 seconds"),
1123	);
1124	expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(
1125	true,
1126	);
1127	} else {
1128	yield* Effect.sync(proxy.release);
1129	const result = yield* Fiber.join(attempt).pipe(
1130	Effect.timeout("180 seconds"),
1131	);
1132	expect(Result.isFailure(result)).toBe(true);
1133	if (Result.isFailure(result))
1134	expect(result.failure._tag).toBe("Fly.ReplicaChecksNotPassing");
1135	}
1136	yield* Effect.sync(proxy.clear);
1137	const pending = yield* census(site.appName);
1138	expect(pending.map((machine) => machine.id)).toEqual([machineId]);
1139	expect(pending[0]!.cordoned).toBe(false);
1140	const metadata = pending[0]!.config!.metadata!;
1141	expect(metadata["alchemy.phase"]).toBe("validating");
1142	expect(metadata["alchemy.checked-instance"]).toBeUndefined();
1143	const read = yield* observeReplicaSet({
1144	appName: site.appName,
1145	id: "Worker",
1146	type: "Fly.Machine",
1147	fqn: metadata["alchemy.fqn"]!,
1148	resourceInstanceId: metadata["alchemy.instance"]!,
1149	machineIds: [machineId],
1150	});
1151	expect(read?.rolloutPending).toBe(true);
1152	expect(read?.machineIds).toEqual([]);
1153
1154	const retryActor = yield* engineActor(
1155	stack,
1156	`FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
1157	file,
1158	proxy.url,
1159	);
1160	const stillBroken = yield* readiness
1161	.deployWorker(retryActor, "one")
1162	.pipe(Effect.scoped, Effect.result);
1163	expect(Result.isFailure(stillBroken)).toBe(true);
1164	if (Result.isFailure(stillBroken))
1165	expect(stillBroken.failure._tag).toBe(
1166	"Fly.ReplicaChecksNotPassing",
1167	);
1168	expect(
1169	(yield* census(site.appName)).map((machine) => machine.id),
1170	).toEqual([machineId]);
1171	expect(
1172	proxy.events.some(
1173	(event) =>
1174	event.phase === "active" && event.path.endsWith("/metadata"),
1175	),
1176	).toBe(false);
1177
1178	yield* repairReadiness(site.appName, machineId);
1179	const recoveryActor = yield* engineActor(
1180	stack,
1181	`FLY-REVIEW-1 first-deploy ${interrupted ? "interrupted" : "failed"} final readiness stays pending and cannot shortcut recovery`,
1182	file,
1183	proxy.url,
1184	);
1185	const recovered = yield* readiness
1186	.deployWorker(recoveryActor, "one")
1187	.pipe(Effect.scoped);
1188	expect(recovered.machineIds).toEqual([machineId]);
1189	yield* assertCommitted(site.appName, recovered.machineIds);
1190	const committed = yield* observeReplicaSet({
1191	appName: site.appName,
1192	id: "Worker",
1193	type: "Fly.Machine",
1194	fqn: metadata["alchemy.fqn"]!,
1195	resourceInstanceId: metadata["alchemy.instance"]!,
1196	machineIds: [machineId],
1197	});
1198	expect(committed?.rolloutPending).toBe(false);
1199	expect(committed?.machineIds).toEqual([machineId]);
1200	yield* stack.destroy();
1201	yield* assertAppGone(site.appName);
1202	}).pipe(Effect.scoped),
1203	{ timeout: 600_000 },
1204	);
1205	}
1206	},
1207	);
1208
1209	describe.sequential(
1210	"concurrency",
1211	{
1212	tags: [
1213	"provider:fly",
1214	"provider:fly:app",
1215	"provider:fly:machine",
1216	"provider:fly:service",
1217	"live",
1218	],
1219	},
1220	() => {
1221	const file = "test/Fly/BlueGreen.test.ts";
1222	const { test } = Test.make({ providers: Fly.providers() });
1223
1224	test.provider(
1225	"F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1226	(stack) =>
1227	Effect.gen(function* () {
1228	yield* stack.destroy();
1229	const initial = yield* deployWorker(stack, "one");
1230	const proxy = yield* transportProxy();
1231	yield* Effect.sync(() =>
1232	proxy.arm({
1233	match: (event) =>
1234	event.method === "GET" && event.path.endsWith("/machines"),
1235	action: "hold-response",
1236	remaining: 1,
1237	}),
1238	);
1239	yield* Effect.gen(function* () {
1240	const staleActor = yield* engineActor(
1241	stack,
1242	"F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1243	file,
1244	proxy.url,
1245	);
1246	const freshActor = yield* engineActor(
1247	stack,
1248	"F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1249	file,
1250	);
1251	const delayed = yield* deployWorker(staleActor, "two").pipe(
1252	Effect.scoped,
1253	Effect.result,
1254	Effect.forkScoped,
1255	);
1256	yield* proxy.wait(
1257	(event) => event.stage === "held" && event.status === 200,
1258	);
1259	const newer = yield* deployWorker(freshActor, "three");
1260	expect(newer.machineId).not.toBe(initial.machineId);
1261	expect(
1262	proxy.events.some(
1263	(event) => event.stage === "request" && event.method !== "GET",
1264	),
1265	).toBe(false);
1266	yield* Effect.sync(proxy.release);
1267	const result = yield* Fiber.join(delayed).pipe(
1268	Effect.timeout("180 seconds"),
1269	);
1270	if (Result.isSuccess(result)) {
1271	// A later valid reconcile may win; native leases do not fence LocalState with a global epoch.
1272	yield* assertCommitted(
1273	initial.appName,
1274	result.success.machineIds,
1275	);
1276	for (const id of newer.machineIds) {
1277	const retire = proxy.events.findIndex(
1278	(event) =>
1279	event.stage === "request" &&
1280	event.machineId === id &&
1281	(event.path.endsWith("/stop") ||
1282	event.path.endsWith("/cordon") ||
1283	(event.method === "DELETE" &&
1284	!event.path.endsWith("/lease"))),
1285	);
1286	expect(retire).toBeGreaterThan(0);
1287	for (const candidate of result.success.machineIds) {
1288	expect(
1289	proxy.events
1290	.slice(0, retire)
1291	.some(
1292	(event) =>
1293	event.stage === "completed" &&
1294	event.machineId === candidate &&
1295	event.method === "GET" &&
1296	event.state === "started" &&
1297	event.checks?.some(
1298	(check) =>
1299	check.name === "ready" &&
1300	check.status === "passing",
1301	),
1302	),
1303	).toBe(true);
1304	}
1305	}
1306	} else {
1307	expect([
1308	"Fly.DeploymentRecoveryAmbiguous",
1309	"Fly.ReplicaOwnershipChanged",
1310	"Fly.MachineLeaseBusy",
1311	"Fly.MachineLeaseLost",
1312	"NotFound",
1313	]).toContain(result.failure._tag);
1314	expect(
1315	(yield* census(initial.appName)).some(
1316	(machine) => machine.id === newer.machineId,
1317	),
1318	).toBe(true);
1319	yield* Effect.logInfo(
1320	"Stale snapshot refused without state-fencing guarantee",
1321	{ outcome: result.failure._tag },
1322	);
1323	}
1324	const resumed = yield* engineActor(
1325	stack,
1326	"F11 an actual delayed Machine-list snapshot cannot retire a newer generation before fresh readiness",
1327	file,
1328	);
1329	const settled = yield* deployWorker(resumed, "four");
1330	yield* assertCommitted(initial.appName, settled.machineIds);
1331	}).pipe(
1332	Effect.ensuring(
1333	Effect.sync(() => {
1334	proxy.clear();
1335	proxy.release();
1336	}),
1337	),
1338	Effect.scoped,
1339	);
1340	yield* stack.destroy();
1341	yield* assertAppGone(initial.appName);
1342	}).pipe(Effect.scoped),
1343	{ timeout: 600_000 },
1344	);
1345
1346	describe.sequential("independent engine contexts", () => {
1347	for (const competitor of ["bluegreen", "rolling", "destroy"] as const) {
1348	test.provider(
1349	`F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1350	(stack) =>
1351	Effect.gen(function* () {
1352	yield* stack.destroy();
1353	const initial = yield* deployWorker(stack, "one");
1354	const holderProxy = yield* transportProxy();
1355	const contenderProxy = yield* transportProxy();
1356	yield* Effect.sync(() =>
1357	holderProxy.arm({
1358	match: (event) =>
1359	event.method === "POST" && event.path.endsWith("/machines"),
1360	action: "hold-response",
1361	remaining: 1,
1362	}),
1363	);
1364	yield* Effect.gen(function* () {
1365	const holderActor = yield* engineActor(
1366	stack,
1367	`F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1368	file,
1369	holderProxy.url,
1370	);
1371	const contenderActor = yield* engineActor(
1372	stack,
1373	`F11 shared-old lease excludes ${competitor} while a candidate response is held`,
1374	file,
1375	contenderProxy.url,
1376	);
1377	expect(contenderActor.state).not.toBe(holderActor.state);
1378	const holder = yield* deployWorker(holderActor, "two").pipe(
1379	Effect.scoped,
1380	Effect.forkScoped,
1381	);
1382	const held = yield* holderProxy.wait(
1383	(event) => event.stage === "held" && event.status! < 300,
1384	);
1385	const operation =
1386	competitor === "destroy"
1387	? contenderActor.destroy()
1388	: deployWorker(contenderActor, "three", {
1389	deploy: {
1390	strategy: competitor,
1391	healthTimeout: "30 seconds",
1392	},
1393	}).pipe(Effect.asVoid);
1394	const result = yield* operation.pipe(
1395	Effect.scoped,
1396	Effect.timeout("75 seconds"),
1397	Effect.result,
1398	);
1399	yield* Effect.sync(holderProxy.release);
1400	expect(Result.isFailure(result)).toBe(true);
1401	if (Result.isFailure(result))
1402	expect(result.failure).not.toMatchObject({
1403	_tag: "TimeoutError",
1404	});
1405	expect(
1406	contenderProxy.events.some(
1407	(event) =>
1408	event.stage === "completed" &&
1409	event.status === 409 &&
1410	event.path.endsWith("/lease") &&
1411	event.machineId === initial.machineId,
1412	),
1413	).toBe(true);
1414	expect(
1415	contenderProxy.events.some(
1416	(event) =>
1417	event.stage === "completed" &&
1418	event.status! < 300 &&
1419	event.method !== "GET" &&
1420	!event.path.endsWith("/lease"),
1421	),
1422	).toBe(false);
1423	const next = yield* Fiber.join(holder).pipe(
1424	Effect.timeout("90 seconds"),
1425	);
1426	expect(next.machineIds).toEqual([held.machineId]);
1427	expect(
1428	(yield* census(initial.appName)).map((machine) => machine.id),
1429	).toEqual(next.machineIds);
1430	}).pipe(
1431	Effect.ensuring(
1432	Effect.sync(() => {
1433	holderProxy.clear();
1434	holderProxy.release();
1435	}),
1436	),
1437	Effect.scoped,
1438	);
1439	yield* stack.destroy();
1440	yield* assertAppGone(initial.appName);
1441	}).pipe(Effect.scoped),
1442	{ timeout: 300_000 },
1443	);
1444	}
1445
1446	test.provider(
1447	"S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
1448	(stack) =>
1449	Effect.gen(function* () {
1450	yield* stack.destroy();
1451	const app = yield* stack.deploy(Fly.App("Site"));
1452	const firstProxy = yield* transportProxy();
1453	const secondProxy = yield* transportProxy();
1454	yield* Effect.sync(() =>
1455	firstProxy.arm({
1456	match: (event) =>
1457	event.method === "POST" && event.path.endsWith("/machines"),
1458	action: "hold-response",
1459	remaining: 1,
1460	}),
1461	);
1462	yield* Effect.gen(function* () {
1463	const firstActor = yield* engineActor(
1464	stack,
1465	"S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
1466	file,
1467	firstProxy.url,
1468	);
1469	const secondActor = yield* engineActor(
1470	stack,
1471	"S02 F11 concurrent first Machine engine deployments preserve ownership without assuming a global lock",
1472	file,
1473	secondProxy.url,
1474	);
1475	const first = yield* deployWorker(firstActor, "one").pipe(
1476	Effect.scoped,
1477	Effect.result,
1478	Effect.forkScoped,
1479	);
1480	const created = yield* firstProxy.wait(
1481	(event) => event.stage === "held" && event.status! < 300,
1482	);
1483	const original = (yield* census(app.appName)).find(
1484	(machine) => machine.id === created.machineId,
1485	);
1486	const owner = original?.config?.metadata;
1487	expect(owner?.["alchemy.fqn"]).toBeTruthy();
1488	expect(owner?.["alchemy.instance"]).toBeTruthy();
1489	expect(
1490	firstProxy.events.some(
1491	(event) =>
1492	event.stage === "completed" &&
1493	event.method === "POST" &&
1494	event.path.endsWith("/lease") &&
1495	event.status! < 300,
1496	),
1497	).toBe(false);
1498	const second = yield* deployWorker(secondActor, "two").pipe(
1499	Effect.scoped,
1500	Effect.timeout("90 seconds"),
1501	Effect.result,
1502	);
1503	const during = yield* census(app.appName);
1504	yield* Effect.sync(firstProxy.release);
1505	const firstResult = yield* Fiber.join(first).pipe(
1506	Effect.timeout("90 seconds"),
1507	);
1508	yield* Effect.logInfo(
1509	"First-deploy results: no global exclusion promised",
1510	{
1511	first: Result.isFailure(firstResult)
1512	? firstResult.failure._tag
1513	: firstResult._tag,
1514	second: Result.isFailure(second)
1515	? second.failure._tag
1516	: second._tag,
1517	},
1518	);
1519	const successfulIds: string[] = [];
1520	for (const result of [firstResult, second]) {
1521	if (Result.isSuccess(result)) {
1522	expect(result.success.machineIds.length).toBe(1);
1523	successfulIds.push(...result.success.machineIds);
1524	} else {
1525	expect([
1526	"Fly.DeploymentRecoveryAmbiguous",
1527	"Fly.ReplicaOwnershipChanged",
1528	"Fly.MachineLeaseBusy",
1529	"Fly.MachineLeaseLost",
1530	"Fly.MachineNotCreated",
1531	"NotFound",
1532	]).toContain(result.failure._tag);
1533	}
1534	}
1535	expect(successfulIds.length).toBeGreaterThan(0);
1536	// A held create response conveys no lease; a leased, ready successor may retire it.
1537	if (Result.isSuccess(second)) {
1538	const committed = yield* assertCommitted(
1539	app.appName,
1540	second.success.machineIds,
1541	);
1542	for (const machine of committed) {
1543	yield* waitHealthy(
1544	app.appName,
1545	machine,
1546	30_000,
1547	machine.config,
1548	);
1549	}
1550	expect(during.map((machine) => machine.id).sort()).toEqual(
1551	[...second.success.machineIds].sort(),
1552	);
1553	expect(
1554	during.every(
1555	(machine) =>
1556	machine.state === "started" &&
1557	machine.cordoned === false &&
1558	machine.config?.metadata?.["alchemy.phase"] === "active",
1559	),
1560	).toBe(true);
1561	} else {
1562	expect(
1563	during.some((machine) => machine.id === created.machineId),
1564	).toBe(true);
1565	}
1566	const createdIds = new Set(
1567	[firstProxy, secondProxy].flatMap((proxy) =>
1568	proxy.events
1569	.filter(
1570	(event) =>
1571	event.stage === "completed" &&
1572	event.method === "POST" &&
1573	event.path.endsWith("/machines") &&
1574	event.status! < 300,
1575	)
1576	.map((event) => event.machineId),
1577	),
1578	);
1579	for (const proxy of [firstProxy, secondProxy]) {
1580	for (const [index, event] of proxy.events.entries()) {
1581	if (
1582	event.stage !== "request" ||
1583	!(
1584	event.path.endsWith("/stop") ||
1585	event.path.endsWith("/cordon") ||
1586	event.phase === "retiring" ||
1587	(event.method === "DELETE" &&
1588	/\/machines\/[^/]+$/.test(event.path))
1589	)
1590	)
1591	continue;
1592	expect(event.machineId).toBeDefined();
1593	expect(createdIds.has(event.machineId)).toBe(true);
1594	const before = proxy.events.slice(0, index);
1595	const lease = before.findIndex(
1596	(prior) =>
1597	prior.stage === "completed" &&
1598	prior.machineId === event.machineId &&
1599	prior.method === "POST" &&
1600	prior.path.endsWith("/lease") &&
1601	prior.status! < 300,
1602	);
1603	expect(lease).toBeGreaterThan(-1);
1604	expect(
1605	before
1606	.slice(lease + 1)
1607	.some(
1608	(prior) =>
1609	prior.stage === "completed" &&
1610	prior.machineId === event.machineId &&
1611	prior.method === "GET" &&
1612	/\/machines\/[^/]+$/.test(prior.path) &&
1613	prior.status === 200,
1614	),
1615	).toBe(true);
1616	expect(
1617	before.some(
1618	(ready, readyIndex) =>
1619	ready.stage === "completed" &&
1620	ready.method === "GET" &&
1621	ready.status === 200 &&
1622	ready.machineId !== event.machineId &&
1623	createdIds.has(ready.machineId) &&
1624	ready.state === "started" &&
1625	ready.cordoned === false &&
1626	ready.checks?.some(
1627	(check) =>
1628	check.name === "ready" &&
1629	check.status === "passing",
1630	) &&
1631	before
1632	.slice(readyIndex + 1)
1633	.some(
1634	(active) =>
1635	active.stage === "completed" &&
1636	active.machineId === ready.machineId &&
1637	active.method === "PUT" &&
1638	active.phase === "active" &&
1639	active.status! < 300,
1640	),
1641	),
1642	).toBe(true);
1643	}
1644	}
1645	const live = yield* census(app.appName);
1646	expect(live.length).toBe(1);
1647	expect(
1648	live.every(
1649	(machine) =>
1650	successfulIds.includes(machine.id!) &&
1651	machine.state === "started" &&
1652	machine.cordoned === false &&
1653	machine.checks?.some(
1654	(check) =>
1655	check.name === "ready" && check.status === "passing",
1656	) &&
1657	machine.config?.metadata?.["alchemy.fqn"] ===
1658	owner?.["alchemy.fqn"] &&
1659	machine.config?.metadata?.["alchemy.instance"] ===
1660	owner?.["alchemy.instance"],
1661	),
1662	).toBe(true);
1663	yield* assertCommitted(
1664	app.appName,
1665	live.map((machine) => machine.id!),
1666	);
1667	}).pipe(
1668	Effect.ensuring(
1669	Effect.sync(() => {
1670	firstProxy.clear();
1671	firstProxy.release();
1672	}),
1673	),
1674	Effect.scoped,
1675	);
1676	yield* stack.destroy();
1677	yield* assertAppGone(app.appName);
1678	}).pipe(Effect.scoped),
1679	{ timeout: 300_000 },
1680	);
1681	});
1682	},
1683	);
1684
1685	describe.sequential(
1686	"create faults",
1687	{
1688	tags: [
1689	"provider:fly",
1690	"provider:fly:app",
1691	"provider:fly:machine",
1692	"provider:fly:service",
1693	"live",
1694	],
1695	},
1696	() => {
1697	let endpoint: string | undefined;
1698	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
1699
1700	test.provider(
1701	"F02 lost completed create reuses the same owned name and real Conflict",
1702	(stack) =>
1703	Effect.gen(function* () {
1704	yield* stack.destroy();
1705	const initial = yield* deployWorker(stack, "one");
1706	const proxy = yield* transportProxy();
1707	yield* Effect.sync(() => {
1708	endpoint = proxy.url;
1709	proxy.arm({
1710	match: (event) =>
1711	event.method === "POST" && event.path.endsWith("/machines"),
1712	action: "drop-response",
1713	remaining: 1,
1714	});
1715	});
1716	try {
1717	const next = yield* deployWorker(stack, "two");
1718	const created = proxy.events.filter(
1719	(event) =>
1720	event.stage === "completed" &&
1721	event.method === "POST" &&
1722	event.path.endsWith("/machines") &&
1723	event.status! < 300,
1724	);
1725	expect(created).toHaveLength(1);
1726	expect(next.machineIds).toEqual([created[0]!.machineId]);
1727	expect(next.machineId).not.toBe(initial.machineId);
1728	expect(
1729	proxy.events.filter((event) => event.stage === "dropped"),
1730	).toHaveLength(1);
1731	const live = yield* assertCommitted(next.appName, next.machineIds);
1732	const conflict = yield* machines
1733	.createMachine({
1734	app_name: next.appName,
1735	name: live[0]!.name,
1736	region: live[0]!.region,
1737	config: live[0]!.config,
1738	})
1739	.pipe(Retry.none, Effect.result);
1740	expect(Result.isFailure(conflict)).toBe(true);
1741	if (Result.isFailure(conflict))
1742	expect(conflict.failure._tag).toBe("Conflict");
1743	yield* assertCommitted(next.appName, next.machineIds);
1744	} finally {
1745	yield* Effect.sync(() => {
1746	endpoint = undefined;
1747	proxy.clear();
1748	});
1749	}
1750	yield* stack.destroy();
1751	yield* assertAppGone(initial.appName);
1752	}).pipe(
1753	Effect.scoped,
1754	Effect.ensuring(
1755	Effect.sync(() => {
1756	endpoint = undefined;
1757	}),
1758	),
1759	),
1760	{ timeout: 300_000 },
1761	);
1762	},
1763	);
1764
1765	describe.sequential(
1766	"engine state",
1767	{
1768	tags: [
1769	"provider:fly",
1770	"provider:fly:app",
1771	"provider:fly:machine",
1772	"provider:fly:service",
1773	"live",
1774	],
1775	},
1776	() => {
1777	const file = "test/Fly/BlueGreen.test.ts";
1778
1779	const { test } = Test.make({ providers: Fly.providers() });
1780
1781	test.provider(
1782	"F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
1783	(stack) =>
1784	Effect.gen(function* () {
1785	yield* stack.destroy();
1786	const proxy = yield* transportProxy();
1787	const firstActor = yield* engineActor(
1788	stack,
1789	"F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
1790	file,
1791	proxy.url,
1792	);
1793	const first = yield* firstActor
1794	.deploy(Fly.App("Site"))
1795	.pipe(Effect.scoped);
1796	const secondActor = yield* engineActor(
1797	stack,
1798	"F07 F11 fresh engine contexts reopen real durable LocalState without recreating the App",
1799	file,
1800	proxy.url,
1801	);
1802	expect(secondActor.state).not.toBe(firstActor.state);
1803	const second = yield* secondActor
1804	.deploy(Fly.App("Site"))
1805	.pipe(Effect.scoped);
1806	expect(second.appName).toBe(first.appName);
1807	expect(second.appId).toBe(first.appId);
1808	expect(
1809	proxy.events.filter(
1810	(event) =>
1811	event.stage === "completed" &&
1812	event.method === "POST" &&
1813	event.path === "/v1/apps" &&
1814	event.status! < 300,
1815	),
1816	).toHaveLength(1);
1817	yield* stack.destroy();
1818	yield* assertAppGone(first.appName);
1819	}).pipe(Effect.scoped),
1820	{ timeout: 180_000 },
1821	);
1822	},
1823	);
1824
1825	describe.sequential(
1826	"exec leases",
1827	{ tags: ["provider:fly", "provider:fly:service"] },
1828	() => {
1829	const { test } = Test.make({ providers: Fly.providers() });
1830
1831	it.effect(
1832	"exec probe sanitizes failures and defects without losing interruption",
1833	() =>
1834	Effect.gen(function* () {
1835	const privateHeader = "fixture-private-lease-header";
1836	const outcome = yield* Effect.failCause(
1837	Cause.fromReasons([
1838	Cause.makeFailReason(new Error(privateHeader)),
1839	Cause.makeDieReason({
1840	headers: { "fly-machine-lease-nonce": privateHeader },
1841	}),
1842	Cause.makeInterruptReason(123),
1843	]),
1844	).pipe(sanitizeExecFailure, Effect.exit);
1845	expect(Exit.isFailure(outcome)).toBe(true);
1846	if (Exit.isFailure(outcome)) {
1847	expect(outcome.cause.reasons.map((reason) => reason._tag)).toEqual([
1848	"Fail",
1849	"Die",
1850	"Interrupt",
1851	]);
1852	expect(Cause.pretty(outcome.cause)).not.toContain(privateHeader);
1853	const interrupted = outcome.cause.reasons.find(
1854	Cause.isInterruptReason,
1855	);
1856	expect(interrupted?.fiberId).toBe(123);
1857	}
1858	}),
1859	{ tags: ["unit", "local"] },
1860	);
1861
1862	test.provider(
1863	"exec remains blocked during a native lease even with its nonce and succeeds after observable release",
1864	(stack) =>
1865	Effect.gen(function* () {
1866	yield* stack.destroy();
1867	const created = yield* stack.deploy(
1868	Effect.gen(function* () {
1869	const app = yield* Fly.App("ExecLeaseSite");
1870	return yield* Fly.Machine("ExecLeaseTarget", {
1871	app,
1872	region: "iad",
1873	image: "nginx:alpine",
1874	guest: { cpus: 1, memoryMb: 256 },
1875	checks,
1876	deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
1877	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
1878	});
1879	}),
1880	);
1881	const target = {
1882	app_name: created.appName,
1883	machine_id: created.machineId,
1884	};
1885	const command = ["/bin/sh", "-c", "printf lease-exec-ok"];
1886	yield* Effect.gen(function* () {
1887	const leases = yield* makeMachineLeases(created.appName);
1888	yield* leases.acquire([created.machineId]);
1889	yield* leases.guard(
1890	Effect.gen(function* () {
1891	const verifyAuthority = Effect.gen(function* () {
1892	const current = yield* machines
1893	.getMachineLease(target)
1894	.pipe(Retry.none, Effect.timeout("15 seconds"));
1895	const nonce = current.data?.nonce;
1896	if (!nonce)
1897	return yield* Effect.fail(
1898	new Error("Owned Machine lease response has no nonce"),
1899	);
1900	expect(
1901	nonce === (yield* leases.nonceIfHeld(created.machineId)),
1902	).toBe(true);
1903	expect(current.data?.expires_at).toBeGreaterThan(
1904	(yield* Clock.currentTimeMillis) / 1000 + 15,
1905	);
1906	return nonce;
1907	});
1908	const client = yield* HttpClient.HttpClient;
1909	for (const header of ["held", "missing", "wrong"] as const) {
1910	const nonce = yield* verifyAuthority;
1911	const value =
1912	header === "held"
1913	? nonce
1914	: `${nonce[0] === "a" ? "b" : "a"}${nonce.slice(1)}`;
1915	let observed = false;
1916	// Probe the general lease header without advertising unsupported exec authorization.
1917	const probeClient = HttpClient.mapRequest(
1918	client,
1919	(request) =>
1920	header === "missing"
1921	? request
1922	: HttpClientRequest.setHeader(
1923	request,
1924	"fly-machine-lease-nonce",
1925	value,
1926	),
1927	).pipe(
1928	HttpClient.tapRequest((request) =>
1929	Effect.sync(() => {
1930	observed = true;
1931	expect(
1932	request.headers["fly-machine-lease-nonce"] ===
1933	(header === "missing" ? undefined : value),
1934	).toBe(true);
1935	}),
1936	),
1937	);
1938	const outcome = yield* machines
1939	.execMachine({
1940	...target,
1941	command,
1942	timeout: 5,
1943	})
1944	.pipe(
1945	Retry.none,
1946	Effect.timeout("15 seconds"),
1947	Effect.provideService(HttpClient.HttpClient, probeClient),
1948	Effect.as("accepted" as const),
1949	Effect.catchTag("Conflict", () =>
1950	Effect.succeed("Conflict" as const),
1951	),
1952	);
1953	expect(observed).toBe(true);
1954	expect(outcome).toBe("Conflict");
1955	yield* verifyAuthority;
1956	}
1957	}).pipe(Effect.timeout("90 seconds")),
1958	);
1959	}).pipe(Effect.scoped);
1960
1961	const released = yield* machines.getMachineLease(target).pipe(
1962	Retry.none,
1963	Effect.timeout("15 seconds"),
1964	Effect.as(false),
1965	Effect.catchTag("NotFound", () => Effect.succeed(true)),
1966	);
1967	expect(released).toBe(true);
1968	expect((yield* machines.getMachine(target)).state).toBe("started");
1969	const unleased = yield* machines
1970	.execMachine({
1971	...target,
1972	command,
1973	timeout: 5,
1974	})
1975	.pipe(Retry.none, Effect.timeout("15 seconds"));
1976	expect(unleased.exit_code).toBe(0);
1977	expect(unleased.stdout).toBe("lease-exec-ok");
1978	yield* stack.destroy();
1979	yield* assertAppGone(created.appName);
1980	}).pipe(sanitizeExecFailure),
1981	{
1982	tags: ["provider:fly:app", "provider:fly:machine", "live"],
1983	timeout: 300_000,
1984	},
1985	);
1986	},
1987	);
1988
1989	describe.sequential(
1990	"failures",
1991	{
1992	tags: [
1993	"provider:fly",
1994	"provider:fly:app",
1995	"provider:fly:machine",
1996	"provider:fly:service",
1997	"live",
1998	],
1999	},
2000	() => {
2001	const { test } = Test.make({ providers: Fly.providers() });
2002
2003	test.provider(
2004	"F01 a rejected candidate create preserves the active generation",
2005	(stack) =>
2006	Effect.gen(function* () {
2007	yield* stack.destroy();
2008	const deploy = (image: string) =>
2009	stack.deploy(
2010	Effect.gen(function* () {
2011	const app = yield* Fly.App("Site");
2012	return yield* Fly.Machine("Worker", {
2013	app,
2014	image,
2015	deploy: {
2016	strategy: "bluegreen",
2017	healthTimeout: "30 seconds",
2018	},
2019	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
2020	checks: {
2021	ready: {
2022	type: "http",
2023	port: 80,
2024	path: "/",
2025	interval: "2s",
2026	timeout: "1s",
2027	},
2028	},
2029	});
2030	}),
2031	);
2032	const initial = yield* deploy("nginx:alpine");
2033	const failed = yield* deploy(
2034	"nginx:alchemy-bluegreen-nonexistent-image",
2035	).pipe(Effect.result);
2036	expect(Result.isFailure(failed)).toBe(true);
2037	if (Result.isFailure(failed))
2038	expect(failed.failure).toMatchObject({ _tag: "BadRequest" });
2039	const live = (yield* machines.listMachines({
2040	app_name: initial.appName,
2041	})).filter((machine) => machine.state !== "destroyed");
2042	expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
2043	expect(live[0]?.state).toBe("started");
2044	expect(live[0]?.cordoned).toBe(false);
2045	expect(live[0]?.config?.metadata?.["alchemy.phase"]).toBe("active");
2046	yield* stack.destroy();
2047	yield* assertAppGone(initial.appName);
2048	}),
2049	{ timeout: 180_000 },
2050	);
2051	},
2052	);
2053
2054	describe.sequential(
2055	"health",
2056	{
2057	tags: [
2058	"provider:fly",
2059	"provider:fly:app",
2060	"provider:fly:machine",
2061	"provider:fly:service",
2062	"live",
2063	],
2064	},
2065	() => {
2066	const { test } = Test.make({ providers: Fly.providers() });
2067
2068	for (const kind of ["named", "multiple published services"] as const) {
2069	test.provider(
2070	`S03 every ${kind} check must pass before blue retirement`,
2071	(stack) =>
2072	Effect.gen(function* () {
2073	yield* stack.destroy();
2074	const initial = yield* deployWorker(stack, "one", {
2075	deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2076	});
2077	const extra = { ...checks.ready, path: "/does-not-exist" };
2078	const failed = yield* deployWorker(stack, "two", {
2079	checks:
2080	kind === "named"
2081	? { ready: checks.ready, dependency: extra }
2082	: checks,
2083	services:
2084	kind === "named"
2085	? undefined
2086	: [
2087	{
2088	protocol: "tcp",
2089	internalPort: 80,
2090	ports: [{ port: 80, handlers: ["http"] }],
2091	checks: [checks.ready],
2092	},
2093	{
2094	protocol: "tcp",
2095	internalPort: 80,
2096	ports: [{ port: 443, handlers: ["tls", "http"] }],
2097	checks: [extra],
2098	},
2099	],
2100	deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
2101	}).pipe(Effect.result);
2102	expect(Result.isFailure(failed)).toBe(true);
2103	if (Result.isFailure(failed))
2104	expect(failed.failure).toMatchObject({
2105	_tag: "Fly.ReplicaChecksNotPassing",
2106	});
2107	const live = yield* census(initial.appName);
2108	expect(live.map((machine) => machine.id)).toEqual(
2109	initial.machineIds,
2110	);
2111	expect(live[0]!.cordoned).toBe(false);
2112	const fixed = yield* deployWorker(stack, "two", {
2113	checks: { ready: checks.ready, dependency: checks.ready },
2114	deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2115	});
2116	const committed = yield* assertCommitted(
2117	initial.appName,
2118	fixed.machineIds,
2119	);
2120	// Active commit metadata can reset reports after readiness validation.
2121	const healthy = yield* waitHealthy(
2122	initial.appName,
2123	committed[0]!,
2124	30_000,
2125	);
2126	expect(healthy.instance_id).toBe(committed[0]!.instance_id);
2127	expect(
2128	["ready", "dependency"].every((name) =>
2129	healthy.checks?.some(
2130	(check) => check.name === name && check.status === "passing",
2131	),
2132	),
2133	).toBe(true);
2134	yield* stack.destroy();
2135	yield* assertAppGone(initial.appName);
2136	}),
2137	{ timeout: 300_000 },
2138	);
2139	}
2140	},
2141	);
2142
2143	describe.sequential(
2144	"health transport faults",
2145	{
2146	tags: [
2147	"provider:fly",
2148	"provider:fly:app",
2149	"provider:fly:machine",
2150	"provider:fly:service",
2151	"live",
2152	],
2153	},
2154	() => {
2155	let endpoint: string | undefined;
2156	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
2157
2158	test.provider(
2159	"S07 health polling recovers a lost real GET and exhausts a persistent connection cut",
2160	(stack) =>
2161	Effect.gen(function* () {
2162	yield* stack.destroy();
2163	const initial = yield* deployWorker(stack, "one", {
2164	deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
2165	});
2166	const observed = yield* machines.getMachine({
2167	app_name: initial.appName,
2168	machine_id: initial.machineId,
2169	});
2170	const proxy = yield* transportProxy();
2171	const match = (event: { method: string; path: string }) =>
2172	event.method === "GET" &&
2173	event.path.endsWith(`/machines/${initial.machineId}`);
2174	yield* Effect.sync(() => {
2175	endpoint = proxy.url;
2176	proxy.arm({ match, action: "drop-response", remaining: 1 });
2177	});
2178	try {
2179	const healthy = yield* waitHealthy(
2180	initial.appName,
2181	observed,
2182	30_000,
2183	observed.config!,
2184	);
2185	expect(healthy.id).toBe(initial.machineId);
2186	expect(healthy.instance_id).toBe(observed.instance_id);
2187	expect(
2188	healthy.checks?.every((check) => check.status === "passing"),
2189	).toBe(true);
2190	expect(
2191	proxy.events.filter(
2192	(event) => event.stage === "dropped" && event.status === 200,
2193	),
2194	).toHaveLength(1);
2195	yield* Effect.sync(() =>
2196	proxy.arm({ match, action: "cut-request", remaining: Infinity }),
2197	);
2198	let attempts = 0;
2199	const client = yield* HttpClient.HttpClient;
2200	const observedClient = HttpClient.tapRequest(client, (request) =>
2201	Effect.sync(() => {
2202	if (
2203	request.method === "GET" &&
2204	request.url.endsWith(`/machines/${initial.machineId}`)
2205	) {
2206	attempts++;
2207	}
2208	}),
2209	);
2210	const started = yield* Clock.currentTimeMillis;
2211	const failed = yield* waitHealthy(
2212	initial.appName,
2213	healthy,
2214	8_000,
2215	healthy.config!,
2216	).pipe(
2217	Effect.provideService(HttpClient.HttpClient, observedClient),
2218	Effect.result,
2219	);
2220	expect(
2221	(yield* Clock.currentTimeMillis) - started,
2222	).toBeLessThanOrEqual(12_000);
2223	expect(Result.isFailure(failed)).toBe(true);
2224	if (Result.isFailure(failed)) {
2225	expect(failed.failure).toMatchObject({
2226	_tag: "Fly.ReplicaChecksNotPassing",
2227	appName: initial.appName,
2228	machineId: initial.machineId,
2229	});
2230	}
2231	expect(
2232	proxy.events.filter((event) => event.stage === "cut").length,
2233	).toBeGreaterThan(0);
2234	// Count logical client calls independently of Bun's physical GET retries.
2235	expect(attempts).toBeGreaterThan(0);
2236	expect(attempts).toBeLessThanOrEqual(11);
2237	const live = yield* census(initial.appName);
2238	expect(live.map((machine) => machine.id)).toEqual(
2239	initial.machineIds,
2240	);
2241	expect(live[0]!.cordoned).toBe(false);
2242	} finally {
2243	yield* Effect.sync(() => {
2244	endpoint = undefined;
2245	proxy.clear();
2246	});
2247	}
2248	yield* stack.destroy();
2249	yield* assertAppGone(initial.appName);
2250	}).pipe(
2251	Effect.scoped,
2252	Effect.ensuring(
2253	Effect.sync(() => {
2254	endpoint = undefined;
2255	}),
2256	),
2257	),
2258	{ timeout: 180_000 },
2259	);
2260	},
2261	);
2262
2263	describe.sequential(
2264	"health deadlines",
2265	{
2266	tags: [
2267	"provider:fly",
2268	"provider:fly:app",
2269	"provider:fly:machine",
2270	"provider:fly:service",
2271	"live",
2272	],
2273	},
2274	() => {
2275	const { test } = Test.make({ providers: Fly.providers() });
2276
2277	for (const budget of [8_000, 90_000]) {
2278	test.provider(
2279	`S06 real delayed readiness beyond the default minute with ${budget}ms budget`,
2280	(stack) =>
2281	Effect.gen(function* () {
2282	yield* stack.destroy();
2283	const initial = yield* deployWorker(stack, "one");
2284	const result = yield* deployWorker(stack, "two", {
2285	init: {
2286	exec: [
2287	"/bin/sh",
2288	"-c",
2289	"sleep 65; exec nginx -g 'daemon off;'",
2290	],
2291	},
2292	checks: { ready: { ...checks.ready, gracePeriod: "65s" } },
2293	deploy: { strategy: "bluegreen", healthTimeout: budget },
2294	}).pipe(Effect.result);
2295	if (budget === 8_000) {
2296	expect(Result.isFailure(result)).toBe(true);
2297	if (Result.isFailure(result))
2298	expect(result.failure).toMatchObject({
2299	_tag: "Fly.ReplicaChecksNotPassing",
2300	});
2301	const live = yield* census(initial.appName);
2302	expect(live.map((machine) => machine.id)).toEqual(
2303	initial.machineIds,
2304	);
2305	expect(live[0]!.cordoned).toBe(false);
2306	} else {
2307	expect(Result.isSuccess(result)).toBe(true);
2308	if (Result.isSuccess(result)) {
2309	expect(result.success.machineId).not.toBe(initial.machineId);
2310	yield* assertCommitted(
2311	initial.appName,
2312	result.success.machineIds,
2313	);
2314	}
2315	}
2316	yield* stack.destroy();
2317	yield* assertAppGone(initial.appName);
2318	}),
2319	{ timeout: 600_000 },
2320	);
2321	}
2322	},
2323	);
2324
2325	describe.sequential(
2326	"idle capacity",
2327	{
2328	tags: [
2329	"provider:fly",
2330	"provider:fly:app",
2331	"provider:fly:machine",
2332	"provider:fly:service",
2333	"live",
2334	],
2335	},
2336	() => {
2337	let endpoint: string | undefined;
2338	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
2339
2340	describe.sequential("idle topology", () => {
2341	for (const mode of ["stop", "suspend"] as const) {
2342	for (const allIdle of [false, true]) {
2343	test.provider(
2344	`S05 S11 ${mode} ${allIdle ? "all-idle" : "mixed"} replacement keeps an idle nonrepresentative`,
2345	(stack) =>
2346	Effect.gen(function* () {
2347	yield* stack.destroy();
2348	const props = {
2349	count: 2,
2350	services: [
2351	{
2352	protocol: "tcp",
2353	internalPort: 80,
2354	autostop: mode,
2355	autostart: true,
2356	minMachinesRunning: 0,
2357	ports: [{ port: 80, handlers: ["http"] }],
2358	checks: [checks.ready],
2359	},
2360	],
2361	};
2362	const initial = yield* deployWorker(stack, "one", {
2363	...props,
2364	deploy: { strategy: "rolling" },
2365	});
2366	for (const id of allIdle
2367	? initial.machineIds
2368	: initial.machineIds.slice(1)) {
2369	const target = { app_name: initial.appName, machine_id: id };
2370	yield* mode === "stop"
2371	? machines.stopMachine({
2372	...target,
2373	signal: "SIGQUIT",
2374	timeout: "5s",
2375	})
2376	: machines.suspendMachine(target);
2377	const observed = yield* machines.getMachine(target).pipe(
2378	Effect.repeat({
2379	schedule: Schedule.spaced("2 seconds"),
2380	times: 8,
2381	until: (machine) =>
2382	machine.state ===
2383	(mode === "stop" ? "stopped" : "suspended"),
2384	}),
2385	);
2386	expect(observed.state).toBe(
2387	mode === "stop" ? "stopped" : "suspended",
2388	);
2389	}
2390	const proxy = yield* transportProxy();
2391	yield* Effect.sync(() => {
2392	endpoint = proxy.url;
2393	});
2394	const next = yield* deployWorker(stack, "two", props);
2395	expect(next.machineIds).toHaveLength(2);
2396	const live = yield* census(initial.appName);
2397	expect(live.map((machine) => machine.id).sort()).toEqual(
2398	[...next.machineIds].sort(),
2399	);
2400	expect(
2401	live.every((machine) => {
2402	const autostop = machine.config?.services?.[0]?.autostop;
2403	return (autostop === true ? "stop" : autostop) === mode;
2404	}),
2405	).toBe(true);
2406	expect(
2407	live.filter((machine) => machine.state === "started").length,
2408	).toBeLessThanOrEqual(1);
2409	expect(
2410	live.some((machine) =>
2411	["stopped", "suspended"].includes(machine.state!),
2412	),
2413	).toBe(true);
2414	expect(
2415	proxy.events.some(
2416	(event) =>
2417	initial.machineIds.includes(event.machineId!) &&
2418	event.path.endsWith("/start"),
2419	),
2420	).toBe(false);
2421	const same = yield* deployWorker(stack, "two", props);
2422	expect(same.machineIds).toEqual(next.machineIds);
2423	yield* Effect.sync(() => {
2424	endpoint = undefined;
2425	});
2426	yield* stack.destroy();
2427	yield* assertAppGone(initial.appName);
2428	}).pipe(
2429	Effect.scoped,
2430	Effect.ensuring(
2431	Effect.sync(() => {
2432	endpoint = undefined;
2433	}),
2434	),
2435	),
2436	{ timeout: 300_000 },
2437	);
2438	}
2439	}
2440	});
2441	},
2442	);
2443
2444	describe.sequential(
2445	"idle topology",
2446	{
2447	tags: [
2448	"provider:fly",
2449	"provider:fly:app",
2450	"provider:fly:machine",
2451	"provider:fly:service",
2452	"live",
2453	],
2454	},
2455	() => {
2456	const file = "test/Fly/BlueGreen.test.ts";
2457	const { test } = Test.make({ providers: Fly.providers() });
2458	type IdleMode = "stop" | "suspend";
2459
2460	const topology = (
2461	mode: IdleMode,
2462	count: number,
2463	floor = 0,
2464	mixed = false,
2465	): Partial<Omit<Extract<MachineProps, { image: string }>, "app">> => ({
2466	count,
2467	deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
2468	services: [
2469	{
2470	protocol: "tcp",
2471	internalPort: 80,
2472	autostop: mode,
2473	autostart: true,
2474	minMachinesRunning: floor,
2475	checks: [checks.ready],
2476	},
2477	...(mixed
2478	? [{ protocol: "tcp", internalPort: 81, autostop: "off" as const }]
2479	: []),
2480	],
2481	});
2482
2483	const waitState = (appName: string, machineId: string, state: string) =>
2484	machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
2485	Effect.repeat({
2486	schedule: Schedule.spaced("2 seconds"),
2487	times: 30,
2488	until: (machine) => machine.state === state,
2489	}),
2490	Effect.tap((machine) =>
2491	Effect.sync(() => expect(machine.state).toBe(state)),
2492	),
2493	Effect.timeout("90 seconds"),
2494	);
2495
2496	// Native stop/suspend is setup only; automatic return-to-idle has its own test.
2497	const idleAll = (appName: string, ids: string[], mode: IdleMode) =>
2498	Effect.forEach(ids, (id) =>
2499	Effect.gen(function* () {
2500	const target = { app_name: appName, machine_id: id };
2501	const current = yield* machines.getMachine(target);
2502	if (current.state !== "started") {
2503	yield* machines.startMachine(target);
2504	yield* waitState(appName, id, "started");
2505	}
2506	if (mode === "stop") {
2507	yield* machines.stopMachine({
2508	...target,
2509	signal: "SIGQUIT",
2510	timeout: "5s",
2511	});
2512	} else {
2513	yield* machines.suspendMachine(target);
2514	}
2515	yield* waitState(
2516	appName,
2517	id,
2518	mode === "stop" ? "stopped" : "suspended",
2519	);
2520	}),
2521	);
2522
2523	// Expected slots are literal scenario inputs, never computed by readinessRoles.
2524	const assertSlots = (
2525	live: machines.Machine[],
2526	runningSlots: readonly number[],
2527	) => {
2528	const ordered = [...live].sort(
2529	(left, right) =>
2530	Number(left.config?.metadata?.["alchemy.replica"]) -
2531	Number(right.config?.metadata?.["alchemy.replica"]),
2532	);
2533	expect(
2534	ordered.map((machine) => machine.config?.metadata?.["alchemy.replica"]),
2535	).toEqual(
2536	Array.from({ length: live.length }, (_, index) => String(index)),
2537	);
2538	expect(
2539	ordered
2540	.filter(
2541	(machine) =>
2542	machine.config?.metadata?.["alchemy.readiness-role"] === "run",
2543	)
2544	.map((machine) =>
2545	Number(machine.config?.metadata?.["alchemy.replica"]),
2546	),
2547	).toEqual([...runningSlots]);
2548	const roles = Array.from({ length: live.length }, (_, slot) =>
2549	runningSlots.includes(slot) ? "run" : "idle",
2550	).join(",");
2551	for (const [slot, machine] of ordered.entries()) {
2552	expect(machine.config?.metadata?.["alchemy.readiness-role"]).toBe(
2553	runningSlots.includes(slot) ? "run" : "idle",
2554	);
2555	expect(machine.config?.metadata?.["alchemy.readiness-roles"]).toBe(
2556	roles,
2557	);
2558	}
2559	return ordered;
2560	};
2561
2562	const assertTopology = (
2563	appName: string,
2564	ids: string[],
2565	mode: IdleMode,
2566	runningSlots: readonly number[],
2567	floor = 0,
2568	mixed = false,
2569	) =>
2570	Effect.gen(function* () {
2571	const live = yield* assertCommitted(appName, ids);
2572	const ordered = assertSlots(live, runningSlots);
2573	expect(
2574	new Set(
2575	live.map(
2576	(machine) => machine.config?.metadata?.["alchemy.generation"],
2577	),
2578	).size,
2579	).toBe(1);
2580	for (const [slot, machine] of ordered.entries()) {
2581	expect(
2582	machine.config?.metadata?.["alchemy.idle-policy-restored"],
2583	).toBe("true");
2584	expect(machine.cordoned).toBe(false);
2585	expect(machine.config?.services).toHaveLength(mixed ? 2 : 1);
2586	expect(autostopMode(machine.config?.services?.[0]?.autostop)).toBe(
2587	mode,
2588	);
2589	expect(machine.config?.services?.[0]?.autostart).toBe(true);
2590	expect(machine.config?.services?.[0]?.min_machines_running).toBe(
2591	floor,
2592	);
2593	if (mixed) {
2594	expect(autostopMode(machine.config?.services?.[1]?.autostop)).toBe(
2595	"off",
2596	);
2597	}
2598	if (!runningSlots.includes(slot)) {
2599	expect(["created", "stopped", "suspended"]).toContain(
2600	machine.state,
2601	);
2602	} else {
2603	expect(
2604	mixed ? ["started"] : ["started", "stopped", "suspended"],
2605	).toContain(machine.state);
2606	expect(machine.instance_id).toBeDefined();
2607	expect(machine.config?.metadata?.["alchemy.checked-instance"]).toBe(
2608	machine.instance_id,
2609	);
2610	}
2611	}
2612	return ordered;
2613	});
2614
2615	const checkedDeployment = (
2616	actor: Test.ScratchStack,
2617	proxy: { events: TransportEvent[]; readiness: ReadinessEvent[] },
2618	version: string,
2619	mode: IdleMode,
2620	count: number,
2621	runningSlots: readonly number[],
2622	priorIds: readonly string[] = [],
2623	floor = 0,
2624	mixed = false,
2625	) =>
2626	Effect.gen(function* () {
2627	const begin = proxy.events.length;
2628	const proofBegin = proxy.readiness.length;
2629	const output = yield* deployWorker(
2630	actor,
2631	version,
2632	topology(mode, count, floor, mixed),
2633	).pipe(Effect.scoped);
2634	const live = yield* assertTopology(
2635	output.appName,
2636	output.machineIds,
2637	mode,
2638	runningSlots,
2639	floor,
2640	mixed,
2641	);
2642	assertReadinessCommit(
2643	proxy.readiness.slice(proofBegin),
2644	priorIds,
2645	live,
2646	runningSlots,
2647	["ready", "servicecheck-00-http-80"],
2648	!mixed,
2649	);
2650	expect(
2651	proxy.events
2652	.slice(begin)
2653	.some(
2654	(event) =>
2655	event.stage === "request" &&
2656	priorIds.includes(event.machineId!) &&
2657	event.path.endsWith("/start"),
2658	),
2659	).toBe(false);
2660	return output;
2661	});
2662
2663	describe.sequential("live idle topology and automatic cadence", () => {
2664	for (const mode of ["stop", "suspend"] as const) {
2665	test.provider(
2666	`S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
2667	(stack) =>
2668	Effect.gen(function* () {
2669	yield* stack.destroy();
2670	const site = yield* stack.deploy(Fly.App("Site"));
2671	try {
2672	const proxy = yield* readinessProxy();
2673	const actor = yield* readinessActor(
2674	stack,
2675	`S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
2676	file,
2677	proxy,
2678	);
2679	const first = yield* checkedDeployment(
2680	actor,
2681	proxy,
2682	"one",
2683	mode,
2684	1,
2685	[0],
2686	);
2687	yield* idleAll(site.appName, first.machineIds, mode);
2688	const up = yield* checkedDeployment(
2689	actor,
2690	proxy,
2691	"two",
2692	mode,
2693	3,
2694	[0],
2695	first.machineIds,
2696	);
2697	expect(up.machineIds).toHaveLength(3);
2698	expect(
2699	up.machineIds.every((id) => !first.machineIds.includes(id)),
2700	).toBe(true);
2701	yield* idleAll(site.appName, up.machineIds, mode);
2702	const down = yield* checkedDeployment(
2703	actor,
2704	proxy,
2705	"three",
2706	mode,
2707	1,
2708	[0],
2709	up.machineIds,
2710	);
2711	expect(down.machineIds).toHaveLength(1);
2712	expect(up.machineIds).not.toContain(down.machineId);
2713	yield* idleAll(site.appName, down.machineIds, mode);
2714	const unchanged = yield* deployWorker(
2715	actor,
2716	"three",
2717	topology(mode, 1),
2718	);
2719	expect(unchanged.machineIds).toEqual(down.machineIds);
2720	expect((yield* census(site.appName))[0]!.state).toBe(
2721	mode === "stop" ? "stopped" : "suspended",
2722	);
2723	} finally {
2724	yield* stack.destroy();
2725	yield* assertAppGone(site.appName);
2726	}
2727	}).pipe(Effect.scoped),
2728	{ timeout: 900_000 },
2729	);
2730
2731	test.provider(
2732	`S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
2733	(stack) =>
2734	Effect.gen(function* () {
2735	yield* stack.destroy();
2736	const site = yield* stack.deploy(Fly.App("Site"));
2737	try {
2738	const proxy = yield* readinessProxy();
2739	const actor = yield* readinessActor(
2740	stack,
2741	`S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
2742	file,
2743	proxy,
2744	);
2745	const first = yield* checkedDeployment(
2746	actor,
2747	proxy,
2748	"one",
2749	mode,
2750	3,
2751	[0],
2752	);
2753	yield* idleAll(site.appName, first.machineIds, mode);
2754	const raised = yield* checkedDeployment(
2755	actor,
2756	proxy,
2757	"one",
2758	mode,
2759	3,
2760	[0, 1],
2761	first.machineIds,
2762	2,
2763	);
2764	yield* idleAll(site.appName, raised.machineIds, mode);
2765	const lowered = yield* checkedDeployment(
2766	actor,
2767	proxy,
2768	"one",
2769	mode,
2770	3,
2771	[0],
2772	raised.machineIds,
2773	);
2774	yield* idleAll(site.appName, lowered.machineIds, mode);
2775	const slots = assertSlots(yield* census(site.appName), [0]);
2776	yield* machines.startMachine({
2777	app_name: site.appName,
2778	machine_id: slots[2]!.id!,
2779	});
2780	yield* waitState(site.appName, slots[2]!.id!, "started");
2781	const mixedOld = (yield* census(site.appName)).sort(
2782	(a, b) =>
2783	Number(a.config?.metadata?.["alchemy.replica"]) -
2784	Number(b.config?.metadata?.["alchemy.replica"]),
2785	);
2786	expect(mixedOld.map((machine) => machine.state)).toEqual([
2787	mode === "stop" ? "stopped" : "suspended",
2788	mode === "stop" ? "stopped" : "suspended",
2789	"started",
2790	]);
2791	const retained = yield* checkedDeployment(
2792	actor,
2793	proxy,
2794	"retained",
2795	mode,
2796	3,
2797	[2],
2798	lowered.machineIds,
2799	);
2800	yield* idleAll(site.appName, retained.machineIds, mode);
2801	const retainedSlots = assertSlots(
2802	yield* census(site.appName),
2803	[2],
2804	);
2805	yield* machines.startMachine({
2806	app_name: site.appName,
2807	machine_id: retainedSlots[2]!.id!,
2808	});
2809	yield* waitState(
2810	site.appName,
2811	retainedSlots[2]!.id!,
2812	"started",
2813	);
2814	expect(
2815	assertSlots(yield* census(site.appName), [2]).map(
2816	(machine) => machine.state,
2817	),
2818	).toEqual([
2819	mode === "stop" ? "stopped" : "suspended",
2820	mode === "stop" ? "stopped" : "suspended",
2821	"started",
2822	]);
2823	const retainedFloor = yield* checkedDeployment(
2824	actor,
2825	proxy,
2826	"retained",
2827	mode,
2828	3,
2829	[0, 2],
2830	retained.machineIds,
2831	2,
2832	);
2833	yield* idleAll(site.appName, retainedFloor.machineIds, mode);
2834	const mixed = yield* checkedDeployment(
2835	actor,
2836	proxy,
2837	"one",
2838	mode,
2839	3,
2840	[0, 1, 2],
2841	retainedFloor.machineIds,
2842	0,
2843	true,
2844	);
2845	yield* idleAll(site.appName, mixed.machineIds, mode);
2846	yield* checkedDeployment(
2847	actor,
2848	proxy,
2849	"one",
2850	mode,
2851	3,
2852	[0],
2853	mixed.machineIds,
2854	);
2855	} finally {
2856	yield* stack.destroy();
2857	yield* assertAppGone(site.appName);
2858	}
2859	}).pipe(Effect.scoped),
2860	{ timeout: 1_200_000 },
2861	);
2862
2863	for (const recoveredCount of [3, 1, 4]) {
2864	test.provider(
2865	`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
2866	(stack) =>
2867	Effect.gen(function* () {
2868	yield* stack.destroy();
2869	const site = yield* stack.deploy(Fly.App("Site"));
2870	try {
2871	const proxy = yield* readinessProxy();
2872	const initial = yield* readinessActor(
2873	stack,
2874	`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
2875	file,
2876	proxy,
2877	);
2878	const first = yield* checkedDeployment(
2879	initial,
2880	proxy,
2881	"one",
2882	mode,
2883	3,
2884	[0],
2885	);
2886	yield* idleAll(site.appName, first.machineIds, mode);
2887	const source = (yield* census(site.appName))[0]!;
2888	const metadata = source.config!.metadata!;
2889	const attemptBegin = proxy.events.length;
2890	yield* Effect.gen(function* () {
2891	try {
2892	const gate = yield* Deferred.make<string>();
2893	const actor = yield* readinessActor(
2894	stack,
2895	`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
2896	file,
2897	proxy,
2898	(event) =>
2899	event.method === "POST" &&
2900	event.path ===
2901	`/v1/apps/${site.appName}/machines/${event.machineId}` &&
2902	event.metadata?.["alchemy.replica"] === "1" &&
2903	event.metadata["alchemy.fqn"] ===
2904	metadata["alchemy.fqn"] &&
2905	event.metadata["alchemy.generation"] !==
2906	metadata["alchemy.generation"] &&
2907	event.metadata["alchemy.readiness-roles"] ===
2908	"run,run,idle" &&
2909	event.metadata["alchemy.idle-policy-restored"] ===
2910	"true" &&
2911	event.phase === "promoting"
2912	? Deferred.succeed(gate, event.machineId!).pipe(
2913	Effect.andThen(Effect.never),
2914	)
2915	: Effect.void,
2916	);
2917	const attempt = yield* deployWorker(
2918	actor,
2919	"two",
2920	topology(mode, 3, 2),
2921	).pipe(Effect.scoped, Effect.forkScoped);
2922	const barrierMachine = yield* Deferred.await(gate).pipe(
2923	Effect.timeout("180 seconds"),
2924	);
2925	const pending = yield* census(site.appName);
2926	expect(
2927	pending.filter((machine) =>
2928	first.machineIds.includes(machine.id!),
2929	),
2930	).toHaveLength(3);
2931	expect(
2932	proxy.events.some((event) =>
2933	retires(event, first.machineIds),
2934	),
2935	).toBe(false);
2936	const candidates = yield* Effect.forEach(
2937	pending.filter(
2938	(machine) => !first.machineIds.includes(machine.id!),
2939	),
2940	(machine) =>
2941	machines.getMachine({
2942	app_name: site.appName,
2943	machine_id: machine.id!,
2944	}),
2945	);
2946	expect(candidates).toHaveLength(3);
2947	expect(
2948	candidates.every(
2949	(machine) =>
2950	machine.config?.metadata?.["alchemy.phase"] ===
2951	"promoting",
2952	),
2953	).toBe(true);
2954	const pendingSlots = assertSlots(candidates, [0, 1]);
2955	expect(
2956	pendingSlots[0]!.config?.metadata?.[
2957	"alchemy.idle-policy-restored"
2958	],
2959	).toBe("true");
2960	expect(
2961	pendingSlots[1]!.config?.metadata?.[
2962	"alchemy.idle-policy-restored"
2963	],
2964	).toBe("false");
2965	expect(
2966	proxy.readiness.some(
2967	(event) =>
2968	event.stage === "forwarded" &&
2969	event.method === "GET" &&
2970	event.machineId === pendingSlots[0]!.id &&
2971	event.instanceId === pendingSlots[0]!.instance_id &&
2972	event.metadata?.["alchemy.idle-policy-restored"] ===
2973	"true" &&
2974	event.phase === "promoting" &&
2975	event.state === "started" &&
2976	readinessChecksPassing(event.checks, [
2977	"ready",
2978	"servicecheck-00-http-80",
2979	]),
2980	),
2981	).toBe(true);
2982	expect(barrierMachine).toBe(pendingSlots[1]!.id);
2983	expect(
2984	autostopMode(
2985	pendingSlots[0]!.config?.services?.[0]?.autostop,
2986	),
2987	).toBe(mode);
2988	expect(
2989	autostopMode(
2990	pendingSlots[1]!.config?.services?.[0]?.autostop,
2991	),
2992	).toBe("off");
2993	expect(
2994	proxy.events
2995	.slice(attemptBegin)
2996	.some(
2997	(event) =>
2998	event.method === "POST" &&
2999	event.path.endsWith(
3000	`/machines/${barrierMachine}`,
3001	) &&
3002	event.phase === "promoting",
3003	),
3004	).toBe(false);
3005	const read = yield* observeReplicaSet({
3006	appName: site.appName,
3007	id: "Worker",
3008	type: "Fly.Machine",
3009	fqn: metadata["alchemy.fqn"]!,
3010	resourceInstanceId: metadata["alchemy.instance"]!,
3011	baseName: first.baseName,
3012	machineIds: first.machineIds,
3013	});
3014	expect(read?.machineIds).toEqual(first.machineIds);
3015	expect(read?.count).toBe(3);
3016	expect(read?.rolloutPending).toBe(true);
3017	const interruption = yield* Fiber.interrupt(attempt).pipe(
3018	Effect.forkScoped,
3019	);
3020	yield* Effect.yieldNow;
3021	yield* Effect.sync(() => {
3022	proxy.clear();
3023	proxy.release();
3024	});
3025	yield* Fiber.join(interruption).pipe(
3026	Effect.timeout("180 seconds"),
3027	);
3028	expect(
3029	Exit.hasInterrupts(yield* Fiber.await(attempt)),
3030	).toBe(true);
3031	expect(
3032	proxy.events
3033	.slice(attemptBegin)
3034	.some(
3035	(event) =>
3036	first.machineIds.includes(event.machineId!) &&
3037	event.path.endsWith("/start"),
3038	),
3039	).toBe(false);
3040	expect(
3041	proxy.readiness.some(
3042	(event) =>
3043	event.stage === "request" &&
3044	event.path.endsWith("/metadata") &&
3045	event.phase === "active" &&
3046	candidates.some(
3047	(machine) => machine.id === event.machineId,
3048	),
3049	),
3050	).toBe(false);
3051	expect(
3052	proxy.events.some((event) =>
3053	retires(event, first.machineIds),
3054	),
3055	).toBe(false);
3056	const resumed = yield* readinessActor(
3057	stack,
3058	`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
3059	file,
3060	proxy,
3061	);
3062	const begin = proxy.readiness.length;
3063	const recovered = yield* deployWorker(
3064	resumed,
3065	"two",
3066	topology(mode, recoveredCount, 2),
3067	).pipe(Effect.scoped);
3068	expect(recovered.machineIds).toHaveLength(recoveredCount);
3069	if (recoveredCount === 3) {
3070	expect([...recovered.machineIds].sort()).toEqual(
3071	candidates.map((machine) => machine.id).sort(),
3072	);
3073	} else {
3074	expect(
3075	recovered.machineIds.every(
3076	(id) =>
3077	!pending.some((machine) => machine.id === id),
3078	),
3079	).toBe(true);
3080	}
3081	const runningSlots = recoveredCount === 1 ? [0] : [0, 1];
3082	const live = yield* assertTopology(
3083	site.appName,
3084	recovered.machineIds,
3085	mode,
3086	runningSlots,
3087	2,
3088	);
3089	assertReadinessCommit(
3090	proxy.readiness.slice(begin),
3091	pending
3092	.map((machine) => machine.id!)
3093	.filter((id) => !recovered.machineIds.includes(id)),
3094	live,
3095	runningSlots,
3096	["ready", "servicecheck-00-http-80"],
3097	true,
3098	);
3099	const unchanged = yield* deployWorker(
3100	resumed,
3101	"two",
3102	topology(mode, recoveredCount, 2),
3103	).pipe(Effect.scoped);
3104	expect(unchanged.machineIds).toEqual(
3105	recovered.machineIds,
3106	);
3107	const committed = yield* observeReplicaSet({
3108	appName: site.appName,
3109	id: "Worker",
3110	type: "Fly.Machine",
3111	fqn: metadata["alchemy.fqn"]!,
3112	resourceInstanceId: metadata["alchemy.instance"]!,
3113	machineIds: recovered.machineIds,
3114	});
3115	expect(committed?.rolloutPending).toBe(false);
3116	expect(committed?.machineIds).toEqual(
3117	recovered.machineIds,
3118	);
3119	} finally {
3120	yield* Effect.sync(() => {
3121	proxy.clear();
3122	proxy.release();
3123	});
3124	}
3125	}).pipe(Effect.scoped);
3126	} finally {
3127	yield* stack.destroy();
3128	yield* assertAppGone(site.appName);
3129	}
3130	}).pipe(Effect.scoped),
3131	{ timeout: 900_000 },
3132	);
3133	}
3134
3135	test.provider(
3136	`S11 public traffic autostarts ${mode} and Fly automatically returns it to idle within 15 minutes`,
3137	(stack) =>
3138	Effect.gen(function* () {
3139	yield* stack.destroy();
3140	const site = yield* stack.deploy(Fly.App("Site"));
3141	try {
3142	const output = yield* stack.deploy(
3143	Effect.gen(function* () {
3144	const app = yield* Fly.App("Site");
3145	yield* Fly.IpAssignment("Public", {
3146	app,
3147	type: "shared_v4",
3148	});
3149	return yield* Fly.Machine("Worker", {
3150	app,
3151	image: "nginx:alpine",
3152	deploy: {
3153	strategy: "bluegreen",
3154	healthTimeout: "60 seconds",
3155	},
3156	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
3157	services: [
3158	{
3159	protocol: "tcp",
3160	internalPort: 80,
3161	autostop: mode,
3162	autostart: true,
3163	minMachinesRunning: 0,
3164	ports: [{ port: 80, handlers: ["http"] }],
3165	checks: [checks.ready],
3166	},
3167	],
3168	});
3169	}),
3170	);
3171	yield* idleAll(site.appName, output.machineIds, mode);
3172	const client = yield* HttpClient.HttpClient;
3173	yield* Effect.gen(function* () {
3174	const response = yield* client.get(
3175	`http://${site.appName}.fly.dev`,
3176	);
3177	const body = yield* response.text;
3178	if (
3179	response.status !== 200 ||
3180	!body.includes("Welcome to nginx")
3181	) {
3182	return yield* Effect.fail(
3183	new Error(
3184	`Public Fly route is not ready: ${response.status}`,
3185	),
3186	);
3187	}
3188	expect(response.status).toBe(200);
3189	expect(body).toContain("Welcome to nginx");
3190	}).pipe(
3191	Effect.retry({
3192	schedule: Schedule.spaced("3 seconds"),
3193	times: 20,
3194	}),
3195	Effect.timeout("90 seconds"),
3196	);
3197	yield* waitState(site.appName, output.machineId, "started");
3198	const started = yield* Clock.currentTimeMillis;
3199	const idleState = mode === "stop" ? "stopped" : "suspended";
3200	// Only native GETs follow the last public request; no stop, suspend, or update substitutes.
3201	const idle = yield* machines
3202	.getMachine({
3203	app_name: site.appName,
3204	machine_id: output.machineId,
3205	})
3206	.pipe(
3207	Effect.tap((machine) =>
3208	Effect.logInfo("Automatic Fly idle observation", {
3209	mode,
3210	state: machine.state,
3211	instanceId: machine.instance_id,
3212	}),
3213	),
3214	Effect.repeat({
3215	schedule: Schedule.spaced("10 seconds"),
3216	times: 90,
3217	until: (machine) => machine.state === idleState,
3218	}),
3219	Effect.timeout("15 minutes"),
3220	);
3221	expect(idle.state).toBe(idleState);
3222	expect(idle.cordoned).toBe(false);
3223	expect(autostopMode(idle.config?.services?.[0]?.autostop)).toBe(
3224	mode,
3225	);
3226	expect(
3227	idle.config?.metadata?.["alchemy.idle-policy-restored"],
3228	).toBe("true");
3229	yield* Effect.logInfo("Automatic return to idle observed", {
3230	mode,
3231	elapsedMs: (yield* Clock.currentTimeMillis) - started,
3232	});
3233	expect(
3234	(yield* census(site.appName)).map((machine) => machine.id),
3235	).toEqual(output.machineIds);
3236	} finally {
3237	yield* stack.destroy();
3238	yield* assertAppGone(site.appName);
3239	}
3240	}),
3241	{ tags: ["provider:fly:ipassignment"], timeout: 1_200_000 },
3242	);
3243	}
3244	});
3245	},
3246	);
3247
3248	describe.sequential(
3249	"image identity",
3250	{
3251	tags: [
3252	"provider:fly",
3253	"provider:fly:app",
3254	"provider:fly:machine",
3255	"provider:fly:service",
3256	"live",
3257	],
3258	},
3259	() => {
3260	const { test } = Test.make({ providers: Fly.providers() });
3261
3262	test.provider(
3263	"S08 mixed real image digests and invalid recovery pin replace the complete generation",
3264	(stack) =>
3265	Effect.gen(function* () {
3266	yield* stack.destroy();
3267	const initial = yield* deployWorker(stack, "one", { count: 2 });
3268	const first = yield* machines.getMachine({
3269	app_name: initial.appName,
3270	machine_id: initial.machineIds[0]!,
3271	});
3272	const target = {
3273	app_name: initial.appName,
3274	machine_id: initial.machineIds[1]!,
3275	};
3276	const second = yield* machines.getMachine(target);
3277	yield* machines.updateMachine({
3278	...target,
3279	config: { ...second.config, image: "nginx:1.26-alpine" },
3280	});
3281	const drifted = yield* machines.getMachine(target).pipe(
3282	Effect.repeat({
3283	schedule: Schedule.spaced("2 seconds"),
3284	times: 8,
3285	until: (machine) =>
3286	machine.state === "started" &&
3287	!!machine.image_ref?.digest &&
3288	machine.image_ref.digest !== first.image_ref?.digest,
3289	}),
3290	);
3291	expect(drifted.state).toBe("started");
3292	expect(drifted.image_ref?.digest).toBeDefined();
3293	expect(drifted.image_ref?.digest).not.toBe(first.image_ref?.digest);
3294	const repaired = yield* deployWorker(stack, "one", {
3295	count: 2,
3296	deploy: { strategy: "bluegreen", healthTimeout: "25 seconds" },
3297	});
3298	expect(
3299	repaired.machineIds.every((id) => !initial.machineIds.includes(id)),
3300	).toBe(true);
3301	const live = yield* assertCommitted(
3302	initial.appName,
3303	repaired.machineIds,
3304	);
3305	expect(
3306	live.every((machine) =>
3307	machine.config?.metadata?.["alchemy.image"]?.endsWith(
3308	machine.image_ref!.digest!,
3309	),
3310	),
3311	).toBe(true);
3312	yield* stack.destroy();
3313	yield* assertAppGone(initial.appName);
3314	}),
3315	{ timeout: 300_000 },
3316	);
3317
3318	test.provider(
3319	"S08 a real Fly registry mutable-tag change after first resolution cannot split the generation",
3320	(stack) =>
3321	Effect.gen(function* () {
3322	yield* stack.destroy();
3323	const initial = yield* deployWorker(stack, "one");
3324	const publisher = yield* Effect.sync(() =>
3325	scratchStack(
3326	{ providers: Docker.providers(), stage: stack.stage },
3327	`${"S08 a real Fly registry mutable-tag change after first resolution cannot split the generation"} publisher`,
3328	"test/Fly/BlueGreen.test.ts",
3329	),
3330	);
3331	yield* publisher.destroy();
3332	yield* Effect.gen(function* () {
3333	const minted = yield* machines.createAppDeployToken({
3334	app_name: initial.appName,
3335	});
3336	expect(minted.token).toBeDefined();
3337	const publish = (tag: string) =>
3338	publisher.deploy(
3339	Docker.RemoteImage("Mutable", {
3340	name: "nginx",
3341	tag,
3342	platform: "linux/amd64",
3343	targetName: initial.appName,
3344	targetTag: "acceptance-mutable",
3345	registry: {
3346	server: "registry.fly.io",
3347	username: "x",
3348	password: Redacted.make(minted.token!),
3349	},
3350	}),
3351	);
3352	// Upload both layer sets before holding a provider call with its own bounded deadline.
3353	yield* publish("1.27-alpine");
3354	const original = yield* publish("1.26-alpine");
3355	expect(original.imageRef).toBe(
3356	`registry.fly.io/${initial.appName}:acceptance-mutable`,
3357	);
3358	const proxy = yield* transportProxy();
3359	yield* Effect.sync(() =>
3360	proxy.arm({
3361	match: (event) =>
3362	event.method === "POST" && event.path.endsWith("/machines"),
3363	action: "hold-response",
3364	remaining: 1,
3365	}),
3366	);
3367	yield* Effect.gen(function* () {
3368	const actor = yield* engineActor(
3369	stack,
3370	"S08 a real Fly registry mutable-tag change after first resolution cannot split the generation",
3371	"test/Fly/BlueGreen.test.ts",
3372	proxy.url,
3373	);
3374	const rollout = yield* deployWorker(actor, "two", {
3375	image: original.imageRef,
3376	count: 3,
3377	}).pipe(Effect.scoped, Effect.forkScoped);
3378	const held = yield* proxy.wait(
3379	(event) => event.stage === "held" && event.status! < 300,
3380	);
3381	expect(held.digest).toMatch(/^sha256:/);
3382	const moved = yield* publish("1.27-alpine");
3383	expect(moved.imageRef).toBe(original.imageRef);
3384	expect(moved.repoDigest).toBeDefined();
3385	expect(moved.repoDigest).not.toBe(original.repoDigest);
3386	expect(
3387	proxy.events.filter(
3388	(event) =>
3389	event.stage === "request" &&
3390	event.method === "POST" &&
3391	event.path.endsWith("/machines"),
3392	),
3393	).toHaveLength(1);
3394	const probe = yield* machines.createMachine({
3395	app_name: initial.appName,
3396	name: "mutable-tag-probe",
3397	region: "iad",
3398	skip_launch: true,
3399	skip_service_registration: true,
3400	config: { image: moved.imageRef },
3401	});
3402	yield* Effect.gen(function* () {
3403	expect(probe.image_ref?.digest).toMatch(/^sha256:/);
3404	expect(probe.image_ref?.digest).not.toBe(held.digest);
3405	yield* Effect.sync(proxy.release);
3406	const next = yield* Fiber.join(rollout).pipe(
3407	Effect.timeout("180 seconds"),
3408	);
3409	const creates = proxy.events.filter(
3410	(event) =>
3411	event.stage === "completed" &&
3412	event.method === "POST" &&
3413	event.path.endsWith("/machines") &&
3414	event.status! < 300,
3415	);
3416	expect(creates).toHaveLength(3);
3417	expect(
3418	creates.every((event) => event.digest === held.digest),
3419	).toBe(true);
3420	expect(
3421	creates
3422	.slice(1)
3423	.every((event) => event.image?.endsWith(`@${held.digest}`)),
3424	).toBe(true);
3425	for (const id of next.machineIds) {
3426	const machine = yield* machines.getMachine({
3427	app_name: initial.appName,
3428	machine_id: id,
3429	});
3430	expect(machine.image_ref?.digest).toBe(held.digest);
3431	expect(
3432	machine.config?.metadata?.["alchemy.image"]?.endsWith(
3433	`@${held.digest}`,
3434	),
3435	).toBe(true);
3436	}
3437	expect(next.machineIds).toContain(held.machineId);
3438	expect(next.machineIds).not.toContain(initial.machineId);
3439	}).pipe(
3440	Effect.ensuring(
3441	machines
3442	.deleteMachine({
3443	app_name: initial.appName,
3444	machine_id: probe.id!,
3445	force: true,
3446	})
3447	.pipe(
3448	Effect.catchTag("NotFound", () => Effect.void),
3449	Effect.orDie,
3450	),
3451	),
3452	);
3453	}).pipe(
3454	Effect.ensuring(
3455	Effect.sync(() => {
3456	proxy.clear();
3457	proxy.release();
3458	}),
3459	),
3460	Effect.scoped,
3461	);
3462	const live = yield* machines.listMachines({
3463	app_name: initial.appName,
3464	});
3465	const active = live.filter(
3466	(machine) => machine.state !== "destroyed",
3467	);
3468	expect(active).toHaveLength(3);
3469	yield* assertCommitted(
3470	initial.appName,
3471	active.map((machine) => machine.id!),
3472	);
3473	}).pipe(Effect.ensuring(publisher.destroy().pipe(Effect.orDie)));
3474	yield* stack.destroy();
3475	yield* assertAppGone(initial.appName);
3476	}).pipe(Effect.scoped),
3477	{
3478	tags: ["provider:docker", "provider:docker:remoteimage"],
3479	timeout: 600_000,
3480	},
3481	);
3482	},
3483	);
3484
3485	describe.sequential(
3486	"fiber interruption",
3487	{
3488	tags: [
3489	"provider:fly",
3490	"provider:fly:app",
3491	"provider:fly:machine",
3492	"provider:fly:service",
3493	"live",
3494	],
3495	},
3496	() => {
3497	const file = "test/Fly/BlueGreen.test.ts";
3498	const { test } = Test.make({ providers: Fly.providers() });
3499
3500	describe.sequential("in-process engine interruption", () => {
3501	for (const phase of ["create", "promotion", "retirement"] as const) {
3502	test.provider(
3503	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
3504	(stack) =>
3505	Effect.gen(function* () {
3506	yield* stack.destroy();
3507	const initial = yield* deployWorker(stack, "one");
3508	const proxy = yield* transportProxy();
3509	const match = (event: TransportEvent) =>
3510	phase === "create"
3511	? event.method === "POST" && event.path.endsWith("/machines")
3512	: phase === "promotion"
3513	? event.path.endsWith("/uncordon")
3514	: event.method === "DELETE" &&
3515	/\/machines\/[^/]+$/.test(event.path) &&
3516	event.machineId === initial.machineId;
3517	yield* Effect.sync(() =>
3518	proxy.arm({ match, action: "hold-response", remaining: 1 }),
3519	);
3520	yield* Effect.gen(function* () {
3521	const actor = yield* engineActor(
3522	stack,
3523	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
3524	file,
3525	proxy.url,
3526	);
3527	const interrupted = yield* deployWorker(actor, "two").pipe(
3528	Effect.scoped,
3529	Effect.forkScoped,
3530	);
3531	const barrier = yield* proxy.wait(
3532	(event) =>
3533	event.stage === "held" &&
3534	event.status! >= 200 &&
3535	event.status! < 300 &&
3536	match(event),
3537	);
3538	expect(barrier.machineId).toBeDefined();
3539	const interruption = yield* Fiber.interrupt(interrupted).pipe(
3540	Effect.forkScoped,
3541	);
3542	yield* Effect.yieldNow;
3543	// Let accepted uninterruptible work and finalizers settle; this is not process-kill evidence.
3544	yield* Effect.sync(() => {
3545	proxy.clear();
3546	proxy.release();
3547	});
3548	yield* Fiber.join(interruption).pipe(
3549	Effect.timeout("90 seconds"),
3550	);
3551	const exit = yield* Fiber.await(interrupted);
3552	expect(Exit.hasInterrupts(exit)).toBe(true);
3553	const surviving = yield* census(initial.appName);
3554	const green = surviving.filter(
3555	(machine) => machine.id !== initial.machineId,
3556	);
3557	if (phase === "create") {
3558	expect(
3559	surviving.some(
3560	(machine) => machine.id === initial.machineId,
3561	),
3562	).toBe(true);
3563	expect(green.length).toBeLessThanOrEqual(1);
3564	} else {
3565	expect(green).toHaveLength(1);
3566	expect(green[0]!.cordoned).toBe(false);
3567	}
3568	const resumed = yield* engineActor(
3569	stack,
3570	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 fiber interruption at a completed ${phase} response barrier`,
3571	file,
3572	);
3573	expect(resumed.state).not.toBe(actor.state);
3574	const recovered = yield* deployWorker(resumed, "two").pipe(
3575	Effect.scoped,
3576	);
3577	if (green.length)
3578	expect(recovered.machineIds).toEqual(
3579	green.map((machine) => machine.id),
3580	);
3581	yield* assertCommitted(initial.appName, recovered.machineIds);
3582	}).pipe(
3583	Effect.ensuring(
3584	Effect.sync(() => {
3585	proxy.clear();
3586	proxy.release();
3587	}),
3588	),
3589	Effect.scoped,
3590	);
3591	yield* stack.destroy();
3592	yield* assertAppGone(initial.appName);
3593	}).pipe(Effect.scoped),
3594	{ timeout: 300_000 },
3595	);
3596	}
3597	});
3598	},
3599	);
3600
3601	describe.sequential(
3602	"native leases",
3603	{ tags: ["provider:fly", "provider:fly:service", "live"] },
3604	() => {
3605	const { test } = Test.make({ providers: Fly.providers() });
3606
3607	type Target = { app_name: string; machine_id: string };
3608
3609	// Transport failures can retain authorization and lease headers.
3610	const sanitizeFailure = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
3611	effect.pipe(
3612	Effect.mapError(
3613	(error) =>
3614	new Error(
3615	error instanceof Error ? error.name : "Fly SDK probe failed",
3616	),
3617	),
3618	);
3619
3620	test(
3621	"pure lease cleanup fails before successful-body teardown and keeps diagnostics private",
3622	Effect.gen(function* () {
3623	let teardownReached = false;
3624	const exit = yield* Effect.gen(function* () {
3625	yield* scopedLeaseCleanup(
3626	Effect.fail({
3627	_tag: "ReleaseProbeFailure",
3628	privateValue: "cleanup-secret",
3629	}),
3630	).pipe(Effect.scoped);
3631	teardownReached = true;
3632	}).pipe(Effect.exit);
3633	expect(teardownReached).toBe(false);
3634	expect(Exit.isFailure(exit)).toBe(true);
3635	if (Exit.isFailure(exit)) {
3636	expect(Cause.hasDies(exit.cause)).toBe(true);
3637	expect(Cause.pretty(exit.cause)).toContain(
3638	"Lease cleanup failed (ReleaseProbeFailure)",
3639	);
3640	expect(Cause.pretty(exit.cause)).not.toContain("cleanup-secret");
3641	}
3642	}),
3643	);
3644
3645	test(
3646	"pure lease cleanup retains the primary failure alongside cleanup failure",
3647	Effect.gen(function* () {
3648	const exit = yield* Effect.gen(function* () {
3649	yield* scopedLeaseCleanup(
3650	Effect.fail({ _tag: "ReleaseProbeFailure" }),
3651	);
3652	return yield* Effect.fail("primary failure");
3653	}).pipe(Effect.scoped, Effect.exit);
3654	expect(Exit.isFailure(exit)).toBe(true);
3655	if (Exit.isFailure(exit)) {
3656	expect(Cause.findError(exit.cause)).toEqual(
3657	Result.succeed("primary failure"),
3658	);
3659	expect(Cause.hasDies(exit.cause)).toBe(true);
3660	expect(Cause.pretty(exit.cause)).toContain(
3661	"Lease cleanup failed (ReleaseProbeFailure)",
3662	);
3663	}
3664	}),
3665	);
3666
3667	test(
3668	"pure lease cleanup timeout remains a test failure",
3669	Effect.gen(function* () {
3670	const exit = yield* scopedLeaseCleanup(
3671	Effect.never.pipe(Effect.timeout("1 millis")),
3672	).pipe(Effect.scoped, Effect.exit);
3673	expect(Exit.isFailure(exit)).toBe(true);
3674	if (Exit.isFailure(exit)) {
3675	expect(Cause.pretty(exit.cause)).toContain(
3676	"Lease cleanup failed (TimeoutError)",
3677	);
3678	}
3679	}),
3680	);
3681
3682	test(
3683	"pure lease cleanup does not repeat a successful explicit release or confirmed absence",
3684	Effect.gen(function* () {
3685	const calls = yield* Ref.make(0);
3686	yield* Effect.gen(function* () {
3687	const cleanup = yield* scopedLeaseCleanup(
3688	Ref.update(calls, (n) => n + 1),
3689	);
3690	yield* cleanup.release;
3691	yield* cleanup.release;
3692	}).pipe(Effect.scoped);
3693	expect(yield* Ref.get(calls)).toBe(1);
3694	yield* Effect.gen(function* () {
3695	const cleanup = yield* scopedLeaseCleanup(
3696	Ref.update(calls, (n) => n + 1),
3697	);
3698	yield* cleanup.complete;
3699	}).pipe(Effect.scoped);
3700	expect(yield* Ref.get(calls)).toBe(1);
3701	}),
3702	);
3703
3704	const differentNonce = (nonce: string) =>
3705	`${nonce[0] === "a" ? "b" : "a"}${nonce.slice(1)}`;
3706
3707	test(
3708	"P4 pure HTTP 408 status mapping agrees with the Fly timeout model",
3709	Effect.sync(() => {
3710	expect(HTTP_STATUS_MAP[408]).toBe(GatewayTimeout);
3711	expect(RETRYABLE_HTTP_STATUSES.has(408)).toBe(true);
3712	}),
3713	);
3714
3715	const observeTransport = Effect.gen(function* () {
3716	const client = yield* HttpClient.HttpClient;
3717	return client.pipe(
3718	HttpClient.transform((response, request) =>
3719	response.pipe(
3720	Effect.tap((actual) =>
3721	Effect.logInfo("Fly SDK probe transport", {
3722	method: request.method,
3723	path: request.url.split("?")[0],
3724	status: actual.status,
3725	ttl:
3726	request.url.match(/[?&]ttl=(\d+)/)?.[1] ??
3727	request.urlParams.params.find(([key]) => key === "ttl")?.[1],
3728	nonceHeader: !!request.headers["fly-machine-lease-nonce"],
3729	}),
3730	),
3731	),
3732	),
3733	);
3734	});
3735
3736	const scopedLeaseCleanup = <A, E extends { readonly _tag: string }, R>(
3737	action: Effect.Effect<A, E, R>,
3738	) =>
3739	Effect.gen(function* () {
3740	const completed = yield* Ref.make(false);
3741	const complete = Ref.set(completed, true);
3742	const release = Effect.uninterruptibleMask((restore) =>
3743	Effect.gen(function* () {
3744	if (yield* Ref.get(completed)) return;
3745	yield* restore(action);
3746	yield* complete;
3747	}),
3748	);
3749	yield* Effect.addFinalizer(() =>
3750	release.pipe(
3751	Effect.interruptible,
3752	Effect.catchCause((cause) => {
3753	const error = Cause.findError(cause);
3754	const tag = Result.isSuccess(error)
3755	? error.success._tag
3756	: Cause.hasInterrupts(cause)
3757	? "Interrupted"
3758	: "Defect";
3759	return Effect.die(new Error(`Lease cleanup failed (${tag})`));
3760	}),
3761	),
3762	);
3763	return { release, complete };
3764	});
3765
3766	const lease = (target: Target, ttl = 120) =>
3767	Effect.gen(function* () {
3768	yield* Effect.logInfo("P1 acquiring native lease", {
3769	machineId: target.machine_id,
3770	ttl,
3771	});
3772	const acquired = yield* machines
3773	.createMachineLease({ ...target, ttl })
3774	.pipe(Effect.timeout("15 seconds"));
3775	const nonce = acquired.data?.nonce;
3776	if (!nonce)
3777	return yield* Effect.fail(new Error("Lease response has no nonce"));
3778	const cleanup = yield* scopedLeaseCleanup(
3779	machines
3780	.machinesReleaseLease({ ...target, lease_nonce: nonce })
3781	.pipe(Retry.none, Effect.timeout("15 seconds")),
3782	);
3783	return {
3784	acquired,
3785	nonce,
3786	release: cleanup.release,
3787	confirmDeleted: expectGone(target).pipe(
3788	Effect.andThen(cleanup.complete),
3789	),
3790	confirmExpired: Effect.gen(function* () {
3791	const now = yield* Effect.sync(() => Math.floor(Date.now() / 1000));
3792	expect(acquired.data?.expires_at).toBeLessThanOrEqual(now);
3793	const current = yield* machines.getMachineLease(target).pipe(
3794	Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
3795	Effect.timeout("15 seconds"),
3796	);
3797	expect(current?.data?.nonce === nonce).toBe(false);
3798	yield* cleanup.complete;
3799	}),
3800	};
3801	});
3802
3803	const expectFailure = <A, E extends { readonly _tag: string }, R>(
3804	attempt: Effect.Effect<A, E, R>,
3805	tag: "Conflict" | "Forbidden" = "Conflict",
3806	) =>
3807	attempt.pipe(
3808	Effect.result,
3809	Effect.tap((result) =>
3810	Effect.sync(() => {
3811	expect(Result.isFailure(result)).toBe(true);
3812	if (Result.isFailure(result)) expect(result.failure._tag).toBe(tag);
3813	}),
3814	),
3815	Effect.asVoid,
3816	);
3817
3818	const waitState = (target: Target, state: string) =>
3819	machines.getMachine(target).pipe(
3820	Retry.none,
3821	Effect.repeat({
3822	schedule: Schedule.spaced("2 seconds"),
3823	until: (machine) => machine.state === state,
3824	times: 8,
3825	}),
3826	Effect.tap((machine) =>
3827	Effect.sync(() => expect(machine.state).toBe(state)),
3828	),
3829	);
3830
3831	const expectGone = (target: Target) =>
3832	machines.getMachine(target).pipe(
3833	Retry.none,
3834	Effect.map((machine) => machine.state === "destroyed"),
3835	Effect.catchTag("NotFound", () => Effect.succeed(true)),
3836	Effect.repeat({
3837	schedule: Schedule.spaced("1 second"),
3838	until: Boolean,
3839	times: 8,
3840	}),
3841	Effect.tap((gone) => Effect.sync(() => expect(gone).toBe(true))),
3842	);
3843
3844	test.provider(
3845	"P1/P2 lease envelope, query TTL, renewal, contention and release on a real Machine",
3846	(stack) =>
3847	Effect.gen(function* () {
3848	yield* stack.destroy();
3849	const created = yield* stack.deploy(
3850	Effect.gen(function* () {
3851	const app = yield* Fly.App("LeaseSite");
3852	return yield* Fly.Machine("LeaseTarget", {
3853	app,
3854	region: "iad",
3855	image: "nginx:alpine",
3856	guest: { cpus: 1, memoryMb: 256 },
3857	skipLaunch: true,
3858	});
3859	}),
3860	);
3861	const target = {
3862	app_name: created.appName,
3863	machine_id: created.machineId,
3864	};
3865	const client = yield* observeTransport;
3866	yield* Effect.gen(function* () {
3867	const before = yield* Effect.sync(() =>
3868	Math.floor(Date.now() / 1000),
3869	);
3870	const held = yield* lease(target);
3871	const { acquired, nonce } = held;
3872	expect(acquired.status).toBe("success");
3873	expect(typeof acquired.data?.owner).toBe("string");
3874	expect(typeof acquired.data?.version).toBe("string");
3875	expect(acquired.data?.expires_at).toBeGreaterThanOrEqual(
3876	before + 115,
3877	);
3878	expect(acquired.data?.expires_at).toBeLessThanOrEqual(before + 130);
3879	yield* Effect.logInfo("P1 lease envelope verified", {
3880	keys: Object.keys(acquired),
3881	dataKeys: Object.keys(acquired.data ?? {}),
3882	ttlSeconds: acquired.data!.expires_at! - before,
3883	});
3884	const observed = yield* machines.getMachineLease(target);
3885	expect(observed.data?.expires_at).toBe(acquired.data?.expires_at);
3886	yield* expectFailure(
3887	machines.createMachineLease({ ...target, ttl: 120 }),
3888	);
3889	yield* expectFailure(
3890	machines.createMachineLease({
3891	...target,
3892	ttl: 120,
3893	lease_nonce: differentNonce(nonce),
3894	}),
3895	);
3896	yield* expectFailure(
3897	machines.machinesReleaseLease({
3898	...target,
3899	lease_nonce: differentNonce(nonce),
3900	}),
3901	"Forbidden",
3902	);
3903	const refreshed = yield* machines.createMachineLease({
3904	...target,
3905	ttl: 120,
3906	lease_nonce: nonce,
3907	});
3908	expect(refreshed.data?.nonce === nonce).toBe(true);
3909	expect(refreshed.data?.expires_at).toBeGreaterThanOrEqual(
3910	acquired.data!.expires_at!,
3911	);
3912	yield* held.release;
3913	const next = yield* lease(target);
3914	expect(next.nonce !== nonce).toBe(true);
3915	}).pipe(
3916	Effect.scoped,
3917	Retry.none,
3918	Effect.provideService(HttpClient.HttpClient, client),
3919	);
3920	yield* stack.destroy();
3921	yield* expectGone(target);
3922	}).pipe(sanitizeFailure),
3923	{ tags: ["provider:fly:app", "provider:fly:machine"], timeout: 120_000 },
3924	);
3925
3926	test.provider(
3927	"P1/P2 native nonce enforcement and metadata boundary on a real Machine",
3928	(stack) =>
3929	Effect.gen(function* () {
3930	yield* stack.destroy();
3931	const created = yield* stack.deploy(
3932	Effect.gen(function* () {
3933	const app = yield* Fly.App("EnforcementSite");
3934	return yield* Fly.Machine("EnforcementTarget", {
3935	app,
3936	region: "iad",
3937	image: "nginx:alpine",
3938	guest: { cpus: 1, memoryMb: 256 },
3939	restart: { policy: "always" },
3940	services: [
3941	{
3942	protocol: "tcp",
3943	internalPort: 80,
3944	ports: [{ port: 80, handlers: ["http"] }],
3945	checks: [
3946	{
3947	type: "http",
3948	port: 80,
3949	path: "/",
3950	interval: "2s",
3951	timeout: "1s",
3952	},
3953	],
3954	},
3955	],
3956	});
3957	}),
3958	);
3959	const target = {
3960	app_name: created.appName,
3961	machine_id: created.machineId,
3962	};
3963	const client = yield* observeTransport;
3964	yield* Effect.gen(function* () {
3965	const held = yield* lease(target);
3966	const { acquired, nonce } = held;
3967	const verifyAuthority = (minimumRemainingSeconds = 0) =>
3968	Effect.gen(function* () {
3969	const held = yield* machines
3970	.getMachineLease(target)
3971	.pipe(Effect.timeout("15 seconds"));
3972	const now = yield* Effect.sync(() =>
3973	Math.floor(Date.now() / 1000),
3974	);
3975	expect(held.data?.nonce === nonce).toBe(true);
3976	expect(held.data?.owner === acquired.data?.owner).toBe(true);
3977	expect(held.data?.expires_at).toBeGreaterThan(
3978	now + minimumRemainingSeconds,
3979	);
3980	});
3981	const refreshAuthority = Effect.gen(function* () {
3982	yield* verifyAuthority(15);
3983	const refreshed = yield* machines
3984	.createMachineLease({ ...target, ttl: 120, lease_nonce: nonce })
3985	.pipe(Effect.timeout("15 seconds"));
3986	expect(refreshed.data?.nonce === nonce).toBe(true);
3987	// A complete request and its authority readback must fit inside the lease.
3988	yield* verifyAuthority(90);
3989	});
3990	const withAuthority = <A, E, R>(phase: Effect.Effect<A, E, R>) =>
3991	Effect.gen(function* () {
3992	yield* refreshAuthority;
3993	const value = yield* phase.pipe(Effect.timeout("60 seconds"));
3994	yield* verifyAuthority();
3995	return value;
3996	});
3997	const initial = yield* withAuthority(machines.getMachine(target));
3998	const outcomes: string[] = [];
3999	for (const lease_nonce of [undefined, differentNonce(nonce)]) {
4000	const request = { ...target, lease_nonce };
4001	const attempts = [
4002	[
4003	"update",
4004	machines.updateMachine({
4005	...request,
4006	config: initial.config,
4007	}),
4008	],
4009	["start", machines.startMachine(request)],
4010	["cordon", machines.cordonMachine(request)],
4011	["uncordon", machines.uncordonMachine(request)],
4012	[
4013	"stop",
4014	machines.stopMachine({
4015	...request,
4016	signal: "SIGTERM",
4017	timeout: "5s",
4018	}),
4019	],
4020	["suspend", machines.suspendMachine(request)],
4021	[
4022	"restart",
4023	machines.restartMachine({
4024	...request,
4025	signal: "SIGTERM",
4026	timeout: "5s",
4027	}),
4028	],
4029	["delete", machines.deleteMachine({ ...request, force: true })],
4030	] as const;
4031	for (const [operation, attempt] of attempts) {
4032	yield* refreshAuthority;
4033	const result = yield* attempt.pipe(
4034	Effect.timeout("60 seconds"),
4035	Effect.result,
4036	);
4037	yield* verifyAuthority();
4038	yield* Effect.logInfo("P1 mutation enforcement", {
4039	operation,
4040	nonce: lease_nonce ? "wrong" : "absent",
4041	outcome: Result.isFailure(result)
4042	? result.failure._tag
4043	: "accepted",
4044	});
4045	outcomes.push(
4046	Result.isFailure(result) ? result.failure._tag : "accepted",
4047	);
4048	expect(Result.isFailure(result)).toBe(true);
4049	}
4050	}
4051	const intact = yield* withAuthority(machines.getMachine(target));
4052	expect(intact.state).toBe("started");
4053	expect(intact.instance_id).toBe(initial.instance_id);
4054	const metadata = yield* withAuthority(
4055	Effect.gen(function* () {
4056	yield* machines.upsertMachineMetadata({
4057	...target,
4058	key: "lease-probe",
4059	value: "metadata-is-not-fenced",
4060	});
4061	const observed = yield* machines.getMachine(target);
4062	expect(observed.config?.metadata?.["lease-probe"]).toBe(
4063	"metadata-is-not-fenced",
4064	);
4065	return observed;
4066	}),
4067	);
4068	yield* Effect.logInfo(
4069	"P2 metadata mutation bypasses target lease",
4070	{
4071	checksAfterMutation: metadata.checks?.length ?? 0,
4072	},
4073	);
4074	const request = { ...target, lease_nonce: nonce };
4075	yield* withAuthority(
4076	Effect.gen(function* () {
4077	yield* machines.updateMachine({
4078	...request,
4079	config: {
4080	...metadata.config,
4081	env: { ...metadata.config?.env, LEASE_PROBE: "updated" },
4082	},
4083	});
4084	yield* waitState(target, "started");
4085	expect(
4086	(yield* machines.getMachine(target)).config?.env?.LEASE_PROBE,
4087	).toBe("updated");
4088	}),
4089	);
4090	yield* withAuthority(
4091	Effect.gen(function* () {
4092	yield* machines.cordonMachine(request);
4093	expect((yield* machines.getMachine(target)).cordoned).toBe(
4094	true,
4095	);
4096	}),
4097	);
4098	yield* withAuthority(
4099	Effect.gen(function* () {
4100	yield* machines.uncordonMachine(request);
4101	expect((yield* machines.getMachine(target)).cordoned).toBe(
4102	false,
4103	);
4104	}),
4105	);
4106	yield* withAuthority(
4107	Effect.gen(function* () {
4108	yield* machines.stopMachine({
4109	...request,
4110	signal: "SIGTERM",
4111	timeout: "5s",
4112	});
4113	yield* waitState(target, "stopped");
4114	}),
4115	);
4116	yield* withAuthority(
4117	Effect.gen(function* () {
4118	yield* machines.startMachine(request);
4119	yield* waitState(target, "started");
4120	}),
4121	);
4122	yield* withAuthority(
4123	Effect.gen(function* () {
4124	yield* machines.restartMachine({
4125	...request,
4126	signal: "SIGTERM",
4127	timeout: "5s",
4128	});
4129	yield* waitState(target, "started");
4130	}),
4131	);
4132	yield* withAuthority(
4133	Effect.gen(function* () {
4134	yield* machines.suspendMachine(request);
4135	yield* waitState(target, "suspended");
4136	}),
4137	);
4138	yield* withAuthority(
4139	Effect.gen(function* () {
4140	yield* machines.stopMachine({
4141	...request,
4142	signal: "SIGTERM",
4143	timeout: "5s",
4144	});
4145	yield* waitState(target, "stopped");
4146	}),
4147	);
4148	yield* refreshAuthority;
4149	yield* machines
4150	.deleteMachine(request)
4151	.pipe(Effect.timeout("60 seconds"));
4152	yield* held.confirmDeleted.pipe(Effect.timeout("30 seconds"));
4153	expect(outcomes).toEqual(Array(16).fill("Conflict"));
4154	}).pipe(
4155	Effect.scoped,
4156	Retry.none,
4157	Effect.provideService(HttpClient.HttpClient, client),
4158	);
4159	yield* stack.destroy();
4160	yield* expectGone(target);
4161	}).pipe(sanitizeFailure),
4162	{
4163	tags: ["provider:fly:app", "provider:fly:machine"],
4164	timeout: 30 * 60_000,
4165	},
4166	);
4167
4168	test.provider(
4169	"P2 partial acquisition cleanup, expiry and lost authority during a real mutation",
4170	(stack) =>
4171	Effect.gen(function* () {
4172	yield* stack.destroy();
4173	const created = yield* stack.deploy(
4174	Effect.gen(function* () {
4175	const app = yield* Fly.App("LifetimeSite");
4176	return yield* Fly.Machine("LifetimeTargets", {
4177	app,
4178	region: "iad",
4179	image: "nginx:alpine",
4180	count: 2,
4181	guest: { cpus: 1, memoryMb: 256 },
4182	skipLaunch: true,
4183	});
4184	}),
4185	);
4186	const first = {
4187	app_name: created.appName,
4188	machine_id: created.machineIds[0]!,
4189	};
4190	const second = {
4191	app_name: created.appName,
4192	machine_id: created.machineIds[1]!,
4193	};
4194	yield* Effect.gen(function* () {
4195	yield* lease(first);
4196	const partial = yield* Effect.gen(function* () {
4197	yield* lease(second);
4198	yield* machines.createMachineLease({ ...first, ttl: 120 });
4199	}).pipe(Effect.scoped, Effect.result);
4200	expect(Result.isFailure(partial)).toBe(true);
4201	if (Result.isFailure(partial))
4202	expect(partial.failure).toMatchObject({ _tag: "Conflict" });
4203	const before = yield* Effect.sync(() =>
4204	Math.floor(Date.now() / 1000),
4205	);
4206	const expired = yield* lease(second, 2);
4207	expect(expired.acquired.data?.expires_at).toBeLessThanOrEqual(
4208	before + 5,
4209	);
4210	yield* Effect.sleep("3 seconds");
4211	const successor = yield* lease(second);
4212	expect(successor.nonce !== expired.nonce).toBe(true);
4213	yield* expired.confirmExpired;
4214	const observed = yield* machines.getMachine(second);
4215	yield* expectFailure(
4216	machines.updateMachine({
4217	...second,
4218	lease_nonce: expired.nonce,
4219	config: observed.config,
4220	skip_launch: true,
4221	}),
4222	);
4223	yield* expectFailure(
4224	machines.createMachineLease({
4225	...second,
4226	lease_nonce: expired.nonce,
4227	ttl: 120,
4228	}),
4229	);
4230	yield* expectFailure(
4231	machines.machinesReleaseLease({
4232	...second,
4233	lease_nonce: expired.nonce,
4234	}),
4235	"Forbidden",
4236	);
4237	const stillHeld = yield* machines.getMachineLease(second);
4238	expect(stillHeld.data?.expires_at).toBe(
4239	successor.acquired.data?.expires_at,
4240	);
4241	}).pipe(Effect.scoped, Retry.none);
4242	yield* stack.destroy();
4243	yield* expectGone(first);
4244	yield* expectGone(second);
4245	}).pipe(sanitizeFailure),
4246	{ tags: ["provider:fly:app", "provider:fly:machine"], timeout: 120_000 },
4247	);
4248
4249	test.provider(
4250	"P4 duplicate create name Conflict permits exact owned Machine readback",
4251	(stack) =>
4252	Effect.gen(function* () {
4253	yield* stack.destroy();
4254	const created = yield* stack.deploy(
4255	Effect.gen(function* () {
4256	const app = yield* Fly.App("IdentitySite");
4257	return yield* Fly.Machine("IdentityTarget", {
4258	app,
4259	region: "iad",
4260	image: "nginx:alpine",
4261	guest: { cpus: 1, memoryMb: 256 },
4262	skipLaunch: true,
4263	});
4264	}),
4265	);
4266	const target = {
4267	app_name: created.appName,
4268	machine_id: created.machineId,
4269	};
4270	const observed = yield* machines.getMachine(target);
4271	yield* expectFailure(
4272	machines
4273	.createMachine({
4274	app_name: created.appName,
4275	name: created.name,
4276	region: "iad",
4277	config: observed.config,
4278	skip_launch: true,
4279	})
4280	.pipe(Retry.none),
4281	);
4282	const matches = (yield* machines.listMachines({
4283	app_name: created.appName,
4284	})).filter(
4285	(machine) =>
4286	machine.name === created.name && machine.state !== "destroyed",
4287	);
4288	expect(matches).toHaveLength(1);
4289	expect(matches[0]?.id).toBe(created.machineId);
4290	expect(matches[0]?.config?.metadata).toEqual(
4291	observed.config?.metadata,
4292	);
4293	expect(matches[0]?.image_ref?.digest).toBe(
4294	observed.image_ref?.digest,
4295	);
4296	yield* stack.destroy();
4297	yield* expectGone(target);
4298	}).pipe(sanitizeFailure),
4299	{ tags: ["provider:fly:app", "provider:fly:machine"], timeout: 120_000 },
4300	);
4301
4302	test.provider(
4303	"P4 lost completed create response reuses its unique owned name",
4304	(stack) =>
4305	Effect.gen(function* () {
4306	yield* stack.destroy();
4307	const app = yield* stack.deploy(Fly.App("LostCreateSite"));
4308	const request: machines.CreateMachineRequest = {
4309	app_name: app.appName,
4310	name: "lost-create-target",
4311	region: "iad",
4312	skip_launch: true,
4313	config: {
4314	image: "nginx:alpine",
4315	guest: { cpu_kind: "shared", cpus: 1, memory_mb: 256 },
4316	metadata: { "sdk-probe-owner": "completed-response-loss" },
4317	},
4318	};
4319	yield* Effect.gen(function* () {
4320	const proxy = yield* dropCompletedCreate(app.appName);
4321	const result = yield* machines
4322	.createMachine(request)
4323	.pipe(
4324	Retry.none,
4325	Effect.provideService(Credentials, proxy.credentials),
4326	Effect.timeout("15 seconds"),
4327	Effect.result,
4328	);
4329	yield* Effect.logInfo("P4 transport fault outcome", {
4330	forwarded: yield* proxy.forwarded,
4331	completedStatus: yield* proxy.completedStatus,
4332	outcome: Result.isFailure(result)
4333	? result.failure._tag
4334	: "accepted",
4335	rejection:
4336	Result.isFailure(result) && result.failure._tag === "BadRequest"
4337	? result.failure.message
4338	: undefined,
4339	});
4340	expect(Result.isFailure(result)).toBe(true);
4341	if (Result.isFailure(result)) {
4342	expect(result.failure._tag).toBe("HttpClientError");
4343	if (result.failure._tag === "HttpClientError")
4344	expect(result.failure.reason._tag).toBe("TransportError");
4345	}
4346	expect(yield* proxy.forwarded).toBe(1);
4347	expect([200, 201]).toContain(yield* proxy.completedStatus);
4348	yield* Effect.logInfo("P4 completed create response dropped", {
4349	status: yield* proxy.completedStatus,
4350	forwarded: yield* proxy.forwarded,
4351	});
4352	}).pipe(Effect.scoped);
4353	const matches = (yield* machines.listMachines({
4354	app_name: app.appName,
4355	})).filter(
4356	(machine) =>
4357	machine.name === request.name && machine.state !== "destroyed",
4358	);
4359	expect(matches).toHaveLength(1);
4360	expect(matches[0]?.config?.metadata?.["sdk-probe-owner"]).toBe(
4361	"completed-response-loss",
4362	);
4363	yield* expectFailure(
4364	machines.createMachine(request).pipe(Retry.none),
4365	);
4366	const after = (yield* machines.listMachines({
4367	app_name: app.appName,
4368	})).filter(
4369	(machine) =>
4370	machine.name === request.name && machine.state !== "destroyed",
4371	);
4372	expect(after.map((machine) => machine.id)).toEqual(
4373	matches.map((machine) => machine.id),
4374	);
4375	yield* stack.destroy();
4376	yield* expectGone({
4377	app_name: app.appName,
4378	machine_id: matches[0]!.id!,
4379	});
4380	}).pipe(sanitizeFailure),
4381	{ tags: ["provider:fly:app", "provider:fly:machine"], timeout: 120_000 },
4382	);
4383
4384	const p3CheckNames = [
4385	"servicecheck-00-http-80",
4386	"servicecheck-00-tcp-80",
4387	"ready",
4388	];
4389
4390	interface RestoredReadiness {
4391	previousInstance: string;
4392	after: number;
4393	checks: ReadonlyMap<string, number>;
4394	}
4395
4396	const readyCheckReports = (
4397	reports: ReadonlyArray<machines.CheckStatus> | undefined,
4398	freshness?: RestoredReadiness,
4399	) =>
4400	p3CheckNames.every((name) => {
4401	const matching = reports?.filter((check) => check.name === name) ?? [];
4402	if (matching.length !== 1 || matching[0]!.status !== "passing")
4403	return false;
4404	const observedAt = Date.parse(matching[0]!.updated_at ?? "");
4405	if (!Number.isFinite(observedAt)) return false;
4406	if (!freshness) return true;
4407	const previous = freshness.checks.get(name);
4408	return (
4409	previous !== undefined &&
4410	Number.isFinite(previous) &&
4411	observedAt > previous &&
4412	observedAt >= freshness.after
4413	);
4414	});
4415
4416	const wireAutostop = (value: string | boolean | undefined) =>
4417	value === true ? "stop" : value === false ? "off" : value;
4418
4419	test(
4420	"P3 pure readiness rejects stale, missing, duplicate and undated check observations",
4421	Effect.sync(() => {
4422	const freshness: RestoredReadiness = {
4423	previousInstance: "prepared",
4424	after: 2_000,
4425	checks: new Map(p3CheckNames.map((name) => [name, 1_000])),
4426	};
4427	const reports = p3CheckNames.map((name) => ({
4428	name,
4429	status: "passing",
4430	updated_at: new Date(3_000).toISOString(),
4431	}));
4432	expect(readyCheckReports(reports, freshness)).toBe(true);
4433	for (const updated_at of [
4434	new Date(1_000).toISOString(),
4435	new Date(1_500).toISOString(),
4436	"invalid",
4437	undefined,
4438	]) {
4439	expect(
4440	readyCheckReports(
4441	[{ ...reports[0]!, updated_at }, ...reports.slice(1)],
4442	freshness,
4443	),
4444	).toBe(false);
4445	}
4446	expect(readyCheckReports(reports.slice(1), freshness)).toBe(false);
4447	expect(readyCheckReports([...reports, reports[0]!], freshness)).toBe(
4448	false,
4449	);
4450	}),
4451	{ tags: ["provider:fly:machine"] },
4452	);
4453
4454	test(
4455	"P3 pure autostop schema accepts strings and legacy booleans",
4456	Effect.sync(() => {
4457	const decode = Schema.decodeUnknownSync(
4458	machines.FlyMachineServiceAutostop,
4459	);
4460	for (const value of ["off", "stop", "suspend", false, true]) {
4461	expect(decode(value)).toBe(value);
4462	}
4463	expect(() => decode({ mode: "stop" })).toThrow();
4464	expect(() => decode(1)).toThrow();
4465	}),
4466	{ tags: ["provider:fly:machine"] },
4467	);
4468
4469	for (const autostop of ["stop", "suspend"] as const) {
4470	test.provider(
4471	`P3 ${autostop} cordoned representative, restored checks and public autostart`,
4472	(stack) =>
4473	Effect.gen(function* () {
4474	yield* stack.destroy();
4475	const created = yield* stack.deploy(
4476	Effect.gen(function* () {
4477	const app = yield* Fly.App("IdleSite");
4478	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
4479	return yield* Fly.Machine("IdleTargets", {
4480	app,
4481	region: "iad",
4482	image: "nginx:alpine",
4483	count: 2,
4484	guest: { cpus: 1, memoryMb: 256 },
4485	skipLaunch: true,
4486	checks: {
4487	ready: {
4488	type: "http",
4489	port: 80,
4490	path: "/",
4491	interval: "2s",
4492	timeout: "1s",
4493	},
4494	},
4495	services: [
4496	{
4497	protocol: "tcp",
4498	internalPort: 80,
4499	autostop,
4500	autostart: true,
4501	minMachinesRunning: 1,
4502	ports: [{ port: 443, handlers: ["tls", "http"] }],
4503	checks: [
4504	{
4505	type: "http",
4506	port: 80,
4507	path: "/",
4508	interval: "2s",
4509	timeout: "1s",
4510	},
4511	],
4512	},
4513	{
4514	protocol: "tcp",
4515	internalPort: 80,
4516	autostop,
4517	autostart: true,
4518	minMachinesRunning: 0,
4519	ports: [{ port: 8080, handlers: ["http"] }],
4520	// Identical HTTP checks reported only one service-check identity.
4521	checks: [
4522	{
4523	type: "tcp",
4524	port: 80,
4525	interval: "2s",
4526	timeout: "1s",
4527	},
4528	],
4529	},
4530	],
4531	});
4532	}),
4533	);
4534	const target = {
4535	app_name: created.appName,
4536	machine_id: created.machineIds[0]!,
4537	};
4538	const idle = {
4539	app_name: created.appName,
4540	machine_id: created.machineIds[1]!,
4541	};
4542	const client = yield* observeTransport;
4543	const healthy = (
4544	mode: "off" | "stop" | "suspend",
4545	freshness?: RestoredReadiness,
4546	) => {
4547	const isReady = (machine: machines.Machine) => {
4548	const services = machine.config?.services;
4549	return (
4550	machine.state === "started" &&
4551	machine.cordoned === true &&
4552	!!machine.instance_id &&
4553	(!freshness ||
4554	machine.instance_id !== freshness.previousInstance) &&
4555	readyCheckReports(machine.checks, freshness) &&
4556	services?.length === 2 &&
4557	services.every(
4558	(service, index) =>
4559	wireAutostop(service.autostop) === mode &&
4560	service.autostart === true &&
4561	service.min_machines_running === (index === 0 ? 1 : 0),
4562	)
4563	);
4564	};
4565	return machines.getMachine(target).pipe(
4566	Retry.none,
4567	Effect.repeat({
4568	schedule: Schedule.spaced("3 seconds"),
4569	times: 8,
4570	until: isReady,
4571	}),
4572	Effect.tap((machine) =>
4573	Effect.logInfo("P3 observed readiness reports", {
4574	state: machine.state,
4575	instance: machine.instance_id,
4576	checks: machine.checks?.map((check) => ({
4577	name: check.name,
4578	status: check.status,
4579	updatedAt: check.updated_at,
4580	})),
4581	named: Object.keys(machine.config?.checks ?? {}),
4582	serviceCheckCounts: machine.config?.services?.map(
4583	(service) => service.checks?.length ?? 0,
4584	),
4585	}),
4586	),
4587	Effect.tap((machine) =>
4588	Effect.sync(() => expect(isReady(machine)).toBe(true)),
4589	),
4590	);
4591	};
4592	yield* Effect.gen(function* () {
4593	const held = yield* lease(target);
4594	const { nonce } = held;
4595	const other = yield* lease(idle);
4596	const request = { ...target, lease_nonce: nonce };
4597	const initial = yield* machines.getMachine(target);
4598	const config = initial.config!;
4599	yield* machines.updateMachine({
4600	...request,
4601	config: {
4602	...config,
4603	services: config.services?.map((service) => ({
4604	...service,
4605	autostop: "off",
4606	})),
4607	},
4608	skip_launch: true,
4609	skip_service_registration: true,
4610	});
4611	yield* machines.cordonMachine({
4612	...idle,
4613	lease_nonce: other.nonce,
4614	});
4615	yield* machines.cordonMachine(request);
4616	yield* machines.startMachine(request).pipe(
4617	Effect.retry({
4618	while: (error) => error._tag === "MachineReplacing",
4619	schedule: Schedule.spaced("2 seconds"),
4620	times: 8,
4621	}),
4622	Effect.timeout("20 seconds"),
4623	);
4624	yield* waitState(target, "started");
4625	const prepared = yield* healthy("off");
4626	expect(prepared.cordoned).toBe(true);
4627	expect(["created", "stopped"]).toContain(
4628	(yield* machines.getMachine(idle)).state,
4629	);
4630	if (!prepared.instance_id) {
4631	return yield* Effect.fail(
4632	new Error("Prepared Machine has no instance ID"),
4633	);
4634	}
4635	const previousChecks = yield* Effect.sync(
4636	() =>
4637	new Map<string, number>(
4638	p3CheckNames.map((name) => [
4639	name,
4640	Date.parse(
4641	prepared.checks?.find((check) => check.name === name)
4642	?.updated_at ?? "",
4643	),
4644	]),
4645	),
4646	);
4647	const freshness: RestoredReadiness = {
4648	previousInstance: prepared.instance_id,
4649	checks: previousChecks,
4650	after: yield* Effect.sync(() => Date.now()),
4651	};
4652	yield* machines.updateMachine({
4653	...request,
4654	config,
4655	skip_service_registration: true,
4656	});
4657	yield* waitState(target, "started");
4658	const restored = yield* healthy(autostop, freshness);
4659	expect(restored.instance_id).not.toBe(prepared.instance_id);
4660	expect(readyCheckReports(restored.checks, freshness)).toBe(true);
4661	yield* Effect.logInfo("P3 restored idle policy", {
4662	autostop,
4663	instanceChanged: prepared.instance_id !== restored.instance_id,
4664	services: restored.config?.services?.map((service) => ({
4665	autostop: service.autostop,
4666	autostart: service.autostart,
4667	minimum: service.min_machines_running,
4668	})),
4669	checks: restored.checks?.map((check) => ({
4670	name: check.name,
4671	status: check.status,
4672	updatedAt: check.updated_at,
4673	})),
4674	});
4675	expect(
4676	restored.config?.services?.map(
4677	(service) => service.min_machines_running,
4678	),
4679	).toEqual([1, 0]);
4680	expect(
4681	restored.config?.services?.map((service) =>
4682	wireAutostop(service.autostop),
4683	),
4684	).toEqual([autostop, autostop]);
4685	expect(
4686	restored.config?.services?.map((service) => service.autostart),
4687	).toEqual([true, true]);
4688	yield* machines.uncordonMachine(request);
4689	yield* machines.uncordonMachine({
4690	...idle,
4691	lease_nonce: other.nonce,
4692	});
4693	if (autostop === "suspend") {
4694	yield* machines
4695	.suspendMachine(request)
4696	.pipe(Effect.timeout("15 seconds"));
4697	yield* waitState(target, "suspended");
4698	} else {
4699	yield* machines.stopMachine({
4700	...request,
4701	signal: "SIGTERM",
4702	timeout: "5s",
4703	});
4704	yield* waitState(target, "stopped");
4705	}
4706	// Fly Proxy must be free to start either Machine without our nonce.
4707	yield* held.release;
4708	yield* other.release;
4709	yield* Effect.logInfo(
4710	"P3 released own leases before proxy autostart",
4711	);
4712	const beforeRequest = (yield* machines.listMachines({
4713	app_name: created.appName,
4714	})).filter(
4715	(machine) =>
4716	machine.id === target.machine_id ||
4717	machine.id === idle.machine_id,
4718	);
4719	expect(beforeRequest.length).toBe(2);
4720	expect(
4721	beforeRequest.every(
4722	(machine) =>
4723	machine.state === "created" ||
4724	machine.state === "stopped" ||
4725	machine.state === "suspended",
4726	),
4727	).toBe(true);
4728	const response = yield* client
4729	.pipe(HttpClient.filterStatusOk)
4730	.get(`https://${created.appName}.fly.dev`)
4731	.pipe(
4732	Effect.timeout("5 seconds"),
4733	Effect.retry({
4734	schedule: Schedule.spaced("2 seconds"),
4735	times: 3,
4736	}),
4737	);
4738	expect(response.status).toBe(200);
4739	const states = yield* machines.listMachines({
4740	app_name: created.appName,
4741	});
4742	expect(
4743	states.some(
4744	(machine) =>
4745	machine.state === "started" &&
4746	beforeRequest.some((before) => before.id === machine.id),
4747	),
4748	).toBe(true);
4749	yield* Effect.logInfo("P3 public proxy autostart verified", {
4750	autostop,
4751	states: states.map((machine) => machine.state),
4752	});
4753	}).pipe(
4754	Effect.scoped,
4755	Retry.none,
4756	Effect.provideService(HttpClient.HttpClient, client),
4757	);
4758	yield* stack.destroy();
4759	yield* expectGone(target);
4760	yield* expectGone(idle);
4761	}).pipe(sanitizeFailure),
4762	{
4763	tags: [
4764	"provider:fly:app",
4765	"provider:fly:ipassignment",
4766	"provider:fly:machine",
4767	],
4768	timeout: 5 * 60_000,
4769	},
4770	);
4771	}
4772	},
4773	);
4774
4775	describe.sequential(
4776	"legacy compatibility",
4777	{
4778	tags: [
4779	"provider:fly",
4780	"provider:fly:app",
4781	"provider:fly:machine",
4782	"provider:fly:service",
4783	"live",
4784	],
4785	},
4786	() => {
4787	const file = "test/Fly/BlueGreen.test.ts";
4788	let endpoint: string | undefined;
4789	let stops: StopRequest[] = [];
4790	const { test } = Test.make({
4791	providers: observeStops(
4792	() => endpoint,
4793	(request) => stops.push(request),
4794	),
4795	});
4796
4797	const runningLegacy = (appName: string, id: string) =>
4798	machines.getMachine({ app_name: appName, machine_id: id }).pipe(
4799	Effect.repeat({
4800	schedule: Schedule.spaced("2 seconds"),
4801	times: 45,
4802	until: (machine) =>
4803	machine.state === "started" &&
4804	machine.config?.metadata?.["alchemy.deployment-protocol"] ===
4805	undefined &&
4806	machine.config?.metadata?.["alchemy.generation"] === undefined &&
4807	machine.config?.stop_config === undefined,
4808	}),
4809	Effect.timeout("120 seconds"),
4810	Effect.tap((machine) =>
4811	Effect.sync(() => {
4812	expect(machine.state).toBe("started");
4813	expect(machine.config?.stop_config).toBeUndefined();
4814	expect(
4815	machine.config?.metadata?.["alchemy.deployment-protocol"],
4816	).toBeUndefined();
4817	expect(
4818	machine.config?.metadata?.["alchemy.generation"],
4819	).toBeUndefined();
4820	}),
4821	),
4822	);
4823
4824	describe.sequential("live legacy protocol compatibility models", () => {
4825	for (const runtimeTimeoutMs of [undefined, 10_000, 60_000]) {
4826	test.provider(
4827	`F14 live legacy missing stop_config ${runtimeTimeoutMs === undefined ? "preserves raw-image defaults" : `uses ${runtimeTimeoutMs}ms runtime-env fallback`}`,
4828	(stack) =>
4829	Effect.gen(function* () {
4830	yield* stack.destroy();
4831	const site = yield* stack.deploy(Fly.App("Site"));
4832	try {
4833	const initial = yield* deployWorker(stack, "one", {
4834	deploy: { strategy: "rolling" },
4835	shutdown: undefined,
4836	});
4837	yield* writeLegacyProtocol(
4838	site.appName,
4839	initial.machineId,
4840	runtimeTimeoutMs,
4841	);
4842	const legacy = yield* runningLegacy(
4843	site.appName,
4844	initial.machineId,
4845	);
4846	expect(
4847	legacy.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS,
4848	).toBe(runtimeTimeoutMs?.toString());
4849	const policy = yield* predecessorShutdown(legacy);
4850	if (runtimeTimeoutMs === undefined) {
4851	expect(policy.signal).toBeUndefined();
4852	expect(policy.timeout).toBeUndefined();
4853	} else {
4854	expect(policy.signal).toBe("SIGTERM");
4855	expect(policy.timeout).toBe(`${runtimeTimeoutMs}ms`);
4856	expect(policy.timeoutMs).toBe(runtimeTimeoutMs);
4857	}
4858	const proxy = yield* transportProxy();
4859	yield* Effect.sync(() => {
4860	endpoint = proxy.url;
4861	stops = [];
4862	});
4863	const upgraded = yield* deployWorker(stack, "two", {
4864	deploy: {
4865	strategy: "bluegreen",
4866	healthTimeout: "60 seconds",
4867	},
4868	});
4869	expect(upgraded.machineId).not.toBe(initial.machineId);
4870	yield* assertCommitted(site.appName, upgraded.machineIds);
4871	const stop = stops.filter(
4872	(request) => request.machineId === initial.machineId,
4873	);
4874	expect(stop).toHaveLength(1);
4875	expect(stop[0]!.signal).toBe(
4876	runtimeTimeoutMs === undefined ? undefined : "SIGTERM",
4877	);
4878	expect(stop[0]!.timeout).toBe(
4879	runtimeTimeoutMs === undefined
4880	? undefined
4881	: `${runtimeTimeoutMs}ms`,
4882	);
4883	expect(
4884	proxy.events.some(
4885	(event) =>
4886	event.stage === "completed" &&
4887	event.machineId === initial.machineId &&
4888	event.path.endsWith("/stop") &&
4889	event.status! >= 200 &&
4890	event.status! < 300,
4891	),
4892	).toBe(true);
4893	expect(
4894	proxy.events.some(
4895	(event) =>
4896	event.stage === "completed" &&
4897	event.machineId === initial.machineId &&
4898	event.method === "DELETE" &&
4899	/\/machines\/[^/]+$/.test(event.path) &&
4900	event.status! >= 200 &&
4901	event.status! < 300,
4902	),
4903	).toBe(true);
4904	const same = yield* deployWorker(stack, "two", {
4905	deploy: {
4906	strategy: "bluegreen",
4907	healthTimeout: "65 seconds",
4908	},
4909	});
4910	expect(same.machineIds).toEqual(upgraded.machineIds);
4911	} finally {
4912	yield* Effect.sync(() => {
4913	endpoint = undefined;
4914	stops = [];
4915	});
4916	yield* stack.destroy();
4917	yield* assertAppGone(site.appName);
4918	}
4919	}).pipe(Effect.scoped),
4920	{ timeout: 900_000 },
4921	);
4922	}
4923
4924	for (const affected of [1, 2]) {
4925	test.provider(
4926	`F14 unknown metadata protocol on ${affected} of 2 real Machines safely refuses without mutation`,
4927	(stack) =>
4928	Effect.gen(function* () {
4929	yield* stack.destroy();
4930	const site = yield* stack.deploy(Fly.App("Site"));
4931	try {
4932	const initial = yield* deployWorker(stack, "one", { count: 2 });
4933	for (const id of initial.machineIds.slice(0, affected)) {
4934	const target = { app_name: site.appName, machine_id: id };
4935	const current = yield* machines.getMachine(target);
4936	yield* machines.patchMachineMetadata({
4937	...target,
4938	metadata: {
4939	...current.config!.metadata,
4940	"alchemy.deployment-protocol": "future-unknown",
4941	},
4942	});
4943	}
4944	const before = yield* census(site.appName);
4945	expect(
4946	before.filter(
4947	(machine) =>
4948	machine.config?.metadata?.[
4949	"alchemy.deployment-protocol"
4950	] === "future-unknown",
4951	),
4952	).toHaveLength(affected);
4953	const metadata = before[0]!.config!.metadata!;
4954	const read = yield* observeReplicaSet({
4955	appName: site.appName,
4956	id: "Worker",
4957	type: "Fly.Machine",
4958	fqn: metadata["alchemy.fqn"]!,
4959	resourceInstanceId: metadata["alchemy.instance"]!,
4960	machineIds: initial.machineIds,
4961	});
4962	expect(read?.rolloutPending).toBe(true);
4963	expect(read?.machineIds).toEqual([]);
4964	const proxy = yield* transportProxy();
4965	yield* Effect.sync(() => {
4966	endpoint = proxy.url;
4967	});
4968	const result = yield* deployWorker(stack, "two", {
4969	count: 2,
4970	}).pipe(Effect.timeout("180 seconds"), Effect.result);
4971	expect(Result.isFailure(result)).toBe(true);
4972	if (Result.isFailure(result)) {
4973	expect(result.failure).toMatchObject({
4974	_tag: "Fly.DeploymentRecoveryAmbiguous",
4975	});
4976	}
4977	expect(
4978	proxy.events.some(
4979	(event) =>
4980	event.stage === "request" &&
4981	event.method !== "GET" &&
4982	!event.path.endsWith("/lease"),
4983	),
4984	).toBe(false);
4985	const after = yield* census(site.appName);
4986	expect(after.map((machine) => machine.id).sort()).toEqual(
4987	initial.machineIds.slice().sort(),
4988	);
4989	for (const machine of after) {
4990	const previous = before.find(
4991	(item) => item.id === machine.id,
4992	)!;
4993	expect(machine.instance_id).toBe(previous.instance_id);
4994	expect(machine.config).toEqual(previous.config);
4995	expect(machine.cordoned).toBe(false);
4996	expect(machine.state).toBe("started");
4997	}
4998	} finally {
4999	yield* Effect.sync(() => {
5000	endpoint = undefined;
5001	});
5002	yield* stack.destroy();
5003	yield* assertAppGone(site.appName);
5004	}
5005	}).pipe(Effect.scoped),
5006	{ timeout: 600_000 },
5007	);
5008	}
5009
5010	test.provider(
5011	"F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
5012	(stack) =>
5013	Effect.gen(function* () {
5014	yield* stack.destroy();
5015	const site = yield* stack.deploy(Fly.App("Site"));
5016	try {
5017	const first = yield* deployWorker(stack, "one", { count: 2 });
5018	yield* writeLegacyProtocol(
5019	site.appName,
5020	first.machineIds[0]!,
5021	60_000,
5022	);
5023	yield* runningLegacy(site.appName, first.machineIds[0]!);
5024	const mixed = yield* census(site.appName);
5025	expect(
5026	mixed.filter(
5027	(machine) =>
5028	machine.config?.metadata?.[
5029	"alchemy.deployment-protocol"
5030	] === "1",
5031	),
5032	).toHaveLength(1);
5033	expect(
5034	mixed.filter(
5035	(machine) =>
5036	machine.config?.metadata?.[
5037	"alchemy.deployment-protocol"
5038	] === undefined,
5039	),
5040	).toHaveLength(1);
5041	const proxy = yield* transportProxy();
5042	try {
5043	yield* Effect.sync(() =>
5044	proxy.arm({
5045	match: (event) =>
5046	event.method === "POST" &&
5047	event.path.endsWith("/machines"),
5048	action: "hold-response",
5049	remaining: 1,
5050	}),
5051	);
5052	yield* Effect.gen(function* () {
5053	const actor = yield* engineActor(
5054	stack,
5055	"F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
5056	file,
5057	proxy.url,
5058	);
5059	const upgrade = yield* deployWorker(actor, "two", {
5060	count: 2,
5061	}).pipe(Effect.scoped, Effect.forkScoped);
5062	yield* proxy.wait(
5063	(event) =>
5064	event.stage === "held" &&
5065	event.status! >= 200 &&
5066	event.status! < 300,
5067	);
5068	const competing = yield* writeLegacyProtocol(
5069	site.appName,
5070	first.machineIds[1]!,
5071	).pipe(Effect.result);
5072	expect(Result.isFailure(competing)).toBe(true);
5073	if (Result.isFailure(competing))
5074	expect(competing.failure._tag).toBe("Conflict");
5075	expect(
5076	(yield* census(site.appName)).filter((machine) =>
5077	first.machineIds.includes(machine.id!),
5078	),
5079	).toHaveLength(2);
5080	yield* Effect.sync(() => {
5081	proxy.clear();
5082	proxy.release();
5083	});
5084	const upgraded = yield* Fiber.join(upgrade).pipe(
5085	Effect.timeout("300 seconds"),
5086	);
5087	yield* assertCommitted(site.appName, upgraded.machineIds);
5088	// A lease-aware legacy writer can still change a successor after lease release.
5089	yield* writeLegacyProtocol(
5090	site.appName,
5091	upgraded.machineIds[0]!,
5092	60_000,
5093	);
5094	yield* runningLegacy(site.appName, upgraded.machineIds[0]!);
5095	const partial = yield* census(site.appName);
5096	expect(partial.map((machine) => machine.id).sort()).toEqual(
5097	upgraded.machineIds.slice().sort(),
5098	);
5099	expect(
5100	partial.filter(
5101	(machine) =>
5102	machine.config?.metadata?.[
5103	"alchemy.deployment-protocol"
5104	] === undefined,
5105	),
5106	).toHaveLength(1);
5107	const resumed = yield* engineActor(
5108	stack,
5109	"F11 F14 partial upgrade with explicit native legacy-protocol writer honors current leases but has no old-binary global fence",
5110	file,
5111	);
5112	const recovered = yield* deployWorker(resumed, "three", {
5113	count: 2,
5114	}).pipe(Effect.scoped);
5115	expect(
5116	recovered.machineIds.every(
5117	(id) => !upgraded.machineIds.includes(id),
5118	),
5119	).toBe(true);
5120	yield* assertCommitted(site.appName, recovered.machineIds);
5121	yield* Effect.logInfo("Partial-upgrade model boundary", {
5122	model:
5123	"native lease-aware legacy protocol writer, not a literal old binary",
5124	globalFence: false,
5125	recoveredCount: recovered.machineIds.length,
5126	});
5127	}).pipe(
5128	Effect.ensuring(
5129	Effect.sync(() => {
5130	proxy.clear();
5131	proxy.release();
5132	}),
5133	),
5134	Effect.scoped,
5135	);
5136	} finally {
5137	yield* Effect.sync(() => {
5138	proxy.clear();
5139	proxy.release();
5140	});
5141	}
5142	} finally {
5143	yield* stack.destroy();
5144	yield* assertAppGone(site.appName);
5145	}
5146	}).pipe(Effect.scoped),
5147	{ timeout: 900_000 },
5148	);
5149	});
5150	},
5151	);
5152
5153	describe.sequential(
5154	"check cadence",
5155	{
5156	tags: [
5157	"provider:fly",
5158	"provider:fly:app",
5159	"provider:fly:machine",
5160	"provider:fly:service",
5161	"live",
5162	],
5163	},
5164	() => {
5165	const file = "test/Fly/BlueGreen.test.ts";
5166	const { test } = Test.make({ providers: Fly.providers() });
5167
5168	const cadenceCheck = {
5169	...checks.ready,
5170	path: "/cadence-ready",
5171	interval: "75s",
5172	gracePeriod: "0s",
5173	};
5174
5175	describe.sequential("live long-cadence readiness and service interval limits", () => {
5176	for (const kind of ["named", "service"] as const) {
5177	for (const sufficient of [false, true]) {
5178	const budget = sufficient ? 120_000 : 35_000;
5179
5180	test.provider(
5181	`S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`,
5182	(stack) =>
5183	Effect.gen(function* () {
5184	yield* stack.destroy();
5185	const site = yield* stack.deploy(Fly.App("Site"));
5186	try {
5187	const first = yield* deployWorker(stack, "one");
5188	const proxy = yield* readinessProxy();
5189	const actor = yield* readinessActor(
5190	stack,
5191	`S06 ${kind === "named" ? "named 75-second check interval" : "service 75-second interval capped by Fly to 60 seconds"} with zero grace ${sufficient ? "passes the next real report within a 120-second health budget" : "fails a 35-second budget without retiring predecessors"}`,
5192	file,
5193	proxy,
5194	);
5195	const props: Partial<
5196	Omit<Extract<MachineProps, { image: string }>, "app">
5197	> = {
5198	init: {
5199	exec: [
5200	"/bin/sh",
5201	"-c",
5202	"rm -f /usr/share/nginx/html/cadence-ready; (sleep 80; touch /usr/share/nginx/html/cadence-ready) & exec nginx -g 'daemon off;'",
5203	],
5204	},
5205	checks: kind === "named" ? { ready: cadenceCheck } : {},
5206	services:
5207	kind === "service"
5208	? [
5209	{
5210	protocol: "tcp",
5211	internalPort: 80,
5212	autostop: "off",
5213	checks: [cadenceCheck],
5214	},
5215	]
5216	: [],
5217	deploy: { strategy: "bluegreen", healthTimeout: budget },
5218	};
5219	yield* Effect.gen(function* () {
5220	const started = yield* Clock.currentTimeMillis;
5221	const attempt = yield* deployWorker(
5222	actor,
5223	"two",
5224	props,
5225	).pipe(Effect.scoped, Effect.result, Effect.forkScoped);
5226	const checkName =
5227	kind === "named" ? "ready" : "servicecheck-00-http-80";
5228	const failed = yield* census(site.appName).pipe(
5229	Effect.map((live) =>
5230	live.find(
5231	(machine) =>
5232	!first.machineIds.includes(machine.id!) &&
5233	machine.state === "started" &&
5234	machine.checks?.some(
5235	(check) =>
5236	check.name === checkName &&
5237	check.status === "critical",
5238	),
5239	),
5240	),
5241	Effect.repeat({
5242	schedule: Schedule.spaced("1 second"),
5243	times: 120,
5244	until: (machine) => machine !== undefined,
5245	}),
5246	Effect.timeout("150 seconds"),
5247	Effect.raceFirst(
5248	Effect.gen(function* () {
5249	const result = yield* Fiber.join(attempt);
5250	if (Result.isFailure(result))
5251	return yield* Effect.fail(result.failure);
5252	return yield* Effect.fail(
5253	new Error(
5254	"Candidate committed before a failing long-cadence report was observed",
5255	),
5256	);
5257	}),
5258	),
5259	);
5260	expect(failed).toBeDefined();
5261	if (!failed)
5262	return yield* Effect.fail(
5263	new Error(
5264	"Fly never emitted the initial failing cadence report",
5265	),
5266	);
5267	const nativeCheck =
5268	kind === "named"
5269	? failed.config?.checks?.ready
5270	: failed.config?.services?.[0]?.checks?.[0];
5271	expect(
5272	kind === "named"
5273	? ["75s", "1m15s"]
5274	: ["60s", "1m0s", "1m"],
5275	).toContain(nativeCheck?.interval);
5276	expect(["0s", "0", undefined]).toContain(
5277	nativeCheck?.grace_period,
5278	);
5279	expect(failed.cordoned).toBe(true);
5280	const failureReport = failed.checks!.find(
5281	(check) => check.name === checkName,
5282	)!;
5283	expect(failureReport.updated_at).toBeDefined();
5284	if (sufficient) {
5285	const passing = yield* machines
5286	.getMachine({
5287	app_name: site.appName,
5288	machine_id: failed.id!,
5289	})
5290	.pipe(
5291	Effect.repeat({
5292	schedule: Schedule.spaced("1 second"),
5293	times: 120,
5294	until: (machine) =>
5295	machine.checks?.some(
5296	(check) =>
5297	check.name === checkName &&
5298	check.status === "passing",
5299	) === true,
5300	}),
5301	Effect.timeout("120 seconds"),
5302	);
5303	const report = passing.checks?.find(
5304	(check) => check.name === checkName,
5305	);
5306	expect(report?.status).toBe("passing");
5307	expect(report?.updated_at).toBeDefined();
5308	expect(failed.instance_id).toBeDefined();
5309	expect(passing.instance_id).toBe(failed.instance_id);
5310	const reportSpacing = yield* Effect.sync(
5311	() =>
5312	Date.parse(report!.updated_at!) -
5313	Date.parse(failureReport.updated_at!),
5314	);
5315	expect(reportSpacing).toBeGreaterThan(60_000);
5316	yield* Effect.logInfo(
5317	"Native cadence reports across autonomous readiness transition",
5318	{
5319	reportSpacingMs: reportSpacing,
5320	failingReport: failureReport.updated_at,
5321	passingReport: report!.updated_at,
5322	},
5323	);
5324	const result = yield* Fiber.join(attempt).pipe(
5325	Effect.timeout("600 seconds"),
5326	);
5327	expect(Result.isSuccess(result)).toBe(true);
5328	if (Result.isFailure(result))
5329	return yield* Effect.fail(result.failure);
5330	expect(result.success.machineIds).toEqual([failed.id]);
5331	const committed = yield* assertCommitted(
5332	site.appName,
5333	result.success.machineIds,
5334	);
5335	expect(committed[0]!.instance_id).toBe(
5336	failed.instance_id,
5337	);
5338	assertReadinessCommit(
5339	proxy.readiness,
5340	first.machineIds,
5341	committed,
5342	[0],
5343	[checkName],
5344	false,
5345	);
5346	} else {
5347	const result = yield* Fiber.join(attempt).pipe(
5348	Effect.timeout("180 seconds"),
5349	);
5350	expect(Result.isFailure(result)).toBe(true);
5351	if (Result.isFailure(result))
5352	expect(result.failure._tag).toBe(
5353	"Fly.ReplicaChecksNotPassing",
5354	);
5355	expect(
5356	(yield* Clock.currentTimeMillis) - started,
5357	).toBeLessThan(180_000);
5358	const live = yield* census(site.appName);
5359	expect(live.map((machine) => machine.id)).toEqual(
5360	first.machineIds,
5361	);
5362	expect(live[0]!.state).toBe("started");
5363	expect(live[0]!.cordoned).toBe(false);
5364	expect(
5365	proxy.events.some(
5366	(event) =>
5367	event.stage === "request" &&
5368	first.machineIds.includes(event.machineId!) &&
5369	(event.path.endsWith("/stop") ||
5370	event.path.endsWith("/cordon") ||
5371	event.path.endsWith("/suspend") ||
5372	(event.path.endsWith("/metadata") &&
5373	event.phase === "retiring") ||
5374	(event.method === "DELETE" &&
5375	/\/machines\/[^/]+$/.test(event.path))),
5376	),
5377	).toBe(false);
5378	expect(
5379	proxy.events.some(
5380	(event) =>
5381	event.machineId === failed.id &&
5382	event.path.endsWith("/uncordon"),
5383	),
5384	).toBe(false);
5385	expect(
5386	proxy.readiness.some(
5387	(event) =>
5388	event.stage === "request" &&
5389	event.machineId === failed.id &&
5390	event.path.endsWith("/metadata") &&
5391	event.phase === "active",
5392	),
5393	).toBe(false);
5394	}
5395	yield* Effect.logInfo("Live long-cadence evidence", {
5396	kind,
5397	interval: nativeCheck?.interval,
5398	grace: nativeCheck?.grace_period,
5399	budget,
5400	firstFailure: failureReport.updated_at,
5401	elapsedMs: (yield* Clock.currentTimeMillis) - started,
5402	});
5403	}).pipe(Effect.scoped);
5404	} finally {
5405	yield* stack.destroy();
5406	yield* assertAppGone(site.appName);
5407	}
5408	}).pipe(Effect.scoped),
5409	{ timeout: 1_200_000 },
5410	);
5411	}
5412	}
5413	});
5414	},
5415	);
5416
5417	describe.sequential(
5418	"ownership",
5419	{
5420	tags: [
5421	"provider:fly",
5422	"provider:fly:app",
5423	"provider:fly:machine",
5424	"provider:fly:service",
5425	"live",
5426	],
5427	},
5428	() => {
5429	const { test } = Test.make({ providers: Fly.providers() });
5430
5431	test.provider(
5432	"S09 foreign routed Machine and sibling survive rollout and stale-instance deletion",
5433	(stack) =>
5434	Effect.gen(function* () {
5435	yield* stack.destroy();
5436	const deploy = (version: string) =>
5437	stack.deploy(
5438	Effect.gen(function* () {
5439	const app = yield* Fly.App("Site");
5440	const sibling = yield* Fly.Machine("Sibling", {
5441	app,
5442	image: "nginx:alpine",
5443	env: { VERSION: "sibling" },
5444	});
5445	const worker = yield* Fly.Machine("Worker", {
5446	app,
5447	image: "nginx:alpine",
5448	env: { VERSION: version },
5449	checks,
5450	deploy: {
5451	strategy: "bluegreen",
5452	healthTimeout: "20 seconds",
5453	},
5454	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
5455	});
5456	return { sibling, worker };
5457	}),
5458	);
5459	const initial = yield* deploy("one");
5460	const foreign = yield* machines.createMachine({
5461	app_name: initial.worker.appName,
5462	name: "foreign-routed",
5463	region: "iad",
5464	config: {
5465	image: "nginx:alpine",
5466	env: { VERSION: "foreign" },
5467	services: [
5468	{
5469	protocol: "tcp",
5470	internal_port: 80,
5471	ports: [{ port: 80, handlers: ["http"] }],
5472	},
5473	],
5474	},
5475	});
5476	const removeForeign = machines
5477	.deleteMachine({
5478	app_name: initial.worker.appName,
5479	machine_id: foreign.id!,
5480	force: true,
5481	})
5482	.pipe(Effect.catchTag("NotFound", () => Effect.void));
5483	yield* Effect.gen(function* () {
5484	const next = yield* deploy("two");
5485	expect(next.worker.machineId).not.toBe(initial.worker.machineId);
5486	expect(next.sibling.machineId).toBe(initial.sibling.machineId);
5487	const before = yield* census(initial.worker.appName);
5488	expect(before.map((machine) => machine.id).sort()).toEqual(
5489	[
5490	next.worker.machineId,
5491	next.sibling.machineId,
5492	foreign.id!,
5493	].sort(),
5494	);
5495	expect(
5496	before.find((machine) => machine.id === foreign.id)?.cordoned,
5497	).toBe(false);
5498	const owned = before.find(
5499	(machine) => machine.id === next.worker.machineId,
5500	)!;
5501	yield* deleteReplicaSet({
5502	appName: next.worker.appName,
5503	id: "Worker",
5504	type: "Fly.Machine",
5505	fqn: owned.config!.metadata!["alchemy.fqn"]!,
5506	resourceInstanceId: "stale-instance-must-not-delete-successor",
5507	machineIds: next.worker.machineIds,
5508	volumeIds: [],
5509	});
5510	expect(
5511	(yield* census(initial.worker.appName))
5512	.map((machine) => machine.id)
5513	.sort(),
5514	).toEqual(before.map((machine) => machine.id).sort());
5515	}).pipe(Effect.ensuring(removeForeign.pipe(Effect.orDie)));
5516	yield* stack.destroy();
5517	yield* assertAppGone(initial.worker.appName);
5518	}),
5519	{ timeout: 300_000 },
5520	);
5521
5522	test.provider(
5523	"S09 F11 stale output and a same-logical-ID foreign FQN cannot authorize successor deletion",
5524	(stack) =>
5525	Effect.gen(function* () {
5526	yield* stack.destroy();
5527	const deploy = (name: string, version: string) =>
5528	stack.deploy(
5529	Effect.gen(function* () {
5530	const app = yield* Fly.App("Site");
5531	return yield* Fly.Machine("Worker", {
5532	app,
5533	name,
5534	image: "nginx:alpine",
5535	checks,
5536	env: { VERSION: version },
5537	deploy: {
5538	strategy: "bluegreen",
5539	healthTimeout: "30 seconds",
5540	},
5541	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
5542	});
5543	}),
5544	);
5545	const initial = yield* deploy("stale-original", "one");
5546	const old = yield* machines.getMachine({
5547	app_name: initial.appName,
5548	machine_id: initial.machineId,
5549	});
5550	const oldMetadata = old.config!.metadata!;
5551	const collision = yield* machines.createMachine({
5552	app_name: initial.appName,
5553	name: "fqn-collision",
5554	region: "iad",
5555	config: {
5556	...old.config,
5557	metadata: {
5558	...oldMetadata,
5559	"alchemy.fqn": `${oldMetadata["alchemy.fqn"]}/OtherScope`,
5560	},
5561	},
5562	});
5563	yield* Effect.gen(function* () {
5564	const next = yield* deploy("stale-original", "two");
5565	const refreshed = yield* observeReplicaSet({
5566	appName: initial.appName,
5567	id: "Worker",
5568	type: "Fly.Machine",
5569	fqn: oldMetadata["alchemy.fqn"]!,
5570	resourceInstanceId: oldMetadata["alchemy.instance"]!,
5571	baseName: initial.baseName,
5572	machineIds: [...initial.machineIds, collision.id!],
5573	});
5574	expect(refreshed?.machineIds).toEqual(next.machineIds);
5575	expect(refreshed?.rolloutPending).toBe(false);
5576	const successor = yield* deploy("stale-successor", "three");
5577	const current = yield* machines.getMachine({
5578	app_name: initial.appName,
5579	machine_id: successor.machineId,
5580	});
5581	expect(current.config?.metadata?.["alchemy.instance"]).not.toBe(
5582	oldMetadata["alchemy.instance"],
5583	);
5584	// Even an ID cache containing the new Machine grants no old-lineage authority.
5585	yield* deleteReplicaSet({
5586	appName: initial.appName,
5587	id: "Worker",
5588	type: "Fly.Machine",
5589	fqn: oldMetadata["alchemy.fqn"]!,
5590	resourceInstanceId: oldMetadata["alchemy.instance"]!,
5591	machineIds: [
5592	...initial.machineIds,
5593	...next.machineIds,
5594	...successor.machineIds,
5595	collision.id!,
5596	],
5597	volumeIds: [],
5598	});
5599	const live = yield* census(initial.appName);
5600	expect(live.map((machine) => machine.id).sort()).toEqual(
5601	[successor.machineId, collision.id!].sort(),
5602	);
5603	const foreign = live.find(
5604	(machine) => machine.id === collision.id,
5605	)!;
5606	expect(foreign.config?.metadata?.["alchemy.fqn"]).toBe(
5607	`${oldMetadata["alchemy.fqn"]}/OtherScope`,
5608	);
5609	expect(foreign.cordoned).toBe(false);
5610	}).pipe(
5611	Effect.ensuring(
5612	machines
5613	.deleteMachine({
5614	app_name: initial.appName,
5615	machine_id: collision.id!,
5616	force: true,
5617	})
5618	.pipe(
5619	Effect.catchTag("NotFound", () => Effect.void),
5620	Effect.orDie,
5621	),
5622	),
5623	);
5624	yield* stack.destroy();
5625	yield* assertAppGone(initial.appName);
5626	}),
5627	{ timeout: 600_000 },
5628	);
5629	},
5630	);
5631
5632	describe.sequential(
5633	"post-promotion health",
5634	{
5635	tags: [
5636	"provider:fly",
5637	"provider:fly:app",
5638	"provider:fly:ipassignment",
5639	"provider:fly:machine",
5640	"provider:fly:secret",
5641	"provider:fly:service",
5642	"live",
5643	],
5644	},
5645	() => {
5646	let endpoint: string | undefined;
5647	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
5648
5649	describe.sequential("post-promotion health", () => {
5650	for (const changed of [false, true]) {
5651	test.provider(
5652	`F09 a real post-uncordon health flip preserves both generations then recovers ${changed ? "changed" : "same"} desired state`,
5653	(stack) =>
5654	Effect.gen(function* () {
5655	yield* stack.destroy();
5656	const readiness = yield* makeReadinessControl();
5657	const initial = yield* readiness.deployWorker(stack, "one");
5658	const proxy = yield* transportProxy();
5659	yield* Effect.sync(() => {
5660	endpoint = proxy.url;
5661	proxy.arm({
5662	match: (event) => event.path.endsWith("/uncordon"),
5663	action: "hold-response",
5664	remaining: 1,
5665	});
5666	});
5667	const update = yield* readiness
5668	.deployWorker(stack, "two")
5669	.pipe(Effect.result, Effect.forkScoped);
5670	const promoted = yield* proxy.wait(
5671	(event) =>
5672	event.stage === "held" &&
5673	event.path.endsWith("/uncordon") &&
5674	event.status! < 300,
5675	);
5676	expect(promoted.machineId).toBeDefined();
5677	yield* readiness.turnOff(initial.appName, promoted.machineId!);
5678	yield* Effect.sync(proxy.release);
5679	const result = yield* Fiber.join(update).pipe(
5680	Effect.timeout("60 seconds"),
5681	);
5682	expect(Result.isFailure(result)).toBe(true);
5683	if (Result.isFailure(result))
5684	expect(result.failure).toMatchObject({
5685	_tag: "Fly.ReplicaChecksNotPassing",
5686	});
5687	const live = yield* census(initial.appName);
5688	expect(live.map((machine) => machine.id).sort()).toEqual(
5689	[initial.machineId, promoted.machineId!].sort(),
5690	);
5691	expect(live.every((machine) => machine.cordoned === false)).toBe(
5692	true,
5693	);
5694	const pending = live.find(
5695	(machine) => machine.id === promoted.machineId,
5696	)!;
5697	const metadata = pending.config!.metadata!;
5698	expect(metadata["alchemy.phase"]).toBe("validating");
5699	expect(metadata["alchemy.checked-instance"]).toBeUndefined();
5700	const read = yield* observeReplicaSet({
5701	appName: initial.appName,
5702	id: "Worker",
5703	type: "Fly.Machine",
5704	fqn: metadata["alchemy.fqn"]!,
5705	resourceInstanceId: metadata["alchemy.instance"]!,
5706	machineIds: initial.machineIds,
5707	});
5708	expect(read?.machineIds).toEqual(initial.machineIds);
5709	expect(read?.rolloutPending).toBe(true);
5710	expect(
5711	proxy.events.some(
5712	(event) =>
5713	event.method === "DELETE" &&
5714	/\/machines\/[^/]+$/.test(event.path),
5715	),
5716	).toBe(false);
5717	if (!changed) {
5718	yield* repairReadiness(initial.appName, promoted.machineId!);
5719	}
5720	const recovered = yield* readiness.deployWorker(
5721	stack,
5722	changed ? "three" : "two",
5723	);
5724	if (changed)
5725	expect(recovered.machineId).not.toBe(promoted.machineId);
5726	else expect(recovered.machineId).toBe(promoted.machineId);
5727	yield* assertCommitted(initial.appName, recovered.machineIds);
5728	yield* Effect.sync(() => {
5729	endpoint = undefined;
5730	});
5731	yield* stack.destroy();
5732	yield* assertAppGone(initial.appName);
5733	}).pipe(
5734	Effect.scoped,
5735	Effect.ensuring(
5736	Effect.sync(() => {
5737	endpoint = undefined;
5738	}),
5739	),
5740	),
5741	{ timeout: 300_000 },
5742	);
5743	}
5744	});
5745	},
5746	);
5747
5748	describe.sequential(
5749	"process death",
5750	{
5751	tags: [
5752	"provider:fly",
5753	"provider:fly:app",
5754	"provider:fly:machine",
5755	"provider:fly:service",
5756	"live",
5757	],
5758	},
5759	() => {
5760	const options = {
5761	providers: Fly.providers(),
5762	profile: "testing",
5763	dev: false,
5764	sidecar: false,
5765	};
5766	const { test } = Test.make(options);
5767	const selected = process.env.FLY_PROCESS_DEATH_CASE;
5768
5769	const crash = (stack: Test.ScratchStack, title: string, phase: Phase) =>
5770	Effect.gen(function* () {
5771	const fs = yield* FileSystem.FileSystem;
5772	const paths = yield* evidencePaths(stack);
5773	// Invalidate an earlier attempt before any cleanup or new cloud work.
5774	for (const file of [paths.witness, paths.finalized, paths.recovered]) {
5775	yield* fs.remove(file, { force: true });
5776	}
5777	yield* stack.destroy();
5778	const initial = yield* deploy(stack, "one").pipe(Effect.scoped);
5779	expect(initial.machineIds).toHaveLength(1);
5780	const predecessor = yield* identity(
5781	yield* machine(initial.appName, initial.machineId),
5782	stack,
5783	);
5784	expect(predecessor.phase).toBe("active");
5785	const proxy = yield* transportProxy();
5786	const actor = yield* Effect.sync(() =>
5787	scratchStack(
5788	{
5789	...options,
5790	providers: throughProxy(() => proxy.url),
5791	stage: stack.stage,
5792	},
5793	title,
5794	processDeathFile,
5795	),
5796	);
5797	expect(actor.name).toBe(stack.name);
5798	expect(actor.stage).toBe(stack.stage);
5799	expect(actor.state).not.toBe(stack.state);
5800	const match = (event: Parameters<typeof matchesBarrier>[3]) =>
5801	matchesBarrier(phase, initial.appName, predecessor.id, event);
5802	yield* Effect.sync(() =>
5803	proxy.arm({ match, action: "hold-response", remaining: 1 }),
5804	);
5805	const pid = yield* Effect.sync(() => process.pid);
5806	// Ordinary failure/interruption invalidates the attempt; SIGKILL cannot run this.
5807	yield* Effect.addFinalizer(() =>
5808	writeEvidence(paths.finalized, { pid, phase }).pipe(Effect.orDie),
5809	);
5810	const attempt = yield* deploy(actor, "two").pipe(
5811	Effect.scoped,
5812	Effect.forkScoped,
5813	);
5814	yield* Effect.gen(function* () {
5815	const barrier = yield* proxy.wait(
5816	(event) =>
5817	event.stage === "held" &&
5818	event.status !== undefined &&
5819	event.status >= 200 &&
5820	event.status < 300 &&
5821	match(event),
5822	);
5823	yield* Effect.gen(function* () {
5824	const live = yield* censusProcessDeath(initial.appName);
5825	const candidates = live.filter(
5826	(value) => value.id !== predecessor.id,
5827	);
5828	expect(candidates).toHaveLength(1);
5829	const candidate = yield* identity(
5830	yield* machine(initial.appName, candidates[0]!.id!),
5831	stack,
5832	);
5833	expect(candidate.generation).not.toBe(predecessor.generation);
5834	expect(candidate.workload).not.toBe(predecessor.workload);
5835	expect(Number(candidate.sequence)).toBe(
5836	Number(predecessor.sequence) + 1,
5837	);
5838	expect(candidate.instance).toBe(predecessor.instance);
5839	expect(candidate.fqn).toBe(predecessor.fqn);
5840	expect(barrier.machineId).toBe(
5841	phase === "retirement" ? predecessor.id : candidate.id,
5842	);
5843	const leases = yield* heldLeases(
5844	initial.appName,
5845	live.map((value) => value.id!),
5846	);
5847	expect(leases.map((value) => value.machineId).sort()).toEqual(
5848	(phase === "create"
5849	? [predecessor.id]
5850	: phase === "promotion"
5851	? [predecessor.id, candidate.id]
5852	: [candidate.id]
5853	).sort(),
5854	);
5855	for (const held of leases) {
5856	expect(
5857	proxy.events.some(
5858	(event) =>
5859	event.stage === "completed" &&
5860	event.method === "POST" &&
5861	event.path.endsWith(`/machines/${held.machineId}/lease`) &&
5862	event.status !== undefined &&
5863	event.status >= 200 &&
5864	event.status < 300,
5865	),
5866	).toBe(true);
5867	}
5868	const row = yield* persistedRow(stack, candidate.fqn);
5869	expect(row.instanceId).toBe(candidate.instance);
5870	expect(row.status).toBe("updating");
5871	const witness = {
5872	version: 1,
5873	signal: "SIGKILL",
5874	phase,
5875	pid,
5876	cwd: paths.cwd,
5877	stack: stack.name,
5878	stage: stack.stage,
5879	appName: initial.appName,
5880	recordedAt: yield* nowSeconds,
5881	predecessor,
5882	candidate,
5883	barrier: {
5884	sequence: barrier.sequence,
5885	method: barrier.method,
5886	path: barrier.path,
5887	status: barrier.status!,
5888	machineId: barrier.machineId!,
5889	},
5890	leases,
5891	row,
5892	} satisfies Witness;
5893	yield* assertBarrierInventory(stack, witness);
5894	yield* writeEvidence(paths.witness, witness);
5895	expect(yield* readWitness(paths.witness)).toEqual(witness);
5896	yield* Effect.sync(() => {
5897	expect(attempt.pollUnsafe() === undefined).toBe(true);
5898	expect(
5899	proxy.events.some(
5900	(event) =>
5901	event.sequence === barrier.sequence &&
5902	["forwarded", "dropped"].includes(event.stage),
5903	),
5904	).toBe(false);
5905	expect(
5906	proxy.events.some(
5907	(event) =>
5908	event.method === "DELETE" && event.path.endsWith("/lease"),
5909	),
5910	).toBe(false);
5911	expect(
5912	proxy.events.some(
5913	(event) =>
5914	event.stage === "request" &&
5915	event.sequence > barrier.sequence &&
5916	event.method !== "GET" &&
5917	!event.path.endsWith("/lease"),
5918	),
5919	).toBe(false);
5920	expect(process.pid).toBe(witness.pid);
5921	process.kill(process.pid, "SIGKILL");
5922	});
5923	return yield* Effect.fail(
5924	new Error("SIGKILL returned without terminating the sole runner"),
5925	);
5926	}).pipe(Effect.timeout("20 seconds"));
5927	}).pipe(
5928	Effect.raceFirst(
5929	Fiber.join(attempt).pipe(
5930	Effect.andThen(
5931	Effect.fail(
5932	new Error(
5933	"Rollout finished before the process-death barrier",
5934	),
5935	),
5936	),
5937	),
5938	),
5939	Effect.ensuring(
5940	Effect.sync(() => {
5941	proxy.clear();
5942	proxy.release();
5943	}),
5944	),
5945	);
5946	}).pipe(Effect.scoped);
5947
5948	const recover = (stack: Test.ScratchStack, phase: Phase) =>
5949	Effect.gen(function* () {
5950	const fs = yield* FileSystem.FileSystem;
5951	const paths = yield* evidencePaths(stack);
5952	const witness = yield* readWitness(paths.witness);
5953	const pid = yield* Effect.sync(() => process.pid);
5954	expect(witness.pid).toBeGreaterThan(0);
5955	expect(pid).not.toBe(witness.pid);
5956	expect(witness.phase).toBe(phase);
5957	expect(witness.cwd).toBe(paths.cwd);
5958	expect(witness.stack).toBe(stack.name);
5959	expect(witness.stage).toBe(stack.stage);
5960	expect(yield* fs.exists(paths.finalized)).toBe(false);
5961	expect(yield* fs.exists(paths.recovered)).toBe(false);
5962	expect(witness.barrier.status).toBeGreaterThanOrEqual(200);
5963	expect(witness.barrier.status).toBeLessThan(300);
5964	expect(witness.barrier.sequence).toBeGreaterThan(0);
5965	expect(
5966	matchesBarrier(phase, witness.appName, witness.predecessor.id, {
5967	...witness.barrier,
5968	stage: "held",
5969	}),
5970	).toBe(true);
5971	expect(witness.barrier.machineId).toBe(
5972	phase === "retirement"
5973	? witness.predecessor.id
5974	: witness.candidate.id,
5975	);
5976	expect(yield* persistedRow(stack, witness.row.fqn)).toEqual(
5977	witness.row,
5978	);
5979	expect(witness.row.status).toBe("updating");
5980	// No deploy, destroy, lease acquisition or release precedes the expiry observations.
5981	yield* assertBarrierInventory(stack, witness);
5982	const expiry = yield* observeLeaseExpiry(witness);
5983	yield* assertBarrierInventory(stack, witness);
5984	const recovered = yield* deploy(stack, "two").pipe(Effect.scoped);
5985	expect(recovered.appName).toBe(witness.appName);
5986	yield* assertConverged(stack, witness, recovered.machineIds);
5987	const unchanged = yield* deploy(stack, "two").pipe(Effect.scoped);
5988	expect(unchanged.machineIds).toEqual(recovered.machineIds);
5989	yield* assertConverged(stack, witness, unchanged.machineIds);
5990	yield* stack.destroy();
5991	yield* assertClean(stack, witness.appName);
5992	yield* writeEvidence(paths.recovered, {
5993	phase,
5994	crashPid: witness.pid,
5995	recoveryPid: pid,
5996	barrier: witness.barrier,
5997	expiry,
5998	machineIds: recovered.machineIds,
5999	generation: witness.candidate.generation,
6000	digest: witness.candidate.digest,
6001	cleaned: true,
6002	});
6003	});
6004
6005	for (const phase of phases) {
6006	const skip =
6007	selected === undefined ||
6008	(phases.some((value) => value === selected) && selected !== phase);
6009	if (skip) {
6010	it.live.skip(
6011	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 process death at completed ${phase}`,
6012	() => Effect.void,
6013	);
6014	continue;
6015	}
6016	// test.provider installs unconditional destroy-on-failure; preserve invalid recovery evidence instead.
6017	test(
6018	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 process death at completed ${phase}`,
6019	Effect.gen(function* () {
6020	yield* assertSingleRunner;
6021	expect(selected).toBe(phase);
6022	const mode = process.env.FLY_PROCESS_DEATH_MODE;
6023	if (mode !== "crash" && mode !== "recovery") {
6024	return yield* Effect.fail(
6025	new Error("FLY_PROCESS_DEATH_MODE must be crash or recovery"),
6026	);
6027	}
6028	const stack = yield* Effect.sync(() =>
6029	scratchStack(
6030	options,
6031	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 process death at completed ${phase}`,
6032	processDeathFile,
6033	),
6034	);
6035	yield* withProviders(
6036	mode === "crash"
6037	? crash(
6038	stack,
6039	`${phase === "create" ? "F07" : phase === "promotion" ? "F08" : "F09"} F10 process death at completed ${phase}`,
6040	phase,
6041	)
6042	: recover(stack, phase),
6043	options,
6044	stack.name,
6045	);
6046	}).pipe(Effect.scoped),
6047	{ timeout: 500_000, retry: 0, exclusive: true },
6048	);
6049	}
6050	},
6051	);
6052
6053	describe.sequential(
6054	"promotion faults",
6055	{
6056	tags: [
6057	"provider:fly",
6058	"provider:fly:app",
6059	"provider:fly:machine",
6060	"provider:fly:service",
6061	"live",
6062	],
6063	},
6064	() => {
6065	let endpoint: string | undefined;
6066	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
6067
6068	test.provider(
6069	"F03 F08 partial promotion loses the first uncordon response without rolling back serving green",
6070	(stack) =>
6071	Effect.gen(function* () {
6072	yield* stack.destroy();
6073	const initial = yield* deployWorker(stack, "one", { count: 2 });
6074	const proxy = yield* transportProxy();
6075	const match = (event: { path: string }) =>
6076	event.path.endsWith("/uncordon");
6077	yield* Effect.sync(() => {
6078	endpoint = proxy.url;
6079	proxy.arm({ match, action: "drop-response", remaining: 1 });
6080	proxy.arm({ match, action: "cut-request", remaining: Infinity });
6081	});
6082	try {
6083	const failed = yield* deployWorker(stack, "two", { count: 2 }).pipe(
6084	Effect.timeout("120 seconds"),
6085	Effect.result,
6086	);
6087	expect(Result.isFailure(failed)).toBe(true);
6088	const lost = proxy.events.find(
6089	(event) =>
6090	event.stage === "dropped" && event.path.endsWith("/uncordon"),
6091	);
6092	expect(lost?.status).toBeGreaterThanOrEqual(200);
6093	expect(lost?.status).toBeLessThan(300);
6094	const live = yield* census(initial.appName);
6095	expect(
6096	initial.machineIds.every((id) =>
6097	live.some(
6098	(machine) => machine.id === id && machine.cordoned === false,
6099	),
6100	),
6101	).toBe(true);
6102	expect(
6103	live.find((machine) => machine.id === lost!.machineId)?.cordoned,
6104	).toBe(false);
6105	const candidateIds = live
6106	.filter((machine) => !initial.machineIds.includes(machine.id!))
6107	.map((machine) => machine.id!);
6108	expect(candidateIds).toHaveLength(2);
6109	expect(
6110	proxy.events.some(
6111	(event) =>
6112	event.method === "DELETE" &&
6113	/\/machines\/[^/]+$/.test(event.path) &&
6114	candidateIds.includes(event.machineId!),
6115	),
6116	).toBe(false);
6117	yield* Effect.sync(proxy.clear);
6118	const recovered = yield* deployWorker(stack, "two", { count: 2 });
6119	expect([...recovered.machineIds].sort()).toEqual(
6120	candidateIds.sort(),
6121	);
6122	yield* assertCommitted(initial.appName, recovered.machineIds);
6123	} finally {
6124	yield* Effect.sync(() => {
6125	endpoint = undefined;
6126	proxy.clear();
6127	});
6128	}
6129	yield* stack.destroy();
6130	yield* assertAppGone(initial.appName);
6131	}).pipe(
6132	Effect.scoped,
6133	Effect.ensuring(
6134	Effect.sync(() => {
6135	endpoint = undefined;
6136	}),
6137	),
6138	),
6139	{ timeout: 300_000 },
6140	);
6141	},
6142	);
6143
6144	describe.sequential(
6145	"protocol branches",
6146	{
6147	tags: [
6148	"unit",
6149	"provider:fly",
6150	"provider:fly:machine",
6151	"provider:fly:service",
6152	"local",
6153	],
6154	},
6155	() => {
6156	it.live(
6157	"S07 P actual SDK GET 408 decoder returns typed GatewayTimeout, not genuine remote 408",
6158	() =>
6159	Effect.gen(function* () {
6160	let requests = 0;
6161	const client = HttpClient.make((request) =>
6162	Effect.sync(() => {
6163	requests++;
6164	expect(request.method).toBe("GET");
6165	expect(new URL(request.url).pathname).toBe(
6166	`/v1/apps/${appName}/machines/${candidateId}`,
6167	);
6168	return reply(
6169	request,
6170	{ error: "controlled request timeout" },
6171	408,
6172	);
6173	}),
6174	);
6175	const result = yield* machines
6176	.getMachine({ app_name: appName, machine_id: candidateId })
6177	.pipe(Retry.none, withControlledClient(client), Effect.result);
6178	expect(Result.isFailure(result)).toBe(true);
6179	if (Result.isFailure(result)) {
6180	expect(result.failure).toBeInstanceOf(GatewayTimeoutErrors);
6181	expect(result.failure._tag).toBe("GatewayTimeout");
6182	if (result.failure._tag === "GatewayTimeout")
6183	expect(result.failure.message).toBe("controlled request timeout");
6184	}
6185	expect(requests).toBe(1);
6186	}),
6187	);
6188
6189	it.live(
6190	"F02 P delayed visibility readback reuses one candidate without replaying create",
6191	() =>
6192	Effect.gen(function* () {
6193	const fixture = yield* protocolClient({
6194	conflict: true,
6195	hiddenLists: 2,
6196	});
6197	const result = yield* reconcile.pipe(
6198	withControlledClient(fixture.client),
6199	);
6200	expect(result.machineIds).toEqual([candidateId]);
6201	const lists = fixture.events.filter(
6202	(event) => event.visible !== undefined,
6203	);
6204	expect(lists.map((event) => event.visible)).toEqual([
6205	false,
6206	false,
6207	false,
6208	true,
6209	]);
6210	const creates = fixture.events.filter(
6211	(event) =>
6212	event.method === "POST" && event.path.endsWith("/machines"),
6213	);
6214	expect(creates).toHaveLength(1);
6215	const visibleAt = fixture.events.indexOf(lists[3]!);
6216	const leaseAt = fixture.events.findIndex(
6217	(event) => event.method === "POST" && event.path.endsWith("/lease"),
6218	);
6219	const promotionAt = fixture.events.findIndex((event) =>
6220	event.path.endsWith("/uncordon"),
6221	);
6222	expect(leaseAt).toBeGreaterThan(visibleAt);
6223	expect(promotionAt).toBeGreaterThan(leaseAt);
6224	expect(
6225	fixture.events.filter((event) => event.phase === "active"),
6226	).toHaveLength(1);
6227	}),
6228	{ timeout: 30_000 },
6229	);
6230
6231	it.live(
6232	"F02 P visibility exhaustion fails bounded readback without replay or promotion",
6233	() =>
6234	Effect.gen(function* () {
6235	const fixture = yield* protocolClient({
6236	conflict: true,
6237	hiddenLists: Infinity,
6238	});
6239	const result = yield* reconcile.pipe(
6240	withControlledClient(fixture.client),
6241	Effect.result,
6242	);
6243	expect(Result.isFailure(result)).toBe(true);
6244	if (Result.isFailure(result))
6245	expect(result.failure).toBeInstanceOf(ReplicaNotCreated);
6246	expect(
6247	fixture.events.filter(
6248	(event) =>
6249	event.method === "POST" && event.path.endsWith("/machines"),
6250	),
6251	).toHaveLength(1);
6252	// Initial observation, nine bounded readback attempts, and failure-path observation.
6253	expect(
6254	fixture.events.filter((event) => event.visible === false),
6255	).toHaveLength(11);
6256	expect(
6257	fixture.events.some(
6258	(event) =>
6259	event.path.endsWith("/lease") ||
6260	event.path.endsWith("/uncordon") ||
6261	event.method === "DELETE",
6262	),
6263	).toBe(false);
6264	}),
6265	{ timeout: 30_000 },
6266	);
6267
6268	for (const missing of ["image_ref", "digest", "repository"] as const) {
6269	it.live(
6270	`S08 P missing ${missing} refuses candidate promotion through the controller`,
6271	() =>
6272	Effect.gen(function* () {
6273	const fixture = yield* protocolClient({ missingImageRef: missing });
6274	const result = yield* reconcile.pipe(
6275	withControlledClient(fixture.client),
6276	Effect.result,
6277	);
6278	expect(Result.isFailure(result)).toBe(true);
6279	if (Result.isFailure(result)) {
6280	expect(result.failure).toBeInstanceOf(
6281	DeploymentRecoveryAmbiguous,
6282	);
6283	if (result.failure._tag === "Fly.DeploymentRecoveryAmbiguous") {
6284	expect(result.failure.appName).toBe(appName);
6285	expect(result.failure.message).toBe(
6286	`Candidate ${candidateId} changed before its lease was acquired. Mismatch: image_ref.`,
6287	);
6288	}
6289	}
6290	expect(
6291	fixture.events.some(
6292	(event) =>
6293	event.method === "POST" && event.path.endsWith("/lease"),
6294	),
6295	).toBe(true);
6296	expect(
6297	fixture.events.some(
6298	(event) =>
6299	event.path.endsWith("/metadata") ||
6300	event.path.endsWith("/uncordon"),
6301	),
6302	).toBe(false);
6303	const current = yield* machines
6304	.getMachine({ app_name: appName, machine_id: candidateId })
6305	.pipe(withControlledClient(fixture.client));
6306	expect(current.cordoned).toBe(true);
6307	expect(current.config?.metadata?.[keys.phase]).toBe("candidate");
6308	if (missing === "image_ref")
6309	expect(current.image_ref).toBeUndefined();
6310	else expect(current.image_ref?.[missing]).toBeUndefined();
6311	}),
6312	);
6313	}
6314
6315	const unreachable = (): machines.Machine => ({
6316	id: "controlled-unreachable",
6317	host_status: "unreachable",
6318	state: "started",
6319	config: { image: "fixture:latest", metadata, mounts: [] },
6320	});
6321
6322	it.live(
6323	"F06 P owned stateless unreachable host skips lease/cordon/stop and force-deletes only with replacement ready",
6324	() =>
6325	Effect.gen(function* () {
6326	const target = unreachable();
6327	const events: Array<{
6328	method: string;
6329	path: string;
6330	force: string | undefined;
6331	}> = [];
6332	const client = HttpClient.make((request) =>
6333	Effect.sync(() => {
6334	const url = new URL(request.url);
6335	const path = url.pathname;
6336	events.push({
6337	method: request.method,
6338	path,
6339	force:
6340	request.urlParams.params.find(
6341	([key]) => key === "force",
6342	)?.[1] ??
6343	url.searchParams.get("force") ??
6344	undefined,
6345	});
6346	if (
6347	request.method === "DELETE" &&
6348	path.endsWith(`/machines/${target.id}`)
6349	)
6350	return reply(request, {});
6351	if (request.method === "GET" && path.endsWith("/wait"))
6352	return reply(request, { error: "not found" }, 404);
6353	throw new Error(
6354	`Unexpected unreachable-host request: ${request.method} ${path}`,
6355	);
6356	}),
6357	);
6358	yield* retireMachines(appName, [target], true).pipe(
6359	withControlledClient(client),
6360	);
6361	expect(events).toEqual([
6362	{
6363	method: "DELETE",
6364	path: `/v1/apps/${appName}/machines/${target.id}`,
6365	force: "true",
6366	},
6367	{
6368	method: "GET",
6369	path: `/v1/apps/${appName}/machines/${target.id}/wait`,
6370	force: undefined,
6371	},
6372	]);
6373	}),
6374	);
6375
6376	const unsafe: Array<{ name: string; machine: () => machines.Machine }> = [
6377	{
6378	name: "missing config",
6379	machine: () => ({ ...unreachable(), config: undefined }),
6380	},
6381	{
6382	name: "incomplete config",
6383	machine: () => ({
6384	...unreachable(),
6385	incomplete_config: { image: "fixture:latest" },
6386	}),
6387	},
6388	{
6389	name: "mounted config",
6390	machine: () => ({
6391	...unreachable(),
6392	config: {
6393	metadata,
6394	mounts: [{ volume: "vol-controlled", path: "/data" }],
6395	},
6396	}),
6397	},
6398	{
6399	name: "missing ownership instance",
6400	machine: () => ({
6401	...unreachable(),
6402	config: { metadata: { ...metadata, [keys.instance]: undefined } },
6403	}),
6404	},
6405	{
6406	name: "missing ownership FQN",
6407	machine: () => ({
6408	...unreachable(),
6409	config: { metadata: { ...metadata, [keys.fqn]: undefined } },
6410	}),
6411	},
6412	];
6413
6414	for (const variant of unsafe) {
6415	it.live(
6416	`F06 P unreachable host with ${variant.name} refuses force retirement`,
6417	() =>
6418	Effect.gen(function* () {
6419	let requests = 0;
6420	const client = HttpClient.make(() =>
6421	Effect.sync(() => {
6422	requests++;
6423	throw new Error(
6424	"Unsafe unreachable target must not reach the transport",
6425	);
6426	}),
6427	);
6428	const target = variant.machine();
6429	const result = yield* retireMachines(appName, [target], true).pipe(
6430	withControlledClient(client),
6431	Effect.result,
6432	);
6433	expect(Result.isFailure(result)).toBe(true);
6434	if (Result.isFailure(result)) {
6435	expect(result.failure).toBeInstanceOf(
6436	ReplicaRetirementIncomplete,
6437	);
6438	if (result.failure._tag === "Fly.ReplicaRetirementIncomplete") {
6439	expect(result.failure.appName).toBe(appName);
6440	expect(result.failure.residuals).toEqual([
6441	{
6442	machineId: target.id,
6443	stage: "Fly.ReplicaOwnershipChanged",
6444	},
6445	]);
6446	}
6447	}
6448	expect(requests).toBe(0);
6449	}),
6450	);
6451	}
6452
6453	it.live(
6454	"F06 P unreachable host without replacement readiness cannot take the force-delete shortcut",
6455	() =>
6456	Effect.gen(function* () {
6457	const events: string[] = [];
6458	const client = HttpClient.make((request) =>
6459	Effect.sync(() => {
6460	const path = new URL(request.url).pathname;
6461	events.push(`${request.method} ${path}`);
6462	if (request.method === "POST" && path.endsWith("/lease"))
6463	return reply(
6464	request,
6465	{ error: "controlled host unavailable" },
6466	503,
6467	);
6468	throw new Error(
6469	`Unexpected unready retirement request: ${request.method} ${path}`,
6470	);
6471	}),
6472	);
6473	const result = yield* retireMachines(
6474	appName,
6475	[unreachable()],
6476	false,
6477	).pipe(withControlledClient(client), Effect.result);
6478	expect(Result.isFailure(result)).toBe(true);
6479	if (
6480	Result.isFailure(result) &&
6481	result.failure._tag === "Fly.ReplicaRetirementIncomplete"
6482	) {
6483	expect(result.failure.residuals).toEqual([
6484	{
6485	machineId: "controlled-unreachable",
6486	stage: "ServiceUnavailable",
6487	},
6488	]);
6489	} else {
6490	throw new Error(
6491	"Expected exact retirement residual for unavailable lease acquisition",
6492	);
6493	}
6494	expect(events).toEqual([
6495	`POST /v1/apps/${appName}/machines/controlled-unreachable/lease`,
6496	]);
6497	}),
6498	);
6499	},
6500	);
6501
6502	describe.sequential(
6503	"readiness oracle",
6504	{
6505	tags: [
6506	"unit",
6507	"provider:fly",
6508	"provider:fly:machine",
6509	"provider:fly:service",
6510	"local",
6511	],
6512	},
6513	() => {
6514	const names = ["ready", "servicecheck-00-http-80"];
6515	const reports = (mirror = false) =>
6516	[...names, ...(mirror ? ["bg_deployments_compat-00-http-80"] : [])].map(
6517	(name) => ({ name, status: "passing" }),
6518	);
6519
6520	// Synthetic journals exercise the oracle only; they are not native Fly evidence.
6521	const trace = (idle = true, retry = false, mirror = false) => {
6522	const events: ReadinessEvent[] = [];
6523	const path = (slot: number) => `/v1/apps/oracle/machines/green-${slot}`;
6524	const metadata = (
6525	slot: number,
6526	phase: string,
6527	): Record<string, string> => ({
6528	"alchemy.stack": "oracle",
6529	"alchemy.stage": "pure",
6530	"alchemy.id": "Worker",
6531	"alchemy.type": "Fly.Machine",
6532	"alchemy.instance": "resource-instance",
6533	"alchemy.fqn": "Worker",
6534	"alchemy.replica": String(slot),
6535	"alchemy.generation": "generation",
6536	"alchemy.workload": "workload",
6537	"alchemy.image": "nginx@sha256:fixture",
6538	"alchemy.phase": phase,
6539	"alchemy.readiness-role": "run",
6540	"alchemy.readiness-roles": "run,run",
6541	"alchemy.idle-policy-restored": "true",
6542	...(phase === "active"
6543	? { "alchemy.checked-instance": `instance-${slot}` }
6544	: {}),
6545	});
6546	const request = (input: Omit<ReadinessEvent, "stage" | "sequence">) => {
6547	const event: ReadinessEvent = {
6548	...input,
6549	stage: "request",
6550	sequence: events.length,
6551	};
6552	events.push(event);
6553	return event;
6554	};
6555	const reply = (
6556	input: ReadinessEvent,
6557	fields: Partial<ReadinessEvent> = {},
6558	) => {
6559	const event: ReadinessEvent = {
6560	...input,
6561	stage: "forwarded",
6562	status: 200,
6563	...fields,
6564	};
6565	events.push(event);
6566	return event;
6567	};
6568	const read = (slot: number, phase: string, state: string) =>
6569	reply(
6570	request({
6571	method: "GET",
6572	path: path(slot),
6573	machineId: `green-${slot}`,
6574	}),
6575	{
6576	instanceId: `instance-${slot}`,
6577	digest: "sha256:fixture",
6578	state,
6579	cordoned: false,
6580	phase,
6581	metadata: metadata(slot, phase),
6582	checks: state === "started" ? reports(mirror) : [],
6583	services: [
6584	{
6585	protocol: "tcp",
6586	port: 80,
6587	autostop: "stop",
6588	autostart: true,
6589	floor: 0,
6590	},
6591	],
6592	},
6593	);
6594	const stamp = (slot: number, phase: string) => {
6595	const input = {
6596	method: "PUT",
6597	path: `${path(slot)}/metadata`,
6598	machineId: `green-${slot}`,
6599	phase,
6600	metadata: metadata(slot, phase),
6601	metadataOnly: true,
6602	};
6603	if (retry)
6604	reply(request(input), { status: phase === "active" ? 503 : 429 });
6605	reply(request(input));
6606	};
6607	for (const slot of [0, 1]) {
6608	reply(
6609	request({
6610	method: "POST",
6611	path: path(slot),
6612	machineId: `green-${slot}`,
6613	}),
6614	);
6615	read(slot, "promoting", "started");
6616	}
6617	for (const slot of [0, 1]) stamp(slot, "validating");
6618	for (const slot of [0, 1])
6619	read(slot, "validating", idle && slot === 0 ? "stopped" : "started");
6620	for (const slot of [1, 0]) stamp(slot, "active");
6621	request({
6622	method: "POST",
6623	path: "/v1/apps/oracle/machines/old/cordon",
6624	machineId: "old",
6625	});
6626	const candidates: Machine[] = [0, 1].map((slot) => ({
6627	id: `green-${slot}`,
6628	instance_id: `instance-${slot}`,
6629	image_ref: { digest: "sha256:fixture" },
6630	state: idle && slot === 0 ? "stopped" : "started",
6631	cordoned: false,
6632	config: {
6633	metadata: metadata(slot, "active"),
6634	services: [
6635	{
6636	protocol: "tcp",
6637	internal_port: 80,
6638	autostop: "stop",
6639	autostart: true,
6640	min_machines_running: 0,
6641	},
6642	],
6643	},
6644	}));
6645	return { events, candidates };
6646	};
6647
6648	it.effect(
6649	"pure readiness trace accepts rejected resume only with fresh started-instance proof",
6650	() =>
6651	Effect.sync(() => {
6652	const rejectedStart = (status: number) => {
6653	const value = trace(false);
6654	const request: ReadinessEvent = {
6655	sequence: 999,
6656	stage: "request",
6657	method: "POST",
6658	path: "/v1/apps/oracle/machines/green-0/start",
6659	machineId: "green-0",
6660	};
6661	value.events.splice(2, 0, request, {
6662	...request,
6663	stage: "forwarded",
6664	status,
6665	});
6666	return value;
6667	};
6668	expect(() => check(rejectedStart(412))).not.toThrow();
6669	expect(() => check(rejectedStart(500))).toThrow();
6670	const missing = rejectedStart(412);
6671	for (const event of missing.events)
6672	if (event.method === "GET" && event.machineId === "green-0")
6673	event.state = "stopped";
6674	expect(() => check(missing)).toThrow();
6675	}),
6676	);
6677
6678	type Trace = ReturnType<typeof trace>;
6679	const check = ({ events, candidates }: Trace, allowIdle = true) =>
6680	assertReadinessCommit(
6681	events,
6682	["old"],
6683	candidates,
6684	[0, 1],
6685	names,
6686	allowIdle,
6687	);
6688	const restored = ({ events }: Trace) =>
6689	events.find(
6690	(event) =>
6691	event.stage === "forwarded" &&
6692	event.method === "GET" &&
6693	event.machineId === "green-0" &&
6694	event.phase === "promoting",
6695	)!;
6696	const pending = ({ events }: Trace) =>
6697	events.find(
6698	(event) =>
6699	event.stage === "forwarded" &&
6700	event.method === "GET" &&
6701	event.machineId === "green-0" &&
6702	event.phase === "validating",
6703	)!;
6704	const active = ({ events }: Trace) =>
6705	events.filter(
6706	(event) =>
6707	event.method === "PUT" &&
6708	event.machineId === "green-0" &&
6709	event.phase === "active",
6710	);
6711
6712	it.effect(
6713	"pure readiness matcher permits only passing corresponding mirrors",
6714	() =>
6715	Effect.sync(() => {
6716	expect(readinessChecksPassing(reports(), names)).toBe(true);
6717	expect(readinessChecksPassing(reports(true), names)).toBe(true);
6718	for (const invalid of [
6719	undefined,
6720	[],
6721	reports(true).filter(
6722	(check) => check.name !== "servicecheck-00-http-80",
6723	),
6724	[...reports(), reports()[0]!],
6725	[...reports(true), reports(true).at(-1)!],
6726	[...reports(), { name: "unrelated", status: "passing" }],
6727	[
6728	...reports(),
6729	{ name: "bg_deployments_compat-00-tcp-80", status: "passing" },
6730	],
6731	[...reports(), { name: undefined, status: "passing" }],
6732	reports(true).map((check) => ({ ...check, status: "warning" })),
6733	reports(true).map((check) =>
6734	check.name.startsWith("bg_")
6735	? { ...check, status: "critical" }
6736	: check,
6737	),
6738	])
6739	expect(readinessChecksPassing(invalid, names)).toBe(false);
6740	expect(readinessChecksPassing(reports(), [...names, names[0]!])).toBe(
6741	false,
6742	);
6743	}),
6744	);
6745
6746	it.effect(
6747	"pure readiness traces accept running and restored-idle proof with settled metadata retries",
6748	() =>
6749	Effect.sync(() => {
6750	for (const idle of [false, true]) {
6751	for (const retry of [false, true]) check(trace(idle, retry, true));
6752	}
6753	const suspended = trace();
6754	pending(suspended).state = "suspended";
6755	suspended.candidates[0]!.state = "suspended";
6756	check(suspended);
6757	}),
6758	);
6759
6760	const invalidTraces: [string, (value: Trace) => void][] = [
6761	[
6762	"missing restored passing proof",
6763	(value) => {
6764	restored(value).checks = [];
6765	},
6766	],
6767	[
6768	"proof from another instance",
6769	(value) => {
6770	restored(value).instanceId = "obsolete";
6771	},
6772	],
6773	[
6774	"proof from another workload",
6775	(value) => {
6776	restored(value).metadata!["alchemy.workload"] = "obsolete";
6777	},
6778	],
6779	[
6780	"proof with the preparation service policy",
6781	(value) => {
6782	restored(value).services![0]!.autostop = "off";
6783	},
6784	],
6785	[
6786	"created representative",
6787	(value) => {
6788	pending(value).state = "created";
6789	},
6790	],
6791	[
6792	"configuration mutation after restored proof",
6793	(value) => {
6794	const index = value.events.indexOf(restored(value)) + 1;
6795	const event: ReadinessEvent = {
6796	sequence: 10_000,
6797	stage: "request",
6798	method: "POST",
6799	path: "/v1/apps/oracle/machines/green-0",
6800	machineId: "green-0",
6801	};
6802	value.events.splice(index, 0, event, {
6803	...event,
6804	stage: "forwarded",
6805	status: 200,
6806	});
6807	},
6808	],
6809	[
6810	"GET requested before restoration completed",
6811	(value) => {
6812	const response = restored(value);
6813	const index = value.events.findIndex(
6814	(event) => event.sequence === response.sequence,
6815	);
6816	const [request] = value.events.splice(index, 1);
6817	value.events.splice(1, 0, request!);
6818	},
6819	],
6820	[
6821	"changed pending metadata after proof",
6822	(value) => {
6823	for (const event of value.events.filter(
6824	(event) =>
6825	event.method === "PUT" &&
6826	event.machineId === "green-0" &&
6827	event.phase === "validating",
6828	)) {
6829	event.metadata = {
6830	...event.metadata,
6831	"alchemy.checked-instance": "unproven",
6832	};
6833	}
6834	},
6835	],
6836	[
6837	"conflicting commit retry",
6838	(value) => {
6839	const retry = active(value)
6840	.filter((event) => event.stage === "request")
6841	.at(-1)!;
6842	retry.metadata = {
6843	...retry.metadata,
6844	"alchemy.checked-instance": "other",
6845	};
6846	},
6847	],
6848	[
6849	"unsettled rejected attempt",
6850	(value) => {
6851	value.events.splice(value.events.indexOf(active(value)[1]!), 1);
6852	},
6853	],
6854	[
6855	"retry begins before rejected receipt",
6856	(value) => {
6857	const attempts = active(value);
6858	const index = value.events.indexOf(attempts[2]!);
6859	value.events.splice(index, 1);
6860	value.events.splice(
6861	value.events.indexOf(attempts[1]!),
6862	0,
6863	attempts[2]!,
6864	);
6865	},
6866	],
6867	[
6868	"terminal commit rejection",
6869	(value) => {
6870	active(value).at(-1)!.status = 429;
6871	},
6872	],
6873	[
6874	"nonretryable metadata rejection",
6875	(value) => {
6876	active(value)[1]!.status = 403;
6877	},
6878	],
6879	[
6880	"duplicate successful commit",
6881	(value) => {
6882	active(value)[1]!.status = 200;
6883	},
6884	],
6885	[
6886	"replica zero starts committing before the other commit settles",
6887	(value) => {
6888	const response = value.events.find(
6889	(event) =>
6890	event.machineId === "green-1" &&
6891	event.phase === "active" &&
6892	event.stage === "forwarded" &&
6893	event.status === 200,
6894	)!;
6895	value.events.splice(value.events.indexOf(response), 1);
6896	value.events.splice(
6897	value.events.indexOf(active(value)[0]!) + 1,
6898	0,
6899	response,
6900	);
6901	},
6902	],
6903	[
6904	"retirement before terminal commit receipt",
6905	(value) => {
6906	const retirement = value.events.pop()!;
6907	value.events.splice(
6908	value.events.indexOf(active(value).at(-1)!),
6909	0,
6910	retirement,
6911	);
6912	},
6913	],
6914	[
6915	"commit before complete topology validation",
6916	(value) => {
6917	const first = value.events.findIndex(
6918	(event) => event.phase === "active",
6919	);
6920	const commit = value.events.splice(first, 4);
6921	const validation = value.events.findIndex(
6922	(event) =>
6923	event.method === "GET" &&
6924	event.machineId === "green-1" &&
6925	event.stage === "forwarded" &&
6926	event.phase === "validating",
6927	);
6928	value.events.splice(validation - 1, 0, ...commit);
6929	},
6930	],
6931	];
6932
6933	for (const [name, corrupt] of invalidTraces) {
6934	it.effect(`pure readiness trace rejects ${name}`, () =>
6935	Effect.sync(() => {
6936	const value = trace(true, true);
6937	corrupt(value);
6938	expect(() => check(value)).toThrow();
6939	}),
6940	);
6941	}
6942
6943	it.effect(
6944	"pure readiness trace refuses idle completion when autostop is disabled",
6945	() =>
6946	Effect.sync(() => {
6947	expect(() => check(trace(), false)).toThrow();
6948	}),
6949	);
6950	},
6951	);
6952
6953	describe.sequential(
6954	"recovery",
6955	{
6956	tags: [
6957	"provider:fly",
6958	"provider:fly:app",
6959	"provider:fly:machine",
6960	"provider:fly:service",
6961	"live",
6962	],
6963	},
6964	() => {
6965	const { test } = Test.make({ providers: Fly.providers() });
6966	const checks = {
6967	ready: {
6968	type: "http" as const,
6969	port: 80,
6970	path: "/",
6971	interval: "2s",
6972	timeout: "1s",
6973	},
6974	};
6975
6976	test.provider(
6977	"recovers partial promotion as one generation and preserves policy-only identity",
6978	(stack) =>
6979	Effect.gen(function* () {
6980	yield* stack.destroy();
6981	const deploy = (healthTimeout: number) =>
6982	stack.deploy(
6983	Effect.gen(function* () {
6984	const app = yield* Fly.App("Site");
6985	return yield* Fly.Machine("Worker", {
6986	app,
6987	count: 2,
6988	image: "nginx:alpine",
6989	checks,
6990	deploy: { strategy: "bluegreen", healthTimeout },
6991	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
6992	});
6993	}),
6994	);
6995	const initial = yield* deploy(30_000);
6996	const source = yield* machines.getMachine({
6997	app_name: initial.appName,
6998	machine_id: initial.machineId,
6999	});
7000	const metadata = source.config!.metadata!;
7001	const candidates = [];
7002	for (let index = 0; index < 2; index++) {
7003	candidates.push(
7004	yield* machines.createMachine({
7005	app_name: initial.appName,
7006	name: `interrupted-promotion-${index}`,
7007	region: "iad",
7008	skip_service_registration: index !== 0,
7009	config: {
7010	...source.config,
7011	image: metadata["alchemy.image"],
7012	metadata: {
7013	...metadata,
7014	"alchemy.generation": "interrupted",
7015	"alchemy.sequence": "2",
7016	"alchemy.replica": String(index),
7017	"alchemy.phase": index === 0 ? "active" : "promoting",
7018	},
7019	},
7020	}),
7021	);
7022	}
7023	const read = yield* observeReplicaSet({
7024	appName: initial.appName,
7025	id: "Worker",
7026	type: "Fly.Machine",
7027	fqn: metadata["alchemy.fqn"]!,
7028	resourceInstanceId: metadata["alchemy.instance"]!,
7029	baseName: initial.baseName,
7030	machineIds: initial.machineIds,
7031	});
7032	expect(read?.machineIds).toEqual(initial.machineIds);
7033	expect(read?.count).toBe(2);
7034	expect(read?.rolloutPending).toBe(true);
7035	const recovered = yield* deploy(40_000);
7036	expect(recovered.machineIds).toEqual(
7037	candidates.map((machine) => machine.id),
7038	);
7039	expect(
7040	(yield* machines.listMachines({
7041	app_name: initial.appName,
7042	})).filter((machine) => machine.state !== "destroyed"),
7043	).toHaveLength(2);
7044	const unchanged = yield* deploy(45_000);
7045	expect(unchanged.machineIds).toEqual(recovered.machineIds);
7046	yield* stack.destroy();
7047	}),
7048	{ timeout: 300_000 },
7049	);
7050
7051	test.provider(
7052	"rolling opt-out retires unfinished candidates without changing the surviving ID",
7053	(stack) =>
7054	Effect.gen(function* () {
7055	yield* stack.destroy();
7056	const deploy = (strategy: "rolling" | "bluegreen") =>
7057	stack.deploy(
7058	Effect.gen(function* () {
7059	const app = yield* Fly.App("Site");
7060	const other = yield* Fly.Machine("Other", {
7061	app,
7062	image: "nginx:alpine",
7063	});
7064	const worker = yield* Fly.Machine("Worker", {
7065	app,
7066	image: "nginx:alpine",
7067	checks,
7068	deploy: { strategy },
7069	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
7070	});
7071	return { other, worker };
7072	}),
7073	);
7074	const initial = yield* deploy("bluegreen");
7075	const source = yield* machines.getMachine({
7076	app_name: initial.worker.appName,
7077	machine_id: initial.worker.machineId,
7078	});
7079	const candidate = yield* machines.createMachine({
7080	app_name: initial.worker.appName,
7081	name: "interrupted-candidate",
7082	region: "iad",
7083	config: {
7084	...source.config,
7085	metadata: {
7086	...source.config?.metadata,
7087	"alchemy.generation": "unfinished",
7088	"alchemy.sequence": "2",
7089	"alchemy.phase": "promoting",
7090	},
7091	},
7092	});
7093	const recovered = yield* deploy("rolling");
7094	expect(recovered.worker.machineId).toBe(initial.worker.machineId);
7095	expect(recovered.other.machineId).toBe(initial.other.machineId);
7096	const live = (yield* machines.listMachines({
7097	app_name: initial.worker.appName,
7098	})).filter((machine) => machine.state !== "destroyed");
7099	expect(live.map((machine) => machine.id).sort()).toEqual(
7100	[initial.worker.machineId, initial.other.machineId].sort(),
7101	);
7102	expect(live.some((machine) => machine.id === candidate.id)).toBe(
7103	false,
7104	);
7105	yield* stack.destroy();
7106	}),
7107	{ timeout: 300_000 },
7108	);
7109
7110	test.provider(
7111	"recovers partial preparation and isolates old engine-instance cleanup",
7112	(stack) =>
7113	Effect.gen(function* () {
7114	yield* stack.destroy();
7115	const deploy = (healthTimeout: number) =>
7116	stack.deploy(
7117	Effect.gen(function* () {
7118	const app = yield* Fly.App("Site");
7119	return yield* Fly.Machine("Worker", {
7120	app,
7121	count: 2,
7122	image: "nginx:alpine",
7123	checks,
7124	deploy: { strategy: "bluegreen", healthTimeout },
7125	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
7126	});
7127	}),
7128	);
7129	const initial = yield* deploy(30_000);
7130	const source = yield* machines.getMachine({
7131	app_name: initial.appName,
7132	machine_id: initial.machineId,
7133	});
7134	const metadata = source.config!.metadata!;
7135	const candidate = yield* machines.createMachine({
7136	app_name: initial.appName,
7137	name: "interrupted-preparation",
7138	region: "iad",
7139	skip_service_registration: true,
7140	config: {
7141	...source.config,
7142	image: metadata["alchemy.image"],
7143	metadata: {
7144	...metadata,
7145	"alchemy.generation": "interrupted",
7146	"alchemy.sequence": "2",
7147	"alchemy.replica": "0",
7148	"alchemy.phase": "candidate",
7149	},
7150	},
7151	});
7152	const read = yield* observeReplicaSet({
7153	appName: initial.appName,
7154	id: "Worker",
7155	type: "Fly.Machine",
7156	fqn: metadata["alchemy.fqn"]!,
7157	resourceInstanceId: metadata["alchemy.instance"]!,
7158	baseName: initial.baseName,
7159	machineIds: initial.machineIds,
7160	});
7161	expect(read?.machineIds).toEqual(initial.machineIds);
7162	expect(read?.rolloutPending).toBe(true);
7163	const recovered = yield* deploy(40_000);
7164	expect(recovered.machineIds[0]).toBe(candidate.id);
7165	expect(recovered.machineIds).toHaveLength(2);
7166	yield* deleteReplicaSet({
7167	appName: initial.appName,
7168	id: "Worker",
7169	type: "Fly.Machine",
7170	fqn: metadata["alchemy.fqn"]!,
7171	resourceInstanceId: "previous-engine-instance",
7172	machineIds: recovered.machineIds,
7173	volumeIds: [],
7174	});
7175	const live = (yield* machines.listMachines({
7176	app_name: initial.appName,
7177	})).filter((machine) => machine.state !== "destroyed");
7178	expect(live.map((machine) => machine.id).sort()).toEqual(
7179	[...recovered.machineIds].sort(),
7180	);
7181	yield* stack.destroy();
7182	}),
7183	{ timeout: 300_000 },
7184	);
7185
7186	test.provider(
7187	"replaces the entire generation when one replica drifts",
7188	(stack) =>
7189	Effect.gen(function* () {
7190	yield* stack.destroy();
7191	const deploy = (healthTimeout: number, count = 2) =>
7192	stack.deploy(
7193	Effect.gen(function* () {
7194	const app = yield* Fly.App("Site");
7195	return yield* Fly.Machine("Worker", {
7196	app,
7197	count,
7198	image: "nginx:alpine",
7199	checks,
7200	deploy: { strategy: "bluegreen", healthTimeout },
7201	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
7202	});
7203	}),
7204	);
7205	const initial = yield* deploy(30_000);
7206	const target = {
7207	app_name: initial.appName,
7208	machine_id: initial.machineIds[1]!,
7209	};
7210	const source = yield* machines.getMachine(target);
7211	const drifted = yield* machines.updateMachine({
7212	...target,
7213	config: { ...source.config, env: { DRIFT: "true" } },
7214	});
7215	expect(drifted.config?.env?.DRIFT).toBe("true");
7216	const observed = yield* machines
7217	.listMachines({ app_name: initial.appName })
7218	.pipe(
7219	Effect.map((listed) =>
7220	listed.find((machine) => machine.id === target.machine_id),
7221	),
7222	Effect.repeat({
7223	schedule: Schedule.spaced("2 seconds"),
7224	until: (machine) =>
7225	machine?.state === "started" &&
7226	machine.config?.env?.DRIFT === "true",
7227	times: 10,
7228	}),
7229	);
7230	expect(observed?.config?.env?.DRIFT).toBe("true");
7231	expect(observed?.state).toBe("started");
7232	const recovered = yield* deploy(40_000);
7233	expect(
7234	recovered.machineIds.every(
7235	(id) => !initial.machineIds.includes(id),
7236	),
7237	).toBe(true);
7238	const live = (yield* machines.listMachines({
7239	app_name: initial.appName,
7240	})).filter((machine) => machine.state !== "destroyed");
7241	expect(live).toHaveLength(2);
7242	expect(
7243	live.every((machine) => machine.config?.env?.DRIFT === undefined),
7244	).toBe(true);
7245	expect(
7246	new Set(live.map((machine) => machine.image_ref?.digest)).size,
7247	).toBe(1);
7248	yield* stack.destroy();
7249	}),
7250	{ timeout: 300_000 },
7251	);
7252	},
7253	);
7254
7255	describe.sequential(
7256	"required metadata writes",
7257	{
7258	tags: [
7259	"provider:fly",
7260	"provider:fly:app",
7261	"provider:fly:machine",
7262	"provider:fly:service",
7263	"live",
7264	],
7265	},
7266	() => {
7267	let endpoint: string | undefined;
7268	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
7269
7270	const props = {
7271	count: 2,
7272	deploy: { strategy: "bluegreen", healthTimeout: "90 seconds" },
7273	} as const;
7274
7275	test.provider(
7276	"F05 real required promotion-intent write fault forbids every uncordon and early old deletion",
7277	(stack) =>
7278	Effect.gen(function* () {
7279	yield* stack.destroy();
7280	const initial = yield* deployWorker(stack, "one", props);
7281	const proxy = yield* transportProxy();
7282	try {
7283	yield* Effect.sync(() => {
7284	endpoint = proxy.url;
7285	proxy.arm({
7286	match: (event) =>
7287	event.method === "PUT" &&
7288	event.path.endsWith("/metadata") &&
7289	event.phase === "promoting",
7290	action: "cut-request",
7291	remaining: Infinity,
7292	});
7293	});
7294	const result = yield* deployWorker(stack, "two", props).pipe(
7295	Effect.result,
7296	);
7297	expect(Result.isFailure(result)).toBe(true);
7298	if (Result.isFailure(result))
7299	expect(result.failure).toMatchObject({
7300	_tag: "Fly.MachineMutationUncertain",
7301	});
7302	const cut = proxy.events.filter(
7303	(event) => event.stage === "cut" && event.phase === "promoting",
7304	);
7305	expect(cut.length).toBeGreaterThan(0);
7306	expect(new Set(cut.map((event) => event.machineId)).size).toBe(1);
7307	const live = yield* census(initial.appName);
7308	const old = live.filter((machine) =>
7309	initial.machineIds.includes(machine.id!),
7310	);
7311	const candidates = live.filter(
7312	(machine) => !initial.machineIds.includes(machine.id!),
7313	);
7314	expect(old.map((machine) => machine.id).sort()).toEqual(
7315	[...initial.machineIds].sort(),
7316	);
7317	expect(
7318	old.every(
7319	(machine) =>
7320	machine.cordoned === false && machine.state === "started",
7321	),
7322	).toBe(true);
7323	expect(candidates).toHaveLength(2);
7324	expect(
7325	candidates.every(
7326	(machine) =>
7327	machine.cordoned === true &&
7328	machine.config?.metadata?.["alchemy.phase"] === "candidate",
7329	),
7330	).toBe(true);
7331	expect(
7332	candidates.some((machine) => machine.id === cut[0]!.machineId),
7333	).toBe(true);
7334	expect(
7335	proxy.events.some((event) => event.path.endsWith("/uncordon")),
7336	).toBe(false);
7337	expect(
7338	proxy.events.some(
7339	(event) =>
7340	initial.machineIds.includes(event.machineId!) &&
7341	(event.path.endsWith("/cordon") ||
7342	event.path.endsWith("/stop") ||
7343	(event.method === "DELETE" &&
7344	/\/machines\/[^/]+$/.test(event.path))),
7345	),
7346	).toBe(false);
7347	expect(
7348	proxy.events.some(
7349	(event) =>
7350	event.phase === "active" || event.phase === "retiring",
7351	),
7352	).toBe(false);
7353	expect(
7354	proxy.events.some(
7355	(event) =>
7356	event.method === "DELETE" &&
7357	/\/machines\/[^/]+$/.test(event.path),
7358	),
7359	).toBe(false);
7360	const candidateIds = candidates
7361	.map((machine) => machine.id!)
7362	.sort();
7363	yield* Effect.sync(proxy.clear);
7364	const recovered = yield* deployWorker(stack, "two", props);
7365	expect([...recovered.machineIds].sort()).toEqual(candidateIds);
7366	yield* assertCommitted(initial.appName, recovered.machineIds);
7367	} finally {
7368	yield* Effect.sync(() => {
7369	endpoint = undefined;
7370	proxy.clear();
7371	});
7372	}
7373	yield* stack.destroy();
7374	yield* assertAppGone(initial.appName);
7375	}).pipe(
7376	Effect.scoped,
7377	Effect.ensuring(
7378	Effect.sync(() => {
7379	endpoint = undefined;
7380	}),
7381	),
7382	),
7383	{ timeout: 900_000 },
7384	);
7385	},
7386	);
7387
7388	describe.sequential(
7389	"retirement residuals",
7390	{
7391	tags: [
7392	"provider:fly",
7393	"provider:fly:app",
7394	"provider:fly:machine",
7395	"provider:fly:service",
7396	"live",
7397	],
7398	},
7399	() => {
7400	let endpoint: string | undefined;
7401	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
7402	const props = {
7403	count: 2,
7404	deploy: { strategy: "bluegreen", healthTimeout: "90 seconds" },
7405	} as const;
7406
7407	describe.sequential("real retirement residual diagnostics", () => {
7408	for (const operation of ["cordon", "stop", "delete"] as const) {
7409	test.provider(
7410	`F04 real ${operation} transport fault reports exact target/stage while sibling retirement completes`,
7411	(stack) =>
7412	Effect.gen(function* () {
7413	yield* stack.destroy();
7414	const initial = yield* deployWorker(stack, "one", props);
7415	const [targetId, siblingId] = [...initial.machineIds].sort();
7416	expect(initial.machineIds).toHaveLength(2);
7417	const proxy = yield* transportProxy();
7418	try {
7419	yield* Effect.sync(() => {
7420	endpoint = proxy.url;
7421	proxy.arm({
7422	match: (event) =>
7423	event.machineId === targetId &&
7424	(operation === "delete"
7425	? event.method === "DELETE" &&
7426	event.path.endsWith(`/machines/${targetId}`)
7427	: event.method === "POST" &&
7428	event.path.endsWith(`/${operation}`)),
7429	action: "cut-request",
7430	remaining: Infinity,
7431	});
7432	});
7433	const result = yield* deployWorker(stack, "two", props).pipe(
7434	Effect.result,
7435	);
7436	expect(Result.isFailure(result)).toBe(true);
7437	if (Result.isFailure(result)) {
7438	expect(result.failure).toBeInstanceOf(
7439	ReplicaRetirementIncomplete,
7440	);
7441	if (result.failure instanceof ReplicaRetirementIncomplete) {
7442	expect(result.failure.appName).toBe(initial.appName);
7443	expect(result.failure.residuals).toEqual([
7444	{
7445	machineId: targetId,
7446	stage: `${operation}: Fly.MachineMutationUncertain`,
7447	},
7448	]);
7449	}
7450	}
7451	const cuts = proxy.events.filter(
7452	(event) => event.stage === "cut",
7453	);
7454	expect(cuts.length).toBeGreaterThan(0);
7455	expect([
7456	...new Set(cuts.map((event) => event.machineId)),
7457	]).toEqual([targetId]);
7458	const live = yield* census(initial.appName);
7459	const old = live.filter((machine) =>
7460	initial.machineIds.includes(machine.id!),
7461	);
7462	const green = live.filter(
7463	(machine) => !initial.machineIds.includes(machine.id!),
7464	);
7465	expect(old.map((machine) => machine.id)).toEqual([targetId]);
7466	expect(live.some((machine) => machine.id === siblingId)).toBe(
7467	false,
7468	);
7469	expect(old[0]!.cordoned).toBe(operation !== "cordon");
7470	expect(old[0]!.state).toBe(
7471	operation === "delete" ? "stopped" : "started",
7472	);
7473	expect(green).toHaveLength(2);
7474	expect(
7475	green.every(
7476	(machine) =>
7477	machine.cordoned === false &&
7478	machine.config?.metadata?.["alchemy.phase"] === "active",
7479	),
7480	).toBe(true);
7481	expect(
7482	proxy.events.some(
7483	(event) =>
7484	event.stage === "completed" &&
7485	event.machineId === siblingId &&
7486	event.method === "DELETE" &&
7487	event.path.endsWith(`/machines/${siblingId}`) &&
7488	event.status! >= 200 &&
7489	event.status! < 300,
7490	),
7491	).toBe(true);
7492	const greenIds = green.map((machine) => machine.id!).sort();
7493	expect(live.map((machine) => machine.id).sort()).toEqual(
7494	[...greenIds, targetId].sort(),
7495	);
7496	yield* Effect.sync(proxy.clear);
7497	const recovered = yield* deployWorker(stack, "two", props);
7498	expect([...recovered.machineIds].sort()).toEqual(greenIds);
7499	yield* assertCommitted(initial.appName, recovered.machineIds);
7500	expect(
7501	(yield* census(initial.appName)).some((machine) =>
7502	initial.machineIds.includes(machine.id!),
7503	),
7504	).toBe(false);
7505	} finally {
7506	yield* Effect.sync(() => {
7507	endpoint = undefined;
7508	proxy.clear();
7509	});
7510	}
7511	yield* stack.destroy();
7512	yield* assertAppGone(initial.appName);
7513	}).pipe(
7514	Effect.scoped,
7515	Effect.ensuring(
7516	Effect.sync(() => {
7517	endpoint = undefined;
7518	}),
7519	),
7520	),
7521	{ timeout: 900_000 },
7522	);
7523	}
7524	});
7525	},
7526	);
7527
7528	describe.sequential(
7529	"retirement faults",
7530	{
7531	tags: [
7532	"provider:fly",
7533	"provider:fly:app",
7534	"provider:fly:machine",
7535	"provider:fly:service",
7536	"live",
7537	],
7538	},
7539	() => {
7540	let endpoint: string | undefined;
7541	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
7542
7543	describe.sequential("retirement faults", () => {
7544	test.provider(
7545	"F04 a real second remover accepts terminal success or typed NotFound and destroy stays idempotent",
7546	(stack) =>
7547	Effect.gen(function* () {
7548	yield* stack.destroy();
7549	const initial = yield* deployWorker(stack, "one");
7550	const target = {
7551	app_name: initial.appName,
7552	machine_id: initial.machineId,
7553	force: true,
7554	};
7555	yield* machines.deleteMachine(target).pipe(Retry.none);
7556	const removedAgain = yield* machines
7557	.deleteMachine(target)
7558	.pipe(Retry.none, Effect.result);
7559	if (Result.isFailure(removedAgain))
7560	expect(removedAgain.failure._tag).toBe("NotFound");
7561	const absent = yield* machines
7562	.getMachine({
7563	app_name: target.app_name,
7564	machine_id: target.machine_id,
7565	})
7566	.pipe(
7567	Retry.none,
7568	Effect.map(
7569	(machine) =>
7570	machine.id === target.machine_id &&
7571	machine.state === "destroyed",
7572	),
7573	Effect.catchTag("NotFound", () => Effect.succeed(true)),
7574	Effect.repeat({
7575	times: 8,
7576	schedule: Schedule.spaced("2 seconds"),
7577	until: (gone) => gone,
7578	}),
7579	Effect.timeout("30 seconds"),
7580	);
7581	expect(absent).toBe(true);
7582	yield* Effect.logInfo("Observed real second-remover outcome", {
7583	appName: initial.appName,
7584	machineId: initial.machineId,
7585	outcome: Result.isSuccess(removedAgain)
7586	? "accepted"
7587	: removedAgain.failure._tag,
7588	absenceConfirmed: absent,
7589	});
7590	expect(yield* census(initial.appName)).toHaveLength(0);
7591	yield* stack.destroy();
7592	yield* assertAppGone(initial.appName);
7593	yield* stack.destroy();
7594	}),
7595	{ timeout: 180_000 },
7596	);
7597	test.provider(
7598	"FLY-REVIEW-2 lost successful DELETE does not cancel a sibling draining across renewal",
7599	(stack) =>
7600	Effect.gen(function* () {
7601	yield* stack.destroy();
7602	const props = {
7603	count: 2,
7604	shutdown: { signal: "SIGTERM", timeout: "90 seconds" },
7605	init: {
7606	exec: [
7607	"/bin/sh",
7608	"-c",
7609	"trap 'if [ -f /slow-drain ]; then sleep 60; fi; exit 0' TERM; nginx -g 'daemon off;' & while :; do sleep 1 & wait $!; done",
7610	],
7611	},
7612	} satisfies Partial<
7613	Omit<Extract<MachineProps, { image: string }>, "app">
7614	>;
7615	const initial = yield* deployWorker(stack, "one", props);
7616	const [fastId, slowId] = [...initial.machineIds].sort();
7617	expect(initial.machineIds).toHaveLength(2);
7618	const configured = yield* machines.execMachine({
7619	app_name: initial.appName,
7620	machine_id: slowId!,
7621	command: ["touch", "/slow-drain"],
7622	timeout: 5,
7623	});
7624	expect(configured.exit_code).toBe(0);
7625	const proxy = yield* transportProxy();
7626	const isFastDelete = (event: TransportEvent) =>
7627	event.machineId === fastId &&
7628	event.method === "DELETE" &&
7629	event.path.endsWith(`/machines/${fastId}`);
7630	yield* Effect.sync(() => {
7631	endpoint = proxy.url;
7632	proxy.arm({
7633	match: isFastDelete,
7634	action: "drop-response",
7635	remaining: 1,
7636	});
7637	proxy.arm({
7638	match: isFastDelete,
7639	action: "cut-request",
7640	remaining: Infinity,
7641	});
7642	});
7643	const update = yield* deployWorker(stack, "two", props).pipe(
7644	Effect.result,
7645	Effect.forkScoped,
7646	);
7647	const dropped = yield* proxy.wait(
7648	(event) =>
7649	isFastDelete(event) &&
7650	event.stage === "dropped" &&
7651	event.status! >= 200 &&
7652	event.status! < 300,
7653	);
7654	const deletedAt = yield* Clock.currentTimeMillis;
7655	// The real old process remains in its shutdown trap across the 25-second renewal tick.
7656	yield* Effect.sleep("30 seconds");
7657	const draining = yield* machines.getMachine({
7658	app_name: initial.appName,
7659	machine_id: slowId!,
7660	});
7661	expect(draining.state).not.toBe("destroyed");
7662	expect(
7663	(yield* Clock.currentTimeMillis) - deletedAt,
7664	).toBeGreaterThanOrEqual(30_000);
7665	expect(
7666	proxy.events.some(
7667	(event) =>
7668	event.sequence > dropped.sequence &&
7669	event.stage === "completed" &&
7670	event.method === "POST" &&
7671	event.machineId === slowId &&
7672	event.path.endsWith("/lease") &&
7673	event.status! >= 200 &&
7674	event.status! < 300,
7675	),
7676	).toBe(true);
7677	const result = yield* Fiber.join(update).pipe(
7678	Effect.timeout("240 seconds"),
7679	);
7680	expect(Result.isSuccess(result)).toBe(true);
7681	if (Result.isSuccess(result))
7682	yield* assertCommitted(
7683	initial.appName,
7684	result.success.machineIds,
7685	);
7686	const live = yield* census(initial.appName);
7687	expect(live).toHaveLength(2);
7688	expect(
7689	live.every(
7690	(machine) => !initial.machineIds.includes(machine.id!),
7691	),
7692	).toBe(true);
7693	expect(
7694	proxy.events.some(
7695	(event) =>
7696	event.stage === "completed" &&
7697	event.method === "GET" &&
7698	event.path.endsWith(`/machines/${fastId}`) &&
7699	event.sequence > dropped.sequence &&
7700	(event.status === 404 || event.state === "destroyed"),
7701	),
7702	).toBe(true);
7703	yield* Effect.logInfo(
7704	"Exact-target retirement transport evidence",
7705	{
7706	machineId: fastId,
7707	events: proxy.events.filter(
7708	(event) =>
7709	isFastDelete(event) ||
7710	(event.method === "GET" &&
7711	event.path.endsWith(`/machines/${fastId}`)),
7712	),
7713	},
7714	);
7715	expect(
7716	proxy.events.some(
7717	(event) =>
7718	isFastDelete(event) &&
7719	event.stage === "request" &&
7720	event.sequence > dropped.sequence,
7721	),
7722	).toBe(false);
7723	expect(
7724	proxy.events.some(
7725	(event) =>
7726	event.stage === "completed" &&
7727	event.machineId === slowId &&
7728	event.method === "DELETE" &&
7729	event.path.endsWith(`/machines/${slowId}`) &&
7730	event.status! >= 200 &&
7731	event.status! < 300,
7732	),
7733	).toBe(true);
7734	yield* Effect.sync(() => {
7735	proxy.clear();
7736	endpoint = undefined;
7737	});
7738	yield* stack.destroy();
7739	yield* assertAppGone(initial.appName);
7740	}).pipe(
7741	Effect.scoped,
7742	Effect.ensuring(
7743	Effect.sync(() => {
7744	endpoint = undefined;
7745	}),
7746	),
7747	),
7748	{ timeout: 600_000 },
7749	);
7750
7751	for (const operation of ["cordon", "stop", "delete"] as const) {
7752	test.provider(
7753	`F04 completed ${operation} response loss preserves green and recovers residuals`,
7754	(stack) =>
7755	Effect.gen(function* () {
7756	yield* stack.destroy();
7757	const initial = yield* deployWorker(stack, "one");
7758	const proxy = yield* transportProxy();
7759	const match = (event: TransportEvent) =>
7760	event.machineId === initial.machineId &&
7761	(operation === "delete"
7762	? event.method === "DELETE" &&
7763	event.path.endsWith(initial.machineId)
7764	: event.path.endsWith(`/${operation}`));
7765	yield* Effect.sync(() => {
7766	endpoint = proxy.url;
7767	proxy.arm({ match, action: "drop-response", remaining: 1 });
7768	proxy.arm({
7769	match,
7770	action: "cut-request",
7771	remaining: Infinity,
7772	});
7773	});
7774	const result = yield* deployWorker(stack, "two").pipe(
7775	Effect.timeout("120 seconds"),
7776	Effect.result,
7777	);
7778	expect(
7779	proxy.events.some(
7780	(event) => event.stage === "dropped" && event.status! < 300,
7781	),
7782	).toBe(true);
7783	const live = yield* census(initial.appName);
7784	const green = live.filter(
7785	(machine) => machine.id !== initial.machineId,
7786	);
7787	expect(green).toHaveLength(1);
7788	expect(green[0]!.cordoned).toBe(false);
7789	expect(green[0]!.config?.metadata?.["alchemy.phase"]).toBe(
7790	"active",
7791	);
7792	if (live.some((machine) => machine.id === initial.machineId))
7793	expect(Result.isFailure(result)).toBe(true);
7794	yield* Effect.sync(proxy.clear);
7795	const recovered = yield* deployWorker(stack, "two");
7796	expect(recovered.machineIds).toEqual(
7797	green.map((machine) => machine.id),
7798	);
7799	yield* assertCommitted(initial.appName, recovered.machineIds);
7800	yield* Effect.sync(() => {
7801	endpoint = undefined;
7802	});
7803	yield* stack.destroy();
7804	yield* assertAppGone(initial.appName);
7805	}).pipe(
7806	Effect.scoped,
7807	Effect.ensuring(
7808	Effect.sync(() => {
7809	endpoint = undefined;
7810	}),
7811	),
7812	),
7813	{ timeout: 300_000 },
7814	);
7815	}
7816
7817	for (const phase of ["retiring", "active"] as const) {
7818	test.provider(
7819	`F05 ${phase === "retiring" ? "advisory old" : "required commit"} metadata connection failure`,
7820	(stack) =>
7821	Effect.gen(function* () {
7822	yield* stack.destroy();
7823	const initial = yield* deployWorker(stack, "one");
7824	const proxy = yield* transportProxy();
7825	yield* Effect.sync(() => {
7826	endpoint = proxy.url;
7827	proxy.arm({
7828	match: (event) =>
7829	event.path.endsWith("/metadata") && event.phase === phase,
7830	action: "cut-request",
7831	remaining: Infinity,
7832	});
7833	});
7834	const result = yield* deployWorker(stack, "two").pipe(
7835	Effect.timeout("120 seconds"),
7836	Effect.result,
7837	);
7838	expect(
7839	proxy.events.some(
7840	(event) => event.stage === "cut" && event.phase === phase,
7841	),
7842	).toBe(true);
7843	const live = yield* census(initial.appName);
7844	const green = live.filter(
7845	(machine) => machine.id !== initial.machineId,
7846	);
7847	expect(green).toHaveLength(1);
7848	expect(green[0]!.cordoned).toBe(false);
7849	if (phase === "retiring") {
7850	expect(Result.isSuccess(result)).toBe(true);
7851	expect(
7852	live.some((machine) => machine.id === initial.machineId),
7853	).toBe(false);
7854	} else {
7855	expect(Result.isFailure(result)).toBe(true);
7856	expect(
7857	live.find((machine) => machine.id === initial.machineId)
7858	?.cordoned,
7859	).toBe(false);
7860	expect(
7861	proxy.events.some(
7862	(event) =>
7863	event.machineId === initial.machineId &&
7864	["POST", "DELETE"].includes(event.method) &&
7865	!event.path.endsWith("/lease"),
7866	),
7867	).toBe(false);
7868	}
7869	yield* Effect.sync(proxy.clear);
7870	const recovered = yield* deployWorker(stack, "two");
7871	expect(recovered.machineIds).toEqual(
7872	green.map((machine) => machine.id),
7873	);
7874	yield* assertCommitted(initial.appName, recovered.machineIds);
7875	yield* Effect.sync(() => {
7876	endpoint = undefined;
7877	});
7878	yield* stack.destroy();
7879	yield* assertAppGone(initial.appName);
7880	}).pipe(
7881	Effect.scoped,
7882	Effect.ensuring(
7883	Effect.sync(() => {
7884	endpoint = undefined;
7885	}),
7886	),
7887	),
7888	{ timeout: 300_000 },
7889	);
7890	}
7891	});
7892	},
7893	);
7894
7895	describe.sequential(
7896	"runtime secrets",
7897	{
7898	tags: [
7899	"provider:fly",
7900	"provider:fly:app",
7901	"provider:fly:ipassignment",
7902	"provider:fly:machine",
7903	"provider:fly:secret",
7904	"provider:fly:service",
7905	"live",
7906	],
7907	},
7908	() => {
7909	const { test } = Test.make({ providers: Fly.providers() });
7910
7911	class WriterProbeFailed extends Data.TaggedError("WriterProbeFailed")<{
7912	stage: "transport" | "status" | "decode";
7913	status?: number;
7914	}> {}
7915
7916	const Receipt = Schema.Struct({
7917	version: Schema.Number,
7918	machineId: Schema.String,
7919	marker: Schema.Literals(["ready", "three"]),
7920	});
7921
7922	const probeWriter = (
7923	appName: string,
7924	token: Redacted.Redacted<string>,
7925	method: "GET" | "POST",
7926	) =>
7927	Effect.gen(function* () {
7928	const client = yield* HttpClient.HttpClient;
7929	const url = `https://${appName}.fly.dev/writer`;
7930	const request = (
7931	method === "POST"
7932	? HttpClientRequest.post(url)
7933	: HttpClientRequest.get(url)
7934	).pipe(HttpClientRequest.bearerToken(token));
7935	const response = yield* client
7936	.execute(request)
7937	.pipe(
7938	Effect.mapError(
7939	() => new WriterProbeFailed({ stage: "transport" }),
7940	),
7941	);
7942	if (response.status !== 200) {
7943	return yield* new WriterProbeFailed({
7944	stage: "status",
7945	status: response.status,
7946	});
7947	}
7948	return yield* response.json.pipe(
7949	Effect.flatMap(
7950	Schema.decodeUnknownEffect(Receipt, { onExcessProperty: "error" }),
7951	),
7952	Effect.mapError(() => new WriterProbeFailed({ stage: "decode" })),
7953	);
7954	}).pipe(
7955	Effect.timeout("30 seconds"),
7956	Effect.catchTag("TimeoutError", () =>
7957	Effect.fail(new WriterProbeFailed({ stage: "transport" })),
7958	),
7959	);
7960
7961	const marker = (appName: string, machineId: string) =>
7962	machines
7963	.execMachine({
7964	app_name: appName,
7965	machine_id: machineId,
7966	command: ["cat", "/usr/share/nginx/html/marker"],
7967	timeout: 5,
7968	})
7969	.pipe(
7970	Effect.map((response) => {
7971	const value = response.stdout?.trim();
7972	return {
7973	code: response.exit_code,
7974	marker: value === "two" || value === "three" ? value : "missing",
7975	};
7976	}),
7977	);
7978
7979	test.provider(
7980	"F12 deployed sibling runtime writer advances the shared vault during a held candidate create",
7981	(stack) =>
7982	Effect.gen(function* () {
7983	yield* stack.destroy();
7984	const proxy = yield* transportProxy();
7985	let appName: string | undefined;
7986	yield* Effect.addFinalizer(() =>
7987	stack.destroy().pipe(
7988	Effect.andThen(() =>
7989	appName === undefined ? Effect.void : assertAppGone(appName),
7990	),
7991	Effect.orDie,
7992	),
7993	);
7994	const actor = yield* engineActor(
7995	stack,
7996	"F12 deployed sibling runtime writer advances the shared vault during a held candidate create",
7997	"test/Fly/BlueGreen.test.ts",
7998	proxy.url,
7999	);
8000	const trigger = yield* Effect.sync(() =>
8001	Redacted.make(
8002	Array.from(randomBytes(32), (byte) =>
8003	byte.toString(16).padStart(2, "0"),
8004	).join(""),
8005	),
8006	);
8007	const deploy = (count: number, floor?: number) =>
8008	actor.deploy(
8009	Effect.gen(function* () {
8010	const app = yield* Site;
8011	const secret = yield* Token;
8012	const auth = yield* Fly.Secret("WriterTrigger", {
8013	app,
8014	name: TRIGGER_SECRET,
8015	value: trigger,
8016	});
8017	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
8018	const writer = yield* Writer.pipe(
8019	Effect.provide(writerLayer(auth.digest)),
8020	);
8021	const consumer = yield* Fly.Machine("Consumer", {
8022	app,
8023	region: "iad",
8024	image: "nginx:alpine",
8025	count,
8026	minSecretsVersion: floor,
8027	env: { SECRET_RESOURCE_NAME: secret.name },
8028	init: {
8029	exec: [
8030	"/bin/sh",
8031	"-c",
8032	"case \"$ACCEPTANCE_RUNTIME_SECRET\" in *-two) marker=two;; *-three) marker=three;; *) marker=missing;; esac; printf '%s' \"$marker\" > /usr/share/nginx/html/marker; exec nginx -g 'daemon off;'",
8033	],
8034	},
8035	checks,
8036	deploy: {
8037	strategy: "bluegreen",
8038	healthTimeout: "60 seconds",
8039	},
8040	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
8041	});
8042	return { app, secret, writer, consumer };
8043	}),
8044	);
8045
8046	const initial = yield* deploy(1);
8047	appName = initial.app.appName;
8048	expect(initial.writer.machineIds).toHaveLength(1);
8049	expect(initial.consumer.machineIds).toHaveLength(1);
8050	expect(initial.consumer.imageRef?.digest).toBeTruthy();
8051	expect(yield* marker(appName, initial.consumer.machineId)).toEqual({
8052	code: 0,
8053	marker: "two",
8054	});
8055	const before = yield* probeWriter(appName, trigger, "GET").pipe(
8056	Effect.retry({
8057	while: (error) =>
8058	error.stage === "transport" ||
8059	(error.stage === "status" &&
8060	[404, 502, 503].includes(error.status ?? 0)),
8061	schedule: Schedule.spaced("2 seconds"),
8062	times: 8,
8063	}),
8064	Effect.timeout("90 seconds"),
8065	);
8066	expect(before).toEqual({
8067	version: 0,
8068	machineId: initial.writer.machineId,
8069	marker: "ready",
8070	});
8071	const client = yield* HttpClient.HttpClient;
8072	const unauthorized = yield* client
8073	.post(`https://${appName}.fly.dev/writer`)
8074	.pipe(
8075	Effect.timeout("30 seconds"),
8076	Effect.mapError(
8077	() => new WriterProbeFailed({ stage: "transport" }),
8078	),
8079	);
8080	expect(unauthorized.status).toBe(401);
8081	const initialCensus = yield* census(appName);
8082	expect(initialCensus.map((machine) => machine.id).sort()).toEqual(
8083	[
8084	...initial.writer.machineIds,
8085	...initial.consumer.machineIds,
8086	].sort(),
8087	);
8088	const writerBefore = initialCensus.find(
8089	(machine) => machine.id === initial.writer.machineId,
8090	);
8091	expect(writerBefore?.instance_id).toBeTruthy();
8092	expect(writerBefore?.image_ref?.digest).toBeTruthy();
8093	const secretWrites = proxy.events.filter(
8094	(event) =>
8095	event.stage === "completed" &&
8096	event.method === "POST" &&
8097	event.path.startsWith(`/v1/apps/${appName}/secrets`) &&
8098	event.status !== undefined &&
8099	event.status >= 200 &&
8100	event.status < 300 &&
8101	event.secretsVersion !== undefined,
8102	);
8103	expect(secretWrites.length).toBeGreaterThan(0);
8104	const floor = Math.max(
8105	...secretWrites.map((event) => event.secretsVersion!),
8106	);
8107	expect(Number.isSafeInteger(floor)).toBe(true);
8108	expect(floor).toBeGreaterThan(0);
8109	const start = proxy.events.length;
8110	const machinePath = `/v1/apps/${appName}/machines`;
8111	yield* Effect.sync(() =>
8112	proxy.arm({
8113	match: (event) =>
8114	event.method === "POST" &&
8115	event.path === machinePath &&
8116	event.phase === "candidate",
8117	action: "hold-response",
8118	remaining: 1,
8119	}),
8120	);
8121	yield* Effect.gen(function* () {
8122	const rollout = yield* deploy(2, floor).pipe(
8123	Effect.scoped,
8124	Effect.forkScoped,
8125	);
8126	const held = yield* proxy.wait(
8127	(event) =>
8128	event.stage === "held" &&
8129	event.path === machinePath &&
8130	event.status !== undefined &&
8131	event.status >= 200 &&
8132	event.status < 300,
8133	);
8134	expect(held.machineId).toBeTruthy();
8135	expect(held.minSecretsVersion).toBe(floor);
8136	// This POST runs the existing WriteSecret binding inside the sibling Machine.
8137	const later = yield* probeWriter(
8138	initial.app.appName,
8139	trigger,
8140	"POST",
8141	);
8142	expect(later.machineId).toBe(initial.writer.machineId);
8143	expect(later.marker).toBe("three");
8144	expect(Number.isSafeInteger(later.version)).toBe(true);
8145	expect(later.version).toBeGreaterThan(floor);
8146	expect(
8147	proxy.events.some(
8148	(event) =>
8149	event.sequence === held.sequence &&
8150	event.stage === "forwarded",
8151	),
8152	).toBe(false);
8153	yield* Effect.sync(proxy.release);
8154	const next = yield* Fiber.join(rollout).pipe(
8155	Effect.timeout("240 seconds"),
8156	);
8157	expect(next.app.appName).toBe(initial.app.appName);
8158	expect(next.secret.name).toBe(initial.secret.name);
8159	expect(next.writer.machineIds).toEqual(initial.writer.machineIds);
8160	expect(next.consumer.machineIds).toHaveLength(2);
8161	expect(next.consumer.machineIds).toContain(held.machineId);
8162	expect(next.consumer.machineIds).not.toContain(
8163	initial.consumer.machineId,
8164	);
8165	const events = proxy.events.slice(start);
8166	const creates = events.filter(
8167	(event) =>
8168	event.stage === "request" &&
8169	event.method === "POST" &&
8170	event.path === machinePath,
8171	);
8172	expect(creates).toHaveLength(2);
8173	expect(creates[0]?.sequence).toBe(held.sequence);
8174	expect(
8175	creates.every((event) => event.minSecretsVersion === floor),
8176	).toBe(true);
8177	const released = events.findIndex(
8178	(event) =>
8179	event.sequence === held.sequence && event.stage === "forwarded",
8180	);
8181	expect(released).toBeGreaterThan(-1);
8182	expect(events.indexOf(creates[1]!)).toBeGreaterThan(released);
8183	expect(
8184	events.some(
8185	(event) =>
8186	event.stage === "request" &&
8187	event.machineId === initial.writer.machineId &&
8188	event.method !== "GET",
8189	),
8190	).toBe(false);
8191	const live = yield* census(initial.app.appName);
8192	expect(live.map((machine) => machine.id).sort()).toEqual(
8193	[...next.writer.machineIds, ...next.consumer.machineIds].sort(),
8194	);
8195	const writerAfter = live.find(
8196	(machine) => machine.id === initial.writer.machineId,
8197	);
8198	expect(writerAfter?.instance_id).toBe(writerBefore?.instance_id);
8199	expect(writerAfter?.image_ref?.digest).toBe(
8200	writerBefore?.image_ref?.digest,
8201	);
8202	for (const id of next.consumer.machineIds) {
8203	const machine = live.find((machine) => machine.id === id);
8204	expect(machine?.config?.metadata?.["alchemy.phase"]).toBe(
8205	"active",
8206	);
8207	expect(machine?.image_ref?.digest).toBe(
8208	initial.consumer.imageRef?.digest,
8209	);
8210	const observed = yield* marker(initial.app.appName, id);
8211	expect(observed.code).toBe(0);
8212	expect(
8213	id === held.machineId ? ["two", "three"] : ["three"],
8214	).toContain(observed.marker);
8215	}
8216	expect(
8217	yield* probeWriter(initial.app.appName, trigger, "GET"),
8218	).toEqual(before);
8219	}).pipe(
8220	Effect.ensuring(
8221	Effect.sync(() => {
8222	proxy.clear();
8223	proxy.release();
8224	}),
8225	),
8226	Effect.scoped,
8227	);
8228	}).pipe(Effect.scoped),
8229	{ timeout: 900_000 },
8230	);
8231	},
8232	);
8233
8234	describe.sequential(
8235	"scaling",
8236	{
8237	tags: [
8238	"provider:fly",
8239	"provider:fly:app",
8240	"provider:fly:machine",
8241	"provider:fly:service",
8242	"live",
8243	],
8244	},
8245	() => {
8246	const { test } = Test.make({ providers: Fly.providers() });
8247
8248	test.provider(
8249	"S01 F13 real 1-to-10-to-10-to-1 generations pin one digest and check every candidate before routing or old retirement",
8250	(stack) =>
8251	Effect.gen(function* () {
8252	yield* stack.destroy();
8253	const proxy = yield* transportProxy();
8254	const actor = yield* engineActor(
8255	stack,
8256	"S01 F13 real 1-to-10-to-10-to-1 generations pin one digest and check every candidate before routing or old retirement",
8257	"test/Fly/BlueGreen.test.ts",
8258	proxy.url,
8259	);
8260	const services = [
8261	{
8262	protocol: "tcp" as const,
8263	internalPort: 80,
8264	autostop: "off" as const,
8265	ports: [{ port: 80, handlers: ["http" as const] }],
8266	checks: [
8267	{
8268	type: "http" as const,
8269	port: 80,
8270	path: "/",
8271	interval: "2s",
8272	timeout: "1s",
8273	},
8274	],
8275	},
8276	];
8277	const deploy = (version: string, count: number) =>
8278	actor.deploy(
8279	Effect.gen(function* () {
8280	const app = yield* Fly.App("Site");
8281	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
8282	return yield* Fly.Machine("Worker", {
8283	app,
8284	image: "nginx:alpine",
8285	count,
8286	services,
8287	checks,
8288	env: { VERSION: version },
8289	init: {
8290	exec: [
8291	"/bin/sh",
8292	"-c",
8293	"printf '%s:%s' \"$VERSION\" \"$FLY_MACHINE_ID\" > /usr/share/nginx/html/version; exec nginx -g 'daemon off;'",
8294	],
8295	},
8296	deploy: {
8297	strategy: "bluegreen",
8298	healthTimeout: "60 seconds",
8299	},
8300	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
8301	});
8302	}),
8303	);
8304	const client = yield* HttpClient.HttpClient;
8305	const traffic = (appName: string) =>
8306	client.get(`http://${appName}.fly.dev/version`).pipe(
8307	Effect.flatMap((response) => {
8308	expect(response.status).toBe(200);
8309	return response.text;
8310	}),
8311	);
8312	let previous = yield* deploy("one", 1);
8313	expect(
8314	yield* traffic(previous.appName).pipe(
8315	Effect.retry({
8316	schedule: Schedule.spaced("2 seconds"),
8317	times: 8,
8318	}),
8319	),
8320	).toBe(`one:${previous.machineId}`);
8321	for (const [version, count] of [
8322	["two", 10],
8323	["three", 10],
8324	["four", 1],
8325	] as const) {
8326	const start = proxy.events.length;
8327	const next = yield* deploy(version, count);
8328	const events = proxy.events.slice(start);
8329	const live = yield* assertCommitted(next.appName, next.machineIds);
8330	expect(live).toHaveLength(count);
8331	expect(
8332	new Set(
8333	live.map(
8334	(machine) => machine.config?.metadata?.["alchemy.generation"],
8335	),
8336	).size,
8337	).toBe(1);
8338	expect(
8339	new Set(
8340	live.map(
8341	(machine) => machine.config?.metadata?.["alchemy.replica"],
8342	),
8343	).size,
8344	).toBe(count);
8345	expect(
8346	live.every(
8347	(machine) =>
8348	machine.config?.metadata?.["alchemy.count"] ===
8349	String(count) && machine.config?.env?.VERSION === version,
8350	),
8351	).toBe(true);
8352	expect(
8353	next.machineIds.every((id) => !previous.machineIds.includes(id)),
8354	).toBe(true);
8355	const creates = events.filter(
8356	(event) =>
8357	event.stage === "completed" &&
8358	event.method === "POST" &&
8359	event.path.endsWith("/machines") &&
8360	event.status! < 300,
8361	);
8362	expect(creates).toHaveLength(count);
8363	const digest = creates[0]!.digest;
8364	expect(digest).toMatch(/^sha256:/);
8365	expect(
8366	live.every((machine) => machine.image_ref?.digest === digest),
8367	).toBe(true);
8368	expect(
8369	creates
8370	.slice(1)
8371	.every((event) => event.image?.endsWith(`@${digest}`)),
8372	).toBe(true);
8373	const firstRouting = events.findIndex(
8374	(event) =>
8375	event.stage === "request" && event.path.endsWith("/uncordon"),
8376	);
8377	expect(firstRouting).toBeGreaterThan(0);
8378	const beforeRouting = events.slice(0, firstRouting);
8379	for (const id of next.machineIds) {
8380	const reads = beforeRouting.filter(
8381	(event) =>
8382	event.stage === "completed" &&
8383	event.method === "GET" &&
8384	event.path.endsWith(`/machines/${id}`) &&
8385	event.status === 200,
8386	);
8387	const ready = reads.at(-1);
8388	const lastConfigWrite = beforeRouting.findLastIndex(
8389	(event) =>
8390	event.stage === "completed" &&
8391	event.machineId === id &&
8392	event.status! < 300 &&
8393	event.method !== "GET" &&
8394	!event.path.endsWith("/lease"),
8395	);
8396	expect(ready).toBeDefined();
8397	expect(beforeRouting.indexOf(ready!)).toBeGreaterThan(
8398	lastConfigWrite,
8399	);
8400	expect(ready?.state).toBe("started");
8401	expect(ready?.cordoned).toBe(true);
8402	expect(ready?.instanceId).toBeDefined();
8403	expect(ready?.digest).toBe(digest);
8404	expect(
8405	ready?.checks?.find((check) => check.name === "ready")?.status,
8406	).toBe("passing");
8407	expect(
8408	ready?.checks?.some((check) =>
8409	check.name?.startsWith("servicecheck-"),
8410	),
8411	).toBe(true);
8412	expect(
8413	ready?.checks?.every((check) => check.status === "passing"),
8414	).toBe(true);
8415	}
8416	expect(
8417	beforeRouting.some(
8418	(event) =>
8419	event.stage === "request" &&
8420	previous.machineIds.includes(event.machineId ?? "") &&
8421	(event.path.endsWith("/stop") ||
8422	event.path.endsWith("/cordon") ||
8423	event.method === "DELETE") &&
8424	!event.path.endsWith("/lease"),
8425	),
8426	).toBe(false);
8427	const served = yield* traffic(next.appName);
8428	expect(next.machineIds.map((id) => `${version}:${id}`)).toContain(
8429	served,
8430	);
8431	previous = next;
8432	}
8433	yield* stack.destroy();
8434	yield* assertAppGone(previous.appName);
8435	}).pipe(Effect.scoped),
8436	{ tags: ["provider:fly:ipassignment"], timeout: 900_000 },
8437	);
8438
8439	for (const [before, after] of [
8440	[1, 2],
8441	[2, 1],
8442	] as const) {
8443	test.provider(
8444	`F13 ${before === 2 ? "F14 " : ""}replaces the complete replica set when scaling from ${before} to ${after}`,
8445	(stack) =>
8446	Effect.gen(function* () {
8447	yield* stack.destroy();
8448	const deploy = (count: number, strategy: "rolling" | "bluegreen") =>
8449	stack.deploy(
8450	Effect.gen(function* () {
8451	const app = yield* Fly.App("Site");
8452	return yield* Fly.Machine("Worker", {
8453	app,
8454	name: "scaling-worker",
8455	image: "nginx:alpine",
8456	count,
8457	deploy: { strategy, healthTimeout: "30 seconds" },
8458	shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
8459	checks: {
8460	ready: {
8461	type: "http",
8462	port: 80,
8463	path: "/",
8464	interval: "2s",
8465	timeout: "1s",
8466	},
8467	},
8468	});
8469	}),
8470	);
8471	const initial = yield* deploy(
8472	before,
8473	before === 2 ? "rolling" : "bluegreen",
8474	);
8475	if (before === 2) {
8476	// Reproduce the ownership metadata written before generation support.
8477	for (const machineId of initial.machineIds) {
8478	for (const key of [
8479	"alchemy.instance",
8480	"alchemy.fqn",
8481	"alchemy.base-name",
8482	]) {
8483	yield* machines.deleteMachineMetadata({
8484	app_name: initial.appName,
8485	machine_id: machineId,
8486	key,
8487	});
8488	}
8489	}
8490	const legacy = yield* machines
8491	.listMachines({ app_name: initial.appName })
8492	.pipe(
8493	Effect.repeat({
8494	schedule: Schedule.spaced("1 second"),
8495	until: (listed) =>
8496	listed.every(
8497	(machine) =>
8498	machine.config?.metadata?.["alchemy.instance"] ===
8499	undefined,
8500	),
8501	times: 8,
8502	}),
8503	);
8504	expect(
8505	legacy.every(
8506	(machine) =>
8507	machine.config?.metadata?.["alchemy.instance"] ===
8508	undefined,
8509	),
8510	).toBe(true);
8511	}
8512	const scaled = yield* deploy(after, "bluegreen");
8513	expect(scaled.count).toBe(after);
8514	expect(scaled.machineIds).toHaveLength(after);
8515	expect(
8516	scaled.machineIds.every((id) => !initial.machineIds.includes(id)),
8517	).toBe(true);
8518	const live = (yield* machines.listMachines({
8519	app_name: initial.appName,
8520	})).filter((machine) => machine.state !== "destroyed");
8521	expect(live).toHaveLength(after);
8522	expect(
8523	live.every(
8524	(machine) =>
8525	machine.cordoned === false &&
8526	machine.config?.metadata?.["alchemy.phase"] === "active",
8527	),
8528	).toBe(true);
8529	expect(
8530	new Set(live.map((machine) => machine.image_ref?.digest)).size,
8531	).toBe(1);
8532	yield* stack.destroy();
8533	yield* assertAppGone(initial.appName);
8534	}),
8535	{ timeout: 300_000 },
8536	);
8537	}
8538	},
8539	);
8540
8541	describe.sequential(
8542	"secrets",
8543	{
8544	tags: [
8545	"provider:fly",
8546	"provider:fly:app",
8547	"provider:fly:machine",
8548	"provider:fly:service",
8549	"live",
8550	],
8551	},
8552	() => {
8553	const { test } = Test.make({ providers: Fly.providers() });
8554
8555	const marker = (appName: string, machineId: string) =>
8556	machines
8557	.execMachine({
8558	app_name: appName,
8559	machine_id: machineId,
8560	command: ["cat", "/usr/share/nginx/html/marker"],
8561	timeout: 5,
8562	})
8563	.pipe(
8564	Effect.map((response) => ({
8565	code: response.exit_code,
8566	marker: response.stdout?.trim(),
8567	})),
8568	);
8569	const init = {
8570	exec: [
8571	"/bin/sh",
8572	"-c",
8573	"case \"$ACCEPTANCE_SECRET\" in *-one) marker=one;; *-two) marker=two;; *-three) marker=three;; *) marker=missing;; esac; printf '%s' \"$marker\" > /usr/share/nginx/html/marker; exec nginx -g 'daemon off;'",
8574	],
8575	};
8576
8577	test.provider(
8578	"F12 staged standalone rotation requires an explicit floor and later vault versions satisfy that floor",
8579	(stack) =>
8580	Effect.gen(function* () {
8581	yield* stack.destroy();
8582	const app = yield* stack.deploy(Fly.App("Site"));
8583	const firstSecret = yield* machines.updateSecrets({
8584	app_name: app.appName,
8585	values: { ACCEPTANCE_SECRET: "fixture-token-one" },
8586	});
8587	const firstFloor = firstSecret.version ?? firstSecret.Version;
8588	expect(firstFloor).toBeGreaterThan(0);
8589	const first = yield* deployWorker(stack, "same-image", {
8590	init,
8591	minSecretsVersion: firstFloor,
8592	});
8593	expect(yield* marker(app.appName, first.machineId)).toEqual({
8594	code: 0,
8595	marker: "one",
8596	});
8597	const rotated = yield* machines.updateSecrets({
8598	app_name: app.appName,
8599	values: { ACCEPTANCE_SECRET: "fixture-token-two" },
8600	});
8601	const floor = rotated.version ?? rotated.Version;
8602	expect(floor).toBeGreaterThan(firstFloor!);
8603	const unchanged = yield* deployWorker(stack, "same-image", {
8604	init,
8605	minSecretsVersion: firstFloor,
8606	});
8607	expect(unchanged.machineIds).toEqual(first.machineIds);
8608	expect(yield* marker(app.appName, first.machineId)).toEqual({
8609	code: 0,
8610	marker: "one",
8611	});
8612	// A later writer advances the shared vault; the requested version is a floor, not a snapshot.
8613	const later = yield* machines.updateSecrets({
8614	app_name: app.appName,
8615	values: { ACCEPTANCE_SECRET: "fixture-token-three" },
8616	});
8617	expect(later.version ?? later.Version).toBeGreaterThan(floor!);
8618	const next = yield* deployWorker(stack, "same-image", {
8619	init,
8620	minSecretsVersion: floor,
8621	});
8622	expect(next.machineId).not.toBe(first.machineId);
8623	expect(yield* marker(app.appName, next.machineId)).toEqual({
8624	code: 0,
8625	marker: "three",
8626	});
8627	yield* assertCommitted(app.appName, next.machineIds);
8628	yield* stack.destroy();
8629	yield* assertAppGone(app.appName);
8630	}),
8631	{ timeout: 300_000 },
8632	);
8633
8634	test.provider(
8635	"F12 bound Config and real Redis attachment rotate before every replacement candidate",
8636	(stack) =>
8637	Effect.gen(function* () {
8638	yield* stack.destroy();
8639	const proxy = yield* transportProxy();
8640	const actor = yield* engineActor(
8641	stack,
8642	"F12 bound Config and real Redis attachment rotate before every replacement candidate",
8643	"test/Fly/BlueGreen.test.ts",
8644	proxy.url,
8645	);
8646	const ambient = yield* ConfigProvider.ConfigProvider;
8647	const deploy = (version: "one" | "two", cache: "one" | "two") =>
8648	actor
8649	.deploy(
8650	Effect.gen(function* () {
8651	const app = yield* SiteBluegreenSecrets;
8652	const one = yield* CacheOne;
8653	const two = yield* CacheTwo;
8654	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
8655	const service = yield* BoundSecrets;
8656	return { app, one, two, service };
8657	}),
8658	)
8659	.pipe(
8660	Effect.provideService(
8661	ConfigProvider.ConfigProvider,
8662	ConfigProvider.orElse(
8663	ConfigProvider.fromUnknown({
8664	ACCEPTANCE_BOUND_SECRET: `fixture-token-${version}`,
8665	ACCEPTANCE_CACHE: cache,
8666	}),
8667	ambient,
8668	),
8669	),
8670	);
8671	const client = yield* HttpClient.HttpClient;
8672	const request = (appName: string, path: string) =>
8673	client.get(`http://${appName}.fly.dev${path}`).pipe(
8674	Effect.flatMap((response) => {
8675	expect(response.status).toBe(200);
8676	return response.json;
8677	}),
8678	);
8679	const initial = yield* deploy("one", "one");
8680	expect(
8681	yield* request(initial.app.appName, "/seed").pipe(
8682	Effect.retry({
8683	schedule: Schedule.spaced("2 seconds"),
8684	times: 8,
8685	}),
8686	),
8687	).toEqual({ config: "one", redis: "one" });
8688	const configOnly = yield* deploy("two", "one");
8689	expect(
8690	configOnly.service.machineIds.every(
8691	(id) => !initial.service.machineIds.includes(id),
8692	),
8693	).toBe(true);
8694	expect(yield* request(initial.app.appName, "/marker")).toEqual({
8695	config: "two",
8696	redis: "one",
8697	});
8698	const start = proxy.events.length;
8699	const next = yield* deploy("two", "two");
8700	expect(next.one.redisId).toBe(initial.one.redisId);
8701	expect(next.two.redisId).toBe(initial.two.redisId);
8702	expect(
8703	next.service.machineIds.every(
8704	(id) => !configOnly.service.machineIds.includes(id),
8705	),
8706	).toBe(true);
8707	expect(yield* request(next.app.appName, "/marker")).toEqual({
8708	config: "two",
8709	redis: "empty",
8710	});
8711	expect(yield* request(next.app.appName, "/seed")).toEqual({
8712	config: "two",
8713	redis: "two",
8714	});
8715	const events = proxy.events.slice(start);
8716	const secretWrites = events.filter(
8717	(event) =>
8718	event.stage === "completed" &&
8719	event.method !== "GET" &&
8720	event.path.endsWith("/secrets") &&
8721	event.secretsVersion !== undefined,
8722	);
8723	expect(secretWrites.length).toBeGreaterThan(0);
8724	const floor = Math.max(
8725	...secretWrites.map((event) => event.secretsVersion!),
8726	);
8727	const creates = events.filter(
8728	(event) =>
8729	event.stage === "request" &&
8730	event.method === "POST" &&
8731	event.path.endsWith("/machines"),
8732	);
8733	expect(creates).toHaveLength(2);
8734	expect(
8735	creates.every(
8736	(event) =>
8737	secretWrites.every(
8738	(write) => events.indexOf(write) < events.indexOf(event),
8739	) &&
8740	event.minSecretsVersion !== undefined &&
8741	event.minSecretsVersion >= floor,
8742	),
8743	).toBe(true);
8744	const live = yield* assertCommitted(
8745	next.app.appName,
8746	next.service.machineIds,
8747	);
8748	expect(
8749	live.every(
8750	(machine) =>
8751	machine.image_ref?.digest === initial.service.imageRef?.digest,
8752	),
8753	).toBe(true);
8754	yield* stack.destroy();
8755	yield* assertAppGone(next.app.appName);
8756	expect(
8757	yield* Fly.findRedisAddOn({
8758	id: next.one.redisId,
8759	name: next.one.name,
8760	}),
8761	).toBeUndefined();
8762	expect(
8763	yield* Fly.findRedisAddOn({
8764	id: next.two.redisId,
8765	name: next.two.name,
8766	}),
8767	).toBeUndefined();
8768	}).pipe(Effect.scoped),
8769	{
8770	tags: ["provider:fly:ipassignment", "provider:fly:redis"],
8771	timeout: 600_000,
8772	},
8773	);
8774
8775	test.provider(
8776	"F12 Fly.Secret resource rotation is staged until an explicit Machine floor rollout",
8777	(stack) =>
8778	Effect.gen(function* () {
8779	yield* stack.destroy();
8780	const deploy = (value: "one" | "two", floor?: number) =>
8781	stack.deploy(
8782	Effect.gen(function* () {
8783	const app = yield* Fly.App("Site");
8784	const secret = yield* Fly.Secret("Token", {
8785	app,
8786	name: "ACCEPTANCE_SECRET",
8787	value: Redacted.make(`fixture-token-${value}`),
8788	});
8789	const worker = yield* Fly.Machine("Worker", {
8790	app,
8791	image: "nginx:alpine",
8792	init,
8793	checks,
8794	minSecretsVersion: floor,
8795	env: { SECRET_RESOURCE_NAME: secret.name },
8796	deploy: {
8797	strategy: "bluegreen",
8798	healthTimeout: "30 seconds",
8799	},
8800	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
8801	});
8802	return { app, secret, worker };
8803	}),
8804	);
8805	const initial = yield* deploy("one");
8806	expect(
8807	yield* marker(initial.app.appName, initial.worker.machineId),
8808	).toEqual({ code: 0, marker: "one" });
8809	const staged = yield* deploy("two");
8810	expect(staged.secret.name).toBe(initial.secret.name);
8811	expect(staged.secret.digest).not.toBe(initial.secret.digest);
8812	expect(staged.worker.machineIds).toEqual(initial.worker.machineIds);
8813	expect(
8814	yield* marker(staged.app.appName, staged.worker.machineId),
8815	).toEqual({ code: 0, marker: "one" });
8816	// A separate vault fence returns a floor that includes the completed Secret reconcile.
8817	const fence = yield* machines.updateSecrets({
8818	app_name: staged.app.appName,
8819	values: { ACCEPTANCE_FENCE: "two" },
8820	});
8821	const floor = fence.version ?? fence.Version;
8822	expect(floor).toBeGreaterThan(0);
8823	const next = yield* deploy("two", floor);
8824	expect(next.worker.machineId).not.toBe(initial.worker.machineId);
8825	expect(
8826	yield* marker(next.app.appName, next.worker.machineId),
8827	).toEqual({
8828	code: 0,
8829	marker: "two",
8830	});
8831	const live = yield* assertCommitted(
8832	next.app.appName,
8833	next.worker.machineIds,
8834	);
8835	expect(live[0]?.image_ref?.digest).toBe(
8836	initial.worker.imageRef?.digest,
8837	);
8838	yield* stack.destroy();
8839	yield* assertAppGone(next.app.appName);
8840	}),
8841	{ tags: ["provider:fly:secret"], timeout: 600_000 },
8842	);
8843
8844	test.provider(
8845	"F12 a real concurrent vault writer advances the floor while the first candidate response is held",
8846	(stack) =>
8847	Effect.gen(function* () {
8848	yield* stack.destroy();
8849	const app = yield* stack.deploy(Fly.App("Site"));
8850	const firstSecret = yield* machines.updateSecrets({
8851	app_name: app.appName,
8852	values: { ACCEPTANCE_SECRET: "fixture-token-one" },
8853	});
8854	const initial = yield* deployWorker(stack, "same-image", {
8855	init,
8856	minSecretsVersion: firstSecret.version ?? firstSecret.Version,
8857	});
8858	const rotated = yield* machines.updateSecrets({
8859	app_name: app.appName,
8860	values: { ACCEPTANCE_SECRET: "fixture-token-two" },
8861	});
8862	const floor = rotated.version ?? rotated.Version;
8863	expect(floor).toBeGreaterThan(0);
8864	const proxy = yield* transportProxy();
8865	yield* Effect.sync(() =>
8866	proxy.arm({
8867	match: (event) =>
8868	event.method === "POST" && event.path.endsWith("/machines"),
8869	action: "hold-response",
8870	remaining: 1,
8871	}),
8872	);
8873	yield* Effect.gen(function* () {
8874	const actor = yield* engineActor(
8875	stack,
8876	"F12 a real concurrent vault writer advances the floor while the first candidate response is held",
8877	"test/Fly/BlueGreen.test.ts",
8878	proxy.url,
8879	);
8880	const rollout = yield* deployWorker(actor, "same-image", {
8881	count: 2,
8882	init,
8883	minSecretsVersion: floor,
8884	}).pipe(Effect.scoped, Effect.forkScoped);
8885	const held = yield* proxy.wait(
8886	(event) => event.stage === "held" && event.status! < 300,
8887	);
8888	const writer = yield* machines
8889	.updateSecrets({
8890	app_name: app.appName,
8891	values: { ACCEPTANCE_SECRET: "fixture-token-three" },
8892	})
8893	.pipe(Effect.forkScoped);
8894	const later = yield* Fiber.join(writer);
8895	expect(later.version ?? later.Version).toBeGreaterThan(floor!);
8896	yield* Effect.sync(proxy.release);
8897	const next = yield* Fiber.join(rollout).pipe(
8898	Effect.timeout("180 seconds"),
8899	);
8900	const live = yield* assertCommitted(app.appName, next.machineIds);
8901	expect(next.machineIds).toContain(held.machineId);
8902	expect(next.machineIds).not.toContain(initial.machineId);
8903	const creates = proxy.events.filter(
8904	(event) =>
8905	event.stage === "request" &&
8906	event.method === "POST" &&
8907	event.path.endsWith("/machines"),
8908	);
8909	expect(creates).toHaveLength(2);
8910	expect(
8911	creates.every((event) => event.minSecretsVersion === floor),
8912	).toBe(true);
8913	for (const machine of live) {
8914	const observed = yield* marker(app.appName, machine.id!);
8915	expect(observed.code).toBe(0);
8916	expect(
8917	machine.id === held.machineId ? ["two", "three"] : ["three"],
8918	).toContain(observed.marker);
8919	}
8920	}).pipe(
8921	Effect.ensuring(
8922	Effect.sync(() => {
8923	proxy.clear();
8924	proxy.release();
8925	}),
8926	),
8927	Effect.scoped,
8928	);
8929	yield* stack.destroy();
8930	yield* assertAppGone(app.appName);
8931	}).pipe(Effect.scoped),
8932	{ timeout: 600_000 },
8933	);
8934
8935	test.provider(
8936	"F12 interrupted promoted candidate cannot satisfy a newer explicit secret floor",
8937	(stack) =>
8938	Effect.gen(function* () {
8939	yield* stack.destroy();
8940	const app = yield* stack.deploy(Fly.App("Site"));
8941	const firstSecret = yield* machines.updateSecrets({
8942	app_name: app.appName,
8943	values: { ACCEPTANCE_SECRET: "fixture-token-one" },
8944	});
8945	yield* deployWorker(stack, "one", {
8946	init,
8947	minSecretsVersion: firstSecret.version ?? firstSecret.Version,
8948	});
8949	const rotated = yield* machines.updateSecrets({
8950	app_name: app.appName,
8951	values: { ACCEPTANCE_SECRET: "fixture-token-two" },
8952	});
8953	const floor = rotated.version ?? rotated.Version;
8954	const proxy = yield* transportProxy();
8955	yield* Effect.sync(() =>
8956	proxy.arm({
8957	match: (event) => event.path.endsWith("/uncordon"),
8958	action: "hold-response",
8959	remaining: 1,
8960	}),
8961	);
8962	yield* Effect.gen(function* () {
8963	const actor = yield* engineActor(
8964	stack,
8965	"F12 interrupted promoted candidate cannot satisfy a newer explicit secret floor",
8966	"test/Fly/BlueGreen.test.ts",
8967	proxy.url,
8968	);
8969	const rollout = yield* deployWorker(actor, "two", {
8970	init,
8971	minSecretsVersion: floor,
8972	}).pipe(Effect.scoped, Effect.forkScoped);
8973	const held = yield* proxy.wait(
8974	(event) => event.stage === "held" && event.status! < 300,
8975	);
8976	const interruption = yield* Fiber.interrupt(rollout).pipe(
8977	Effect.forkScoped,
8978	);
8979	yield* Effect.yieldNow;
8980	yield* Effect.sync(() => {
8981	proxy.clear();
8982	proxy.release();
8983	});
8984	yield* Fiber.join(interruption).pipe(Effect.timeout("120 seconds"));
8985	expect(Exit.hasInterrupts(yield* Fiber.await(rollout))).toBe(true);
8986	expect(
8987	(yield* census(app.appName)).some(
8988	(machine) => machine.id === held.machineId,
8989	),
8990	).toBe(true);
8991	const later = yield* machines.updateSecrets({
8992	app_name: app.appName,
8993	values: { ACCEPTANCE_SECRET: "fixture-token-three" },
8994	});
8995	const nextFloor = later.version ?? later.Version;
8996	expect(nextFloor).toBeGreaterThan(floor!);
8997	const resumed = yield* engineActor(
8998	stack,
8999	"F12 interrupted promoted candidate cannot satisfy a newer explicit secret floor",
9000	"test/Fly/BlueGreen.test.ts",
9001	);
9002	const next = yield* deployWorker(resumed, "two", {
9003	init,
9004	minSecretsVersion: nextFloor,
9005	});
9006	expect(next.machineIds).not.toContain(held.machineId);
9007	expect(yield* marker(app.appName, next.machineId)).toEqual({
9008	code: 0,
9009	marker: "three",
9010	});
9011	yield* assertCommitted(app.appName, next.machineIds);
9012	}).pipe(
9013	Effect.ensuring(
9014	Effect.sync(() => {
9015	proxy.clear();
9016	proxy.release();
9017	}),
9018	),
9019	Effect.scoped,
9020	);
9021	yield* stack.destroy();
9022	yield* assertAppGone(app.appName);
9023	}).pipe(Effect.scoped),
9024	{ timeout: 600_000 },
9025	);
9026	},
9027	);
9028
9029	describe.sequential(
9030	"managed HTTP services",
9031	{
9032	tags: [
9033	"provider:fly",
9034	"provider:fly:app",
9035	"provider:fly:ipassignment",
9036	"provider:fly:machine",
9037	"provider:fly:redis",
9038	"provider:fly:secret",
9039	"provider:fly:service",
9040	"live",
9041	],
9042	},
9043	() => {
9044	const { test } = Test.make({ providers: Fly.providers() });
9045
9046	describe.sequential("Fly live HTTP and worker drain", () => {
9047	for (const policy of [
9048	{
9049	name: "managed old60/new10 SIGTERM",
9050	raw: false,
9051	old: "60 seconds",
9052	next: "10 seconds",
9053	signal: "SIGTERM",
9054	nextSignal: "SIGINT",
9055	rawSignal: undefined,
9056	nextRawSignal: undefined,
9057	delay: 12_000,
9058	},
9059	{
9060	name: "managed old10/new60 SIGINT",
9061	raw: false,
9062	old: "10 seconds",
9063	next: "60 seconds",
9064	signal: "SIGINT",
9065	nextSignal: "SIGTERM",
9066	rawSignal: undefined,
9067	nextRawSignal: undefined,
9068	delay: 3_000,
9069	},
9070	{
9071	name: "raw old60/new10 SIGQUIT to SIGTERM",
9072	raw: true,
9073	old: "60 seconds",
9074	next: "10 seconds",
9075	signal: "SIGTERM",
9076	nextSignal: "SIGTERM",
9077	rawSignal: "SIGQUIT",
9078	nextRawSignal: "SIGTERM",
9079	delay: 12_000,
9080	},
9081	{
9082	name: "raw old10/new60 SIGTERM to SIGQUIT",
9083	raw: true,
9084	old: "10 seconds",
9085	next: "60 seconds",
9086	signal: "SIGTERM",
9087	nextSignal: "SIGTERM",
9088	rawSignal: "SIGTERM",
9089	nextRawSignal: "SIGQUIT",
9090	delay: 3_000,
9091	},
9092	] as const) {
9093	test.provider(
9094	`R01/R02/R03/R04 ${policy.name} preserves traffic, jobs and streaming after the actual stop barrier`,
9095	(stack) =>
9096	Effect.gen(function* () {
9097	yield* stack.destroy();
9098	const scenario = yield* makeScenario(stack);
9099	yield* Effect.gen(function* () {
9100	const first = yield* scenario.deploy({
9101	raw: policy.raw,
9102	version: "one",
9103	timeout: policy.old,
9104	signal: policy.signal,
9105	rawSignal: policy.rawSignal,
9106	afterSignalMs: policy.delay,
9107	});
9108	const oldWorker = yield* requireValue(
9109	first.worker,
9110	"initial worker output",
9111	);
9112	const oldId = oldWorker.machineId;
9113	const url = `https://${first.workerApp.appName}.fly.dev`;
9114	const get = (route: string) =>
9115	HttpClient.get(`${url}${route}`, {
9116	headers: { connection: "close" },
9117	}).pipe(
9118	Effect.flatMap((response) =>
9119	response.status === 200
9120	? response.text
9121	: Effect.fail(new Error(`HTTP ${response.status}`)),
9122	),
9123	Effect.timeout(route === "/" ? "5 seconds" : "240 seconds"),
9124	);
9125	expect(
9126	JSON.parse(
9127	yield* get("/").pipe(
9128	Effect.retry({
9129	times: 8,
9130	schedule: Schedule.spaced("1 second"),
9131	}),
9132	),
9133	),
9134	).toEqual({ machine: oldId, version: "one" });
9135	const observed = yield* machines.getMachine({
9136	app_name: first.workerApp.appName,
9137	machine_id: oldId,
9138	});
9139	expect(observed.config?.stop_config?.signal).toBe(
9140	policy.rawSignal ?? policy.signal,
9141	);
9142	expect(
9143	policy.old === "60 seconds"
9144	? ["60000ms", "60s", "1m", "1m0s"]
9145	: ["10000ms", "10s"],
9146	).toContain(observed.config?.stop_config?.timeout);
9147	yield* scenario.call(first.ledgerUrl, "enqueue", [
9148	"in-flight-job",
9149	"hold",
9150	]);
9151	yield* scenario.wait(first.ledgerUrl, (ledger) =>
9152	ledger.events.some(
9153	(event) =>
9154	event.machine === oldId &&
9155	event.event === "claimed" &&
9156	event.job === "in-flight-job",
9157	),
9158	);
9159	const slow = yield* get("/hold").pipe(Effect.forkScoped);
9160	const streamed = yield* get("/stream").pipe(Effect.forkScoped);
9161	yield* scenario.wait(
9162	first.ledgerUrl,
9163	(ledger) =>
9164	ledger.events.filter(
9165	(event) =>
9166	event.machine === oldId &&
9167	event.event === "request-started",
9168	).length === 2,
9169	);
9170	const responses = yield* Ref.make<
9171	Array<
9172	{ machine: string; version: string } | { failure: string }
9173	>
9174	>([]);
9175	const finished = yield* Ref.make(false);
9176	yield* Effect.addFinalizer(() =>
9177	Ref.get(responses).pipe(
9178	Effect.flatMap((samples) =>
9179	Effect.logInfo(
9180	"Unretried public traffic samples",
9181	samples,
9182	),
9183	),
9184	),
9185	);
9186	const traffic = yield* Stream.range(0, 599).pipe(
9187	Stream.mapEffect(() =>
9188	get("/").pipe(
9189	Effect.result,
9190	Effect.flatMap((result) =>
9191	Ref.update(responses, (values) => [
9192	...values,
9193	Result.isSuccess(result)
9194	? (JSON.parse(result.success) as {
9195	machine: string;
9196	version: string;
9197	})
9198	: { failure: String(result.failure) },
9199	]),
9200	),
9201	Effect.andThen(Effect.sleep("500 millis")),
9202	Effect.andThen(Ref.get(finished)),
9203	),
9204	),
9205	Stream.takeUntil((done) => done),
9206	Stream.runDrain,
9207	Effect.forkScoped,
9208	);
9209	const second = yield* scenario.deploy({
9210	raw: policy.raw,
9211	version: "two",
9212	timeout: policy.next,
9213	signal: policy.nextSignal,
9214	rawSignal: policy.nextRawSignal,
9215	afterSignalMs: 1000,
9216	});
9217	const newWorker = yield* requireValue(
9218	second.worker,
9219	"replacement worker output",
9220	);
9221	expect(JSON.parse(yield* Fiber.join(slow))).toEqual({
9222	machine: oldId,
9223	version: "one",
9224	});
9225	const body = yield* Fiber.join(streamed);
9226	const expected =
9227	"first\n".repeat(32768) + "last\n".repeat(32768);
9228	const hashes = yield* Effect.sync(() =>
9229	[body, expected].map((value) =>
9230	createHash("sha256").update(value).digest("hex"),
9231	),
9232	);
9233	expect(body.length).toBe(360448);
9234	expect(hashes[0]).toBe(hashes[1]);
9235	expect(JSON.parse(yield* get("/"))).toEqual({
9236	machine: newWorker.machineId,
9237	version: "two",
9238	});
9239	yield* Effect.sleep("600 millis");
9240	yield* Ref.set(finished, true);
9241	yield* Fiber.join(traffic);
9242	const samples = yield* Ref.get(responses);
9243	expect(samples.filter((sample) => "failure" in sample)).toEqual(
9244	[],
9245	);
9246	expect(
9247	samples.some(
9248	(sample) =>
9249	"version" in sample &&
9250	sample.version === "one" &&
9251	sample.machine === oldId,
9252	),
9253	).toBe(true);
9254	expect(
9255	samples.some(
9256	(sample) =>
9257	"version" in sample &&
9258	sample.version === "two" &&
9259	sample.machine === newWorker.machineId,
9260	),
9261	).toBe(true);
9262	yield* assertReplacement(
9263	first.workerApp.appName,
9264	oldId,
9265	newWorker.machineId,
9266	);
9267	const ledger = yield* scenario.settle(first.ledgerUrl);
9268	assertStopped(ledger.events, oldId);
9269	assertOrder(
9270	ledger.events,
9271	oldId,
9272	"request-finalized",
9273	"shared-closed",
9274	);
9275	assertOrder(
9276	ledger.events,
9277	oldId,
9278	"client-released",
9279	"shared-closed",
9280	);
9281	const stop = yield* requireValue(
9282	ledger.events.find(
9283	(event) =>
9284	event.machine === oldId && event.event === "stop-started",
9285	),
9286	"old worker stop event",
9287	);
9288	if (policy.raw) expect(stop.signal).toBe(policy.rawSignal);
9289	const completed = ledger.events.filter(
9290	(event) =>
9291	event.machine === oldId &&
9292	event.event === "response-finished",
9293	);
9294	expect(completed).toHaveLength(2);
9295	for (const event of completed) {
9296	expect(event.at - stop.at).toBeGreaterThan(
9297	policy.delay - 500,
9298	);
9299	}
9300	const ack = yield* requireValue(
9301	ledger.events.find(
9302	(event) =>
9303	event.machine === oldId &&
9304	event.event === "acked" &&
9305	event.job === "in-flight-job",
9306	),
9307	"in-flight job ACK event",
9308	);
9309	expect(ack.ack).toBe(1);
9310	expect(ack.at - stop.at).toBeGreaterThan(policy.delay - 500);
9311	if (policy.old === "10 seconds")
9312	expect(ack.at - stop.at).toBeLessThan(10_000);
9313	}).pipe(Effect.scoped, Effect.ensuring(scenario.cleanup));
9314	}),
9315	{
9316	// Includes cold publication, both readiness phases, proxy overlap, post-signal work and census.
9317	timeout: 480_000,
9318	},
9319	);
9320	}
9321	});
9322	},
9323	);
9324
9325	describe.sequential(
9326	"shutdown policy",
9327	{ tags: ["provider:fly", "provider:fly:service"] },
9328	() => {
9329	const { test } = Test.make({ providers: Fly.providers() });
9330
9331	it.effect(
9332	"R02 predecessor policy preserves raw overrides and managed legacy grace",
9333	() =>
9334	Effect.gen(function* () {
9335	for (const timeout of ["10s", "60s"]) {
9336	for (const signal of ["SIGQUIT", "SIGTERM"] as const) {
9337	const policy = yield* predecessorShutdown({
9338	config: { stop_config: { signal, timeout } },
9339	});
9340	expect(policy.signal).toBe(signal);
9341	expect(policy.timeout).toBe(timeout);
9342	}
9343	}
9344	const raw = yield* predecessorShutdown({ config: {} });
9345	expect(raw.signal).toBeUndefined();
9346	expect(raw.timeout).toBeUndefined();
9347	const legacy = yield* predecessorShutdown({
9348	config: {
9349	stop_config: { signal: "SIGINT" },
9350	env: { ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS: "60000" },
9351	},
9352	});
9353	expect(legacy).toEqual({
9354	signal: "SIGINT",
9355	timeout: "60000ms",
9356	timeoutMs: 60000,
9357	});
9358	for (const injected of ["broken", "0", "300001", "60000"]) {
9359	const error = yield* predecessorShutdown({
9360	config: {
9361	stop_config: { signal: "SIGTERM", timeout: "10s" },
9362	env: { ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS: injected },
9363	},
9364	}).pipe(Effect.flip);
9365	expect(error._tag).toBe("Fly.ShutdownPolicyMismatch");
9366	}
9367	}),
9368	{ tags: ["unit", "local"] },
9369	);
9370
9371	test.provider(
9372	"R02 observed predecessor policy survives replacement and retires suspended without resume",
9373	(stack) =>
9374	Effect.gen(function* () {
9375	yield* stack.destroy();
9376	const current = yield* stack.deploy(
9377	Effect.gen(function* () {
9378	const app = yield* Fly.App("Site");
9379	return yield* Fly.Machine("Worker", {
9380	app,
9381	image: "nginx:alpine",
9382	shutdown: { signal: "SIGQUIT", timeout: "60 seconds" },
9383	});
9384	}),
9385	);
9386	const request = {
9387	app_name: current.appName,
9388	machine_id: current.machineId,
9389	};
9390	const observed = yield* machines.getMachine(request);
9391	expect((yield* predecessorShutdown(observed)).timeoutMs).toBe(60_000);
9392	yield* machines.suspendMachine(request);
9393	yield* machines.waitMachine({
9394	...request,
9395	state: "suspended",
9396	timeout: 8,
9397	});
9398	yield* retireMachine(current.appName, current.machineId);
9399	expect(
9400	yield* machines.getMachine(request).pipe(
9401	Effect.map((machine) => machine.state === "destroyed"),
9402	Effect.catchTag("NotFound", () => Effect.succeed(true)),
9403	),
9404	).toBe(true);
9405	yield* stack.destroy();
9406	}),
9407	{
9408	tags: ["provider:fly:app", "provider:fly:machine", "live"],
9409	timeout: 120_000,
9410	},
9411	);
9412	},
9413	);
9414
9415	describe.sequential(
9416	"process signals and overlap",
9417	{
9418	tags: [
9419	"provider:fly",
9420	"provider:fly:app",
9421	"provider:fly:machine",
9422	"provider:fly:service",
9423	"live",
9424	],
9425	},
9426	() => {
9427	const options = {
9428	providers: Fly.providers(),
9429	profile: "testing",
9430	dev: false,
9431	sidecar: false,
9432	};
9433	const { test, beforeEach, afterEach } = Test.make(options);
9434	const selected = process.env.FLY_SIGNAL_OVERLAP_CASE;
9435	let signaled:
9436	| {
9437	witness: WitnessSignalOverlap;
9438	paths: Effect.Success<ReturnType<typeof pathsFor>>;
9439	}
9440	| undefined;
9441
9442	// Observe the runner's own teardown; never bridge SIGINT into a manual fiber interrupt.
9443	afterEach(
9444	Effect.gen(function* () {
9445	if (!signaled || signaled.witness.signal !== "SIGINT") return;
9446	const { witness, paths } = signaled;
9447	const fs = yield* FileSystem.FileSystem;
9448	yield* writeEvidence(paths.runnerInterrupted, {
9449	pid: yield* Effect.sync(() => process.pid),
9450	case: witness.case,
9451	signal: "SIGINT",
9452	witnessRecordedAt: witness.recordedAt,
9453	at: yield* nowSeconds,
9454	deployFinalized: yield* fs.exists(paths.finalized),
9455	} satisfies typeof RunnerInterrupted.Type);
9456	}),
9457	{ timeout: 30_000 },
9458	);
9459
9460	const crash = (
9461	stack: Test.ScratchStack,
9462	title: string,
9463	selectedCase: SignalCase,
9464	) =>
9465	Effect.gen(function* () {
9466	const fs = yield* FileSystem.FileSystem;
9467	const paths = yield* pathsFor(stack);
9468	// Never erase an incomplete signal attempt to make a retry green.
9469	if (
9470	(yield* fs.exists(paths.witness)) &&
9471	!(yield* fs.exists(paths.recovered))
9472	) {
9473	return yield* Effect.fail(
9474	new Error(
9475	"An unrecovered signal witness exists; preserve it and use the recovery leg",
9476	),
9477	);
9478	}
9479	for (const file of [
9480	paths.witness,
9481	paths.finalized,
9482	paths.runnerInterrupted,
9483	paths.blocked,
9484	paths.recovered,
9485	]) {
9486	yield* fs.remove(file, { force: true });
9487	}
9488	yield* stack.destroy();
9489	const initial = yield* deploy(stack, "one").pipe(Effect.scoped);
9490	expect(initial.machineIds).toHaveLength(1);
9491	const predecessor = yield* identity(
9492	yield* machine(initial.appName, initial.machineId),
9493	stack,
9494	);
9495	expect(predecessor.phase).toBe("active");
9496	const proxy = yield* transportProxy();
9497	const actor = yield* Effect.sync(() =>
9498	scratchStack(
9499	{
9500	...options,
9501	providers: throughProxy(() => proxy.url),
9502	stage: stack.stage,
9503	},
9504	title,
9505	signalOverlapFile,
9506	),
9507	);
9508	expect(actor.name).toBe(stack.name);
9509	expect(actor.stage).toBe(stack.stage);
9510	expect(actor.state).not.toBe(stack.state);
9511	const match = (event: Parameters<typeof matches>[4]) =>
9512	matches(
9513	selectedCase.phase,
9514	initial.appName,
9515	predecessor.id,
9516	proxy.events,
9517	event,
9518	);
9519	yield* Effect.sync(() =>
9520	proxy.arm({ match, action: "hold-response", remaining: 1 }),
9521	);
9522	const pid = yield* Effect.sync(() => process.pid);
9523	let armed: WitnessSignalOverlap | undefined;
9524	const attempt = yield* deploy(actor, "two").pipe(
9525	Effect.scoped,
9526	Effect.onExit((exit) =>
9527	Effect.gen(function* () {
9528	if (!armed) {
9529	yield* writeEvidence(paths.finalized, {
9530	pid,
9531	case: selectedCase.name,
9532	armed: false,
9533	});
9534	return;
9535	}
9536	// The deploy scope has closed, including native lease-release finalizers.
9537	const leases = yield* observeLeases(armed);
9538	const releases = yield* Effect.sync(() =>
9539	proxy.events
9540	.filter(
9541	(event) =>
9542	event.stage === "completed" &&
9543	event.sequence > armed!.barrier.sequence &&
9544	event.method === "DELETE" &&
9545	event.path.endsWith("/lease") &&
9546	successful(event),
9547	)
9548	.map(boundary),
9549	);
9550	yield* writeEvidence(paths.finalized, {
9551	pid,
9552	case: selectedCase.name,
9553	witnessRecordedAt: armed.recordedAt,
9554	interruptedOnly:
9555	Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause),
9556	at: yield* nowSeconds,
9557	releases,
9558	leases,
9559	} satisfies typeof Finalized.Type);
9560	}),
9561	),
9562	Effect.forkScoped,
9563	);
9564	yield* Effect.gen(function* () {
9565	const barrier = yield* proxy.wait(
9566	(event) =>
9567	event.stage === "held" && successful(event) && match(event),
9568	);
9569	yield* Effect.gen(function* () {
9570	const live = yield* censusProcessDeath(initial.appName);
9571	const candidates = live.filter(
9572	(value) => value.id !== predecessor.id,
9573	);
9574	expect(candidates).toHaveLength(1);
9575	const candidate = yield* identity(
9576	yield* machine(initial.appName, candidates[0]!.id!),
9577	stack,
9578	);
9579	expect(candidate.generation).not.toBe(predecessor.generation);
9580	expect(candidate.workload).not.toBe(predecessor.workload);
9581	expect(Number(candidate.sequence)).toBe(
9582	Number(predecessor.sequence) + 1,
9583	);
9584	expect(candidate.instance).toBe(predecessor.instance);
9585	expect(candidate.fqn).toBe(predecessor.fqn);
9586	const leases = yield* heldLeases(
9587	initial.appName,
9588	live.map((value) => value.id!),
9589	);
9590	expect(leases.map((value) => value.machineId).sort()).toEqual(
9591	(selectedCase.phase === "create"
9592	? [predecessor.id]
9593	: selectedCase.phase === "retirement"
9594	? [candidate.id]
9595	: [predecessor.id, candidate.id]
9596	).sort(),
9597	);
9598	for (const held of leases) {
9599	expect(
9600	proxy.events.some(
9601	(event) =>
9602	event.stage === "completed" &&
9603	event.method === "POST" &&
9604	event.path.endsWith(`/machines/${held.machineId}/lease`) &&
9605	successful(event),
9606	),
9607	).toBe(true);
9608	}
9609	const row = yield* persistedRow(stack, candidate.fqn);
9610	expect(row.instanceId).toBe(candidate.instance);
9611	expect(row.status).toBe("updating");
9612	const returned = firstReturnedUncordon(
9613	proxy.events,
9614	initial.appName,
9615	);
9616	const witness = {
9617	version: 1,
9618	case: selectedCase.name,
9619	signal: selectedCase.signal,
9620	phase: selectedCase.phase,
9621	pid,
9622	cwd: paths.cwd,
9623	stack: stack.name,
9624	stage: stack.stage,
9625	appName: initial.appName,
9626	recordedAt: yield* nowSeconds,
9627	predecessor,
9628	candidate,
9629	barrier: boundary(barrier),
9630	...(selectedCase.phase === "overlap"
9631	? { returnedUncordon: boundary(returned!) }
9632	: {}),
9633	leases,
9634	row,
9635	} satisfies WitnessSignalOverlap;
9636	yield* assertBoundary(witness);
9637	yield* assertInventory(stack, witness);
9638	yield* writeEvidence(paths.witness, witness);
9639	expect(
9640	yield* readEvidence(paths.witness, WitnessSignalOverlap),
9641	).toEqual(witness);
9642	yield* Effect.sync(() => {
9643	expect(attempt.pollUnsafe() === undefined).toBe(true);
9644	expect(
9645	proxy.events.some(
9646	(event) =>
9647	event.sequence === barrier.sequence &&
9648	["forwarded", "dropped"].includes(event.stage),
9649	),
9650	).toBe(false);
9651	expect(
9652	proxy.events.some(
9653	(event) =>
9654	event.method === "DELETE" && event.path.endsWith("/lease"),
9655	),
9656	).toBe(false);
9657	expect(
9658	proxy.events.some(
9659	(event) =>
9660	event.stage === "request" &&
9661	event.sequence > barrier.sequence &&
9662	event.method !== "GET" &&
9663	!event.path.endsWith("/lease"),
9664	),
9665	).toBe(false);
9666	if (selectedCase.phase === "overlap") {
9667	expect(returned).toBeDefined();
9668	expect(
9669	proxy.events.filter(
9670	(event) =>
9671	event.stage === "forwarded" &&
9672	event.path.endsWith("/uncordon"),
9673	),
9674	).toHaveLength(1);
9675	expect(
9676	proxy.events.some(
9677	(event) =>
9678	event.sequence === returned!.sequence &&
9679	["held", "dropped"].includes(event.stage),
9680	),
9681	).toBe(false);
9682	expect(
9683	proxy.events.some(
9684	(event) =>
9685	event.stage === "request" &&
9686	event.path.endsWith("/cordon"),
9687	),
9688	).toBe(false);
9689	expect(
9690	proxy.events.some(
9691	(event) =>
9692	event.stage === "request" &&
9693	event.sequence === barrier.sequence &&
9694	event.method === "GET" &&
9695	event.machineId === candidate.id,
9696	),
9697	).toBe(true);
9698	}
9699	if (selectedCase.signal === "SIGINT") {
9700	expect(process.listenerCount("SIGINT")).toBeGreaterThan(0);
9701	}
9702	expect(process.pid).toBe(witness.pid);
9703	armed = witness;
9704	signaled = { witness, paths };
9705	process.kill(process.pid, selectedCase.signal);
9706	});
9707	if (selectedCase.signal === "SIGKILL") {
9708	return yield* Effect.fail(
9709	new Error("SIGKILL did not terminate the sole runner"),
9710	);
9711	}
9712	}).pipe(Effect.timeout("20 seconds"));
9713	// Only the runner may interrupt the deploy; the held response stays held.
9714	return yield* Effect.never;
9715	}).pipe(
9716	Effect.raceFirst(
9717	Fiber.join(attempt).pipe(
9718	Effect.andThen(
9719	Effect.fail(
9720	new Error(
9721	"Rollout completed instead of terminating at the signal barrier",
9722	),
9723	),
9724	),
9725	),
9726	),
9727	);
9728	}).pipe(Effect.scoped);
9729
9730	const recover = (stack: Test.ScratchStack, selectedCase: SignalCase) =>
9731	Effect.gen(function* () {
9732	const fs = yield* FileSystem.FileSystem;
9733	const paths = yield* pathsFor(stack);
9734	const witness = yield* readEvidence(
9735	paths.witness,
9736	WitnessSignalOverlap,
9737	);
9738	const pid = yield* Effect.sync(() => process.pid);
9739	expect(witness.pid).toBeGreaterThan(0);
9740	expect(pid).not.toBe(witness.pid);
9741	expect(witness.case).toBe(selectedCase.name);
9742	expect(witness.signal).toBe(selectedCase.signal);
9743	expect(witness.phase).toBe(selectedCase.phase);
9744	expect(witness.cwd).toBe(paths.cwd);
9745	expect(witness.stack).toBe(stack.name);
9746	expect(witness.stage).toBe(stack.stage);
9747	expect(yield* fs.exists(paths.recovered)).toBe(false);
9748	yield* assertBoundary(witness);
9749	expect(yield* persistedRow(stack, witness.row.fqn)).toEqual(
9750	witness.row,
9751	);
9752	expect(witness.row.status).toBe("updating");
9753	yield* assertInventory(stack, witness);
9754	// No deploy, destroy, acquire or release can manufacture either lease proof.
9755	const authority = yield* Effect.gen(function* () {
9756	if (selectedCase.signal === "SIGKILL") {
9757	expect(yield* fs.exists(paths.finalized)).toBe(false);
9758	expect(yield* fs.exists(paths.runnerInterrupted)).toBe(false);
9759	return { mode: "expiry", evidence: yield* observeExpiry(witness) };
9760	}
9761	const runner = yield* readEvidence(
9762	paths.runnerInterrupted,
9763	RunnerInterrupted,
9764	);
9765	expect(runner.pid).toBe(witness.pid);
9766	expect(runner.case).toBe(witness.case);
9767	expect(runner.witnessRecordedAt).toBe(witness.recordedAt);
9768	expect(runner.at).toBeGreaterThanOrEqual(witness.recordedAt);
9769	if (!(yield* fs.exists(paths.finalized))) {
9770	yield* writeEvidence(paths.blocked, {
9771	case: witness.case,
9772	crashPid: witness.pid,
9773	recoveryPid: pid,
9774	runner,
9775	leases: yield* observeLeases(witness),
9776	nativeFinalizerProved: false,
9777	recovered: false,
9778	});
9779	return yield* Effect.fail(
9780	new Error(
9781	"SIGINT reached runner teardown but the scoped deploy did not finalize; native lease RELEASE is unproved. Preserve this attempt; expiry is not SIGINT acceptance.",
9782	),
9783	);
9784	}
9785	expect(runner.deployFinalized).toBe(true);
9786	const finalized = yield* readEvidence(paths.finalized, Finalized);
9787	return {
9788	mode: "release",
9789	runner,
9790	evidence: yield* assertReleased(witness, finalized),
9791	};
9792	});
9793	yield* assertInventory(stack, witness);
9794	const recovered = yield* deploy(stack, "two").pipe(Effect.scoped);
9795	expect(recovered.appName).toBe(witness.appName);
9796	yield* assertConvergedSignalOverlap(
9797	stack,
9798	witness,
9799	recovered.machineIds,
9800	);
9801	const unchanged = yield* deploy(stack, "two").pipe(Effect.scoped);
9802	expect(unchanged.machineIds).toEqual(recovered.machineIds);
9803	yield* assertConvergedSignalOverlap(
9804	stack,
9805	witness,
9806	unchanged.machineIds,
9807	);
9808	yield* stack.destroy();
9809	yield* assertClean(stack, witness.appName);
9810	yield* writeEvidence(paths.recovered, {
9811	case: witness.case,
9812	signal: witness.signal,
9813	phase: witness.phase,
9814	crashPid: witness.pid,
9815	recoveryPid: pid,
9816	barrier: witness.barrier,
9817	returnedUncordon: witness.returnedUncordon,
9818	authority,
9819	machineIds: recovered.machineIds,
9820	generation: witness.candidate.generation,
9821	digest: witness.candidate.digest,
9822	cleaned: true,
9823	});
9824	});
9825
9826	// Runner bodies are detached; ordinary setup hooks remain attached to runMain.
9827	beforeEach(
9828	Effect.gen(function* () {
9829	if (process.env.FLY_SIGNAL_OVERLAP_MODE !== "crash") return;
9830	yield* assertSingleRunnerSignalOverlap;
9831	const selectedCase = cases.find((value) => value.name === selected);
9832	if (!selectedCase) {
9833	return yield* Effect.fail(new Error("Unknown signal case selector"));
9834	}
9835
9836	const stack = yield* Effect.sync(() =>
9837	scratchStack(
9838	options,
9839	`F10 deployer ${selectedCase.signal} at ${selectedCase.phase}`,
9840	signalOverlapFile,
9841	),
9842	);
9843	yield* withProviders(
9844	crash(
9845	stack,
9846	`F10 deployer ${selectedCase.signal} at ${selectedCase.phase}`,
9847	selectedCase,
9848	),
9849	options,
9850	stack.name,
9851	);
9852	}).pipe(Effect.scoped),
9853	{ timeout: 750_000 },
9854	);
9855
9856	for (const selectedCase of cases) {
9857	const skip =
9858	selected === undefined ||
9859	(cases.some((value) => value.name === selected) &&
9860	selected !== selectedCase.name);
9861	if (skip) {
9862	it.live.skip(
9863	`F10 deployer ${selectedCase.signal} at ${selectedCase.phase}`,
9864	() => Effect.void,
9865	);
9866	continue;
9867	}
9868	// No destroy-on-failure wrapper: failed recovery must preserve the forensic state.
9869	test(
9870	`F10 deployer ${selectedCase.signal} at ${selectedCase.phase}`,
9871	Effect.gen(function* () {
9872	yield* assertSingleRunnerSignalOverlap;
9873	expect(selected).toBe(selectedCase.name);
9874	const mode = process.env.FLY_SIGNAL_OVERLAP_MODE;
9875	if (mode !== "crash" && mode !== "recovery") {
9876	return yield* Effect.fail(
9877	new Error("FLY_SIGNAL_OVERLAP_MODE must be crash or recovery"),
9878	);
9879	}
9880	if (mode === "crash") {
9881	return yield* Effect.fail(
9882	new Error(
9883	"The crash setup returned without terminating the runner",
9884	),
9885	);
9886	}
9887	const stack = yield* Effect.sync(() =>
9888	scratchStack(
9889	options,
9890	`F10 deployer ${selectedCase.signal} at ${selectedCase.phase}`,
9891	signalOverlapFile,
9892	),
9893	);
9894	yield* withProviders(
9895	recover(stack, selectedCase),
9896	options,
9897	stack.name,
9898	);
9899	}).pipe(Effect.scoped),
9900	{ timeout: 750_000, retry: 0, exclusive: true },
9901	);
9902	}
9903	},
9904	);
9905
9906	describe.sequential(
9907	"state persistence",
9908	{
9909	tags: [
9910	"provider:fly",
9911	"provider:fly:app",
9912	"provider:fly:machine",
9913	"provider:fly:service",
9914	"live",
9915	],
9916	},
9917	() => {
9918	const stackName = "Fly-BlueGreenStatePersistence-delayed-durable-write";
9919	const options = {
9920	providers: Fly.providers(),
9921	dev: false,
9922	sidecar: false,
9923	stage: undefined,
9924	};
9925	const { test } = Test.make(options);
9926
9927	const actor = (state = localState(), endpoint?: string) => {
9928	const providers = throughProxy(() => endpoint);
9929	const stack = (version: string) =>
9930	Alchemy.Stack(
9931	stackName,
9932	{ providers, state },
9933	Effect.gen(function* () {
9934	const app = yield* Fly.App("Site");
9935	return yield* Fly.Machine("Worker", {
9936	app,
9937	image: "nginx:alpine",
9938	env: { VERSION: version },
9939	checks,
9940	deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
9941	shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
9942	});
9943	}),
9944	);
9945	return {
9946	state,
9947	// Private execution scopes avoid Test.make's file-shared destroy scope.
9948	deploy: (version: string) => TestCore.deploy(options, stack(version)),
9949	destroy: () => TestCore.destroy(options, stack("three")),
9950	};
9951	};
9952
9953	test(
9954	"F11 delayed durable LocalState rename cannot authorize deletion of the live successor",
9955	TestCore.withProviders(
9956	Effect.gen(function* () {
9957	const target = {
9958	stack: stackName,
9959	stage: Test.resolveStage(options),
9960	};
9961	const readWorker = () =>
9962	Effect.gen(function* () {
9963	const state = yield* makeLocalState();
9964	return yield* state.get({ ...target, fqn: "Worker" }).pipe(
9965	Effect.flatMap(Schema.decodeUnknownEffect(ResourceRow)),
9966	Effect.catchTag("SchemaError", () =>
9967	Effect.fail(
9968	new Error("Missing or invalid durable Worker row"),
9969	),
9970	),
9971	);
9972	});
9973	const assertEmptyState = () =>
9974	Effect.gen(function* () {
9975	const state = yield* makeLocalState();
9976	expect(yield* state.list(target)).toEqual([]);
9977	expect(
9978	(yield* state.get({ ...target, fqn: "Worker" })) === undefined,
9979	).toBe(true);
9980	expect(
9981	(yield* state.get({ ...target, fqn: "Site" })) === undefined,
9982	).toBe(true);
9983	expect((yield* state.getOutput(target)) === undefined).toBe(true);
9984	expect(yield* state.listStages(target.stack)).not.toContain(
9985	target.stage,
9986	);
9987	});
9988	yield* actor().destroy();
9989	yield* assertEmptyState();
9990	let appName: string | undefined;
9991	const cleanup = Effect.gen(function* () {
9992	yield* actor().destroy();
9993	if (appName !== undefined) {
9994	yield* assertAppGone(appName);
9995	const remainingMachines = yield* machines
9996	.listMachines({ app_name: appName })
9997	.pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
9998	const remainingVolumes = yield* machines
9999	.listVolumes({ app_name: appName })
10000	.pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
10001	expect(
10002	remainingMachines
10003	.filter((machine) => machine.state !== "destroyed")
10004	.map((machine) => machine.id),
10005	).toEqual([]);
10006	expect(remainingVolumes.map((volume) => volume.id)).toEqual([]);
10007	}
10008	yield* assertEmptyState();
10009	});
10010	yield* Effect.gen(function* () {
10011	const initial = yield* actor().deploy("one");
10012	yield* Effect.sync(() => {
10013	appName = initial.appName;
10014	});
10015	const initialRow = yield* readWorker();
10016	expect(initialRow.status).toBe("created");
10017	expect(initialRow.attr?.machineIds).toEqual(initial.machineIds);
10018	const gate = yield* delayedResourceWrite({
10019	...target,
10020	fqn: "Worker",
10021	});
10022	const actorA = actor(gate.state);
10023	const actorB = actor();
10024	expect(actorA.state).not.toBe(actorB.state);
10025	yield* Effect.gen(function* () {
10026	const delayed = yield* actorA
10027	.deploy("two")
10028	.pipe(Effect.forkScoped);
10029	yield* Effect.gen(function* () {
10030	const held = yield* gate.wait.pipe(
10031	Effect.raceFirst(
10032	Fiber.join(delayed).pipe(
10033	Effect.flatMap(() =>
10034	Effect.fail(
10035	new Error(
10036	"Actor A finished without holding its final rename",
10037	),
10038	),
10039	),
10040	),
10041	),
10042	);
10043	expect(held.status).toBe("updated");
10044	expect(held.instanceId).toBe(initialRow.instanceId);
10045	expect(held.attr?.machineIds).toHaveLength(1);
10046	expect(held.attr?.machineId).not.toBe(initial.machineId);
10047	const heldIds = [...held.attr!.machineIds];
10048	const prepared = yield* assertCommitted(
10049	initial.appName,
10050	heldIds,
10051	);
10052	expect(prepared[0]?.config?.env?.VERSION).toBe("two");
10053	expect(
10054	prepared[0]?.config?.metadata?.["alchemy.instance"],
10055	).toBe(held.instanceId);
10056	expect(prepared[0]?.config?.metadata?.["alchemy.fqn"]).toBe(
10057	"Worker",
10058	);
10059	const beforeRename = yield* readWorker();
10060	expect(beforeRename.status).toBe("updating");
10061	expect(beforeRename.attr?.machineIds).not.toEqual(heldIds);
10062
10063	// B observes A's completed rollout while A's terminal row is still a temp file.
10064	const newer = yield* actorB.deploy("three");
10065	expect(newer.appName).toBe(initial.appName);
10066	expect(newer.machineIds).toHaveLength(1);
10067	expect(newer.machineId).not.toBe(held.attr?.machineId);
10068	const newerRow = yield* readWorker();
10069	expect(newerRow.status).toBe("updated");
10070	expect(newerRow.instanceId).toBe(held.instanceId);
10071	expect(newerRow.attr?.machineIds).toEqual(newer.machineIds);
10072	const successor = yield* assertCommitted(
10073	newer.appName,
10074	newer.machineIds,
10075	);
10076	const successorMetadata = successor[0]!.config!.metadata!;
10077	expect(successor[0]?.config?.env?.VERSION).toBe("three");
10078	expect(successorMetadata["alchemy.instance"]).toBe(
10079	held.instanceId,
10080	);
10081	expect(successorMetadata["alchemy.fqn"]).toBe("Worker");
10082	expect(successorMetadata["alchemy.generation"]).not.toBe(
10083	prepared[0]?.config?.metadata?.["alchemy.generation"],
10084	);
10085	expect(
10086	Number(successorMetadata["alchemy.sequence"]),
10087	).toBeGreaterThan(
10088	Number(prepared[0]?.config?.metadata?.["alchemy.sequence"]),
10089	);
10090
10091	yield* gate.release;
10092	const late = yield* Fiber.join(delayed).pipe(
10093	Effect.timeout("600 seconds"),
10094	);
10095	expect(late.machineIds).toEqual(heldIds);
10096	expect((yield* gate.written).attr?.machineIds).toEqual(heldIds);
10097	const stale = yield* readWorker();
10098	// This controlled late rename wins on disk, not in the cloud; there is no fence.
10099	expect(stale.status).toBe("updated");
10100	expect(stale.instanceId).toBe(held.instanceId);
10101	expect(stale.attr?.machineIds).toEqual(heldIds);
10102	yield* assertCommitted(newer.appName, newer.machineIds);
10103
10104	const proxy = yield* transportProxy();
10105	const actorC = actor(localState(), proxy.url);
10106	expect(actorC.state).not.toBe(actorA.state);
10107	expect(actorC.state).not.toBe(actorB.state);
10108	const recovered = yield* actorC.deploy("three");
10109	expect(
10110	proxy.events.some(
10111	(event) =>
10112	event.stage === "completed" &&
10113	event.method === "GET" &&
10114	event.path === `/v1/apps/${newer.appName}/machines` &&
10115	event.status === 200,
10116	),
10117	).toBe(true);
10118	expect(
10119	proxy.events.filter(
10120	(event) =>
10121	event.stage === "request" &&
10122	newer.machineIds.includes(event.machineId ?? "") &&
10123	(event.path.endsWith("/cordon") ||
10124	event.path.endsWith("/stop") ||
10125	(event.method === "DELETE" &&
10126	!event.path.endsWith("/lease"))),
10127	),
10128	).toEqual([]);
10129	const preserved = yield* assertCommitted(
10130	newer.appName,
10131	newer.machineIds,
10132	);
10133	expect(
10134	preserved[0]?.config?.metadata?.["alchemy.instance"],
10135	).toBe(successorMetadata["alchemy.instance"]);
10136	expect(
10137	preserved[0]?.config?.metadata?.["alchemy.generation"],
10138	).toBe(successorMetadata["alchemy.generation"]);
10139	const recoveredRow = yield* readWorker();
10140	expect(recoveredRow.instanceId).toBe(held.instanceId);
10141	expect(recovered.machineIds).toEqual(newer.machineIds);
10142	expect(recoveredRow.status).toBe("updated");
10143	expect(recoveredRow.attr?.machineIds).toEqual(newer.machineIds);
10144	}).pipe(
10145	Effect.ensuring(
10146	gate.release.pipe(
10147	Effect.andThen(
10148	Fiber.await(delayed).pipe(
10149	Effect.timeout("600 seconds"),
10150	Effect.orDie,
10151	),
10152	),
10153	),
10154	),
10155	);
10156	}).pipe(Effect.scoped);
10157	}).pipe(Effect.ensuring(cleanup.pipe(Effect.orDie)));
10158	}).pipe(Effect.scoped),
10159	options,
10160	stackName,
10161	),
10162	{ timeout: 1_800_000, retry: 0 },
10163	);
10164	},
10165	);
10166
10167	describe.sequential(
10168	"transport",
10169	{
10170	tags: [
10171	"provider:fly",
10172	"provider:fly:app",
10173	"provider:fly:machine",
10174	"provider:fly:service",
10175	"live",
10176	],
10177	},
10178	() => {
10179	let endpoint: string | undefined;
10180	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
10181
10182	test.provider(
10183	"F02 S07 harness forwards real Fly responses, loses completed responses and cuts connections",
10184	(stack) =>
10185	Effect.gen(function* () {
10186	yield* stack.destroy();
10187	const app = yield* stack.deploy(Fly.App("Site"));
10188	const proxy = yield* transportProxy();
10189	yield* Effect.sync(() => {
10190	endpoint = proxy.url;
10191	});
10192	try {
10193	const request = machines
10194	.getApp({ app_name: app.appName })
10195	.pipe(Retry.none);
10196	const forwarded = yield* request;
10197	expect(forwarded.name).toBe(app.appName);
10198	expect(
10199	proxy.events.some(
10200	(event) => event.stage === "forwarded" && event.status === 200,
10201	),
10202	).toBe(true);
10203	yield* Effect.sync(() =>
10204	proxy.arm({
10205	match: (event) => event.path.endsWith(`/apps/${app.appName}`),
10206	action: "drop-response",
10207	remaining: 1,
10208	}),
10209	);
10210	const dropped = yield* request.pipe(Effect.result);
10211	// The HTTP runtime may transparently replay an idempotent GET after a socket reset.
10212	if (Result.isFailure(dropped))
10213	expect(dropped.failure._tag).toBe("HttpClientError");
10214	else expect(dropped.success.name).toBe(app.appName);
10215	expect(
10216	proxy.events.filter(
10217	(event) => event.stage === "dropped" && event.status === 200,
10218	),
10219	).toHaveLength(1);
10220	yield* Effect.sync(() =>
10221	proxy.arm({
10222	match: () => true,
10223	action: "cut-request",
10224	remaining: Infinity,
10225	}),
10226	);
10227	const cut = yield* request.pipe(Effect.result);
10228	expect(Result.isFailure(cut)).toBe(true);
10229	const cutEvent = proxy.events.find(
10230	(event) => event.stage === "cut",
10231	)!;
10232	expect(
10233	proxy.events.some(
10234	(event) =>
10235	event.sequence === cutEvent.sequence &&
10236	event.stage === "completed",
10237	),
10238	).toBe(false);
10239	yield* Effect.sync(() => {
10240	proxy.clear();
10241	proxy.arm({
10242	match: () => true,
10243	action: "hold-response",
10244	remaining: 1,
10245	});
10246	});
10247	const child = yield* request.pipe(Effect.forkScoped);
10248	const held = yield* proxy.wait((event) => event.stage === "held");
10249	expect(held.status).toBe(200);
10250	yield* Effect.sync(proxy.release);
10251	expect(
10252	(yield* Fiber.join(child).pipe(Effect.timeout("10 seconds")))
10253	.name,
10254	).toBe(app.appName);
10255	expect((yield* request).name).toBe(app.appName);
10256	} finally {
10257	yield* Effect.sync(() => {
10258	endpoint = undefined;
10259	proxy.clear();
10260	proxy.release();
10261	});
10262	}
10263	yield* stack.destroy();
10264	yield* assertAppGone(app.appName);
10265	}).pipe(
10266	Effect.scoped,
10267	Effect.ensuring(
10268	Effect.sync(() => {
10269	endpoint = undefined;
10270	}),
10271	),
10272	),
10273	{ timeout: 180_000 },
10274	);
10275	},
10276	);
10277
10278	describe.sequential(
10279	"validation",
10280	{
10281	tags: [
10282	"provider:fly",
10283	"provider:fly:app",
10284	"provider:fly:machine",
10285	"provider:fly:service",
10286	"live",
10287	],
10288	},
10289	() => {
10290	let endpoint: string | undefined;
10291	const { test } = Test.make({ providers: throughProxy(() => endpoint) });
10292	const invalid: Array<
10293	[string, Partial<Extract<MachineProps, { image: string }>>]
10294	> = [
10295	["direct volume", { mounts: [{ path: "/data", sizeGb: 1 }] }],
10296	["skipLaunch", { skipLaunch: true }],
10297	["autoDestroy", { autoDestroy: true }],
10298	["restart=no", { restart: { policy: "no" } }],
10299	["missing checks", { checks: {} }],
10300	[
10301	"unchecked public service",
10302	{ services: [{ internalPort: 80, ports: [{ port: 80 }] }] },
10303	],
10304	];
10305
10306	describe.sequential("pre-mutation validation", () => {
10307	for (const [label, props] of invalid) {
10308	test.provider(
10309	`${label === "direct volume" ? "S10" : "S11"} ${label} rejects before any Machine mutation`,
10310	(stack) =>
10311	Effect.gen(function* () {
10312	yield* stack.destroy();
10313	const app = yield* stack.deploy(Fly.App("Site"));
10314	const proxy = yield* transportProxy();
10315	yield* Effect.sync(() => {
10316	endpoint = proxy.url;
10317	});
10318	const failed = yield* stack
10319	.deploy(
10320	Effect.gen(function* () {
10321	const app = yield* Fly.App("Site");
10322	return yield* Fly.Machine("Worker", {
10323	app,
10324	image: "nginx:alpine",
10325	checks,
10326	deploy: { strategy: "bluegreen" },
10327	...props,
10328	});
10329	}),
10330	)
10331	.pipe(Effect.result);
10332	expect(Result.isFailure(failed)).toBe(true);
10333	if (Result.isFailure(failed))
10334	expect(failed.failure).toMatchObject({
10335	_tag: "Fly.InvalidDeployment",
10336	});
10337	expect(
10338	proxy.events.some(
10339	(event) =>
10340	event.method !== "GET" &&
10341	/\/(machines|volumes)(\/|$)/.test(event.path),
10342	),
10343	).toBe(false);
10344	expect(yield* census(app.appName)).toHaveLength(0);
10345	expect(
10346	yield* machines.listVolumes({ app_name: app.appName }),
10347	).toHaveLength(0);
10348	yield* Effect.sync(() => {
10349	endpoint = undefined;
10350	});
10351	yield* stack.destroy();
10352	yield* assertAppGone(app.appName);
10353	}).pipe(
10354	Effect.scoped,
10355	Effect.ensuring(
10356	Effect.sync(() => {
10357	endpoint = undefined;
10358	}),
10359	),
10360	),
10361	{ timeout: 180_000 },
10362	);
10363	}
10364
10365	test.provider(
10366	"S10 resolved MountVolume binding rejects before Machine or Volume mutation",
10367	(stack) =>
10368	Effect.gen(function* () {
10369	yield* stack.destroy();
10370	const app = yield* stack.deploy(Fly.App("Site"));
10371	const proxy = yield* transportProxy();
10372	yield* Effect.sync(() => {
10373	endpoint = proxy.url;
10374	});
10375	const failed = yield* stack
10376	.deploy(MountedBlueGreen)
10377	.pipe(Effect.result);
10378	expect(Result.isFailure(failed)).toBe(true);
10379	if (Result.isFailure(failed))
10380	expect(failed.failure).toMatchObject({
10381	_tag: "Fly.InvalidDeployment",
10382	});
10383	expect(
10384	proxy.events.some(
10385	(event) =>
10386	event.method !== "GET" &&
10387	/\/(machines|volumes)(\/|$)/.test(event.path),
10388	),
10389	).toBe(false);
10390	expect(yield* census(app.appName)).toHaveLength(0);
10391	expect(
10392	yield* machines.listVolumes({ app_name: app.appName }),
10393	).toHaveLength(0);
10394	yield* Effect.sync(() => {
10395	endpoint = undefined;
10396	});
10397	yield* stack.destroy();
10398	yield* assertAppGone(app.appName);
10399	}).pipe(
10400	Effect.scoped,
10401	Effect.ensuring(
10402	Effect.sync(() => {
10403	endpoint = undefined;
10404	}),
10405	),
10406	),
10407	{ tags: ["provider:fly:volume"], timeout: 300_000 },
10408	);
10409	});
10410	},
10411	);
10412
10413	describe.sequential(
10414	"workers",
10415	{
10416	tags: [
10417	"provider:fly",
10418	"provider:fly:app",
10419	"provider:fly:ipassignment",
10420	"provider:fly:machine",
10421	"provider:fly:redis",
10422	"provider:fly:secret",
10423	"provider:fly:service",
10424	"live",
10425	],
10426	},
10427	() => {
10428	const { test } = Test.make({ providers: Fly.providers() });
10429
10430	// Cold gateway/worker publication, readiness, replacement and census can exceed three minutes.
10431	const timeout = 480_000;
10432
10433	describe.sequential("Fly durable worker acceptance", () => {
10434	for (const mode of ["stop-delay", "stop-fail", "stop-hang"]) {
10435	test.provider(
10436	`R05 live ${mode} preserves another worker and bounds managed shutdown`,
10437	(stack) =>
10438	Effect.gen(function* () {
10439	yield* stack.destroy();
10440	const scenario = yield* makeScenario(stack);
10441	yield* Effect.gen(function* () {
10442	const first = yield* scenario.deploy({
10443	version: "one",
10444	timeout: "10 seconds",
10445	signal: "SIGTERM",
10446	runOnly: true,
10447	mode,
10448	workers: 2,
10449	});
10450	const worker = yield* requireValue(
10451	first.worker,
10452	"worker output",
10453	);
10454	const machineId = worker.machineId;
10455	const observe = <A, E, R>(
10456	phase: string,
10457	effect: Effect.Effect<A, E, R>,
10458	) =>
10459	Effect.gen(function* () {
10460	const started = yield* Clock.currentTimeMillis;
10461	yield* Effect.logInfo("R05 phase started", {
10462	mode,
10463	machineId,
10464	phase,
10465	});
10466	return yield* effect.pipe(
10467	Effect.onExit((exit) =>
10468	Clock.currentTimeMillis.pipe(
10469	Effect.flatMap((finished) =>
10470	Effect.logInfo("R05 phase finished", {
10471	mode,
10472	machineId,
10473	phase,
10474	success: Exit.isSuccess(exit),
10475	elapsedMs: finished - started,
10476	}),
10477	),
10478	),
10479	),
10480	);
10481	});
10482	yield* observe(
10483	"both-workers-ready-ledger",
10484	scenario.wait(
10485	first.ledgerUrl,
10486	(ledger) =>
10487	ledger.events.filter(
10488	(event) =>
10489	event.machine === machineId &&
10490	event.event === "worker-ready",
10491	).length === 2,
10492	),
10493	);
10494	yield* observe(
10495	"stop-machine-request",
10496	machines.stopMachine({
10497	app_name: first.workerApp.appName,
10498	machine_id: machineId,
10499	}),
10500	);
10501	const stopped = yield* observe(
10502	"stopped-machine-observation",
10503	machines
10504	.getMachine({
10505	app_name: first.workerApp.appName,
10506	machine_id: machineId,
10507	})
10508	.pipe(
10509	Effect.repeat({
10510	until: (value) => value.state === "stopped",
10511	times: 8,
10512	schedule: Schedule.spaced("2 seconds"),
10513	}),
10514	),
10515	);
10516	expect(stopped.state).toBe("stopped");
10517	const ledger = yield* observe(
10518	"post-stop-ledger-snapshot",
10519	scenario.snapshot(first.ledgerUrl),
10520	);
10521	assertOrder(
10522	ledger.events,
10523	machineId,
10524	"stopped",
10525	"drained",
10526	"b",
10527	);
10528	assertOrder(
10529	ledger.events,
10530	machineId,
10531	"drained",
10532	"client-released",
10533	"b",
10534	);
10535	const stopStarted = yield* requireValue(
10536	ledger.events.find(
10537	(event) =>
10538	event.machine === machineId &&
10539	event.event === "stop-started",
10540	),
10541	"worker stop-started event",
10542	);
10543	const independentRelease = yield* requireValue(
10544	ledger.events.find(
10545	(event) =>
10546	event.machine === machineId &&
10547	event.worker === "b" &&
10548	event.event === "client-released",
10549	),
10550	"worker b client release event",
10551	);
10552	expect(
10553	independentRelease.at - stopStarted.at,
10554	).toBeGreaterThanOrEqual(0);
10555	expect(independentRelease.at - stopStarted.at).toBeLessThan(
10556	10_000,
10557	);
10558	const selected = ledger.events.filter(
10559	(event) =>
10560	event.machine === machineId && event.worker === "a",
10561	);
10562	if (mode === "stop-delay") {
10563	const a = yield* requireValue(
10564	selected.find((event) => event.event === "stopped"),
10565	"worker a stopped event",
10566	);
10567	const b = yield* requireValue(
10568	ledger.events.find(
10569	(event) =>
10570	event.machine === machineId &&
10571	event.worker === "b" &&
10572	event.event === "stopped",
10573	),
10574	"worker b stopped event",
10575	);
10576	expect(a.at - b.at).toBeGreaterThan(700);
10577	assertOrder(
10578	ledger.events,
10579	machineId,
10580	"stopped",
10581	"drained",
10582	"a",
10583	);
10584	assertStopped(ledger.events, machineId);
10585	} else {
10586	expect(
10587	selected.some(
10588	(event) =>
10589	event.event === "stopped" || event.event === "drained",
10590	),
10591	).toBe(false);
10592	expect(
10593	selected.some(
10594	(event) =>
10595	event.event ===
10596	(mode === "stop-fail" ? "stop-failed" : "stop-started"),
10597	),
10598	).toBe(true);
10599	}
10600	if (mode === "stop-hang") {
10601	expect(
10602	selected.some(
10603	(event) =>
10604	event.event === "work-closed" ||
10605	event.event === "client-released",
10606	),
10607	).toBe(false);
10608	}
10609	expect(
10610	ledger.events.some(
10611	(event) =>
10612	event.machine === machineId &&
10613	event.event === "shared-closed",
10614	),
10615	).toBe(mode !== "stop-hang");
10616	const exit = stopped.events?.find(
10617	(event) => event.type === "exit",
10618	)?.request as
10619	| { exit_event?: { exit_code?: number } }
10620	| undefined;
10621	yield* Effect.logInfo("Observed managed worker exit", {
10622	mode,
10623	exitCode: exit?.exit_event?.exit_code,
10624	});
10625	expect(exit?.exit_event?.exit_code).toBe(
10626	mode === "stop-delay" ? 0 : 1,
10627	);
10628	}).pipe(Effect.ensuring(scenario.cleanup));
10629	}),
10630	{ timeout },
10631	);
10632	}
10633
10634	test.provider(
10635	"R07 authenticated durable Redis Streams gateway",
10636	(stack) =>
10637	Effect.gen(function* () {
10638	yield* stack.destroy();
10639	const scenario = yield* makeScenario(stack);
10640	yield* Effect.gen(function* () {
10641	const first = yield* scenario.deploy({
10642	version: "one",
10643	timeout: "30 seconds",
10644	signal: "SIGTERM",
10645	runOnly: true,
10646	gatewayOnly: true,
10647	});
10648	expect((yield* HttpClient.post(first.ledgerUrl)).status).toBe(
10649	401,
10650	);
10651	const added = yield* scenario
10652	.call(first.ledgerUrl, "enqueue", ["durable-probe", "quick"])
10653	.pipe(
10654	Effect.retry({
10655	times: 8,
10656	schedule: Schedule.spaced("1 second"),
10657	}),
10658	);
10659	expect(typeof added).toBe("string");
10660	expect(
10661	yield* scenario.call(first.ledgerUrl, "enqueue", [
10662	"durable-probe",
10663	"quick",
10664	]),
10665	).toBe(0);
10666	const claimed = JSON.parse(
10667	String(
10668	yield* scenario.call(first.ledgerUrl, "claim", [
10669	"probe-client",
10670	"one",
10671	]),
10672	),
10673	) as { id: string; job: string };
10674	expect(claimed.job).toBe("durable-probe");
10675	expect(
10676	yield* scenario.call(first.ledgerUrl, "finish", [
10677	claimed.id,
10678	claimed.job,
10679	"probe-client",
10680	]),
10681	).toBe(1);
10682	const ledger = yield* scenario.settle(first.ledgerUrl);
10683	expect(ledger.pending).toBe(0);
10684	expect(ledger.results).toContain("durable-probe");
10685	}).pipe(Effect.ensuring(scenario.cleanup));
10686	}),
10687	{ timeout },
10688	);
10689
10690	for (const raw of [false, true]) {
10691	test.provider(
10692	`R07 ${raw ? "raw" : "managed"} private worker checkpoints and reclaims through replacement with producer overlap`,
10693	(stack) =>
10694	Effect.gen(function* () {
10695	yield* stack.destroy();
10696	const scenario = yield* makeScenario(stack);
10697	yield* Effect.gen(function* () {
10698	const options = {
10699	raw,
10700	runOnly: true,
10701	timeout: "30 seconds" as const,
10702	signal: "SIGTERM" as const,
10703	mode: "checkpoint",
10704	};
10705	const first = yield* scenario.deploy({
10706	...options,
10707	version: "one",
10708	});
10709	const oldWorker = yield* requireValue(
10710	first.worker,
10711	"initial worker output",
10712	);
10713	const oldId = oldWorker.machineId;
10714	yield* scenario.call(first.ledgerUrl, "enqueue", [
10715	"checkpoint-job",
10716	"checkpoint",
10717	]);
10718	yield* scenario.wait(first.ledgerUrl, (ledger) =>
10719	ledger.events.some(
10720	(event) =>
10721	event.machine === oldId &&
10722	event.event === "claimed" &&
10723	event.job === "checkpoint-job",
10724	),
10725	);
10726	const second = yield* scenario.deploy({
10727	...options,
10728	version: "two",
10729	mode: "drain",
10730	signal: "SIGINT",
10731	});
10732	const newWorker = yield* requireValue(
10733	second.worker,
10734	"replacement worker output",
10735	);
10736	const newId = newWorker.machineId;
10737	yield* assertReplacement(first.workerApp.appName, oldId, newId);
10738	const ledger = yield* scenario.settle(first.ledgerUrl);
10739	expect(ledger.results).toContain("checkpoint-job");
10740	expect(ledger.checkpoints).toContain("saved-step-1");
10741	expect(
10742	ledger.events.some(
10743	(event) =>
10744	event.event === "reclaimed" &&
10745	event.machine === newId &&
10746	event.job === "checkpoint-job",
10747	),
10748	).toBe(true);
10749	const ack = ledger.events.filter(
10750	(event) =>
10751	event.event === "acked" && event.job === "checkpoint-job",
10752	);
10753	expect(ack).toHaveLength(1);
10754	expect(ack[0]?.ack).toBe(1);
10755	expect(ack[0]?.fresh).toBe(1);
10756	assertStopped(ledger.events, oldId);
10757	assertOrder(ledger.events, oldId, "stopped", "checkpoint");
10758	assertOrder(
10759	ledger.events,
10760	oldId,
10761	"checkpoint",
10762	"client-released",
10763	);
10764	assertOrder(
10765	ledger.events,
10766	oldId,
10767	"client-released",
10768	"shared-closed",
10769	);
10770	const oldSlots = new Set(
10771	ledger.events
10772	.filter(
10773	(event) =>
10774	event.machine === oldId && event.event === "producer",
10775	)
10776	.map((event) => event.job),
10777	);
10778	const overlaps = ledger.events.filter(
10779	(event) =>
10780	event.machine === newId &&
10781	event.event === "producer" &&
10782	oldSlots.has(event.job),
10783	);
10784	expect(overlaps.length).toBeGreaterThan(0);
10785	for (const event of overlaps) {
10786	expect(
10787	ledger.events.filter(
10788	(row) =>
10789	row.event === "producer" &&
10790	row.job === event.job &&
10791	row.fresh === 1,
10792	),
10793	).toHaveLength(1);
10794	}
10795	}).pipe(Effect.ensuring(scenario.cleanup));
10796	}),
10797	{ timeout },
10798	);
10799	}
10800	});
10801	},
10802	);
10803
10804	describe.sequential(
10805	"multi-container",
10806	{
10807	tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"],
10808	},
10809	() => {
10810	const { test } = Test.make({ providers: Fly.providers() });
10811	const firstImage =
10812	"docker-hub-mirror.fly.io/library/nginx@sha256:7396be67b6f53012a5cf955fa9040619294c25ccacf11e22af5de1b572fc756e";
10813	const nextImage =
10814	"docker-hub-mirror.fly.io/library/nginx@sha256:ef8676b33d681f272ba429b27658bdd7e640963279714c96bddf1dc76307f7b6";
10815
10816	interface GroupOptions {
10817	count?: number;
10818	/** Reverse declaration order and add empty optional fields. */
10819	reordered?: boolean;
10820	/** Add a third container to the group. */
10821	extra?: boolean;
10822	badHealth?: boolean;
10823	strategy?: "rolling" | "bluegreen";
10824	}
10825
10826	const containers = (
10827	sidecarImage: string,
10828	{ reordered = false, extra = false }: GroupOptions,
10829	): MachineContainer[] => {
10830	const group: MachineContainer[] = [
10831	{
10832	name: "web",
10833	image: firstImage,
10834	healthChecks: [
10835	{
10836	name: "web-tcp",
10837	kind: "readiness",
10838	tcp: { port: 80 },
10839	interval: 5,
10840	timeout: 2,
10841	},
10842	],
10843	},
10844	{
10845	name: "sidecar",
10846	image: sidecarImage,
10847	cmd: [
10848	"sh",
10849	"-c",
10850	"sed -i 's/80/8080/g' /etc/nginx/conf.d/default.conf; exec nginx -g 'daemon off;'",
10851	],
10852	dependsOn: [{ name: "web", condition: "healthy" }],
10853	healthChecks: [
10854	{
10855	name: "sidecar-tcp",
10856	kind: "readiness",
10857	tcp: { port: 8080 },
10858	interval: 5,
10859	timeout: 2,
10860	},
10861	],
10862	},
10863	...(extra
10864	? [
10865	{
10866	name: "extra",
10867	image: firstImage,
10868	cmd: ["sh", "-c", "sleep infinity"],
10869	},
10870	]
10871	: []),
10872	];
10873	return reordered
10874	? group.reverse().map((container) => ({ ...container, env: {} }))
10875	: group;
10876	};
10877
10878	const machineChecks = (badHealth: boolean) => ({
10879	web: { type: "tcp" as const, port: 80, interval: "2s", timeout: "1s" },
10880	sidecar: {
10881	type: "tcp" as const,
10882	port: badHealth ? 9999 : 8080,
10883	interval: "2s",
10884	timeout: "1s",
10885	},
10886	});
10887
10888	const deployGroup = (
10889	stack: ScratchStack,
10890	sidecarImage: string,
10891	options: GroupOptions = {},
10892	) =>
10893	stack.deploy(
10894	Effect.gen(function* () {
10895	const app = yield* Fly.App("MultiContainerBlueGreenSite");
10896	return yield* Fly.Machine("Group", {
10897	app,
10898	region: "fra",
10899	count: options.count ?? 1,
10900	guest: { cpus: 1, memoryMb: 256 },
10901	containers: containers(sidecarImage, options),
10902	checks: machineChecks(options.badHealth ?? false),
10903	deploy: {
10904	strategy: options.strategy ?? "bluegreen",
10905	healthTimeout: options.badHealth ? "15 seconds" : "30 seconds",
10906	},
10907	shutdown: { signal: "SIGTERM", timeout: "1 second" },
10908	});
10909	}),
10910	);
10911
10912	const images = (machine: Machine) =>
10913	(machine.config?.containers ?? [])
10914	.map(({ name, image }) => ({ name, image }))
10915	.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? ""));
10916
10917	const appGone = (appName: string) =>
10918	machines.getApp({ app_name: appName }).pipe(
10919	Effect.as(false),
10920	Effect.catchTag("NotFound", () => Effect.succeed(true)),
10921	);
10922
10923	const machineGone = (appName: string, machineId: string) =>
10924	machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
10925	Effect.map((machine) => machine.state === "destroyed"),
10926	Effect.catchTag("NotFound", () => Effect.succeed(true)),
10927	Effect.repeat({
10928	schedule: Schedule.spaced("2 seconds"),
10929	times: 10,
10930	until: (gone) => gone,
10931	}),
10932	);
10933
10934	test.provider(
10935	"checks two named-container replicas and ignores declaration order and empty fields",
10936	(stack) =>
10937	Effect.gen(function* () {
10938	yield* stack.destroy();
10939	const first = yield* deployGroup(stack, firstImage, { count: 2 });
10940	expect(first.machineIds).toHaveLength(2);
10941	const observed = yield* Effect.forEach(first.machineIds, (id) =>
10942	machines.getMachine({ app_name: first.appName, machine_id: id }),
10943	);
10944	for (const machine of observed) {
10945	expect(images(machine)).toEqual([
10946	{ name: "sidecar", image: firstImage },
10947	{ name: "web", image: firstImage },
10948	]);
10949	expect(machine.cordoned).toBe(false);
10950	expect(machine.config?.metadata?.[keys.phase]).toBe("active");
10951	expect(machine.config?.metadata?.[keys.protocol]).toBe("2");
10952	const ready = yield* waitHealthy(first.appName, machine, 30_000);
10953	for (const check of ["web", "sidecar"])
10954	expect(
10955	ready.checks?.some(
10956	({ name, status }) => name === check && status === "passing",
10957	),
10958	).toBe(true);
10959	}
10960	const reordered = yield* deployGroup(stack, firstImage, {
10961	count: 2,
10962	reordered: true,
10963	});
10964	expect([...reordered.machineIds].sort()).toEqual(
10965	[...first.machineIds].sort(),
10966	);
10967	for (const machine of observed) {
10968	const after = yield* machines.getMachine({
10969	app_name: first.appName,
10970	machine_id: machine.id!,
10971	});
10972	expect(after.instance_id).toBe(machine.instance_id);
10973	}
10974	yield* stack.destroy();
10975	expect(yield* appGone(first.appName)).toBe(true);
10976	}),
10977	{ timeout: 180_000 },
10978	);
10979
10980	test.provider(
10981	"named group transitions from rolling to blue/green and back",
10982	(stack) =>
10983	Effect.gen(function* () {
10984	yield* stack.destroy();
10985	const rolling = yield* deployGroup(stack, firstImage, {
10986	strategy: "rolling",
10987	});
10988	const promoted = yield* deployGroup(stack, firstImage);
10989	expect(promoted.machineId).not.toBe(rolling.machineId);
10990	const promotedMachine = yield* machines.getMachine({
10991	app_name: promoted.appName,
10992	machine_id: promoted.machineId,
10993	});
10994	expect(promotedMachine.config?.metadata?.[keys.protocol]).toBe("2");
10995	expect(images(promotedMachine)).toEqual([
10996	{ name: "sidecar", image: firstImage },
10997	{ name: "web", image: firstImage },
10998	]);
10999	const optedOut = yield* deployGroup(stack, firstImage, {
11000	strategy: "rolling",
11001	});
11002	expect(optedOut.machineId).toBe(promoted.machineId);
11003	const live = yield* machines.getMachine({
11004	app_name: optedOut.appName,
11005	machine_id: optedOut.machineId,
11006	});
11007	expect(live.config?.metadata?.[keys.protocol]).toBeUndefined();
11008	expect(images(live)).toEqual([
11009	{ name: "sidecar", image: firstImage },
11010	{ name: "web", image: firstImage },
11011	]);
11012	yield* stack.destroy();
11013	expect(yield* appGone(rolling.appName)).toBe(true);
11014	}),
11015	{ timeout: 180_000 },
11016	);
11017
11018	test.provider(
11019	"a single-image blue/green generation is replaced by a named group",
11020	(stack) =>
11021	Effect.gen(function* () {
11022	yield* stack.destroy();
11023	const single = yield* stack.deploy(
11024	Effect.gen(function* () {
11025	const app = yield* Fly.App("MultiContainerBlueGreenSite");
11026	return yield* Fly.Machine("Group", {
11027	app,
11028	region: "fra",
11029	guest: { cpus: 1, memoryMb: 256 },
11030	image: firstImage,
11031	checks: { web: machineChecks(false).web },
11032	deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
11033	shutdown: { signal: "SIGTERM", timeout: "1 second" },
11034	});
11035	}),
11036	);
11037	const before = yield* machines.getMachine({
11038	app_name: single.appName,
11039	machine_id: single.machineId,
11040	});
11041	expect(before.config?.metadata?.[keys.protocol]).toBe("1");
11042	const group = yield* deployGroup(stack, firstImage);
11043	expect(group.machineId).not.toBe(single.machineId);
11044	const after = yield* machines.getMachine({
11045	app_name: group.appName,
11046	machine_id: group.machineId,
11047	});
11048	expect(after.config?.metadata?.[keys.protocol]).toBe("2");
11049	expect(after.config?.metadata?.[keys.phase]).toBe("active");
11050	expect(images(after)).toEqual([
11051	{ name: "sidecar", image: firstImage },
11052	{ name: "web", image: firstImage },
11053	]);
11054	expect(yield* machineGone(single.appName, single.machineId)).toBe(
11055	true,
11056	);
11057	yield* stack.destroy();
11058	expect(yield* appGone(single.appName)).toBe(true);
11059	}),
11060	{ timeout: 180_000 },
11061	);
11062
11063	test.provider(
11064	"changing one image or adding a container replaces the whole group",
11065	(stack) =>
11066	Effect.gen(function* () {
11067	yield* stack.destroy();
11068	const first = yield* deployGroup(stack, firstImage);
11069	const updated = yield* deployGroup(stack, nextImage, {
11070	reordered: true,
11071	});
11072	expect(updated.machineId).not.toBe(first.machineId);
11073	const replacement = yield* machines.getMachine({
11074	app_name: updated.appName,
11075	machine_id: updated.machineId,
11076	});
11077	expect(images(replacement)).toEqual([
11078	{ name: "sidecar", image: nextImage },
11079	{ name: "web", image: firstImage },
11080	]);
11081	expect(replacement.cordoned).toBe(false);
11082	expect(replacement.config?.metadata?.[keys.phase]).toBe("active");
11083	expect(yield* machineGone(first.appName, first.machineId)).toBe(true);
11084
11085	const extended = yield* deployGroup(stack, nextImage, {
11086	extra: true,
11087	});
11088	expect(extended.machineId).not.toBe(updated.machineId);
11089	expect(
11090	images(
11091	yield* machines.getMachine({
11092	app_name: extended.appName,
11093	machine_id: extended.machineId,
11094	}),
11095	),
11096	).toEqual([
11097	{ name: "extra", image: firstImage },
11098	{ name: "sidecar", image: nextImage },
11099	{ name: "web", image: firstImage },
11100	]);
11101	expect(yield* machineGone(updated.appName, updated.machineId)).toBe(
11102	true,
11103	);
11104	yield* stack.destroy();
11105	expect(yield* appGone(first.appName)).toBe(true);
11106	}),
11107	{ timeout: 240_000 },
11108	);
11109
11110	test.provider(
11111	"bad secondary readiness preserves the serving predecessor",
11112	(stack) =>
11113	Effect.gen(function* () {
11114	yield* stack.destroy();
11115	const first = yield* deployGroup(stack, firstImage);
11116	const failed = yield* deployGroup(stack, nextImage, {
11117	badHealth: true,
11118	}).pipe(Effect.result);
11119	expect(Result.isFailure(failed)).toBe(true);
11120	const predecessor = yield* machines.getMachine({
11121	app_name: first.appName,
11122	machine_id: first.machineId,
11123	});
11124	expect(predecessor.state).toBe("started");
11125	expect(predecessor.cordoned).toBe(false);
11126	expect(predecessor.config?.metadata?.[keys.phase]).toBe("active");
11127	expect(images(predecessor)).toEqual([
11128	{ name: "sidecar", image: firstImage },
11129	{ name: "web", image: firstImage },
11130	]);
11131	yield* stack.destroy();
11132	expect(yield* appGone(first.appName)).toBe(true);
11133	}),
11134	{ timeout: 180_000 },
11135	);
11136
11137	test.provider(
11138	"tampered deployment metadata preserves the serving group",
11139	(stack) =>
11140	Effect.gen(function* () {
11141	yield* stack.destroy();
11142	const first = yield* deployGroup(stack, firstImage);
11143	const target = {
11144	app_name: first.appName,
11145	machine_id: first.machineId,
11146	};
11147	const original = yield* machines.getMachine(target);
11148	const tamperings: Array<[string, Record<string, string>]> = [
11149	["unknown protocol", { [keys.protocol]: "future" }],
11150	[
11151	"incomplete image set",
11152	{
11153	[keys.containerImageSet]: JSON.stringify([
11154	{ name: "web", image: firstImage },
11155	]),
11156	},
11157	],
11158	];
11159	for (const [label, overrides] of tamperings) {
11160	// Rewrite Alchemy's ownership metadata directly through the Fly API.
11161	const current = yield* machines.getMachine(target);
11162	const tampered = yield* machines.updateMachine({
11163	...target,
11164	config: {
11165	...current.config,
11166	metadata: { ...original.config?.metadata, ...overrides },
11167	},
11168	});
11169	yield* machines.waitMachine({
11170	...target,
11171	state: "started",
11172	instance_id: tampered.instance_id,
11173	timeout: 30,
11174	});
11175	const failed = yield* deployGroup(stack, nextImage).pipe(
11176	Effect.flip,
11177	);
11178	expect(failed, label).toBeInstanceOf(DeploymentRecoveryAmbiguous);
11179	const preserved = yield* machines.getMachine(target);
11180	expect(preserved.instance_id, label).toBe(tampered.instance_id);
11181	expect(preserved.state, label).toBe("started");
11182	expect(images(preserved), label).toEqual([
11183	{ name: "sidecar", image: firstImage },
11184	{ name: "web", image: firstImage },
11185	]);
11186	const inventory = (yield* machines.listMachines({
11187	app_name: first.appName,
11188	})).filter((machine) => machine.state !== "destroyed");
11189	expect(
11190	inventory.map(({ id }) => id),
11191	label,
11192	).toEqual([first.machineId]);
11193	}
11194	const restored = yield* machines.updateMachine({
11195	...target,
11196	config: {
11197	...(yield* machines.getMachine(target)).config,
11198	metadata: original.config?.metadata,
11199	},
11200	});
11201	yield* machines.waitMachine({
11202	...target,
11203	state: "started",
11204	instance_id: restored.instance_id,
11205	timeout: 30,
11206	});
11207	yield* stack.destroy();
11208	expect(yield* appGone(first.appName)).toBe(true);
11209	}),
11210	{ timeout: 180_000 },
11211	);
11212	},
11213	);
11214
11215	describe.sequential(
11216	"multi-container public traffic",
11217	{
11218	tags: ["provider:fly", "provider:fly:app", "provider:fly:machine", "live"],
11219	},
11220	() => {
11221	const { test } = Test.make({ providers: Fly.providers() });
11222	const image =
11223	"docker-hub-mirror.fly.io/library/node@sha256:b6f26b36c8ff49624cfdac716b8ea1138d606df02586a77d364bb5536a634f85";
11224
11225	const scenario = (stack: ScratchStack) =>
11226	Effect.gen(function* () {
11227	const fs = yield* FileSystem.FileSystem;
11228	const path = yield* Path.Path;
11229	const script = yield* fs.readFileString(
11230	yield* path.fromFileUrl(
11231	new URL("./fixtures/multi-container-http.ts", import.meta.url),
11232	),
11233	);
11234	return (version: string, badHealth = false) =>
11235	stack.deploy(
11236	Effect.gen(function* () {
11237	const app = yield* Fly.App("Traffic");
11238	yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
11239	return yield* Fly.Machine("Group", {
11240	app,
11241	region: "fra",
11242	guest: { cpus: 1, memoryMb: 256 },
11243	containers: [
11244	{
11245	name: "web",
11246	image,
11247	cmd: [
11248	"node",
11249	"--input-type=module-typescript",
11250	"-e",
11251	script,
11252	],
11253	env: {
11254	PORT: "3000",
11255	CONTAINER_NAME: "web",
11256	VERSION: version,
11257	},
11258	},
11259	{
11260	name: "sidecar",
11261	image,
11262	cmd: [
11263	"node",
11264	"--input-type=module-typescript",
11265	"-e",
11266	script,
11267	],
11268	env: {
11269	PORT: "3001",
11270	CONTAINER_NAME: "sidecar",
11271	VERSION: version,
11272	BAD_HEALTH: String(badHealth),
11273	},
11274	dependsOn: [{ name: "web", condition: "started" }],
11275	},
11276	],
11277	checks: {
11278	sidecar: {
11279	type: "http",
11280	port: 3001,
11281	path: "/health",
11282	interval: "2s",
11283	timeout: "1s",
11284	},
11285	},
11286	services: [
11287	{
11288	protocol: "tcp",
11289	internalPort: 3000,
11290	ports: [{ port: 443, handlers: ["tls", "http"] }],
11291	checks: [
11292	{
11293	type: "http",
11294	port: 3000,
11295	path: "/health",
11296	interval: "2s",
11297	timeout: "1s",
11298	},
11299	],
11300	autostop: "off",
11301	},
11302	],
11303	deploy: {
11304	strategy: "bluegreen",
11305	healthTimeout: badHealth ? "15 seconds" : "30 seconds",
11306	},
11307	shutdown: { signal: "SIGTERM", timeout: "10 seconds" },
11308	});
11309	}),
11310	);
11311	});
11312
11313	const request = (appName: string, route = "/") =>
11314	HttpClient.get(`https://${appName}.fly.dev${route}`, {
11315	headers: { connection: "close" },
11316	}).pipe(
11317	Effect.flatMap((response) =>
11318	response.status === 200
11319	? Effect.succeed(response)
11320	: Effect.fail(new Error(`Public HTTP ${response.status}`)),
11321	),
11322	Effect.timeout("5 seconds"),
11323	);
11324	const version = (appName: string) =>
11325	request(appName).pipe(
11326	Effect.flatMap((response) => response.text),
11327	Effect.timeout("5 seconds"),
11328	);
11329	const sampleTraffic = (appName: string) =>
11330	Effect.gen(function* () {
11331	const samples = yield* Ref.make<string[]>([]);
11332	const finished = yield* Ref.make(false);
11333	const fiber = yield* Stream.range(0, 359).pipe(
11334	Stream.mapEffect(() =>
11335	version(appName).pipe(
11336	Effect.result,
11337	Effect.flatMap((result) =>
11338	Ref.update(samples, (values) => [
11339	...values,
11340	Result.isSuccess(result) ? result.success : "HTTP failure",
11341	]),
11342	),
11343	Effect.andThen(Effect.sleep("250 millis")),
11344	Effect.andThen(Ref.get(finished)),
11345	),
11346	),
11347	Stream.takeUntil((done) => done),
11348	Stream.runDrain,
11349	Effect.forkScoped,
11350	);
11351	return Effect.gen(function* () {
11352	yield* Ref.set(finished, true);
11353	yield* Fiber.join(fiber);
11354	return yield* Ref.get(samples);
11355	});
11356	});
11357	const appGone = (appName: string) =>
11358	machines.getApp({ app_name: appName }).pipe(
11359	Effect.as(false),
11360	Effect.catchTag("NotFound", () => Effect.succeed(true)),
11361	);
11362	const deployServing = (stack: ScratchStack) =>
11363	Effect.gen(function* () {
11364	yield* stack.destroy();
11365	const deploy = yield* scenario(stack);
11366	const first = yield* deploy("old");
11367	expect(
11368	yield* version(first.appName).pipe(
11369	Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") }),
11370	),
11371	).toBe("old");
11372	return { first, deploy };
11373	});
11374
11375	test.provider(
11376	"public replacement drains in-flight requests in both containers after SIGTERM",
11377	(stack) =>
11378	Effect.gen(function* () {
11379	const { first, deploy } = yield* deployServing(stack);
11380	const finishTraffic = yield* sampleTraffic(first.appName);
11381	// Headers arrive only once the request is held inside each container.
11382	const web = yield* request(first.appName, "/hold");
11383	const sidecar = yield* request(first.appName, "/sidecar/hold");
11384	const webBody = yield* web.text.pipe(
11385	Effect.timeout("90 seconds"),
11386	Effect.forkScoped,
11387	);
11388	const sidecarBody = yield* sidecar.text.pipe(
11389	Effect.timeout("90 seconds"),
11390	Effect.forkScoped,
11391	);
11392	const second = yield* deploy("new");
11393	expect(second.machineId).not.toBe(first.machineId);
11394	for (const [name, body] of [
11395	["web", yield* Fiber.join(webBody)],
11396	["sidecar", yield* Fiber.join(sidecarBody)],
11397	]) {
11398	expect(body).toBe(
11399	"waiting\n" +
11400	JSON.stringify({
11401	machine: first.machineId,
11402	name,
11403	version: "old",
11404	signal: "SIGTERM",
11405	}),
11406	);
11407	}
11408	expect(yield* version(first.appName)).toBe("new");
11409	// Include a post-cutover sample before joining the continuous probe.
11410	yield* Effect.sleep("500 millis");
11411	const samples = yield* finishTraffic;
11412	expect(samples).toContain("old");
11413	expect(samples).toContain("new");
11414	expect(
11415	samples.every((sample) => sample === "old" || sample === "new"),
11416	).toBe(true);
11417	const oldGone = yield* machines
11418	.getMachine({
11419	app_name: first.appName,
11420	machine_id: first.machineId,
11421	})
11422	.pipe(
11423	Effect.map((machine) => machine.state === "destroyed"),
11424	Effect.catchTag("NotFound", () => Effect.succeed(true)),
11425	);
11426	expect(oldGone).toBe(true);
11427	yield* stack.destroy();
11428	expect(yield* appGone(first.appName)).toBe(true);
11429	}),
11430	{ timeout: 300_000 },
11431	);
11432
11433	test.provider(
11434	"unhealthy container candidates never receive public traffic",
11435	(stack) =>
11436	Effect.gen(function* () {
11437	const { first, deploy } = yield* deployServing(stack);
11438	const finishTraffic = yield* sampleTraffic(first.appName);
11439	const failed = yield* deploy("unready", true).pipe(Effect.flip);
11440	expect(failed).toBeInstanceOf(ReplicaChecksNotPassing);
11441	if (failed instanceof ReplicaChecksNotPassing) {
11442	expect(failed.machineId).not.toBe(first.machineId);
11443	expect(
11444	failed.checks.some(
11445	(check) =>
11446	check.name === "sidecar" && check.status !== "passing",
11447	),
11448	).toBe(true);
11449	}
11450	expect(yield* version(first.appName)).toBe("old");
11451	const samples = yield* finishTraffic;
11452	expect(samples.length).toBeGreaterThan(1);
11453	expect([...new Set(samples)]).toEqual(["old"]);
11454	const old = yield* machines.getMachine({
11455	app_name: first.appName,
11456	machine_id: first.machineId,
11457	});
11458	expect(old.state).toBe("started");
11459	expect(old.cordoned).toBe(false);
11460	yield* stack.destroy();
11461	expect(yield* appGone(first.appName)).toBe(true);
11462	}),
11463	{ timeout: 300_000 },
11464	);
11465	},
11466	);