1import * as machines from "@distilled.cloud/fly-io/machines";
2import * as Retry from "@distilled.cloud/fly-io/Retry";
3import * as Fly from "@/Fly";
4import { State } from "@/State/State";
5import type { ScratchStack } from "@/Test/Alchemy";
6import { expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8import * as FileSystem from "effect/FileSystem";
9import * as Path from "effect/Path";
10import * as Schedule from "effect/Schedule";
11import * as Schema from "effect/Schema";
12import type { TransportEvent } from "./transport.ts";
13
14export const processDeathFile = "test/Fly/BlueGreen.test.ts";
15export const phases = ["create", "promotion", "retirement"] as const;
16export type Phase = (typeof phases)[number];
17
18const Identity = Schema.Struct({
19 id: Schema.String,
20 generation: Schema.String,
21 workload: Schema.String,
22 sequence: Schema.String,
23 digest: Schema.String,
24 instance: Schema.String,
25 fqn: Schema.String,
26 phase: Schema.String,
27});
28
29export const Witness = Schema.Struct({
30 version: Schema.Literal(1),
31 signal: Schema.Literal("SIGKILL"),
32 phase: Schema.Literals(phases),
33 pid: Schema.Number,
34 cwd: Schema.String,
35 stack: Schema.String,
36 stage: Schema.String,
37 appName: Schema.String,
38 recordedAt: Schema.Number,
39 predecessor: Identity,
40 candidate: Identity,
41 barrier: Schema.Struct({
42 sequence: Schema.Number,
43 method: Schema.String,
44 path: Schema.String,
45 status: Schema.Number,
46 machineId: Schema.String,
47 }),
48 leases: Schema.Array(
49 Schema.Struct({
50 machineId: Schema.String,
51 expiresAt: Schema.Number,
52 }),
53 ),
54 row: Schema.Struct({
55 fqn: Schema.String,
56 instanceId: Schema.String,
57 status: Schema.String,
58 }),
59});
60export type Witness = typeof Witness.Type;
61
62// SDK transport failures can retain headers; never persist or print those objects.
63const read = <A, E extends { readonly _tag: string }, R>(
64 effect: Effect.Effect<A, E, R>,
65) =>
66 effect.pipe(
67 Retry.none,
68 Effect.timeout("10 seconds"),
69 Effect.mapError(
70 (error) =>
71 new Error(`Process-death SDK observation failed (${error._tag})`),
72 ),
73 );
74
75export const nowSeconds = Effect.sync(() => Math.floor(Date.now() / 1000));
76
77export const evidencePaths = (stack: ScratchStack) =>
78 Effect.gen(function* () {
79 const path = yield* Path.Path;
80 const cwd = yield* Effect.sync(() => process.cwd());
81 const directory = path.join(
82 cwd,
83 ".alchemy",
84 "fly-process-death",
85 stack.name,
86 stack.stage,
87 );
88 return {
89 cwd,
90 directory,
91 witness: path.join(directory, "witness.json"),
92 finalized: path.join(directory, "finalized.json"),
93 recovered: path.join(directory, "recovered.json"),
94 };
95 });
96
97/** File and containing directory are synced before the caller can signal itself. */
98export const writeEvidence = (file: string, value: unknown) =>
99 Effect.gen(function* () {
100 const fs = yield* FileSystem.FileSystem;
101 const path = yield* Path.Path;
102 const directory = path.dirname(file);
103 yield* fs.makeDirectory(directory, { recursive: true });
104 const temporary = `${file}.tmp`;
105 const bytes = yield* Effect.sync(() =>
106 new TextEncoder().encode(JSON.stringify(value)),
107 );
108 yield* Effect.gen(function* () {
109 const handle = yield* fs.open(temporary, { flag: "w", mode: 0o600 });
110 yield* handle.writeAll(bytes);
111 yield* handle.sync;
112 }).pipe(Effect.scoped);
113 yield* fs.rename(temporary, file);
114 yield* Effect.gen(function* () {
115 const handle = yield* fs.open(directory, { flag: "r" });
116 yield* handle.sync;
117 }).pipe(Effect.scoped);
118 });
119
120export const readWitness = (file: string) =>
121 Effect.gen(function* () {
122 const fs = yield* FileSystem.FileSystem;
123 return yield* fs.readFileString(file).pipe(
124 Effect.flatMap(
125 Schema.decodeUnknownEffect(Schema.fromJsonString(Witness), {
126 onExcessProperty: "error",
127 }),
128 ),
129 Effect.mapError(
130 () =>
131 new Error(
132 "Missing or invalid process-death witness; preserve state and inspect the crash leg",
133 ),
134 ),
135 );
136 });
137
138export const assertSingleRunner = Effect.gen(function* () {
139 const args = yield* Effect.sync(() => process.argv.slice(2));
140 // pnpm test adds this exclusion before the explicitly focused file.
141 const focused =
142 args[0] === "--exclude" && args[1] === "test/Railway"
143 ? args.slice(2)
144 : args;
145 expect(focused).toEqual([
146 processDeathFile,
147 "-t",
148 " > process death > ",
149 "--profile",
150 "testing",
151 "--retry",
152 "0",
153 "--concurrency",
154 "1",
155 "--sequential",
156 "--timeout",
157 "500000",
158 ]);
159 expect(process.env.FLY_PROCESS_DEATH_ONLY_RUNNER).toBe("1");
160 expect(process.env.ALCHEMY_PROFILE).toBe("testing");
161 expect(process.env.ALCHEMY_TEST_DEV).toBe("0");
162 expect(process.env.ALCHEMY_DEV).toBe("0");
163 const fs = yield* FileSystem.FileSystem;
164 for (const file of [
165 processDeathFile,
166 "test/Fly/fixtures/process-death.ts",
167 "test/Fly/fixtures/transport.ts",
168 ]) {
169 const source = yield* fs.readFileString(file);
170 expect(source).not.toMatch(
171 /(?:from\s+|import\s*(?:\(\s*)?|require\s*\()\s*["'](?:node:)?child_process["']/,
172 );
173 expect(source).not.toMatch(
174 /\b(?:Bun|Deno)\s*\.\s*(?:spawn|spawnSync|Command)\s*\(/,
175 );
176 expect(source).not.toMatch(
177 /from\s+["'][^"']*(?:ChildProcess|CommandExecutor)[^"']*["']/,
178 );
179 }
180});
181
182export const deploy = (stack: ScratchStack, version: "one" | "two") =>
183 stack.deploy(
184 Effect.gen(function* () {
185 const app = yield* Fly.App("Site");
186 return yield* Fly.Machine("Worker", {
187 app,
188 region: "iad",
189 count: 1,
190 image: "nginx:alpine",
191 guest: { cpus: 1, memoryMb: 256 },
192 env: { VERSION: version },
193 checks: {
194 ready: {
195 type: "http",
196 port: 80,
197 path: "/",
198 interval: "2s",
199 timeout: "1s",
200 },
201 },
202 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
203 shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
204 });
205 }),
206 );
207
208export const census = (appName: string) =>
209 read(machines.listMachines({ app_name: appName })).pipe(
210 Effect.map((all) => all.filter((machine) => machine.state !== "destroyed")),
211 );
212
213export const machine = (appName: string, machineId: string) =>
214 read(
215 machines.getMachine({
216 app_name: appName,
217 machine_id: machineId,
218 }),
219 );
220
221export const identity = (value: machines.Machine, stack: ScratchStack) =>
222 Effect.sync(() => {
223 const metadata = value.config?.metadata ?? {};
224 expect(metadata["alchemy.stack"]).toBe(stack.name);
225 expect(metadata["alchemy.stage"]).toBe(stack.stage);
226 expect(metadata["alchemy.id"]).toBe("Worker");
227 expect(metadata["alchemy.type"]).toBe("Fly.Machine");
228 expect(metadata["alchemy.count"]).toBe("1");
229 expect(metadata["alchemy.replica"]).toBe("0");
230 expect(metadata["alchemy.deployment-protocol"]).toBe("1");
231 const result = {
232 id: value.id!,
233 generation: metadata["alchemy.generation"]!,
234 workload: metadata["alchemy.workload"]!,
235 sequence: metadata["alchemy.sequence"]!,
236 digest: value.image_ref?.digest!,
237 instance: metadata["alchemy.instance"]!,
238 fqn: metadata["alchemy.fqn"]!,
239 phase: metadata["alchemy.phase"]!,
240 };
241 for (const value of Object.values(result)) {
242 expect(typeof value).toBe("string");
243 expect(value.length).toBeGreaterThan(0);
244 }
245 expect(result.digest).toMatch(/^sha256:/);
246 return result;
247 });
248
249export const persistedRow = (stack: ScratchStack, fqn: string) =>
250 Effect.gen(function* () {
251 const state = yield* yield* State;
252 expect(state.id).toBe("local");
253 const row = yield* state.get({
254 stack: stack.name,
255 stage: stack.stage,
256 fqn,
257 });
258 if (!row || row.kind === "action") {
259 return yield* Effect.fail(
260 new Error(
261 "Expected the original durable resource row; refusing to recreate evidence",
262 ),
263 );
264 }
265 return { fqn: row.fqn, instanceId: row.instanceId, status: row.status };
266 }).pipe(Effect.provide(stack.state));
267
268export const matchesBarrier = (
269 phase: Phase,
270 appName: string,
271 predecessorId: string,
272 event: TransportEvent,
273) => {
274 const base = `/v1/apps/${appName}/machines`;
275 return phase === "create"
276 ? event.method === "POST" && event.path === base
277 : phase === "promotion"
278 ? event.method === "POST" &&
279 event.path.startsWith(`${base}/`) &&
280 event.path.endsWith("/uncordon")
281 : event.method === "DELETE" && event.path === `${base}/${predecessorId}`;
282};
283
284const lease = (appName: string, machineId: string) =>
285 read(
286 machines
287 .getMachineLease({
288 app_name: appName,
289 machine_id: machineId,
290 })
291 .pipe(
292 Effect.map((value) => ({
293 present: !!value.data?.nonce,
294 expiresAt: value.data?.expires_at ?? 0,
295 })),
296 Effect.catchTag("NotFound", () =>
297 Effect.succeed({ present: false, expiresAt: 0 }),
298 ),
299 ),
300 );
301
302export const heldLeases = (appName: string, ids: string[]) =>
303 Effect.gen(function* () {
304 const leases = yield* Effect.forEach(
305 ids,
306 (machineId) =>
307 lease(appName, machineId).pipe(
308 Effect.map((value) => ({ machineId, ...value })),
309 ),
310 { concurrency: 2 },
311 );
312 const now = yield* nowSeconds;
313 const held = leases.filter((value) => value.present);
314 expect(held.length).toBeGreaterThan(0);
315 for (const value of held) {
316 expect(value.expiresAt).toBeGreaterThan(now + 20);
317 expect(value.expiresAt).toBeLessThanOrEqual(now + 130);
318 }
319 return held.map(({ machineId, expiresAt }) => ({ machineId, expiresAt }));
320 });
321
322/** Observe live authority in the new process, then native expiry without release/acquire. */
323export const observeLeaseExpiry = (witness: Witness) =>
324 Effect.gen(function* () {
325 expect(witness.leases.length).toBeGreaterThan(0);
326 const initial = yield* Effect.forEach(
327 witness.leases,
328 (held) =>
329 Effect.gen(function* () {
330 const observed = yield* lease(witness.appName, held.machineId);
331 const now = yield* nowSeconds;
332 expect(observed.present).toBe(true);
333 expect(observed.expiresAt).toBeGreaterThan(now);
334 expect(observed.expiresAt).toBeGreaterThanOrEqual(held.expiresAt);
335 expect(observed.expiresAt).toBeLessThanOrEqual(
336 witness.recordedAt + 130,
337 );
338 return { machineId: held.machineId, expiresAt: observed.expiresAt };
339 }),
340 { concurrency: 2 },
341 );
342 const samples: {
343 machineId: string;
344 present: boolean;
345 expiresAt: number;
346 at: number;
347 }[] = [];
348 const expired = yield* Effect.forEach(
349 initial,
350 (held) =>
351 Effect.gen(function* () {
352 const value = yield* lease(witness.appName, held.machineId);
353 const at = yield* nowSeconds;
354 yield* Effect.sync(() =>
355 samples.push({ machineId: held.machineId, ...value, at }),
356 );
357 if (value.present) expect(value.expiresAt).toBe(held.expiresAt);
358 if (!value.present) expect(at).toBeGreaterThanOrEqual(held.expiresAt);
359 return !value.present;
360 }),
361 { concurrency: 2 },
362 ).pipe(
363 Effect.repeat({
364 schedule: Schedule.spaced("3 seconds"),
365 times: 65,
366 until: (all) => all.every(Boolean),
367 }),
368 Effect.timeout("230 seconds"),
369 );
370 expect(expired.every(Boolean)).toBe(true);
371 // A deleted Machine is not evidence of lease expiration.
372 for (const held of initial) {
373 const surviving = yield* machine(witness.appName, held.machineId);
374 expect(surviving.state).not.toBe("destroyed");
375 }
376 return { initial, samples };
377 });
378
379export const assertBarrierInventory = (stack: ScratchStack, witness: Witness) =>
380 Effect.gen(function* () {
381 const live = yield* census(witness.appName);
382 expect(live.map((value) => value.id).sort()).toEqual(
383 (witness.phase === "retirement"
384 ? [witness.candidate.id]
385 : [witness.predecessor.id, witness.candidate.id]
386 ).sort(),
387 );
388 const candidate = yield* machine(witness.appName, witness.candidate.id);
389 expect(yield* identity(candidate, stack)).toEqual(witness.candidate);
390 expect(candidate.config?.env?.VERSION).toBe("two");
391 expect(candidate.cordoned).toBe(witness.phase === "create");
392 expect(witness.candidate.phase).toBe(
393 witness.phase === "create"
394 ? "candidate"
395 : witness.phase === "promotion"
396 ? "promoting"
397 : "active",
398 );
399 if (witness.phase !== "retirement") {
400 const predecessor = yield* machine(
401 witness.appName,
402 witness.predecessor.id,
403 );
404 expect(yield* identity(predecessor, stack)).toEqual(witness.predecessor);
405 expect(predecessor.cordoned).toBe(false);
406 }
407 });
408
409export const assertConverged = (
410 stack: ScratchStack,
411 witness: Witness,
412 ids: string[],
413) =>
414 Effect.gen(function* () {
415 expect(ids).toEqual([witness.candidate.id]);
416 const live = yield* census(witness.appName);
417 expect(live.map((value) => value.id)).toEqual(ids);
418 const current = yield* machine(witness.appName, witness.candidate.id).pipe(
419 Effect.repeat({
420 schedule: Schedule.spaced("2 seconds"),
421 times: 20,
422 until: (value) =>
423 value.state === "started" &&
424 value.cordoned === false &&
425 value.checks?.some(
426 (check) => check.name === "ready" && check.status === "passing",
427 ) === true,
428 }),
429 Effect.timeout("60 seconds"),
430 );
431 expect(yield* identity(current, stack)).toEqual({
432 ...witness.candidate,
433 phase: "active",
434 });
435 expect(current.state).toBe("started");
436 expect(current.cordoned).toBe(false);
437 expect(current.config?.env?.VERSION).toBe("two");
438 const ready =
439 current.checks?.filter((check) => check.name === "ready") ?? [];
440 expect(ready.length).toBeGreaterThan(0);
441 expect(ready.every((check) => check.status === "passing")).toBe(true);
442 expect(current.config?.metadata?.["alchemy.checked-instance"]).toBe(
443 current.instance_id,
444 );
445 expect(current.instance_id).toBeDefined();
446 expect(current.config?.metadata?.["alchemy.predecessors"]).toBe(
447 witness.predecessor.id,
448 );
449 expect(
450 current.config?.metadata?.["alchemy.image"]?.endsWith(
451 `@${witness.candidate.digest}`,
452 ),
453 ).toBe(true);
454 const row = yield* persistedRow(stack, witness.row.fqn);
455 expect(row.instanceId).toBe(witness.row.instanceId);
456 expect(["created", "updated"]).toContain(row.status);
457 });
458
459export const assertClean = (stack: ScratchStack, appName: string) =>
460 Effect.gen(function* () {
461 const gone = yield* read(
462 machines.getApp({ app_name: appName }).pipe(
463 Effect.as(false),
464 Effect.catchTag("NotFound", () => Effect.succeed(true)),
465 ),
466 ).pipe(
467 Effect.repeat({
468 schedule: Schedule.spaced("1 second"),
469 times: 8,
470 until: Boolean,
471 }),
472 );
473 expect(gone).toBe(true);
474 const remaining = yield* read(
475 machines
476 .listMachines({ app_name: appName })
477 .pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
478 );
479 expect(
480 remaining.filter((value) => value.state !== "destroyed"),
481 ).toHaveLength(0);
482 const rows = yield* Effect.gen(function* () {
483 const state = yield* yield* State;
484 return yield* state.list({ stack: stack.name, stage: stack.stage });
485 }).pipe(Effect.provide(stack.state));
486 expect(rows).toHaveLength(0);
487 });