1import { expect, it } from "alchemy-test";
2import { mkdtemp, rm, writeFile } from "node:fs/promises";
3import { tmpdir } from "node:os";
4import { dirname, resolve } from "node:path";
5import { fileURLToPath, pathToFileURL } from "node:url";
6
7const here = dirname(fileURLToPath(import.meta.url));
8const cli = resolve(here, "../bin/alchemy-test.ts");
9const api = JSON.stringify(
10 pathToFileURL(resolve(here, "../src/index.ts")).href,
11);
12
13const run = async (root: string, plan: unknown, args: string[] = []) => {
14 const child = Bun.spawn(
15 [
16 process.execPath,
17 cli,
18 root,
19 "--retry",
20 "0",
21 "--timeout",
22 "2000",
23 "--concurrency",
24 "1",
25 "--plan",
26 typeof plan === "string" ? plan : JSON.stringify(plan),
27 ...args,
28 ],
29 {
30 cwd: root,
31 stdout: "pipe",
32 stderr: "pipe",
33 env: { ...process.env, NO_COLOR: "1" },
34 },
35 );
36 const timer = setTimeout(() => child.kill(), 15_000);
37 try {
38 const [code, stdout, stderr] = await Promise.all([
39 child.exited,
40 new Response(child.stdout).text(),
41 new Response(child.stderr).text(),
42 ]);
43 return { code, output: stdout + stderr };
44 } finally {
45 clearTimeout(timer);
46 child.kill();
47 }
48};
49
50it(
51 "plans sequence phases, overlap branches, honor file concurrency, and claim tests once",
52 async () => {
53 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-"));
54 try {
55 await writeFile(
56 resolve(root, "state.ts"),
57 `
58 export const local = new Set();
59 export const cleaned = new Set();
60 export const live = new Set();
61 export const calls = new Set();
62 const releases = {};
63 const arrivals = {};
64 export async function barrier(name) {
65 const promise = new Promise(resolve => (releases[name] ??= []).push(resolve));
66 arrivals[name] = (arrivals[name] ?? 0) + 1;
67 if (arrivals[name] === 2) for (const release of releases[name]) release();
68 await promise;
69 }
70 `,
71 );
72 for (const name of ["a", "b"])
73 await writeFile(
74 resolve(root, `local-${name}.test.ts`),
75 `
76 import { it, expect, afterAll } from ${api};
77 import {local, cleaned, barrier} from './state.ts';
78 it('local ${name}', async () => { await barrier('local'); local.add('${name}'); }, {tags:['local']});
79 afterAll(() => { cleaned.add('${name}'); });
80 `,
81 );
82 for (const provider of ["aws", "cloudflare"])
83 await writeFile(
84 resolve(root, `${provider}.test.ts`),
85 `
86 import { it, expect } from ${api};
87 import {local, cleaned, live, calls, barrier} from './state.ts';
88 it('${provider}', async () => {
89 expect(local.size).toBe(2); expect(cleaned.size).toBe(2);
90 expect(calls.has('${provider}')).toBe(false); calls.add('${provider}');
91 await barrier('providers'); live.add('${provider}');
92 }, {tags:['live','provider:${provider}']});
93 `,
94 );
95 await writeFile(
96 resolve(root, "shared.test.ts"),
97 `
98 import { it, expect, beforeAll, afterAll } from ${api};
99 import {local, live} from './state.ts';
100 let setups = 0, early = false, late = false;
101 beforeAll(() => { setups++; expect(setups).toBe(1); });
102 afterAll(() => { expect(early).toBe(true); expect(late).toBe(true); });
103 it('early', () => { early = true; }, {tags:['local']});
104 it('remaining live', () => { expect(early).toBe(true); expect(live.size).toBe(2); late = true; }, {tags:['live']});
105 it.skip('selected skipped', () => { throw new Error('skip ignored'); }, {tags:['live']});
106 it('unmatched', () => { throw new Error('unmatched ran'); }, {tags:['manual']});
107 it('opt in', () => { throw new Error('opt-in bypassed'); }, {tags:['live'], optInTags:['paid']});
108 `,
109 );
110 const result = await run(root, [
111 { tags: ["local"], concurrency: 3 },
112 [
113 { tags: ["provider:aws", "live"], concurrency: 1 },
114 { tags: ["provider:cloudflare", "live"], concurrency: 1 },
115 ],
116 { tags: ["live"] },
117 ]);
118 expect(result.output).toContain("6 passed");
119 expect(result.output).toContain("1 skipped");
120 expect(result.output).toContain("9 found");
121 expect(result.output).toContain("7 selected by plan");
122 expect(result.output).toContain("2 excluded by plan");
123 expect(result.code).toBe(0);
124 } finally {
125 await rm(root, { recursive: true, force: true });
126 }
127 },
128 { tags: ["unit", "local"] },
129);
130
131it(
132 "plan branches sharing a file serialize setup and preserve it until final teardown",
133 async () => {
134 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-shared-"));
135 try {
136 await writeFile(
137 resolve(root, "shared.test.ts"),
138 `
139 import { it, expect, beforeAll, afterAll } from ${api};
140 let ready = false, setup = 0, done = 0, active = false;
141 beforeAll(() => { expect(++setup).toBe(1); ready = true; });
142 afterAll(() => { expect(done).toBe(2); ready = false; });
143 for (const tag of ['aws','cf']) it(tag, async () => {
144 expect(ready).toBe(true); expect(active).toBe(false); active = true;
145 await new Promise(resolve => setTimeout(resolve, 20));
146 active = false; done++;
147 }, {tags:[tag]});
148 `,
149 );
150 const result = await run(root, [
151 [{ tags: ["aws"] }, { tags: ["cf"] }],
152 { tags: [] },
153 ]);
154 expect(result.output).toContain("2 passed");
155 expect(result.output).toContain("0 excluded by plan");
156 expect(result.code).toBe(0);
157 } finally {
158 await rm(root, { recursive: true, force: true });
159 }
160 },
161 { tags: ["unit", "local"] },
162);
163
164it(
165 "plans do not rerun failures and still run later phases and all teardown hooks",
166 async () => {
167 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-fail-"));
168 try {
169 await writeFile(
170 resolve(root, "failure.test.ts"),
171 `
172 import { it, expect, afterAll } from ${api};
173 let attempts = 0, later = false;
174 it('failure', () => { attempts++; throw new Error('expected failure'); }, {tags:['early','live']});
175 it('later', () => { expect(attempts).toBe(1); later = true; }, {tags:['live']});
176 afterAll(() => { expect(later).toBe(true); throw new Error('teardown sentinel'); });
177 afterAll(() => console.log('cleanup sentinel'));
178 `,
179 );
180 const result = await run(root, [{ tags: ["early"] }, { tags: ["live"] }]);
181 expect(result.code).not.toBe(0);
182 expect(result.output).toContain("2 failed");
183 expect(result.output).toContain("1 passed");
184 expect(result.output).toContain("teardown sentinel");
185 // The runner writes hook output to its persistent log even on failure.
186 const { readdir, readFile } = await import("node:fs/promises");
187 const dir = resolve(root, ".alchemy/log/test");
188 const logs = await readdir(dir);
189 expect(await readFile(resolve(dir, logs[0]!), "utf8")).toContain(
190 "cleanup sentinel",
191 );
192 } finally {
193 await rm(root, { recursive: true, force: true });
194 }
195 },
196 { tags: ["unit", "local"] },
197);
198
199it(
200 "validates plan JSON and tag expressions before importing tests",
201 async () => {
202 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-invalid-"));
203 try {
204 await writeFile(
205 resolve(root, "bad.test.ts"),
206 `throw new Error('IMPORTED_SENTINEL');`,
207 );
208 for (const plan of [
209 "{",
210 [],
211 [{}],
212 [{ tags: ["unit"], concurrency: 0 }],
213 [{ tags: ["unit"], concurrency: 1.5 }],
214 [{ tags: ["unit"], concurency: 2 }],
215 [[]],
216 [[[{ tags: [] }]]],
217 [{ tags: ["unit &&"] }],
218 ]) {
219 const result = await run(root, plan);
220 expect(result.code).not.toBe(0);
221 expect(result.output).toContain("Invalid --plan");
222 expect(result.output).not.toContain("IMPORTED_SENTINEL");
223 }
224 } finally {
225 await rm(root, { recursive: true, force: true });
226 }
227 },
228 { tags: ["unit", "local"] },
229);
230
231it(
232 "plans compose with global tags, names and explicit opt-ins",
233 async () => {
234 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-filters-"));
235 try {
236 await writeFile(
237 resolve(root, "filters.test.ts"),
238 `
239 import { it } from ${api};
240 it('chosen', () => {}, {tags:['live','provider:aws'], optInTags:['paid']});
241 it('other name', () => { throw new Error('name bypassed'); }, {tags:['live','provider:aws'], optInTags:['paid']});
242 it('chosen cf', () => { throw new Error('global tags bypassed'); }, {tags:['live','provider:cloudflare'], optInTags:['paid']});
243 `,
244 );
245 const result = await run(
246 root,
247 [{ tags: ["paid", "live"] }],
248 ["--tags", "provider:aws", "-t", "^.* > chosen$"],
249 );
250 expect(result.output).toContain("1 passed");
251 expect(result.output).toContain("1 found");
252 expect(result.code).toBe(0);
253 } finally {
254 await rm(root, { recursive: true, force: true });
255 }
256 },
257 { tags: ["unit", "local"] },
258);
259
260it(
261 "failed shared setup is not retried by later phases and still tears down once",
262 async () => {
263 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-setup-"));
264 try {
265 await writeFile(
266 resolve(root, "setup.test.ts"),
267 `
268 import { it, beforeAll, afterAll, expect } from ${api};
269 let setups = 0;
270 beforeAll(() => { setups++; throw new Error('setup sentinel'); });
271 afterAll(() => { expect(setups).toBe(1); });
272 it('first', () => { throw new Error('body ran'); }, {tags:['first']});
273 it('last', () => { throw new Error('body ran'); }, {tags:['last']});
274 `,
275 );
276 const result = await run(root, [{ tags: ["first"] }, { tags: ["last"] }]);
277 expect(result.code).not.toBe(0);
278 expect(result.output).toContain("2 failed");
279 expect(result.output).toContain("setup sentinel");
280 expect(result.output).not.toContain("body ran");
281 expect(result.output).not.toContain("afterAll hook failed");
282 } finally {
283 await rm(root, { recursive: true, force: true });
284 }
285 },
286 { tags: ["unit", "local"] },
287);
288
289it(
290 "plans honor only and wildcard phases leave opt-in tests for an explicit later phase",
291 async () => {
292 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-only-"));
293 try {
294 await writeFile(
295 resolve(root, "only.test.ts"),
296 `
297 import { describe, it, expect, beforeAll } from ${api};
298 let ready = false;
299 describe.only('selected suite', () => {
300 it('normal', () => { ready = true; }, {tags:['live']});
301 it('opt-in', () => { expect(ready).toBe(true); }, {tags:['live'], optInTags:['paid']});
302 });
303 describe('excluded suite', () => {
304 beforeAll(() => { throw new Error('excluded setup'); });
305 it('excluded', () => { throw new Error('only bypassed'); });
306 });
307 `,
308 );
309 const result = await run(root, [{ tags: ["*"] }, { tags: ["paid"] }]);
310 expect(result.code).toBe(0);
311 expect(result.output).toContain("2 passed");
312 expect(result.output).toContain("Plan phase 1/2: 1 tests");
313 expect(result.output).toContain("Plan phase 2/2: 1 tests");
314 expect(result.output).toContain("2 found");
315 expect(result.output).toContain("0 excluded by plan");
316 } finally {
317 await rm(root, { recursive: true, force: true });
318 }
319 },
320 { tags: ["unit", "local"] },
321);
322
323it(
324 "unmatched plans do not run hooks but still report import errors",
325 async () => {
326 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-empty-"));
327 try {
328 await writeFile(
329 resolve(root, "good.test.ts"),
330 `
331 import { it, beforeAll, afterAll } from ${api};
332 beforeAll(() => { throw new Error('unmatched setup'); });
333 afterAll(() => { throw new Error('unmatched teardown'); });
334 it('unmatched', () => { throw new Error('unmatched body'); }, {tags:['live']});
335 `,
336 );
337 await writeFile(
338 resolve(root, "bad.test.ts"),
339 `throw new Error('import sentinel');`,
340 );
341 const result = await run(root, [{ tags: ["local"] }, { tags: ["unit"] }]);
342 expect(result.code).not.toBe(0);
343 expect(result.output).toContain("1 failed");
344 expect(result.output).toContain("import sentinel");
345 expect(result.output).toContain("1 excluded by plan");
346 expect(result.output).not.toContain("unmatched setup");
347 expect(result.output).not.toContain("unmatched teardown");
348 } finally {
349 await rm(root, { recursive: true, force: true });
350 }
351 },
352 { tags: ["unit", "local"] },
353);
354
355it(
356 "dry-run prints first-match assignments and concurrency without executing bodies or hooks",
357 async () => {
358 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-"));
359 try {
360 await writeFile(
361 resolve(root, "dry.test.ts"),
362 `
363 import { it, beforeAll, afterAll } from ${api};
364 beforeAll(() => { throw new Error('DRY_SETUP_RAN'); });
365 afterAll(() => { throw new Error('DRY_TEARDOWN_RAN'); });
366 it('local aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['local','live','provider:aws']});
367 it('aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['live','provider:aws']});
368 it.skip('cf', () => {}, {tags:['live','provider:cloudflare']});
369 it('left over', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['manual']});
370 `,
371 );
372 const result = await run(
373 root,
374 [
375 { tags: ["local"], concurrency: 64 },
376 [
377 { tags: ["live", "provider:aws"], concurrency: 2 },
378 { tags: ["live", "provider:cloudflare"], concurrency: "unbounded" },
379 ],
380 { tags: ["live"] },
381 ],
382 ["--dry-run"],
383 );
384 expect(result.code).toBe(0);
385 expect(result.output).toContain("Dry run — no tests or hooks executed");
386 expect(result.output).toContain("Phase 2 (parallel)");
387 expect(result.output).toContain(
388 'tags=["local"], concurrency=64 — 1 tests in 1 files',
389 );
390 expect(result.output).toContain(
391 'tags=["live","provider:aws"], concurrency=2 — 1 tests in 1 files',
392 );
393 expect(result.output).toContain(
394 "concurrency=unbounded — 1 tests in 1 files (1 skipped/todo)",
395 );
396 expect(result.output).toContain(
397 'tags=["live"], concurrency=1 — 0 tests in 0 files',
398 );
399 expect(result.output).toContain("4 found");
400 expect(result.output).toContain("3 selected by plan");
401 expect(result.output).toContain("1 excluded by plan");
402 expect(result.output).not.toContain("DRY_SETUP_RAN");
403 expect(result.output).not.toContain("DRY_TEARDOWN_RAN");
404 expect(result.output).not.toContain("DRY_BODY_RAN");
405 expect(result.output).not.toContain("running 3 tests");
406 } finally {
407 await rm(root, { recursive: true, force: true });
408 }
409 },
410 { tags: ["unit", "local"] },
411);
412
413it(
414 "dry-run reports collection failures",
415 async () => {
416 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-error-"));
417 try {
418 await writeFile(
419 resolve(root, "bad.test.ts"),
420 `throw new Error('dry import sentinel');`,
421 );
422 const result = await run(root, [{ tags: [] }], ["--dry-run"]);
423 expect(result.code).not.toBe(0);
424 expect(result.output).toContain("dry import sentinel");
425 expect(result.output).toContain("1 failed");
426 } finally {
427 await rm(root, { recursive: true, force: true });
428 }
429 },
430 { tags: ["unit", "local"] },
431);