1//! This program executes compiler tests found in this directory. Additionally,
2//! it can be run as a standalone program to compile and execute a single test.
3//!
4//! See the "Testing" section in `architecture.md` for more information on the
5//! structure of these tests, and `zig build test-compiler -- --help` for more
6//! details on the arguments this supports.
7
8const cli_config: Cli = .{
9 .options = &.{
10 .{ .name = "--test-root", .desc = "root directory of all tests", .kind = .string },
11 .{ .name = "--all-logs", .desc = "Always print compiler logs" },
12 .{ .name = "--no-threads", .desc = "Avoid using a thread pool" },
13 .{ .name = "--dump-ir", .desc = "Dump intermediate representations" },
14
15 .{ .name = "--listen", .desc = "Integrate with the Zig Build System IPC", .kind = .string },
16 .{ .name = "--seed", .desc = "Set random number seed (unused)", .kind = .string },
17 .{ .name = "--cache-dir", .desc = "Set cache dir (unused)", .kind = .string },
18 },
19 .positionals = .single,
20 .help = .{
21 .usage = "[options...] [test filter]",
22 },
23};
24
25/// This file is included for all behavior tests for the JavaScript backend.
26const helpers =
27 \\(function (self) {
28 \\ function assert(condition) {
29 \\ if (!condition) {
30 \\ throw new Error("Assertion failure");
31 \\ }
32 \\ }
33 \\
34 \\ function assertEqual(actual, expected) {
35 \\ if (actual !== expected) {
36 \\ var err;
37 \\ try {
38 \\ err = new Error("Expected " + expected + ", but got " + actual);
39 \\ } catch (e) {
40 \\ err = new Error("Assertion failure");
41 \\ }
42 \\ throw err;
43 \\ }
44 \\ }
45 \\
46 \\ function print(str) {
47 \\ console.log(str);
48 \\ }
49 \\
50 \\ self.assert = assert;
51 \\ self.assertEqual = assertEqual;
52 \\ self.print = print;
53 \\})(typeof globalThis !== "undefined" ? globalThis : this);
54 \\
55;
56
57const CaseOptions = struct {
58 dump_ir: bool = false,
59 sync: bool = false,
60};
61
62const Case = struct {
63 path: []const u8,
64 display: []const u8,
65 success: bool = false,
66 logs: []const u8 = "",
67 failure: ?struct {
68 err: anyerror,
69 stack: core.StoredTrace,
70 } = null,
71
72 const Kind = enum {
73 behavior,
74 analysis,
75 transform_error,
76 };
77};
78
79fn runTestCase(
80 backing_allocator: std.mem.Allocator,
81 file_path: []const u8,
82 case_options: CaseOptions,
83 out: std.io.AnyWriter,
84) !void {
85 var gpa = std.heap.GeneralPurposeAllocator(.{
86 .stack_trace_frames = 100,
87 }){ .backing_allocator = backing_allocator };
88 const alloc = gpa.allocator();
89
90 var backend = core.Js{};
91 var comp = try core.Comp.init(.{
92 .gpa = tracer.tracingAllocator(&alloc),
93 .options = .{ .dump_ir = case_options.dump_ir },
94 .backend = .{ .js = &backend },
95 .out = .{ .writer = out, .tty = .escape_codes },
96 });
97
98 const main_file = try comp.getFile(file_path);
99
100 var fail_report: core.Report = undefined;
101 const hir = main_file.getHir(&comp, &fail_report) catch |err| switch (err) {
102 error.TransformFail => {
103 try out.print("Transform failed:\n", .{});
104 try fail_report.dump(.{ .comp = .{ .comp = &comp, .file = main_file } }, comp.out);
105 return error.TransformFail;
106 },
107 error.ParseFail => {
108 // Already printed.
109 return error.ParseFail;
110 },
111 else => return err,
112 };
113
114 if (case_options.dump_ir) {
115 try hir.dumpToWriter(out, .escape_codes);
116 }
117
118 const entry_point = try comp.enqueueEntryModule(main_file);
119 try comp.runAllJobs();
120
121 if (comp.temp_flag_had_errors)
122 return error.SemaErrors;
123
124 if (hir.report.warnings.len > 0)
125 return error.HirWarnings;
126
127 var js_out: std.ArrayListUnmanaged(u8) = .{};
128 defer js_out.deinit(alloc);
129
130 try js_out.appendSlice(alloc, helpers);
131 try js_out.appendSlice(alloc, "\n\n(function() {\n\n");
132
133 const start_i = js_out.items.len;
134 try backend.link(&comp, entry_point, js_out.writer(alloc).any());
135
136 try out.writeAll("--- COMPILED CODE ---\n");
137 try out.writeAll(js_out.items[start_i..]);
138 try out.writeAll("--- RUNTIME OUTPUT ---\n");
139
140 if (isOnlyComments(js_out.items[start_i..])) {
141 try out.writeAll("Behavior tests should output code!");
142 return error.EmptyFile;
143 }
144
145 try js_out.appendSlice(alloc, "\n\n})();");
146 try js_out.append(alloc, 0);
147
148 const runtime = qjs.JS_NewRuntime() orelse
149 return error.UnexpectedError;
150 defer qjs.JS_FreeRuntime(runtime);
151 const js_ctx = qjs.JS_NewContext(runtime) orelse
152 return error.UnexpectedError;
153 defer qjs.JS_FreeContext(js_ctx);
154
155 const result = qjs.JS_Eval(
156 js_ctx,
157 js_out.items.ptr,
158 js_out.items.len - 1,
159 "test.js",
160 qjs.JS_EVAL_TYPE_GLOBAL,
161 );
162 defer qjs.JS_FreeValue(js_ctx, result);
163
164 if (qjs.JS_IsException(result) != 0) {
165 const exception = qjs.JS_GetException(js_ctx);
166 defer qjs.JS_FreeValue(js_ctx, exception);
167
168 const exception_str = qjs.JS_ToCString(js_ctx, exception);
169 defer qjs.JS_FreeCString(js_ctx, exception_str);
170
171 const slice = std.mem.span(exception_str);
172
173 try out.print("Exception: {s}\n", .{slice});
174
175 return error.RuntimeFailed;
176 }
177}
178
179fn isOnlyComments(input: []const u8) bool {
180 var rest = input;
181 while (rest.len > 0) {
182 if (rest.len == 1) return false;
183 if (rest[0] == '/' and rest[1] == '/') {
184 rest = rest[std.mem.indexOfScalar(u8, rest, '\n') orelse return true ..];
185 } else {
186 return false;
187 }
188 }
189 return true;
190}
191
192const threadlocal_state = struct {
193 threadlocal var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
194};
195
196pub fn runTestCaseWrap(
197 gpa: std.mem.Allocator,
198 case: *Case,
199 case_options: CaseOptions,
200 progress: std.Progress.Node,
201) void {
202 const node = progress.start(case.display, 0);
203 defer node.end();
204
205 const alloc = threadlocal_state.arena.allocator();
206 defer _ = threadlocal_state.arena.reset(.{ .retain_with_limit = std.mem.page_size * 8 });
207
208 var logs: std.ArrayListUnmanaged(u8) = .{};
209 defer logs.deinit(alloc);
210
211 defer case.logs = gpa.dupe(u8, logs.items) catch
212 "[logs failed to allocate]";
213
214 std.debug.assert(current_case == null);
215 current_case = case.*;
216 defer current_case = null;
217
218 runTestCase(
219 alloc,
220 case.path,
221 case_options,
222 if (case_options.sync)
223 std.io.getStdErr().writer().any()
224 else
225 logs.writer(alloc).any(),
226 ) catch |err| {
227 const stack = switch (err) {
228 error.TransformFail,
229 error.ParseFail,
230 error.HirWarnings,
231 error.SemaErrors,
232 error.AnalysisFail,
233 => core.StoredTrace.empty,
234
235 else => core.StoredTrace.from(@errorReturnTrace()),
236 };
237
238 case.failure = .{
239 .err = err,
240 .stack = stack,
241 };
242
243 return;
244 };
245
246 case.success = true;
247}
248
249pub fn main() !void {
250 // unicode props is asserted to be initialized before core.Tokenizer is used
251 // this will result in a static page allocation, which is never cleaned up.
252 try core.unicode_props.init();
253 var gpa = std.heap.GeneralPurposeAllocator(.{
254 .stack_trace_frames = 100,
255 .thread_safe = true,
256 }){};
257
258 defer _ = gpa.deinit();
259 const alloc = gpa.allocator();
260
261 const program_name = "zig build test-compiler --";
262 _, const args = try Cli.programArgsAlloc(alloc, "");
263 defer Cli.programArgsFree(alloc, args);
264
265 const opts = cli_config.parse(program_name, args);
266
267 const root_path = opts.test_root orelse "test/behavior";
268 const case_options: CaseOptions = .{
269 .dump_ir = opts.dump_ir orelse false,
270 .sync = opts.no_threads orelse false,
271 };
272 const all_logs = opts.all_logs orelse false;
273 const filter: ?[]const u8 = opts.positionals;
274
275 const some_logs = all_logs or args.len > 1;
276
277 const root = try std.fs.cwd().openDir(root_path, .{ .iterate = true });
278
279 var test_cases: std.ArrayListUnmanaged(Case) = .{};
280 defer test_cases.deinit(alloc);
281 defer for (test_cases.items) |test_case| {
282 alloc.free(test_case.path);
283 };
284
285 var path_buf: std.BoundedArray(u8, std.fs.max_path_bytes) = .{};
286 path_buf.len = @intCast((try std.os.getFdPath(root.fd, &path_buf.buffer)).len);
287 try iterateTestCases(alloc, root, path_buf.len, &path_buf, &test_cases, filter);
288
289 if (test_cases.items.len == 0) {
290 if (filter) |f| {
291 std.debug.print("No test cases match filter \"{s}\"\n", .{f});
292 } else {
293 std.debug.print("No test cases in root \"{s}\"", .{root_path});
294 }
295 std.process.exit(1);
296 }
297
298 const progress = std.Progress.start(.{ .root_name = "Behavior Tests" });
299 defer progress.end();
300 progress.setEstimatedTotalItems(test_cases.items.len);
301
302 var wg: std.Thread.WaitGroup = .{};
303
304 var pool: std.Thread.Pool = undefined;
305 if (!case_options.sync) try pool.init(.{ .allocator = alloc });
306 defer if (!case_options.sync) pool.deinit();
307
308 for (test_cases.items) |*test_case| {
309 const call_args = .{
310 alloc,
311 test_case,
312 case_options,
313 progress,
314 };
315
316 if (case_options.sync) {
317 @call(.auto, runTestCaseWrap, call_args);
318 } else {
319 pool.spawnWg(&wg, runTestCaseWrap, call_args);
320 }
321 }
322
323 wg.wait();
324
325 if (opts.listen) |listen| {
326 try serveResults(gpa.allocator(), listen, test_cases.items);
327 return;
328 }
329
330 var num_pass: usize = 0;
331 for (test_cases.items) |*case| {
332 if (case.success) {
333 num_pass += 1;
334 if (some_logs) {
335 std.debug.print("PASS: {s}\n", .{case.display});
336 }
337 if (all_logs) {
338 std.debug.print("{s}", .{case.logs});
339 }
340 } else if (case.failure) |*fail| {
341 std.debug.print("FAIL: {s}\n", .{case.display});
342 std.debug.print("{s}", .{case.logs});
343 if (fail.stack.index > 0) {
344 std.debug.print("Error return trace:\n", .{});
345 std.debug.dumpStackTrace(fail.stack.trace());
346 }
347 } else {
348 // success was never set true but failure was left null
349 // this means runTestCaseWrap was never called
350 std.debug.print("INVALID STATE: {s}\n", .{case.path});
351 }
352 if (case.logs.len > 0)
353 alloc.free(case.logs);
354 }
355
356 if (some_logs) {
357 std.debug.print("Complete: {d} PASS / {d} TOTAL\n", .{ num_pass, test_cases.items.len });
358 }
359}
360
361const builtin = @import("builtin");
362
363var log_err_count: usize = 0;
364var fba_buffer: [8192]u8 = undefined;
365var fba = std.heap.FixedBufferAllocator.init(&fba_buffer);
366
367/// This is the minimum amount of work to report to the Zig Build System the
368/// results of the unit tests. Instead of running tests on-demand, it runs
369/// them in a thread-pool, then reports the results.
370fn serveResults(gpa: std.mem.Allocator, listen: []const u8, finished_cases: []Case) !void {
371 if (!std.mem.eql(u8, listen, "-")) @panic("Server only supports --listen=-");
372
373 var server = try std.zig.Server.init(.{
374 .gpa = fba.allocator(),
375 .in = std.io.getStdIn(),
376 .out = std.io.getStdOut(),
377 .zig_version = builtin.zig_version_string,
378 });
379 defer server.deinit();
380
381 while (true) {
382 const hdr = try server.receiveMessage();
383 switch (hdr.tag) {
384 .exit => {
385 return std.process.exit(0);
386 },
387 .query_test_metadata => {
388 var string_bytes: std.ArrayListUnmanaged(u8) = .{};
389 defer string_bytes.deinit(gpa);
390 try string_bytes.append(gpa, 0); // Reserve 0 for null.
391
392 const names = try gpa.alloc(u32, finished_cases.len);
393 defer gpa.free(names);
394 const expected_panic_msgs = try gpa.alloc(u32, finished_cases.len);
395 defer gpa.free(expected_panic_msgs);
396
397 for (finished_cases, names, expected_panic_msgs) |case, *name, *expected_panic_msg| {
398 name.* = @as(u32, @intCast(string_bytes.items.len));
399 const prefix = "compiler test: ";
400 try string_bytes.ensureUnusedCapacity(gpa, prefix.len + case.display.len + 1);
401 string_bytes.appendSliceAssumeCapacity(prefix);
402 string_bytes.appendSliceAssumeCapacity(case.display);
403 string_bytes.appendAssumeCapacity(0);
404 expected_panic_msg.* = 0;
405 }
406
407 try server.serveTestMetadata(.{
408 .names = names,
409 .expected_panic_msgs = expected_panic_msgs,
410 .string_bytes = string_bytes.items,
411 });
412 },
413 .run_test => {
414 log_err_count = 0;
415 const index = try server.receiveBody_u32();
416 const case = &finished_cases[index];
417 if (case.failure) |*fail| {
418 std.debug.print("{s}\n{s}", .{ @errorName(fail.err), case.logs });
419 if (fail.stack.index > 0) {
420 std.debug.print("Error return trace:\n", .{});
421 std.debug.dumpStackTrace(fail.stack.trace());
422 }
423 }
424 try server.serveTestResults(.{
425 .index = index,
426 .flags = .{
427 .fail = case.failure != null,
428 .skip = false,
429 .leak = false,
430 .fuzz = false,
431 .log_err_count = std.math.lossyCast(
432 @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count),
433 log_err_count,
434 ),
435 },
436 });
437 },
438 else => {
439 std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)});
440 std.process.exit(1);
441 },
442 }
443 }
444}
445
446fn iterateTestCases(
447 alloc: std.mem.Allocator,
448 root: Dir,
449 root_path_len: usize,
450 path_buf: *std.BoundedArray(u8, std.fs.max_path_bytes),
451 test_cases: *std.ArrayListUnmanaged(Case),
452 filter: ?[]const u8,
453) !void {
454 var it = root.iterate();
455 while (try it.next()) |entry| {
456 switch (entry.kind) {
457 .file => {
458 if (entry.name.len > 0 and entry.name[0] == '.')
459 continue;
460
461 const path = try std.fs.path.join(alloc, &.{
462 path_buf.slice(),
463 entry.name,
464 });
465
466 if (filter) |f| {
467 if (std.mem.indexOf(u8, path, f) == null) {
468 alloc.free(path);
469 continue;
470 }
471 }
472
473 try test_cases.append(alloc, .{
474 .path = path,
475 .display = path[root_path_len + 1 ..],
476 });
477 },
478 .directory => {
479 const top = path_buf.len;
480 defer path_buf.len = top;
481
482 try path_buf.append(std.fs.path.sep);
483 try path_buf.appendSlice(entry.name);
484
485 const sub_dir = try root.openDir(entry.name, .{ .iterate = true });
486 try iterateTestCases(alloc, sub_dir, root_path_len, path_buf, test_cases, filter);
487 },
488 else => return error.UnsupportedFileType,
489 }
490 }
491}
492
493fn getFlag(name: []const u8, full_arg: []const u8) ?[]const u8 {
494 if (full_arg.len > name.len + 1 and
495 full_arg[name.len] == '=' and
496 std.mem.startsWith(u8, name, full_arg))
497 {
498 return full_arg[name.len + 1 ..];
499 }
500
501 return null;
502}
503
504threadlocal var current_case: ?Case = null;
505pub fn panic(msg: []const u8, trace: ?*const std.builtin.StackTrace, first_trace_addr: ?usize) noreturn {
506 if (current_case) |case| {
507 std.debug.print("Crash during case: {s}\n", .{case.display});
508 }
509 std.debug.defaultPanic(msg, trace, first_trace_addr);
510}
511
512const std = @import("std");
513const Dir = std.fs.Dir;
514
515const Cli = @import("Cli");
516const core = @import("core");
517const qjs = @import("quickjs");
518
519const tracer = @import("tracer");
520pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode));