authorgravatar for me@paperdave.netdave caruso <me@paperdave.net> 2024-09-16 17:48:35-04:00
committergravatar for me@paperdave.netdave caruso <me@paperdave.net> 2024-09-16 17:48:35-04:00
log4470c3b285523251143a844ca7821d457fdae8af
tree7c02d9a545a90efb3c8aacb284b5a33753539eaa
parent0497aa7eeb1b2ee00840b66313dc49eb65ffac0d
signature Commit is signed but in an unrecognized format.

feat: integrate compiler tests with Zig build system

7 files changed, 548 insertions(+), 437 deletions(-)

build.zig+36-19
...@@ -30,6 +30,7 @@ const ArtifactGroup = struct {...@@ -30,6 +30,7 @@ const ArtifactGroup = struct {
30 transform_cli: *Step.Compile,30 transform_cli: *Step.Compile,
31 loader_cli: *Step.Compile,31 loader_cli: *Step.Compile,
3232
33 cli_mod: *Module,
33 core_mod: *Module,34 core_mod: *Module,
34 loader_mod: *Module,35 loader_mod: *Module,
35 config_mod: *Module,36 config_mod: *Module,
...@@ -59,14 +60,6 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -59,14 +60,6 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
59 break :config_mod config.createModule();60 break :config_mod config.createModule();
60 };61 };
6162
62 // Cli helper
63 const cli_mod = b.createModule(.{
64 .root_source_file = b.path("src/Cli.zig"),
65 .optimize = options.optimize,
66 .target = options.target,
67 .strip = false,
68 });
69
70 // Core63 // Core
71 const core_mod = b.createModule(.{64 const core_mod = b.createModule(.{
72 .root_source_file = b.path("src/core/mod.zig"),65 .root_source_file = b.path("src/core/mod.zig"),
...@@ -103,10 +96,18 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -103,10 +96,18 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
103 const core_mod_tests = createModuleTests(core_mod, options);96 const core_mod_tests = createModuleTests(core_mod, options);
104 const loader_mod_tests = createModuleTests(loader_mod, options);97 const loader_mod_tests = createModuleTests(loader_mod, options);
10598
99 // Cli helper module
100 const cli_mod = b.createModule(.{
101 .root_source_file = b.path("src/Cli.zig"),
102 .optimize = options.optimize,
103 .target = options.target,
104 .strip = false,
105 });
106
106 // Compiler test CLI107 // Compiler test CLI
107 const compiler_tests = b.addExecutable(.{108 const compiler_tests = b.addExecutable(.{
108 .name = "compiler_tests",109 .name = "compiler_tests",
109 .root_source_file = b.path("test/compiler.zig"),110 .root_source_file = b.path("test/compiler_tests.zig"),
110 .target = options.target,111 .target = options.target,
111 .optimize = options.optimize,112 .optimize = options.optimize,
112 .use_lld = use_lld,113 .use_lld = use_lld,
...@@ -134,6 +135,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -134,6 +135,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
134 if (options.config.have_embedded_quickjs) {135 if (options.config.have_embedded_quickjs) {
135 cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc"));136 cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc"));
136 }137 }
138 cli.root_module.addImport("Cli", cli_mod);
137139
138 // Compiler Dev CLI140 // Compiler Dev CLI
139 const compiler_cli = b.addExecutable(.{141 const compiler_cli = b.addExecutable(.{
...@@ -151,6 +153,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -151,6 +153,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
151 if (options.config.have_embedded_quickjs) {153 if (options.config.have_embedded_quickjs) {
152 cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc"));154 cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc"));
153 }155 }
156 compiler_cli.root_module.addImport("Cli", cli_mod);
154157
155 const format_cli = b.addExecutable(.{158 const format_cli = b.addExecutable(.{
156 .name = "format",159 .name = "format",
...@@ -165,6 +168,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -165,6 +168,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
165 format_cli.root_module.addImport("format", format_mod);168 format_cli.root_module.addImport("format", format_mod);
166 format_cli.root_module.addImport("config", config_mod);169 format_cli.root_module.addImport("config", config_mod);
167 format_cli.root_module.addImport("tracer", tracer.module("tracer"));170 format_cli.root_module.addImport("tracer", tracer.module("tracer"));
171 format_cli.root_module.addImport("Cli", cli_mod);
168172
169 const loader_cli = b.addExecutable(.{173 const loader_cli = b.addExecutable(.{
170 .name = "loader",174 .name = "loader",
...@@ -175,9 +179,10 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -175,9 +179,10 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
175 .use_llvm = use_llvm,179 .use_llvm = use_llvm,
176 .single_threaded = true,180 .single_threaded = true,
177 });181 });
178 format_cli.root_module.addImport("loader", loader_mod);182 loader_cli.root_module.addImport("loader", loader_mod);
179 format_cli.root_module.addImport("config", config_mod);183 loader_cli.root_module.addImport("config", config_mod);
180 format_cli.root_module.addImport("tracer", tracer.module("tracer"));184 loader_cli.root_module.addImport("tracer", tracer.module("tracer"));
185 loader_cli.root_module.addImport("Cli", cli_mod);
181186
182 const transform_cli = b.addExecutable(.{187 const transform_cli = b.addExecutable(.{
183 .name = "transform",188 .name = "transform",
...@@ -191,12 +196,14 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {...@@ -191,12 +196,14 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup {
191 transform_cli.root_module.addImport("core", core_mod);196 transform_cli.root_module.addImport("core", core_mod);
192 transform_cli.root_module.addImport("config", config_mod);197 transform_cli.root_module.addImport("config", config_mod);
193 transform_cli.root_module.addImport("tracer", tracer.module("tracer"));198 transform_cli.root_module.addImport("tracer", tracer.module("tracer"));
199 transform_cli.root_module.addImport("Cli", cli_mod);
194200
195 return .{201 return .{
196 .build = b,202 .build = b,
197 .opts = options,203 .opts = options,
198204
199 .cli = cli,205 .cli = cli,
206 .cli_mod = cli_mod,
200 .compiler_cli = compiler_cli,207 .compiler_cli = compiler_cli,
201 .format_cli = format_cli,208 .format_cli = format_cli,
202 .loader_cli = loader_cli,209 .loader_cli = loader_cli,
...@@ -399,13 +406,14 @@ pub fn build(b: *Build) !void {...@@ -399,13 +406,14 @@ pub fn build(b: *Build) !void {
399406
400 b.installArtifact(main.cli);407 b.installArtifact(main.cli);
401408
402 inline for (.{409 const run_templates = .{
403 .{ main.cli, "run" },410 .{ main.cli, "run" },
404 .{ minimal.compiler_cli, "compiler" },411 .{ minimal.compiler_cli, "compiler" },
405 .{ minimal.format_cli, "format" },412 .{ minimal.format_cli, "format" },
406 .{ minimal.loader_cli, "loader" },413 .{ minimal.loader_cli, "loader" },
407 .{ minimal.transform_cli, "transform" },414 .{ minimal.transform_cli, "transform" },
408 }) |entry| {415 };
416 inline for (run_templates) |entry| {
409 const artifact, const name = entry;417 const artifact, const name = entry;
410418
411 const run_cmd = b.addRunArtifact(artifact);419 const run_cmd = b.addRunArtifact(artifact);
...@@ -425,8 +433,12 @@ pub fn build(b: *Build) !void {...@@ -425,8 +433,12 @@ pub fn build(b: *Build) !void {
425 }) |entry| {433 }) |entry| {
426 const artifact, const name, const description = entry;434 const artifact, const name, const description = entry;
427 const run_cmd = b.addRunArtifact(artifact);435 const run_cmd = b.addRunArtifact(artifact);
428 if (b.args) |args| {436 if (artifact == main.compiler_tests) {
429 run_cmd.addArgs(args);437 if (b.args) |args| {
438 run_cmd.addArgs(args);
439 } else {
440 run_cmd.enableTestRunnerMode();
441 }
430 }442 }
431 const run_step = b.step("test-" ++ name, "Test " ++ description);443 const run_step = b.step("test-" ++ name, "Test " ++ description);
432 run_step.dependOn(&run_cmd.step);444 run_step.dependOn(&run_cmd.step);
...@@ -550,9 +562,14 @@ pub fn build(b: *Build) !void {...@@ -550,9 +562,14 @@ pub fn build(b: *Build) !void {
550 {562 {
551 _ = b.top_level_steps.swapRemove("uninstall"); // this step does not work563 _ = b.top_level_steps.swapRemove("uninstall"); // this step does not work
552564
553 const all_step = b.step("all", "Execute all of the above");565 const all_step = b.step("all", "Build everything (used by CI)");
554 const steps = b.top_level_steps.values();566 all_step.dependOn(&b.top_level_steps.get("install").?.step);
555 for (steps[0 .. steps.len - 1]) |step| all_step.dependOn(&step.step);567 all_step.dependOn(&b.top_level_steps.get("package").?.step);
568 all_step.dependOn(&b.top_level_steps.get("test").?.step);
569 all_step.dependOn(&b.top_level_steps.get("manual").?.step);
570 inline for (run_templates) |run| {
571 all_step.dependOn(&run.@"0".step);
572 }
556 }573 }
557}574}
558575
ci+1-1
...@@ -10,7 +10,7 @@ commit_short=$(echo "$commit" | cut -c -9)...@@ -10,7 +10,7 @@ commit_short=$(echo "$commit" | cut -c -9)
1010
11typos11typos
12dprint check12dprint check
13./build test package --summary all13./build all --summary new
1414
15if [ -n "$BUILDKITE" ]; then15if [ -n "$BUILDKITE" ]; then
16 pushd zig-out/package16 pushd zig-out/package
src/cli_compiler.zig+1-1
...@@ -130,7 +130,7 @@ pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode))...@@ -130,7 +130,7 @@ pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode))
130130
131const std = @import("std");131const std = @import("std");
132const builtin = @import("builtin");132const builtin = @import("builtin");
133const Cli = @import("Cli.zig");133const Cli = @import("Cli");
134134
135fn dumpState(comp: *const Comp, dump_path: []const u8) !void {135fn dumpState(comp: *const Comp, dump_path: []const u8) !void {
136 const file = try std.fs.cwd().createFile(dump_path, .{});136 const file = try std.fs.cwd().createFile(dump_path, .{});
src/cli_transform.zig+1-1
...@@ -94,4 +94,4 @@ pub fn main() !u8 {...@@ -94,4 +94,4 @@ pub fn main() !u8 {
9494
95const std = @import("std");95const std = @import("std");
96const core = @import("core");96const core = @import("core");
97const Cli = @import("Cli.zig");97const Cli = @import("Cli");
src/main.zig+1-1
...@@ -434,7 +434,7 @@ const tracer = @import("tracer");...@@ -434,7 +434,7 @@ const tracer = @import("tracer");
434pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode));434pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode));
435435
436const builtin = @import("builtin");436const builtin = @import("builtin");
437const Cli = @import("Cli.zig");437const Cli = @import("Cli");
438438
439fn dumpState(comp: *const Comp, dump_path: []const u8) !void {439fn dumpState(comp: *const Comp, dump_path: []const u8) !void {
440 const file = try std.fs.cwd().createFile(dump_path, .{});440 const file = try std.fs.cwd().createFile(dump_path, .{});
test/compiler.zig deleted-414
...@@ -1,414 +0,0 @@
1//! This program executes compiler tests found in this directory. Additionally,
2//! it can be run as a standalone program to compile and execute a single test.
3//!
4//! See the "Testing" section in `architecture.md` for more information on the
5//! structure of these tests, and `zig build test-compiler -- --help` for more
6//! details on the arguments this supports.
7
8const cli_config: Cli = .{
9 .options = &.{
10 .{ .name = "--test-root", .desc = "root directory of all tests", .kind = .string },
11 .{ .name = "--all-logs", .desc = "Always print compiler logs" },
12 .{ .name = "--no-threads", .desc = "Avoid using a thread pool" },
13 .{ .name = "--dump-ir", .desc = "Dump intermediate representations" },
14 },
15 .positionals = .single,
16 .help = .{
17 .usage = "[options...] [test filter]",
18 },
19};
20
21/// This file is included for all behavior tests for the JavaScript backend.
22const helpers =
23 \\(function (self) {
24 \\ function assert(condition) {
25 \\ if (!condition) {
26 \\ throw new Error("Assertion failure");
27 \\ }
28 \\ }
29 \\
30 \\ function assertEqual(actual, expected) {
31 \\ if (actual !== expected) {
32 \\ var err;
33 \\ try {
34 \\ err = new Error("Expected " + expected + ", but got " + actual);
35 \\ } catch (e) {
36 \\ err = new Error("Assertion failure");
37 \\ }
38 \\ throw err;
39 \\ }
40 \\ }
41 \\
42 \\ function print(str) {
43 \\ console.log(str);
44 \\ }
45 \\
46 \\ self.assert = assert;
47 \\ self.assertEqual = assertEqual;
48 \\ self.print = print;
49 \\})(typeof globalThis !== "undefined" ? globalThis : this);
50 \\
51;
52
53const CaseOptions = struct {
54 dump_ir: bool = false,
55 sync: bool = false,
56};
57
58const Case = struct {
59 path: []const u8,
60 display: []const u8,
61 success: bool = false,
62 logs: []const u8 = "",
63 failure: ?struct {
64 err: anyerror,
65 stack: core.StoredTrace,
66 } = null,
67
68 const Kind = enum {
69 behavior,
70 analysis,
71 transform_error,
72 };
73};
74
75fn runTestCase(
76 backing_allocator: std.mem.Allocator,
77 file_path: []const u8,
78 case_options: CaseOptions,
79 out: std.io.AnyWriter,
80) !void {
81 var gpa = std.heap.GeneralPurposeAllocator(.{
82 .stack_trace_frames = 100,
83 }){ .backing_allocator = backing_allocator };
84 const alloc = gpa.allocator();
85
86 var backend = core.Js{};
87 var comp = try core.Comp.init(.{
88 .gpa = tracer.tracingAllocator(&alloc),
89 .options = .{ .dump_ir = case_options.dump_ir },
90 .backend = .{ .js = &backend },
91 .out = .{ .writer = out, .tty = .escape_codes },
92 });
93
94 const main_file = try comp.getFile(file_path);
95
96 var fail_report: core.Report = undefined;
97 const teg = main_file.getTeg(&comp, &fail_report) catch |err| switch (err) {
98 error.TransformFail => {
99 try out.print("Transform failed:\n", .{});
100 try fail_report.dump(.{ .comp = .{ .comp = &comp, .file = main_file } }, comp.out);
101 return error.TransformFail;
102 },
103 error.ParseFail => {
104 // Already printed.
105 return error.ParseFail;
106 },
107 else => return err,
108 };
109
110 if (case_options.dump_ir) {
111 try teg.dumpToWriter(out, .escape_codes);
112 }
113
114 const entry_point = try comp.enqueueEntryModule(main_file);
115 try comp.runAllJobs();
116
117 if (comp.temp_flag_had_errors)
118 return error.SemaErrors;
119
120 if (teg.report.warnings.len > 0)
121 return error.TegWarnings;
122
123 var js_out: std.ArrayListUnmanaged(u8) = .{};
124 defer js_out.deinit(alloc);
125
126 try js_out.appendSlice(alloc, helpers);
127 try js_out.appendSlice(alloc, "\n\n(function() {\n\n");
128
129 const start_i = js_out.items.len;
130 try backend.link(&comp, entry_point, js_out.writer(alloc).any());
131
132 try out.writeAll("--- JS CODE ---\n");
133 try out.writeAll(js_out.items[start_i..]);
134 try out.writeAll("--- JS CODE ---\n");
135
136 if (isOnlyComments(js_out.items[start_i..])) {
137 try out.writeAll("Behavior tests should output code!");
138 return error.EmptyFile;
139 }
140
141 try js_out.appendSlice(alloc, "\n\n})();");
142 try js_out.append(alloc, 0);
143
144 const runtime = qjs.JS_NewRuntime() orelse
145 return error.UnexpectedError;
146 defer qjs.JS_FreeRuntime(runtime);
147 const js_ctx = qjs.JS_NewContext(runtime) orelse
148 return error.UnexpectedError;
149 defer qjs.JS_FreeContext(js_ctx);
150
151 const result = qjs.JS_Eval(
152 js_ctx,
153 js_out.items.ptr,
154 js_out.items.len - 1,
155 "test.js",
156 qjs.JS_EVAL_TYPE_GLOBAL,
157 );
158 defer qjs.JS_FreeValue(js_ctx, result);
159
160 if (qjs.JS_IsException(result) != 0) {
161 const exception = qjs.JS_GetException(js_ctx);
162 defer qjs.JS_FreeValue(js_ctx, exception);
163
164 const exception_str = qjs.JS_ToCString(js_ctx, exception);
165 defer qjs.JS_FreeCString(js_ctx, exception_str);
166
167 const slice = std.mem.span(exception_str);
168
169 try out.print("Exception: {s}\n", .{slice});
170
171 return error.RuntimeFailed;
172 }
173}
174
175fn isOnlyComments(input: []const u8) bool {
176 var rest = input;
177 while (rest.len > 0) {
178 if (rest.len == 1) return false;
179 if (rest[0] == '/' and rest[1] == '/') {
180 rest = rest[std.mem.indexOfScalar(u8, rest, '\n') orelse return true ..];
181 } else {
182 return false;
183 }
184 }
185 return true;
186}
187
188const threadlocal_state = struct {
189 threadlocal var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
190};
191
192pub fn runTestCaseWrap(
193 gpa: std.mem.Allocator,
194 case: *Case,
195 case_options: CaseOptions,
196 progress: std.Progress.Node,
197) void {
198 const node = progress.start(case.display, 0);
199 defer node.end();
200
201 const alloc = threadlocal_state.arena.allocator();
202 defer _ = threadlocal_state.arena.reset(.{ .retain_with_limit = std.mem.page_size * 8 });
203
204 var logs: std.ArrayListUnmanaged(u8) = .{};
205 defer logs.deinit(alloc);
206
207 defer case.logs = gpa.dupe(u8, logs.items) catch
208 "[logs failed to allocate]";
209
210 runTestCase(
211 alloc,
212 case.path,
213 case_options,
214 if (case_options.sync)
215 std.io.getStdErr().writer().any()
216 else
217 logs.writer(alloc).any(),
218 ) catch |err| {
219 const stack = switch (err) {
220 error.TransformFail,
221 error.ParseFail,
222 error.TegWarnings,
223 error.SemaErrors,
224 error.AnalysisFail,
225 => core.StoredTrace.empty,
226
227 else => core.StoredTrace.from(@errorReturnTrace()),
228 };
229
230 case.failure = .{
231 .err = err,
232 .stack = stack,
233 };
234
235 return;
236 };
237
238 case.success = true;
239}
240
241pub fn main() !void {
242 // unicode props is asserted to be initialized before core.Tokenizer is used
243 // this will result in a static page allocation, which is never cleaned up.
244 try core.unicode_props.init();
245 var gpa = std.heap.GeneralPurposeAllocator(.{
246 .stack_trace_frames = 100,
247 .thread_safe = true,
248 }){};
249
250 defer _ = gpa.deinit();
251 const alloc = gpa.allocator();
252
253 const program_name = "zig build test-compiler --";
254 _, const args = try Cli.programArgsAlloc(alloc, "");
255 defer Cli.programArgsFree(alloc, args);
256
257 const opts = cli_config.parse(program_name, args);
258
259 const test_root = opts.test_root orelse "test/behavior";
260 const case_options: CaseOptions = .{
261 .dump_ir = opts.dump_ir orelse false,
262 .sync = opts.no_threads orelse false,
263 };
264 const all_logs = opts.all_logs orelse false;
265 const filter: ?[]const u8 = opts.positionals;
266
267 const some_logs = all_logs or args.len > 1;
268
269 const root = try std.fs.cwd().openDir(test_root, .{ .iterate = true });
270
271 var test_cases: std.ArrayListUnmanaged(Case) = .{};
272 defer test_cases.deinit(alloc);
273 defer for (test_cases.items) |test_case| {
274 alloc.free(test_case.path);
275 };
276
277 var path_buf: std.BoundedArray(u8, std.fs.max_path_bytes) = .{};
278 path_buf.len = @intCast((try std.os.getFdPath(root.fd, &path_buf.buffer)).len);
279 try iterateTestCases(alloc, root, path_buf.len, &path_buf, &test_cases, filter);
280
281 if (test_cases.items.len == 0) {
282 if (filter) |f| {
283 std.debug.print("No test cases match filter \"{s}\"\n", .{f});
284 } else {
285 std.debug.print("No test cases in root \"{s}\"", .{test_root});
286 }
287 std.process.exit(1);
288 }
289
290 const progress = std.Progress.start(.{ .root_name = "Behavior Tests" });
291 defer progress.end();
292 progress.setEstimatedTotalItems(test_cases.items.len);
293
294 var wg: std.Thread.WaitGroup = .{};
295
296 var pool: std.Thread.Pool = undefined;
297 if (!case_options.sync) try pool.init(.{ .allocator = alloc });
298 defer if (!case_options.sync) pool.deinit();
299
300 for (test_cases.items) |*test_case| {
301 const call_args = .{
302 alloc,
303 test_case,
304 case_options,
305 progress,
306 };
307
308 if (case_options.sync) {
309 @call(.auto, runTestCaseWrap, call_args);
310 } else {
311 pool.spawnWg(&wg, runTestCaseWrap, call_args);
312 }
313 }
314
315 wg.wait();
316
317 var num_pass: usize = 0;
318 for (test_cases.items) |*case| {
319 if (case.success) {
320 num_pass += 1;
321 if (some_logs) {
322 std.debug.print("PASS: {s}\n", .{case.display});
323 }
324 if (all_logs) {
325 std.debug.print("{s}", .{case.logs});
326 }
327 } else if (case.failure) |*fail| {
328 std.debug.print("FAIL: {s}\n", .{case.display});
329 std.debug.print("{s}", .{case.logs});
330 if (fail.stack.index > 0) {
331 std.debug.print("Error return trace:\n", .{});
332 std.debug.dumpStackTrace(fail.stack.trace());
333 }
334 } else {
335 // success was never set true but failure was left null
336 // this means runTestCaseWrap was never called
337 std.debug.print("INVALID STATE: {s}\n", .{case.path});
338 }
339 if (case.logs.len > 0)
340 alloc.free(case.logs);
341 }
342
343 if (some_logs) {
344 std.debug.print("Complete: {d} PASS / {d} TOTAL\n", .{ num_pass, test_cases.items.len });
345 }
346}
347
348fn iterateTestCases(
349 alloc: std.mem.Allocator,
350 root: Dir,
351 root_path_len: usize,
352 path_buf: *std.BoundedArray(u8, std.fs.max_path_bytes),
353 test_cases: *std.ArrayListUnmanaged(Case),
354 filter: ?[]const u8,
355) !void {
356 var it = root.iterate();
357 while (try it.next()) |entry| {
358 switch (entry.kind) {
359 .file => {
360 if (entry.name.len > 0 and entry.name[0] == '.')
361 continue;
362
363 const path = try std.fs.path.join(alloc, &.{
364 path_buf.slice(),
365 entry.name,
366 });
367
368 if (filter) |f| {
369 if (std.mem.indexOf(u8, path, f) == null) {
370 alloc.free(path);
371 continue;
372 }
373 }
374
375 try test_cases.append(alloc, .{
376 .path = path,
377 .display = path[root_path_len + 1 ..],
378 });
379 },
380 .directory => {
381 const top = path_buf.len;
382 defer path_buf.len = top;
383
384 try path_buf.append(std.fs.path.sep);
385 try path_buf.appendSlice(entry.name);
386
387 const sub_dir = try root.openDir(entry.name, .{ .iterate = true });
388 try iterateTestCases(alloc, sub_dir, root_path_len, path_buf, test_cases, filter);
389 },
390 else => return error.UnsupportedFileType,
391 }
392 }
393}
394
395fn getFlag(name: []const u8, full_arg: []const u8) ?[]const u8 {
396 if (full_arg.len > name.len + 1 and
397 full_arg[name.len] == '=' and
398 std.mem.startsWith(u8, name, full_arg))
399 {
400 return full_arg[name.len + 1 ..];
401 }
402
403 return null;
404}
405
406const std = @import("std");
407const Dir = std.fs.Dir;
408
409const Cli = @import("Cli");
410const core = @import("core");
411const qjs = @import("quickjs");
412
413const tracer = @import("tracer");
414pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode));
test/compiler_tests.zig created+508
...@@ -0,0 +1,508 @@
1//! This program executes compiler tests found in this directory. Additionally,
2//! it can be run as a standalone program to compile and execute a single test.
3//!
4//! See the "Testing" section in `architecture.md` for more information on the
5//! structure of these tests, and `zig build test-compiler -- --help` for more
6//! details on the arguments this supports.
7
8const cli_config: Cli = .{
9 .options = &.{
10 .{ .name = "--test-root", .desc = "root directory of all tests", .kind = .string },
11 .{ .name = "--all-logs", .desc = "Always print compiler logs" },
12 .{ .name = "--no-threads", .desc = "Avoid using a thread pool" },
13 .{ .name = "--dump-ir", .desc = "Dump intermediate representations" },
14
15 .{ .name = "--listen", .desc = "Integrate with the Zig Build System IPC", .kind = .string },
16 .{ .name = "--seed", .desc = "Set random number seed (unused)", .kind = .string },
17 .{ .name = "--cache-dir", .desc = "Set cache dir (unused)", .kind = .string },
18 },
19 .positionals = .single,
20 .help = .{
21 .usage = "[options...] [test filter]",
22 },
23};
24
25/// This file is included for all behavior tests for the JavaScript backend.
26const helpers =
27 \\(function (self) {
28 \\ function assert(condition) {
29 \\ if (!condition) {
30 \\ throw new Error("Assertion failure");
31 \\ }
32 \\ }
33 \\
34 \\ function assertEqual(actual, expected) {
35 \\ if (actual !== expected) {
36 \\ var err;
37 \\ try {
38 \\ err = new Error("Expected " + expected + ", but got " + actual);
39 \\ } catch (e) {
40 \\ err = new Error("Assertion failure");
41 \\ }
42 \\ throw err;
43 \\ }
44 \\ }
45 \\
46 \\ function print(str) {
47 \\ console.log(str);
48 \\ }
49 \\
50 \\ self.assert = assert;
51 \\ self.assertEqual = assertEqual;
52 \\ self.print = print;
53 \\})(typeof globalThis !== "undefined" ? globalThis : this);
54 \\
55;
56
57const CaseOptions = struct {
58 dump_ir: bool = false,
59 sync: bool = false,
60};
61
62const Case = struct {
63 path: []const u8,
64 display: []const u8,
65 success: bool = false,
66 logs: []const u8 = "",
67 failure: ?struct {
68 err: anyerror,
69 stack: core.StoredTrace,
70 } = null,
71
72 const Kind = enum {
73 behavior,
74 analysis,
75 transform_error,
76 };
77};
78
79fn runTestCase(
80 backing_allocator: std.mem.Allocator,
81 file_path: []const u8,
82 case_options: CaseOptions,
83 out: std.io.AnyWriter,
84) !void {
85 var gpa = std.heap.GeneralPurposeAllocator(.{
86 .stack_trace_frames = 100,
87 }){ .backing_allocator = backing_allocator };
88 const alloc = gpa.allocator();
89
90 var backend = core.Js{};
91 var comp = try core.Comp.init(.{
92 .gpa = tracer.tracingAllocator(&alloc),
93 .options = .{ .dump_ir = case_options.dump_ir },
94 .backend = .{ .js = &backend },
95 .out = .{ .writer = out, .tty = .escape_codes },
96 });
97
98 const main_file = try comp.getFile(file_path);
99
100 var fail_report: core.Report = undefined;
101 const teg = main_file.getTeg(&comp, &fail_report) catch |err| switch (err) {
102 error.TransformFail => {
103 try out.print("Transform failed:\n", .{});
104 try fail_report.dump(.{ .comp = .{ .comp = &comp, .file = main_file } }, comp.out);
105 return error.TransformFail;
106 },
107 error.ParseFail => {
108 // Already printed.
109 return error.ParseFail;
110 },
111 else => return err,
112 };
113
114 if (case_options.dump_ir) {
115 try teg.dumpToWriter(out, .escape_codes);
116 }
117
118 const entry_point = try comp.enqueueEntryModule(main_file);
119 try comp.runAllJobs();
120
121 if (comp.temp_flag_had_errors)
122 return error.SemaErrors;
123
124 if (teg.report.warnings.len > 0)
125 return error.TegWarnings;
126
127 var js_out: std.ArrayListUnmanaged(u8) = .{};
128 defer js_out.deinit(alloc);
129
130 try js_out.appendSlice(alloc, helpers);
131 try js_out.appendSlice(alloc, "\n\n(function() {\n\n");
132
133 const start_i = js_out.items.len;
134 try backend.link(&comp, entry_point, js_out.writer(alloc).any());
135
136 try out.writeAll("--- COMPILED CODE ---\n");
137 try out.writeAll(js_out.items[start_i..]);
138 try out.writeAll("--- RUNTIME OUTPUT ---\n");
139
140 if (isOnlyComments(js_out.items[start_i..])) {
141 try out.writeAll("Behavior tests should output code!");
142 return error.EmptyFile;
143 }
144
145 try js_out.appendSlice(alloc, "\n\n})();");
146 try js_out.append(alloc, 0);
147
148 const runtime = qjs.JS_NewRuntime() orelse
149 return error.UnexpectedError;
150 defer qjs.JS_FreeRuntime(runtime);
151 const js_ctx = qjs.JS_NewContext(runtime) orelse
152 return error.UnexpectedError;
153 defer qjs.JS_FreeContext(js_ctx);
154
155 const result = qjs.JS_Eval(
156 js_ctx,
157 js_out.items.ptr,
158 js_out.items.len - 1,
159 "test.js",
160 qjs.JS_EVAL_TYPE_GLOBAL,
161 );
162 defer qjs.JS_FreeValue(js_ctx, result);
163
164 if (qjs.JS_IsException(result) != 0) {
165 const exception = qjs.JS_GetException(js_ctx);
166 defer qjs.JS_FreeValue(js_ctx, exception);
167
168 const exception_str = qjs.JS_ToCString(js_ctx, exception);
169 defer qjs.JS_FreeCString(js_ctx, exception_str);
170
171 const slice = std.mem.span(exception_str);
172
173 try out.print("Exception: {s}\n", .{slice});
174
175 return error.RuntimeFailed;
176 }
177}
178
179fn isOnlyComments(input: []const u8) bool {
180 var rest = input;
181 while (rest.len > 0) {
182 if (rest.len == 1) return false;
183 if (rest[0] == '/' and rest[1] == '/') {
184 rest = rest[std.mem.indexOfScalar(u8, rest, '\n') orelse return true ..];
185 } else {
186 return false;
187 }
188 }
189 return true;
190}
191
192const threadlocal_state = struct {
193 threadlocal var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
194};
195
196pub fn runTestCaseWrap(
197 gpa: std.mem.Allocator,
198 case: *Case,
199 case_options: CaseOptions,
200 progress: std.Progress.Node,
201) void {
202 const node = progress.start(case.display, 0);
203 defer node.end();
204
205 const alloc = threadlocal_state.arena.allocator();
206 defer _ = threadlocal_state.arena.reset(.{ .retain_with_limit = std.mem.page_size * 8 });
207
208 var logs: std.ArrayListUnmanaged(u8) = .{};
209 defer logs.deinit(alloc);
210
211 defer case.logs = gpa.dupe(u8, logs.items) catch
212 "[logs failed to allocate]";
213
214 runTestCase(
215 alloc,
216 case.path,
217 case_options,
218 if (case_options.sync)
219 std.io.getStdErr().writer().any()
220 else
221 logs.writer(alloc).any(),
222 ) catch |err| {
223 const stack = switch (err) {
224 error.TransformFail,
225 error.ParseFail,
226 error.TegWarnings,
227 error.SemaErrors,
228 error.AnalysisFail,
229 => core.StoredTrace.empty,
230
231 else => core.StoredTrace.from(@errorReturnTrace()),
232 };
233
234 case.failure = .{
235 .err = err,
236 .stack = stack,
237 };
238
239 return;
240 };
241
242 case.success = true;
243}
244
245pub fn main() !void {
246 // unicode props is asserted to be initialized before core.Tokenizer is used
247 // this will result in a static page allocation, which is never cleaned up.
248 try core.unicode_props.init();
249 var gpa = std.heap.GeneralPurposeAllocator(.{
250 .stack_trace_frames = 100,
251 .thread_safe = true,
252 }){};
253
254 defer _ = gpa.deinit();
255 const alloc = gpa.allocator();
256
257 const program_name = "zig build test-compiler --";
258 _, const args = try Cli.programArgsAlloc(alloc, "");
259 defer Cli.programArgsFree(alloc, args);
260
261 const opts = cli_config.parse(program_name, args);
262
263 const root_path = opts.test_root orelse "test/behavior";
264 const case_options: CaseOptions = .{
265 .dump_ir = opts.dump_ir orelse false,
266 .sync = opts.no_threads orelse false,
267 };
268 const all_logs = opts.all_logs orelse false;
269 const filter: ?[]const u8 = opts.positionals;
270
271 const some_logs = all_logs or args.len > 1;
272
273 const root = try std.fs.cwd().openDir(root_path, .{ .iterate = true });
274
275 var test_cases: std.ArrayListUnmanaged(Case) = .{};
276 defer test_cases.deinit(alloc);
277 defer for (test_cases.items) |test_case| {
278 alloc.free(test_case.path);
279 };
280
281 var path_buf: std.BoundedArray(u8, std.fs.max_path_bytes) = .{};
282 path_buf.len = @intCast((try std.os.getFdPath(root.fd, &path_buf.buffer)).len);
283 try iterateTestCases(alloc, root, path_buf.len, &path_buf, &test_cases, filter);
284
285 if (test_cases.items.len == 0) {
286 if (filter) |f| {
287 std.debug.print("No test cases match filter \"{s}\"\n", .{f});
288 } else {
289 std.debug.print("No test cases in root \"{s}\"", .{root_path});
290 }
291 std.process.exit(1);
292 }
293
294 const progress = std.Progress.start(.{ .root_name = "Behavior Tests" });
295 defer progress.end();
296 progress.setEstimatedTotalItems(test_cases.items.len);
297
298 var wg: std.Thread.WaitGroup = .{};
299
300 var pool: std.Thread.Pool = undefined;
301 if (!case_options.sync) try pool.init(.{ .allocator = alloc });
302 defer if (!case_options.sync) pool.deinit();
303
304 for (test_cases.items) |*test_case| {
305 const call_args = .{
306 alloc,
307 test_case,
308 case_options,
309 progress,
310 };
311
312 if (case_options.sync) {
313 @call(.auto, runTestCaseWrap, call_args);
314 } else {
315 pool.spawnWg(&wg, runTestCaseWrap, call_args);
316 }
317 }
318
319 wg.wait();
320
321 if (opts.listen) |listen| {
322 try serveResults(gpa.allocator(), listen, test_cases.items);
323 return;
324 }
325
326 var num_pass: usize = 0;
327 for (test_cases.items) |*case| {
328 if (case.success) {
329 num_pass += 1;
330 if (some_logs) {
331 std.debug.print("PASS: {s}\n", .{case.display});
332 }
333 if (all_logs) {
334 std.debug.print("{s}", .{case.logs});
335 }
336 } else if (case.failure) |*fail| {
337 std.debug.print("FAIL: {s}\n", .{case.display});
338 std.debug.print("{s}", .{case.logs});
339 if (fail.stack.index > 0) {
340 std.debug.print("Error return trace:\n", .{});
341 std.debug.dumpStackTrace(fail.stack.trace());
342 }
343 } else {
344 // success was never set true but failure was left null
345 // this means runTestCaseWrap was never called
346 std.debug.print("INVALID STATE: {s}\n", .{case.path});
347 }
348 if (case.logs.len > 0)
349 alloc.free(case.logs);
350 }
351
352 if (some_logs) {
353 std.debug.print("Complete: {d} PASS / {d} TOTAL\n", .{ num_pass, test_cases.items.len });
354 }
355}
356
357const builtin = @import("builtin");
358
359var log_err_count: usize = 0;
360var fba_buffer: [8192]u8 = undefined;
361var fba = std.heap.FixedBufferAllocator.init(&fba_buffer);
362
363/// This is the minimum amount of work to report to the Zig Build System the
364/// results of the unit tests. Instead of running tests on-demand, it runs
365/// them in a thread-pool, then reports the results.
366fn serveResults(gpa: std.mem.Allocator, listen: []const u8, finished_cases: []Case) !void {
367 if (!std.mem.eql(u8, listen, "-")) @panic("Server only supports --listen=-");
368
369 var server = try std.zig.Server.init(.{
370 .gpa = fba.allocator(),
371 .in = std.io.getStdIn(),
372 .out = std.io.getStdOut(),
373 .zig_version = builtin.zig_version_string,
374 });
375 defer server.deinit();
376
377 while (true) {
378 const hdr = try server.receiveMessage();
379 switch (hdr.tag) {
380 .exit => {
381 return std.process.exit(0);
382 },
383 .query_test_metadata => {
384 var string_bytes: std.ArrayListUnmanaged(u8) = .{};
385 defer string_bytes.deinit(gpa);
386 try string_bytes.append(gpa, 0); // Reserve 0 for null.
387
388 const names = try gpa.alloc(u32, finished_cases.len);
389 defer gpa.free(names);
390 const expected_panic_msgs = try gpa.alloc(u32, finished_cases.len);
391 defer gpa.free(expected_panic_msgs);
392
393 for (finished_cases, names, expected_panic_msgs) |case, *name, *expected_panic_msg| {
394 name.* = @as(u32, @intCast(string_bytes.items.len));
395 const prefix = "compiler test: ";
396 try string_bytes.ensureUnusedCapacity(gpa, prefix.len + case.display.len + 1);
397 string_bytes.appendSliceAssumeCapacity(prefix);
398 string_bytes.appendSliceAssumeCapacity(case.display);
399 string_bytes.appendAssumeCapacity(0);
400 expected_panic_msg.* = 0;
401 }
402
403 try server.serveTestMetadata(.{
404 .names = names,
405 .expected_panic_msgs = expected_panic_msgs,
406 .string_bytes = string_bytes.items,
407 });
408 },
409 .run_test => {
410 log_err_count = 0;
411 const index = try server.receiveBody_u32();
412 const case = &finished_cases[index];
413 if (case.failure) |*fail| {
414 std.debug.print("{s}\n{s}", .{ @errorName(fail.err), case.logs });
415 if (fail.stack.index > 0) {
416 std.debug.print("Error return trace:\n", .{});
417 std.debug.dumpStackTrace(fail.stack.trace());
418 }
419 }
420 try server.serveTestResults(.{
421 .index = index,
422 .flags = .{
423 .fail = case.failure != null,
424 .skip = false,
425 .leak = false,
426 .fuzz = false,
427 .log_err_count = std.math.lossyCast(
428 @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count),
429 log_err_count,
430 ),
431 },
432 });
433 },
434 else => {
435 std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)});
436 std.process.exit(1);
437 },
438 }
439 }
440}
441
442fn iterateTestCases(
443 alloc: std.mem.Allocator,
444 root: Dir,
445 root_path_len: usize,
446 path_buf: *std.BoundedArray(u8, std.fs.max_path_bytes),
447 test_cases: *std.ArrayListUnmanaged(Case),
448 filter: ?[]const u8,
449) !void {
450 var it = root.iterate();
451 while (try it.next()) |entry| {
452 switch (entry.kind) {
453 .file => {
454 if (entry.name.len > 0 and entry.name[0] == '.')
455 continue;
456
457 const path = try std.fs.path.join(alloc, &.{
458 path_buf.slice(),
459 entry.name,
460 });
461
462 if (filter) |f| {
463 if (std.mem.indexOf(u8, path, f) == null) {
464 alloc.free(path);
465 continue;
466 }
467 }
468
469 try test_cases.append(alloc, .{
470 .path = path,
471 .display = path[root_path_len + 1 ..],
472 });
473 },
474 .directory => {
475 const top = path_buf.len;
476 defer path_buf.len = top;
477
478 try path_buf.append(std.fs.path.sep);
479 try path_buf.appendSlice(entry.name);
480
481 const sub_dir = try root.openDir(entry.name, .{ .iterate = true });
482 try iterateTestCases(alloc, sub_dir, root_path_len, path_buf, test_cases, filter);
483 },
484 else => return error.UnsupportedFileType,
485 }
486 }
487}
488
489fn getFlag(name: []const u8, full_arg: []const u8) ?[]const u8 {
490 if (full_arg.len > name.len + 1 and
491 full_arg[name.len] == '=' and
492 std.mem.startsWith(u8, name, full_arg))
493 {
494 return full_arg[name.len + 1 ..];
495 }
496
497 return null;
498}
499
500const std = @import("std");
501const Dir = std.fs.Dir;
502
503const Cli = @import("Cli");
504const core = @import("core");
505const qjs = @import("quickjs");
506
507const tracer = @import("tracer");
508pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode));