From 4470c3b285523251143a844ca7821d457fdae8af Mon Sep 17 00:00:00 2001 From: dave caruso Date: Mon, 16 Sep 2024 17:48:35 -0400 Subject: [PATCH] feat: integrate compiler tests with Zig build system --- build.zig | 55 ++++++++---- ci | 2 +- src/cli_compiler.zig | 2 +- src/cli_transform.zig | 2 +- src/main.zig | 2 +- test/{compiler.zig => compiler_tests.zig} | 104 ++++++++++++++++++++-- 6 files changed, 139 insertions(+), 28 deletions(-) rename test/{compiler.zig => compiler_tests.zig} (73%) diff --git a/build.zig b/build.zig index de25ba27b57aa2359c760f449487f958c7a52965..18e0ca831f79de13881cd284060a88d42a0c1385 100644 --- a/build.zig +++ b/build.zig @@ -30,6 +30,7 @@ const ArtifactGroup = struct { transform_cli: *Step.Compile, loader_cli: *Step.Compile, + cli_mod: *Module, core_mod: *Module, loader_mod: *Module, config_mod: *Module, @@ -59,14 +60,6 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { break :config_mod config.createModule(); }; - // Cli helper - const cli_mod = b.createModule(.{ - .root_source_file = b.path("src/Cli.zig"), - .optimize = options.optimize, - .target = options.target, - .strip = false, - }); - // Core const core_mod = b.createModule(.{ .root_source_file = b.path("src/core/mod.zig"), @@ -103,10 +96,18 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { const core_mod_tests = createModuleTests(core_mod, options); const loader_mod_tests = createModuleTests(loader_mod, options); + // Cli helper module + const cli_mod = b.createModule(.{ + .root_source_file = b.path("src/Cli.zig"), + .optimize = options.optimize, + .target = options.target, + .strip = false, + }); + // Compiler test CLI const compiler_tests = b.addExecutable(.{ .name = "compiler_tests", - .root_source_file = b.path("test/compiler.zig"), + .root_source_file = b.path("test/compiler_tests.zig"), .target = options.target, .optimize = options.optimize, .use_lld = use_lld, @@ -134,6 +135,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { if (options.config.have_embedded_quickjs) { cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc")); } + cli.root_module.addImport("Cli", cli_mod); // Compiler Dev CLI const compiler_cli = b.addExecutable(.{ @@ -151,6 +153,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { if (options.config.have_embedded_quickjs) { cli.root_module.addImport("quickjs", quickjs.module("quickjs-libc")); } + compiler_cli.root_module.addImport("Cli", cli_mod); const format_cli = b.addExecutable(.{ .name = "format", @@ -165,6 +168,7 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { format_cli.root_module.addImport("format", format_mod); format_cli.root_module.addImport("config", config_mod); format_cli.root_module.addImport("tracer", tracer.module("tracer")); + format_cli.root_module.addImport("Cli", cli_mod); const loader_cli = b.addExecutable(.{ .name = "loader", @@ -175,9 +179,10 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { .use_llvm = use_llvm, .single_threaded = true, }); - format_cli.root_module.addImport("loader", loader_mod); - format_cli.root_module.addImport("config", config_mod); - format_cli.root_module.addImport("tracer", tracer.module("tracer")); + loader_cli.root_module.addImport("loader", loader_mod); + loader_cli.root_module.addImport("config", config_mod); + loader_cli.root_module.addImport("tracer", tracer.module("tracer")); + loader_cli.root_module.addImport("Cli", cli_mod); const transform_cli = b.addExecutable(.{ .name = "transform", @@ -191,12 +196,14 @@ fn createArtifactGroup(b: *Build, options: Options) ArtifactGroup { transform_cli.root_module.addImport("core", core_mod); transform_cli.root_module.addImport("config", config_mod); transform_cli.root_module.addImport("tracer", tracer.module("tracer")); + transform_cli.root_module.addImport("Cli", cli_mod); return .{ .build = b, .opts = options, .cli = cli, + .cli_mod = cli_mod, .compiler_cli = compiler_cli, .format_cli = format_cli, .loader_cli = loader_cli, @@ -399,13 +406,14 @@ pub fn build(b: *Build) !void { b.installArtifact(main.cli); - inline for (.{ + const run_templates = .{ .{ main.cli, "run" }, .{ minimal.compiler_cli, "compiler" }, .{ minimal.format_cli, "format" }, .{ minimal.loader_cli, "loader" }, .{ minimal.transform_cli, "transform" }, - }) |entry| { + }; + inline for (run_templates) |entry| { const artifact, const name = entry; const run_cmd = b.addRunArtifact(artifact); @@ -425,8 +433,12 @@ pub fn build(b: *Build) !void { }) |entry| { const artifact, const name, const description = entry; const run_cmd = b.addRunArtifact(artifact); - if (b.args) |args| { - run_cmd.addArgs(args); + if (artifact == main.compiler_tests) { + if (b.args) |args| { + run_cmd.addArgs(args); + } else { + run_cmd.enableTestRunnerMode(); + } } const run_step = b.step("test-" ++ name, "Test " ++ description); run_step.dependOn(&run_cmd.step); @@ -550,9 +562,14 @@ pub fn build(b: *Build) !void { { _ = b.top_level_steps.swapRemove("uninstall"); // this step does not work - const all_step = b.step("all", "Execute all of the above"); - const steps = b.top_level_steps.values(); - for (steps[0 .. steps.len - 1]) |step| all_step.dependOn(&step.step); + const all_step = b.step("all", "Build everything (used by CI)"); + all_step.dependOn(&b.top_level_steps.get("install").?.step); + all_step.dependOn(&b.top_level_steps.get("package").?.step); + all_step.dependOn(&b.top_level_steps.get("test").?.step); + all_step.dependOn(&b.top_level_steps.get("manual").?.step); + inline for (run_templates) |run| { + all_step.dependOn(&run.@"0".step); + } } } diff --git a/ci b/ci index e721010b36f812794e57fd93ec02f5643b10fea4..619452c747148e93756c6c6bf71355707ced34ad 100755 --- a/ci +++ b/ci @@ -10,7 +10,7 @@ commit_short=$(echo "$commit" | cut -c -9) typos dprint check -./build test package --summary all +./build all --summary new if [ -n "$BUILDKITE" ]; then pushd zig-out/package diff --git a/src/cli_compiler.zig b/src/cli_compiler.zig index 55374767c5dd3648247599a0d1f820849206d0c2..ef054ca3a20d5f4d6ede392a4f2189b4a483fc80 100644 --- a/src/cli_compiler.zig +++ b/src/cli_compiler.zig @@ -130,7 +130,7 @@ pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode)) const std = @import("std"); const builtin = @import("builtin"); -const Cli = @import("Cli.zig"); +const Cli = @import("Cli"); fn dumpState(comp: *const Comp, dump_path: []const u8) !void { const file = try std.fs.cwd().createFile(dump_path, .{}); diff --git a/src/cli_transform.zig b/src/cli_transform.zig index 76200d6457a22fefd34076def4e44cfe3c58c97e..501353608d31b9e0e2523c4a608263a56929a8f9 100644 --- a/src/cli_transform.zig +++ b/src/cli_transform.zig @@ -94,4 +94,4 @@ pub fn main() !u8 { const std = @import("std"); const core = @import("core"); -const Cli = @import("Cli.zig"); +const Cli = @import("Cli"); diff --git a/src/main.zig b/src/main.zig index 5311b43e52c814f5079d392495488fca9408db44..2a2c61c5049c039497e9d8693d30c3871f2970cb 100644 --- a/src/main.zig +++ b/src/main.zig @@ -434,7 +434,7 @@ const tracer = @import("tracer"); pub const tracer_impl = @field(tracer, @tagName(@import("config").tracing_mode)); const builtin = @import("builtin"); -const Cli = @import("Cli.zig"); +const Cli = @import("Cli"); fn dumpState(comp: *const Comp, dump_path: []const u8) !void { const file = try std.fs.cwd().createFile(dump_path, .{}); diff --git a/test/compiler.zig b/test/compiler_tests.zig similarity index 73% rename from test/compiler.zig rename to test/compiler_tests.zig index d5b1cd5ca6f55e1142232dcf922325538513cf1f..f643ca113822df469b333379aaf3f7d9201b523d 100644 --- a/test/compiler.zig +++ b/test/compiler_tests.zig @@ -11,6 +11,10 @@ const cli_config: Cli = .{ .{ .name = "--all-logs", .desc = "Always print compiler logs" }, .{ .name = "--no-threads", .desc = "Avoid using a thread pool" }, .{ .name = "--dump-ir", .desc = "Dump intermediate representations" }, + + .{ .name = "--listen", .desc = "Integrate with the Zig Build System IPC", .kind = .string }, + .{ .name = "--seed", .desc = "Set random number seed (unused)", .kind = .string }, + .{ .name = "--cache-dir", .desc = "Set cache dir (unused)", .kind = .string }, }, .positionals = .single, .help = .{ @@ -129,9 +133,9 @@ fn runTestCase( const start_i = js_out.items.len; try backend.link(&comp, entry_point, js_out.writer(alloc).any()); - try out.writeAll("--- JS CODE ---\n"); + try out.writeAll("--- COMPILED CODE ---\n"); try out.writeAll(js_out.items[start_i..]); - try out.writeAll("--- JS CODE ---\n"); + try out.writeAll("--- RUNTIME OUTPUT ---\n"); if (isOnlyComments(js_out.items[start_i..])) { try out.writeAll("Behavior tests should output code!"); @@ -256,7 +260,7 @@ pub fn main() !void { const opts = cli_config.parse(program_name, args); - const test_root = opts.test_root orelse "test/behavior"; + const root_path = opts.test_root orelse "test/behavior"; const case_options: CaseOptions = .{ .dump_ir = opts.dump_ir orelse false, .sync = opts.no_threads orelse false, @@ -266,7 +270,7 @@ pub fn main() !void { const some_logs = all_logs or args.len > 1; - const root = try std.fs.cwd().openDir(test_root, .{ .iterate = true }); + const root = try std.fs.cwd().openDir(root_path, .{ .iterate = true }); var test_cases: std.ArrayListUnmanaged(Case) = .{}; defer test_cases.deinit(alloc); @@ -282,7 +286,7 @@ pub fn main() !void { if (filter) |f| { std.debug.print("No test cases match filter \"{s}\"\n", .{f}); } else { - std.debug.print("No test cases in root \"{s}\"", .{test_root}); + std.debug.print("No test cases in root \"{s}\"", .{root_path}); } std.process.exit(1); } @@ -314,6 +318,11 @@ pub fn main() !void { wg.wait(); + if (opts.listen) |listen| { + try serveResults(gpa.allocator(), listen, test_cases.items); + return; + } + var num_pass: usize = 0; for (test_cases.items) |*case| { if (case.success) { @@ -345,6 +354,91 @@ pub fn main() !void { } } +const builtin = @import("builtin"); + +var log_err_count: usize = 0; +var fba_buffer: [8192]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); + +/// This is the minimum amount of work to report to the Zig Build System the +/// results of the unit tests. Instead of running tests on-demand, it runs +/// them in a thread-pool, then reports the results. +fn serveResults(gpa: std.mem.Allocator, listen: []const u8, finished_cases: []Case) !void { + if (!std.mem.eql(u8, listen, "-")) @panic("Server only supports --listen=-"); + + var server = try std.zig.Server.init(.{ + .gpa = fba.allocator(), + .in = std.io.getStdIn(), + .out = std.io.getStdOut(), + .zig_version = builtin.zig_version_string, + }); + defer server.deinit(); + + while (true) { + const hdr = try server.receiveMessage(); + switch (hdr.tag) { + .exit => { + return std.process.exit(0); + }, + .query_test_metadata => { + var string_bytes: std.ArrayListUnmanaged(u8) = .{}; + defer string_bytes.deinit(gpa); + try string_bytes.append(gpa, 0); // Reserve 0 for null. + + const names = try gpa.alloc(u32, finished_cases.len); + defer gpa.free(names); + const expected_panic_msgs = try gpa.alloc(u32, finished_cases.len); + defer gpa.free(expected_panic_msgs); + + for (finished_cases, names, expected_panic_msgs) |case, *name, *expected_panic_msg| { + name.* = @as(u32, @intCast(string_bytes.items.len)); + const prefix = "compiler test: "; + try string_bytes.ensureUnusedCapacity(gpa, prefix.len + case.display.len + 1); + string_bytes.appendSliceAssumeCapacity(prefix); + string_bytes.appendSliceAssumeCapacity(case.display); + string_bytes.appendAssumeCapacity(0); + expected_panic_msg.* = 0; + } + + try server.serveTestMetadata(.{ + .names = names, + .expected_panic_msgs = expected_panic_msgs, + .string_bytes = string_bytes.items, + }); + }, + .run_test => { + log_err_count = 0; + const index = try server.receiveBody_u32(); + const case = &finished_cases[index]; + if (case.failure) |*fail| { + std.debug.print("{s}\n{s}", .{ @errorName(fail.err), case.logs }); + if (fail.stack.index > 0) { + std.debug.print("Error return trace:\n", .{}); + std.debug.dumpStackTrace(fail.stack.trace()); + } + } + try server.serveTestResults(.{ + .index = index, + .flags = .{ + .fail = case.failure != null, + .skip = false, + .leak = false, + .fuzz = false, + .log_err_count = std.math.lossyCast( + @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count), + log_err_count, + ), + }, + }); + }, + else => { + std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)}); + std.process.exit(1); + }, + } + } +} + fn iterateTestCases( alloc: std.mem.Allocator, root: Dir, -- 2.54.0