Commit 84d69ae4e43f53b2b0b243cff4f1f67303825b4b | ix

author	dave caruso <me@paperdave.net> 2024-07-27 22:14:55-07:00
committer	dave caruso <me@paperdave.net> 2024-07-27 22:15:52-07:00
log	84d69ae4e43f53b2b0b243cff4f1f67303825b4b
tree	c91c6ebf541e54f44cbf45cc59ef256404c59bde
parent	650159166b283698ecbe363d79403d00164acf06
signature	Commit is signed but in an unrecognized format.

chore: add behavior test runner (#82)

all new compiler behavior should be tested via `zig build test-behavior`

22 files changed, 1400 insertions(+), 373 deletions(-)

CHANGELOG.md
build.zig
build.zig.zon
src
- BuildConfig.zig
- cli.zig
- core
- deps
  - quickjs
    build.zig
    build.zig.zon
- lsp
  - lsp.zig
- manual
  - generate-man.zig
test
- behavior.zig
- behavior
  - arithmetic

CHANGELOG.md+2

...	...	@@ -3,6 +3,8 @@
3	3	- fix: crash when printing unused struct declaration error
4	4	- fix: provide array result type to sema
5	5	- fix: crash when analyzing an empty array without a result type
	6	- fix: binary operands with float and float would not resolve
	7	- fix: assertion false-positive when using an extern global more than once
6	8
7	9	## 0.1.5
8	10

build.zig+120-70

...	...	@@ -1,12 +1,9 @@
1		const std = @import("std");
2		const Build = std.Build;
3		const Target = std.Target;
4		const OsVersion = Target.Query.OsVersion;
	1	//! See `architecture.md` for a codebase overview, and `zig-out/doc/README.html`
	2	//! for information on the project.
5	3
6		pub fn build(b: *std.Build) !void {
	4	pub fn build(b: *Build) !void {
7	5	b.reference_trace = 20;
8	6
9		const native_target = b.resolveTargetQuery(.{});
10	7	const target = b.standardTargetOptions(.{});
11	8
12	9	// I disagree with how the standard optimize flag works
...	...	@@ -28,6 +25,7 @@ pub fn build(b: *std.Build) !void {
28	25	break :brk .ReleaseSafe;
29	26	},
30	27	};
	28
31	29	const tracing = b.option(
32	30	TracingMode,
33	31	"trace",
...	...	@@ -52,14 +50,26 @@ pub fn build(b: *std.Build) !void {
52	50	break :version std.SemanticVersion.parse(version_text) catch unreachable;
53	51	};
54	52
55		const native_debug = buildWithTarget(b, native_target, .Debug, .none, version, no_llvm);
	53	const js_runtime: PathOrString = if (b.option(
	54	[]const u8,
	55	"js_runtime",
	56	"JavaScript runtime to use during behavior tests and manual generation",
	57	)) \|path\|
	58	.{ .string = path }
	59	else
	60	// Default to building QuickJS
	61	.{ .path = b.dependency("quickjs", .{
	62	.target = b.graph.host,
	63	.optimize = .ReleaseSafe,
	64	}).artifact("qjs").getEmittedBin() };
	65
	66	const native_debug = buildWithTarget(b, b.graph.host, .Debug, .none, version, no_llvm);
56	67	const main = buildWithTarget(b, target, optimize, tracing, version, no_llvm);
57	68
58		b.installArtifact(main.compiler);
	69	b.installArtifact(main.cli(b, .{}));
59	70
60	71	inline for (.{
61		.{ main.compiler, "compiler" },
62		.{ main.lsp, "lsp" },
	72	.{ main.cli(b, BuildConfig.minimal), "compiler" },
63	73	}) \|entry\| {
64	74	const artifact, const name = entry;
65	75
...	...	@@ -73,16 +83,17 @@ pub fn build(b: *std.Build) !void {
73	83
74	84	const test_all = b.step("test", "Run all tests");
75	85	inline for (.{
76		.{ main.core_tests, "core" },
	86	.{ main.core_tests, "core", "core module" },
	87	.{ main.behavior_tests, "behavior", "compiler behavior" },
77	88	}) \|entry\| {
78		const artifact, const name = entry;
	89	const artifact, const name, const description = entry;
79	90	const run_cmd = b.addRunArtifact(artifact);
80		run_cmd.step.dependOn(b.getInstallStep());
81	91	if (b.args) \|args\| {
82	92	run_cmd.addArgs(args);
83	93	}
84		const run_step = b.step("test-" ++ name, "Test " ++ name ++ " module");
	94	const run_step = b.step("test-" ++ name, "Test " ++ description);
85	95	run_step.dependOn(&run_cmd.step);
	96	run_step.dependOn(&b.addInstallArtifact(artifact, .{}).step);
86	97	test_all.dependOn(run_step);
87	98	}
88	99
...	...	@@ -90,19 +101,23 @@ pub fn build(b: *std.Build) !void {
90	101	const manual_generator = b.addExecutable(.{
91	102	.name = "generate-man",
92	103	.root_source_file = b.path("src/manual/generate-man.zig"),
93		.target = native_target,
	104	.target = b.graph.host,
94	105	.optimize = .Debug,
	106	.link_libc = true,
95	107	});
96	108	manual_generator.root_module.addImport("build_options", native_debug.opts_mod);
97	109	manual_generator.root_module.addImport("core", native_debug.core);
98	110	const manual_run = b.addRunArtifact(manual_generator);
99		manual_run.addFileArg(native_debug.compiler.getEmittedBin());
	111	manual_run.addFileArg(native_debug.cli(b, BuildConfig.minimal).getEmittedBin());
	112	switch (js_runtime) {
	113	.path => \|path\| manual_run.addFileArg(path),
	114	.string => \|str\| manual_run.addArg(str),
	115	}
100	116	manual_run.addFileArg(b.path("doc/language-manual.md"));
101	117	manual_run.addFileArg(b.path("src/manual/template.html"));
102	118	const manual_out = manual_run.addOutputFileArg("language-reference.html");
103	119	const install_manual = b.addInstallFile(manual_out, "doc/README.html");
104	120	b.step("manual", "build language reference").dependOn(&install_manual.step);
105		b.install_tls.step.dependOn(&install_manual.step);
106	121
107	122	// const wasm_api = b.addExecutable(.{
108	123	// .name = "compiler",
...	...	@@ -129,41 +144,43 @@ pub fn build(b: *std.Build) !void {
129	144	const win_version: OsVersion = .{ .windows = .win7 };
130	145
131	146	inline for (.{
132		b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl, .os_version_min = linux_version, .cpu_model = .baseline }),
133		b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .linux, .abi = .musl, .os_version_min = linux_version, .cpu_model = .baseline }),
134		b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .macos, .os_version_min = mac_version, .cpu_model = .baseline }),
135		b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .macos, .os_version_min = mac_version, .cpu_model = .baseline }),
136		b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .windows, .os_version_min = win_version, .abi = .gnu, .cpu_model = .baseline }),
137		b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .windows, .os_version_min = win_version, .abi = .gnu, .cpu_model = .baseline }),
	147	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl, .os_version_min = linux_version }),
	148	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .linux, .abi = .musl, .os_version_min = linux_version }),
	149	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .macos, .os_version_min = mac_version }),
	150	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .macos, .os_version_min = mac_version }),
	151	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .windows, .os_version_min = win_version, .abi = .gnu }),
	152	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .windows, .os_version_min = win_version, .abi = .gnu }),
138	153	}) \|cross_target\| {
139	154	const name = std.fmt.allocPrint(
140		b.allocator,
	155	b.graph.arena,
141	156	"{s}-{s}",
142	157	.{
143	158	@tagName(cross_target.result.cpu.arch),
144	159	@tagName(cross_target.result.os.tag),
145	160	},
146	161	) catch @panic("OOM");
	162	_ = name; // autofix
147	163
148	164	const artifacts = buildWithTarget(b, cross_target, .ReleaseSafe, .none, version, no_llvm);
	165	_ = artifacts; // autofix
149	166
150		inline for (.{
151		.{ artifacts.compiler, "compiler" },
152		// .{ artifacts.lsp, "lsp" },
153		}) \|entry\| {
154		const artifact, const bin_name = entry;
155		package.dependOn(&b.addInstallFile(
156		artifact.getEmittedBin(),
157		b.pathJoin(&.{
158		"package",
159		name,
160		std.mem.concat(b.allocator, u8, &.{
161		bin_name,
162		cross_target.result.exeFileExt(),
163		}) catch @panic("OOM"),
164		}),
165		).step);
166		}
	167	// inline for (.{
	168	// .{ artifacts.compiler, "compiler" },
	169	// // .{ artifacts.lsp, "lsp" },
	170	// }) \|entry\| {
	171	// const artifact, const bin_name = entry;
	172	// package.dependOn(&b.addInstallFile(
	173	// artifact.getEmittedBin(),
	174	// b.pathJoin(&.{
	175	// "package",
	176	// name,
	177	// std.mem.concat(b.graph.arena, u8, &.{
	178	// bin_name,
	179	// cross_target.result.exeFileExt(),
	180	// }) catch @panic("OOM"),
	181	// }),
	182	// ).step);
	183	// }
167	184
168	185	package.dependOn(&b.addInstallFile(b.path("CHANGELOG.md"), "package/CHANGELOG.md").step);
169	186	package.dependOn(&b.addInstallFile(manual_out, "package/README.html").step);
...	...	@@ -173,11 +190,48 @@ pub fn build(b: *std.Build) !void {
173	190	const TracingMode = enum { none, spall, chrome, log };
174	191
175	192	const Artifacts = struct {
	193	no_llvm: bool,
	194	target: Build.ResolvedTarget,
	195	optimize: std.builtin.OptimizeMode,
	196
176	197	core: *Build.Module,
177	198	core_tests: *Build.Step.Compile,
178		compiler: *Build.Step.Compile,
179		lsp: *Build.Step.Compile,
180	199	opts_mod: *Build.Module,
	200	behavior_tests: *Build.Step.Compile,
	201
	202	/// Construct a CLI with certain features.
	203	/// When developing on a small component, such as the compiler core, you
	204	/// don't want to compile every part of the compiler. This returns a CLI
	205	/// compilation configured with only the features that are desired.
	206	fn cli(a: Artifacts, b: Build, config: BuildConfig) Build.Step.Compile {
	207	const dep_opts = .{ .target = a.target, .optimize = a.optimize };
	208	const use_llvm = if (a.no_llvm) false else null;
	209	const use_lld = if (a.no_llvm) false else null;
	210
	211	const tracer = b.dependency("tracer", dep_opts);
	212
	213	// Options
	214	const build_config = b.addOptions();
	215	build_config.addOption(BuildConfig, "config", config);
	216	const build_config_mod = build_config.createModule();
	217
	218	const exe = b.addExecutable(.{
	219	.name = "compiler",
	220	.root_source_file = b.path("src/cli.zig"),
	221	.target = a.target,
	222	.optimize = a.optimize,
	223	.use_lld = use_lld,
	224	.use_llvm = use_llvm,
	225	.strip = false,
	226	.omit_frame_pointer = false,
	227	.single_threaded = true,
	228	});
	229	exe.root_module.addImport("build_options", a.opts_mod);
	230	exe.root_module.addImport("build_config", build_config_mod);
	231	exe.root_module.addImport("core", a.core);
	232	exe.root_module.addImport("tracer", tracer.module("tracer"));
	233	return exe;
	234	}
181	235	};
182	236
183	237	fn buildWithTarget(
...	...	@@ -226,43 +280,39 @@ fn buildWithTarget(
226	280	core_tests.root_module.addImport("tracer", tracer.module("tracer"));
227	281	core_tests.root_module.addImport("zg:PropsData", zg.module("PropsData"));
228	282
229		// Compiler Demo Exe
230		const compiler_exe = b.addExecutable(.{
231		.name = "compiler",
232		.root_source_file = b.path("src/core/main.zig"),
	283	const behavior_tests = b.addExecutable(.{
	284	.name = "run_behavior_tests",
	285	.root_source_file = b.path("test/behavior.zig"),
233	286	.target = target,
234	287	.optimize = optimize,
235	288	.use_lld = use_lld,
236	289	.use_llvm = use_llvm,
237		.strip = false,
238		.omit_frame_pointer = false,
239		.single_threaded = true,
240	290	});
241		compiler_exe.root_module.addImport("build_options", opts_mod);
242		compiler_exe.root_module.addImport("core", core);
243		compiler_exe.root_module.addImport("tracer", tracer.module("tracer"));
244
245		// Language Server
246		const lsp_exe = b.addExecutable(.{
247		.name = "lsp",
248		.root_source_file = b.path("src/lsp/lsp.zig"),
	291	behavior_tests.root_module.addImport("core", core);
	292	behavior_tests.root_module.addImport("build_options", opts_mod);
	293	behavior_tests.root_module.addImport("tracer", tracer.module("tracer"));
	294	behavior_tests.root_module.addImport("quickjs", b.dependency("quickjs", dep_opts).module("quickjs"));
	295
	296	return .{
	297	.no_llvm = no_llvm,
249	298	.target = target,
250	299	.optimize = optimize,
251		.use_lld = use_lld,
252		.use_llvm = use_llvm,
253		.strip = false,
254		.omit_frame_pointer = false,
255		.unwind_tables = false,
256		});
257		lsp_exe.root_module.addImport("build_options", opts_mod);
258		lsp_exe.root_module.addImport("core", core);
259		lsp_exe.root_module.addImport("tracer", tracer.module("tracer"));
260	300
261		return .{
262	301	.core = core,
263	302	.core_tests = core_tests,
264		.compiler = compiler_exe,
265		.lsp = lsp_exe,
266	303	.opts_mod = opts_mod,
	304	.behavior_tests = behavior_tests,
267	305	};
268	306	}
	307
	308	pub const PathOrString = union(enum) {
	309	path: LazyPath,
	310	string: []const u8,
	311	};
	312
	313	const std = @import("std");
	314	const Build = std.Build;
	315	const LazyPath = Build.LazyPath;
	316	const Target = std.Target;
	317	const OsVersion = Target.Query.OsVersion;
	318	const BuildConfig = @import("src/BuildConfig.zig");

build.zig.zon+6

...	...	@@ -13,10 +13,16 @@
13	13	.url = "https://github.com/paperdave/zig-tracer/archive/d4c2eff040e9e69b9f08d2175b315f3055ad7ba3.tar.gz",
14	14	.hash = "12202d60766a46ceb4074ce5651a58a11f2f6b32927476a55c6889805073e8e3d261",
15	15	},
	16	.quickjs = .{
	17	.path = "src/deps/quickjs",
	18	},
16	19	},
	20
17	21	.paths = .{
18	22	"build.zig",
19	23	"build.zig.zon",
	24	"doc",
20	25	"src",
	26	"test",
21	27	},
22	28	}

src/BuildConfig.zig created+9

...	...	@@ -0,0 +1,9 @@
	1	have_embedded_quickjs: bool = false,
	2
	3	pub const minimal: BuildConfig = .{};
	4
	5	pub const all: BuildConfig = .{
	6	.have_embedded_quickjs = true,
	7	};
	8
	9	const BuildConfig = @This();

src/cli.zig+182-4

...	...	@@ -1,6 +1,184 @@
1	1	//! This is the `lang` command line interface. It combines all components
2		//! of the software development kit into one. For development, it may
3		//! be more reasonable to hack on individual components, which each
4		//! expose their own CLIs
	2	//! of the software development kit into one. Multiple versions of this
	3	//! file are built with different subsets of features, allowing you to,
	4	//! for example, test the `core` module's compilation features without
	5	//! needing to compile the formatter.
5	6
6		pub fn main() !void {}
	7	const have_debug_features = builtin.mode == .Debug;
	8
	9	pub fn main() !u8 {
	10	try tracer.init();
	11	defer tracer.deinit();
	12	try tracer.init_thread(null);
	13	defer tracer.deinit_thread();
	14
	15	// unicode props is asserted to be initialized before core.Tokenizer is used
	16	// this will result in a static page allocation, which is never cleaned up.
	17	try core.unicode_props.init();
	18
	19	const use_gpa = builtin.mode == .Debug;
	20	var gpa = if (use_gpa)
	21	std.heap.GeneralPurposeAllocator(.{
	22	.stack_trace_frames = 100,
	23	}){}
	24	else
	25	std.heap.ArenaAllocator.init(std.heap.page_allocator);
	26	defer _ = if (use_gpa) gpa.deinit();
	27	const alloc = gpa.allocator();
	28
	29	// Current working directory is required for CLI resolution.
	30	var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
	31	const cwd = try std.posix.getcwd(&cwd_buf);
	32
	33	// Command line argument parsing.
	34	var maybe_file_path: ?[]const u8 = null;
	35	defer if (maybe_file_path) \|path\| alloc.free(path);
	36	var exe_name: ?[]const u8 = null;
	37	defer if (exe_name) \|path\| alloc.free(path);
	38	var dump_ir = false;
	39	var dump_state: ?[]const u8 = null;
	40	var has_errors = false;
	41	{
	42	var args = try std.process.argsWithAllocator(alloc);
	43	defer args.deinit();
	44
	45	exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
	46
	47	while (args.next()) \|arg\| {
	48	if (std.mem.startsWith(u8, arg, "-")) {
	49	if (have_debug_features and std.mem.eql(u8, arg, "--report-trace")) {
	50	core.Report.debug_show_report_trace = true;
	51	continue;
	52	}
	53
	54	if (std.mem.eql(u8, arg, "--dump-ir")) {
	55	dump_ir = true;
	56	continue;
	57	}
	58
	59	if (std.mem.startsWith(u8, arg, "--dump-state=")) {
	60	dump_state = arg["--dump-state=".len..];
	61	continue;
	62	}
	63
	64	std.debug.print("error: unknown flag '{s}'\n", .{arg});
	65	has_errors = true;
	66	break;
	67	}
	68
	69	if (maybe_file_path == null) {
	70	maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
	71	} else {
	72	std.debug.print("error: multiple files specified\n", .{});
	73	maybe_file_path = null;
	74	break;
	75	}
	76	}
	77	}
	78
	79	if (maybe_file_path == null) has_errors = true;
	80	if (has_errors) {
	81	std.debug.print("Usage: {s} [flags] <input file>\n", .{exe_name.?});
	82	{
	83	std.debug.print(
	84	// \| 60 chars wrap point\|
	85	\\
	86	\\Flags:
	87	\\ --dump-ir Dump intermediate representations
	88	\\ --dump-state=<path> Dump compiler state to JSON
	89	\\
	90	, .{});
	91	}
	92	if (builtin.mode == .Debug) {
	93	std.debug.print(
	94	\\
	95	\\Debug Mode Flags:
	96	\\ --report-trace Print stack traces per error
	97	\\
	98	, .{});
	99	}
	100	return 1;
	101	}
	102	const file_path = maybe_file_path.?;
	103
	104	// Execute compiler
	105	{
	106	var js_backend = core.Js{};
	107
	108	var comp: core.Comp = .{
	109	.gpa = tracer.tracingAllocator(&alloc),
	110	.options = .{
	111	.dump_ir = dump_ir,
	112	},
	113	.backend = .{ .js = &js_backend },
	114	.out = core.OutStream.init(std.io.getStdOut().writer().any()),
	115	};
	116	try comp.init();
	117	defer comp.deinit();
	118
	119	defer if (dump_state) \|path\| {
	120	dumpState(&comp, path) catch \|err\| {
	121	std.debug.print("Failed to dump compilation state: {}", .{err});
	122	};
	123	};
	124
	125	const main_file = try comp.getFile(file_path);
	126
	127	var fail_report: Report = undefined;
	128	const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
	129	error.TransformFail => {
	130	std.debug.print("Transform failed:\n", .{});
	131	try fail_report.dumpToStdErr(&comp);
	132	return 1;
	133	},
	134	error.ParseFail => {
	135	// Already printed.
	136	return 1;
	137	},
	138	else => return err,
	139	};
	140
	141	if (comp.options.dump_ir) {
	142	const stderr = std.io.getStdErr().writer();
	143	try teg.dumpToWriter(stderr, null);
	144	}
	145
	146	const entry_point = comp.semaEntryPoint(main_file) catch \|err\| switch (err) {
	147	error.AnalysisFail => std.process.exit(1),
	148	else => return err,
	149	};
	150
	151	const output = std.io.getStdOut().writer();
	152	var bufw = std.io.bufferedWriter(output);
	153	const writer = bufw.writer();
	154	try js_backend.link(&comp, entry_point, writer.any());
	155	try bufw.flush();
	156	}
	157
	158	std.process.cleanExit();
	159	return 0;
	160	}
	161
	162	const core = @import("core");
	163	const Comp = core.Comp;
	164	const Report = core.Report;
	165
	166	const tracer = @import("tracer");
	167	pub const tracer_impl = @field(tracer, @import("build_options").tracing);
	168
	169	const std = @import("std");
	170	const builtin = @import("builtin");
	171
	172	fn dumpState(comp: *const Comp, dump_path: []const u8) !void {
	173	const file = try std.fs.cwd().createFile(dump_path, .{});
	174	defer file.close();
	175
	176	var bufw = std.io.bufferedWriter(file.writer());
	177	var jw = std.json.writeStream(bufw.writer(), .{
	178	.whitespace = .indent_2,
	179	});
	180
	181	try comp.jsonStringify(&jw);
	182	try bufw.writer().writeAll("\n");
	183	try bufw.flush();
	184	}

src/core/Air.Dump.zig+2-1

...	...	@@ -1,6 +1,7 @@
1	1	//! This implements a debug formatter for Teg
2	2	air: Air,
3	3	ip: *const InternPool,
	4	tty: std.io.tty.Config,
4	5
5	6	pub const State = struct {
6	7	arena: std.mem.Allocator,
...	...	@@ -423,7 +424,7 @@ pub fn format(
423	424	var state = State{
424	425	.air = &air,
425	426	.ip = dump.ip,
426		.tty = core.detectTTYFromWriter(writer),
	427	.tty = dump.tty,
427	428	.arena = arena.allocator(),
428	429	};
429	430	try state.dumpInst(writer, .main_block);

src/core/Air.zig+4-4

...	...	@@ -396,8 +396,8 @@ pub fn paramCount(air: Air) u32 {
396	396	return @intCast(air.insts.len);
397	397	}
398	398
399		pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
400		try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
	399	pub fn dumpToWriter(air: Air, ip: *InternPool, out: core.OutStream) !void {
	400	try out.print("{}", .{Dump{ .air = air, .ip = ip, .tty = out.tty }});
401	401	}
402	402
403	403	pub const Dump = @import("Air.Dump.zig");
...	...	@@ -407,11 +407,11 @@ const std = @import("std");
407	407	const mem = std.mem;
408	408	const assert = std.debug.assert;
409	409
410		const InternPool = @import("InternPool.zig");
	410	const core = @import("mod.zig");
	411	const InternPool = core.InternPool;
411	412	const Value = InternPool.Value;
412	413	const Type = InternPool.Type;
413	414
414		const Teg = @import("Teg.zig");
415	415	const dod = @import("array.zig");
416	416
417	417	const Unused = enum(u32) { zero = 0 };

src/core/Comp.zig+6-1

...	...	@@ -11,10 +11,15 @@ intern_pool: InternPool = .{},
11	11	/// TODO: turn this into a HashMap(*File, void) as File owns it's key
12	12	files: std.StringHashMapUnmanaged(*File) = .{},
13	13
	14	out: core.OutStream,
	15
14	16	options: Options = .{},
15	17
16	18	backend: Backend,
17	19
	20	// TODO: remove when it is possible to return `Report`s nicely
	21	temp_flag_had_errors: bool = false,
	22
18	23	pub fn init(comp: *Comp) !void {
19	24	try comp.intern_pool.init(comp.gpa);
20	25	}
...	...	@@ -37,7 +42,7 @@ pub fn semaEntryPoint(comp: Comp, file: File) !Decl.Index {
37	42
38	43	// TODO, move this elsewhere. we should print all errors and warnings at once
39	44	if (teg.report.warnings.len > 0) {
40		try teg.report.dumpToStdErr(comp);
	45	try teg.report.dumpToWriter(comp, comp.out.writer, comp.out.tty);
41	46	}
42	47
43	48	const ns = try comp.indexTegModule(file, .main_module_inst);

src/core/InternPool.zig+12

...	...	@@ -245,6 +245,10 @@ pub const Value = enum(u32) {
245	245	return ip.valueToKey(value);
246	246	}
247	247
	248	pub fn assertValid(value: Value, ip: *const InternPool) void {
	249	assert(@intFromEnum(value) < ip.items.len);
	250	}
	251
248	252	pub fn dump(value: Value, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) {
249	253	return .{ .data = .{ .val = value, .ip = ip } };
250	254	}
...	...	@@ -375,6 +379,10 @@ pub const Type = enum(u32) {
375	379	return @enumFromInt(@intFromEnum(ty));
376	380	}
377	381
	382	pub fn assertValid(ty: Type, ip: *const InternPool) void {
	383	ty.value().assertValid(ip);
	384	}
	385
378	386	pub fn tag(ty: Type, ip: *const InternPool) Type.Tag {
379	387	if (ty == .void) return .simple_type;
380	388	return @enumFromInt(@intFromEnum(ty.value().valueTag(ip)));
...	...	@@ -1256,6 +1264,8 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value {
1256	1264	break :brk Item.encode(.func, extra);
1257	1265	},
1258	1266	.function_type => \|f\| brk: {
	1267	for (f.param_types) \|ty\| ty.assertValid(ip);
	1268
1259	1269	const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
1260	1270	.params_len = @intCast(f.param_types.len),
1261	1271	.return_type = f.return_type,
...	...	@@ -1265,6 +1275,8 @@ pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value {
1265	1275	break :brk Item.encode(.function_type, extra);
1266	1276	},
1267	1277	.struct_type => \|s\| brk: {
	1278	for (s.field_types) \|ty\| ty.assertValid(ip);
	1279
1268	1280	assert(s.field_keys.len == s.field_types.len);
1269	1281	const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.StructType, .{
1270	1282	.fields_len = @intCast(s.field_keys.len),

src/core/Sema.zig+14-6

...	...	@@ -63,6 +63,7 @@ pub const Error = error{
63	63	// TODO: remove this from the primary error set
64	64	HitNoReturn,
65	65
	66	OutStream,
66	67	OutOfMemory,
67	68	} \|\| Comp.File.GetTegError;
68	69
...	...	@@ -123,6 +124,9 @@ pub fn init(s: *Sema) !void {
123	124	const t = tracer.trace(@src());
124	125	defer t.end();
125	126
	127	assert(s.ip.items.len > 0); // did not initialize intern pool
	128	_ = s.ip.items.bytes[0]; // segfault if intern pool is undefined memory
	129
126	130	s.report = .{
127	131	.gpa = s.gpa,
128	132	.current_file = s.ip.namespacePtr(s.namespace_index).file_scope,
...	...	@@ -140,7 +144,10 @@ pub fn deinit(s: *Sema) void {
140	144	defer t.end();
141	145
142	146	const report = s.report.finalize() catch @panic("OOM");
143		report.dumpToStdErr(s.comp) catch @panic("Error");
	147	if ((report.errors.len + report.warnings.len) > 0) {
	148	s.comp.temp_flag_had_errors = true;
	149	}
	150	report.dumpToWriter(s.comp, s.comp.out.writer, s.comp.out.tty) catch @panic("Error");
144	151	report.deinit(s.gpa);
145	152
146	153	defer s.* = undefined;
...	...	@@ -158,9 +165,9 @@ pub fn finalize(s: *Sema) !void {
158	165
159	166	if (s.decl.value().valueTag(s.ip) == .func) {
160	167	if (s.comp.options.dump_ir) {
161		std.debug.print("=" ** 20, .{});
162		std.debug.print("\nAir for {}\n", .{s.decl.value().dump(s.ip)});
163		air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
	168	try s.comp.out.print("=" ** 20, .{});
	169	try s.comp.out.print("\nAir for {}\n", .{s.decl.value().dump(s.ip)});
	170	try air.dumpToWriter(s.ip, s.comp.out);
164	171	}
165	172
166	173	try s.comp.codeGenDecl(s.decl.value(), &air);
...	...	@@ -215,7 +222,7 @@ pub fn analyzeFunctionDeclaration(s: *Sema) Error!Value {
215	222	s.params = try s.gpa.alloc(Type, fn_decl.params.len);
216	223	defer s.gpa.free(s.params);
217	224
218		for (fn_decl.params.slice(&s.code), 0..) \|param, i\| {
	225	for (fn_decl.params.slice(&s.code), s.params, 0..) \|param, *dest, i\| {
219	226	const src = s.inst_map.start.op(operandArgument(@intCast(i)));
220	227	const ty = try s.resolveType(src, param);
221	228	if (!ty.hasPossibleRepresentation(s.ip)) {
...	...	@@ -224,7 +231,7 @@ pub fn analyzeFunctionDeclaration(s: *Sema) Error!Value {
224	231	.type = ty,
225	232	});
226	233	}
227		s.params[i] = ty;
	234	dest.* = ty;
228	235	}
229	236
230	237	var return_type, const early_value = if (fn_decl.return_type.unwrap()) \|ty\| brk: {
...	...	@@ -1462,6 +1469,7 @@ fn binTypeResolution(s: *Sema, src: Teg.Inst.Index, op: BinaryOp, lhs: Type, rhs
1462	1469	.float => switch (rhs) {
1463	1470	.int32 => return .float,
1464	1471	.numeric_literal => return .float,
	1472	.float => return .float,
1465	1473	else => {},
1466	1474	},
1467	1475	else => {},

src/core/StoredTrace.zig+2-2

...	...	@@ -5,7 +5,7 @@
5	5	data: [31]usize,
6	6	index: usize,
7	7
8		pub const empty = .{
	8	pub const empty: StoredTrace = .{
9	9	.data = .{0} ** 31,
10	10	.index = 0,
11	11	};
...	...	@@ -29,7 +29,7 @@ pub fn from(stack_trace: ?*std.builtin.StackTrace) StoredTrace {
29	29	if (stack_trace) \|stack\| {
30	30	var data: [31]usize = undefined;
31	31	@memset(&data, 0);
32		const items = @min(stack.instruction_addresses.len, 32);
	32	const items = @min(stack.instruction_addresses.len, 31);
33	33	@memcpy(data[0..items], stack.instruction_addresses[0..items]);
34	34	return .{
35	35	.data = data,

src/core/Teg.Dump.zig+2-1

...	...	@@ -1,5 +1,6 @@
1	1	//! This implements a debug formatter for Teg
2	2	teg: Teg,
	3	tty: ?std.io.tty.Config = null,
3	4
4	5	const State = struct {
5	6	arena: std.mem.Allocator,
...	...	@@ -410,7 +411,7 @@ pub fn format(
410	411	const teg = dump.teg;
411	412	var state = State{
412	413	.teg = &teg,
413		.tty = detectTTYFromWriter(writer),
	414	.tty = dump.tty orelse detectTTYFromWriter(writer),
414	415	.arena = arena.allocator(),
415	416	};
416	417	try state.dumpInst(writer, .main_module_inst);

src/core/Teg.zig+2-2

...	...	@@ -619,8 +619,8 @@ pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
619	619	return decl_data.name.slice(teg.string_bytes);
620	620	}
621	621
622		pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
623		try writer.print("{}", .{Dump{ .teg = teg }});
	622	pub fn dumpToWriter(teg: Teg, writer: anytype, tty: ?std.io.tty.Config) !void {
	623	try writer.print("{}", .{Dump{ .teg = teg, .tty = tty }});
624	624	}
625	625
626	626	const Teg = @This();

src/core/js/NodeGen.zig+7-1

...	...	@@ -478,7 +478,13 @@ fn fetchInstExpression(c: *NodeGen, index: Air.Inst.Index) !Node.Index {
478	478	});
479	479	} else unreachable,
480	480	.single => {},
481		.multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
	481	.multi => {
	482	// multi-references can only duplicate identifier nodes.
	483	assert(switch (c.nodes.items(.tag)[@intFromEnum(data.node)]) {
	484	.local, .global => true,
	485	else => false,
	486	});
	487	},
482	488	}
483	489	}
484	490	return data.node;

src/core/main.zig deleted-170

...	...	@@ -1,170 +0,0 @@
1		//! This file is used to test the compiler implementation without
2		//! needing to build the language server and other tooling.
3		const std = @import("std");
4		const builtin = @import("builtin");
5		const core = @import("core");
6
7		const Tokenizer = core.Tokenizer;
8		const Parse = core.Parse;
9		const Transform = core.Transform;
10		const Comp = core.Comp;
11		const unicode_props = core.unicode_props;
12		pub const Report = core.Report;
13
14		const tracer = @import("tracer");
15		pub const tracer_impl = @field(tracer, @import("build_options").tracing);
16
17		pub fn main() !void {
18		try tracer.init();
19		defer tracer.deinit();
20		try tracer.init_thread(null);
21		defer tracer.deinit_thread();
22
23		try unicode_props.init();
24
25		const use_gpa = builtin.mode == .Debug;
26		var gpa = if (use_gpa)
27		std.heap.GeneralPurposeAllocator(.{
28		.stack_trace_frames = 100,
29		}){}
30		else
31		std.heap.ArenaAllocator.init(std.heap.page_allocator);
32		defer _ = if (use_gpa) gpa.deinit();
33		const alloc = gpa.allocator();
34
35		var maybe_file_path: ?[]const u8 = null;
36		defer if (maybe_file_path) \|path\| alloc.free(path);
37
38		var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
39		const cwd = try std.posix.getcwd(&cwd_buf);
40
41		var exe_name: []const u8 = undefined;
42		var dump_ir = false;
43		var dump_state: ?[]const u8 = null;
44
45		{
46		var args = try std.process.argsWithAllocator(alloc);
47		defer args.deinit();
48
49		exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
50		errdefer alloc.free(exe_name);
51
52		while (args.next()) \|arg\| {
53		if (std.mem.startsWith(u8, arg, "-")) {
54		if (builtin.mode == .Debug and std.mem.eql(u8, arg, "--report-trace")) {
55		Report.debug_show_report_trace = true;
56		continue;
57		}
58		if (std.mem.eql(u8, arg, "--dump-ir")) {
59		dump_ir = true;
60		continue;
61		}
62		if (std.mem.startsWith(u8, arg, "--dump-state=")) {
63		dump_state = arg["--dump-state=".len..];
64		continue;
65		}
66		std.debug.print("error: unknown flag '{s}'\n", .{arg});
67		maybe_file_path = null;
68		break;
69		}
70
71		if (maybe_file_path == null) {
72		maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
73		} else {
74		std.debug.print("error: multiple files specified\n", .{});
75		maybe_file_path = null;
76		break;
77		}
78		}
79		}
80		defer alloc.free(exe_name);
81
82		const file_path = maybe_file_path orelse {
83		std.debug.print("Usage: {s} [flags] <input file>\n", .{exe_name});
84		{
85		std.debug.print(
86		// \| 60 chars wrap point\|
87		\\
88		\\Flags:
89		\\ --dump-ir Dump intermediate representations
90		\\ --dump-state=<path> Dump compiler state to JSON
91		\\
92		, .{});
93		}
94		if (builtin.mode == .Debug) {
95		std.debug.print(
96		\\
97		\\Debug Mode Flags:
98		\\ --report-trace Print stack traces per error
99		\\
100		, .{});
101		}
102		std.process.exit(1);
103		};
104
105		var js_backend = core.Js{};
106
107		var comp: Comp = .{
108		.gpa = tracer.tracingAllocator(&alloc),
109		.options = .{
110		.dump_ir = dump_ir,
111		},
112		.backend = .{ .js = &js_backend },
113		};
114		try comp.init();
115		defer comp.deinit();
116
117		defer if (dump_state) \|path\| {
118		dumpState(&comp, path) catch \|err\| {
119		std.debug.print("Failed to dump compilation state: {}", .{err});
120		};
121		};
122
123		const main_file = try comp.getFile(file_path);
124
125		var fail_report: Report = undefined;
126		const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
127		error.TransformFail => {
128		std.debug.print("Transform failed:\n", .{});
129		try fail_report.dumpToStdErr(&comp);
130		std.process.exit(1);
131		},
132		error.ParseFail => {
133		// Already printed.
134		std.process.exit(1);
135		},
136		else => return err,
137		};
138
139		if (comp.options.dump_ir) {
140		const stderr = std.io.getStdErr().writer();
141		try teg.dumpToWriter(stderr);
142		}
143
144		const entry_point = comp.semaEntryPoint(main_file) catch \|err\| switch (err) {
145		error.AnalysisFail => std.process.exit(1),
146		else => return err,
147		};
148
149		const output = std.io.getStdOut().writer();
150		var bufw = std.io.bufferedWriter(output);
151		const writer = bufw.writer();
152		try js_backend.link(&comp, entry_point, writer.any());
153		try bufw.flush();
154
155		std.process.cleanExit();
156		}
157
158		fn dumpState(comp: *const Comp, dump_path: []const u8) !void {
159		const file = try std.fs.cwd().createFile(dump_path, .{});
160		defer file.close();
161
162		var bufw = std.io.bufferedWriter(file.writer());
163		var jw = std.json.writeStream(bufw.writer(), .{
164		.whitespace = .indent_2,
165		});
166
167		try comp.jsonStringify(&jw);
168		try bufw.writer().writeAll("\n");
169		try bufw.flush();
170		}

src/core/mod.zig+35

...	...	@@ -98,6 +98,7 @@ pub fn detectTTYFromWriter(writer: std.io.AnyWriter) std.io.tty.Config {
98	98	const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
99	99	return std.io.tty.detectConfig(file.*);
100	100	}
	101
101	102	return .no_color;
102	103	}
103	104
...	...	@@ -114,6 +115,40 @@ pub fn jsonWriterCast(json_writer: anytype) *std.json.WriteStream(
114	115	return json_writer;
115	116	}
116	117
	118	/// A writer with TTY information. This abstraction is used so that test runners and wasm
	119	/// builds can specify an output writer that supports TTY, even if the underlying writer
	120	/// is an ArrayList writer (detectTTYFromWriter would report .no_color)
	121	pub const OutStream = struct {
	122	writer: std.io.AnyWriter,
	123	tty: std.io.tty.Config,
	124
	125	/// To make it possible to have reasonable error sets, all logging
	126	/// IO-related errors will be categorized as error.OutStream. We preserve
	127	/// OutOfMemory so that it is still detectable.
	128	pub const Error = error{ OutStream, OutOfMemory };
	129
	130	pub fn init(writer: std.io.AnyWriter) OutStream {
	131	return .{
	132	.writer = writer,
	133	.tty = detectTTYFromWriter(writer),
	134	};
	135	}
	136
	137	pub fn print(out: OutStream, comptime fmt: []const u8, args: anytype) Error!void {
	138	out.writer.print(fmt, args) catch \|e\| switch (e) {
	139	error.OutOfMemory => return Error.OutOfMemory,
	140	else => return Error.OutStream,
	141	};
	142	}
	143
	144	pub fn setColor(out: OutStream, color: std.io.tty.Color) !void {
	145	out.tty.setColor(out.writer, color) catch \|e\| switch (e) {
	146	error.OutOfMemory => return Error.OutOfMemory,
	147	else => return Error.OutStream,
	148	};
	149	}
	150	};
	151
117	152	const std = @import("std");
118	153	const assert = std.debug.assert;
119	154

src/deps/quickjs/build.zig created+128

...	...	@@ -0,0 +1,128 @@
	1	pub fn build(b: *Build) !void {
	2	const target = b.standardTargetOptions(.{});
	3	const optimize = b.standardOptimizeOption(.{});
	4
	5	const src = b.dependency("quickjs", .{});
	6
	7	const version = std.mem.trim(
	8	u8,
	9	try src.builder.build_root.handle.readFileAlloc(b.graph.arena, "VERSION", 4096),
	10	" \t\r\n",
	11	);
	12
	13	const c_flags: []const []const u8 = &.{
	14	"-Wall",
	15	"-Wextra",
	16	"-Wno-sign-compare",
	17	"-Wno-missing-field-initializers",
	18	"-Wundef",
	19	"-Wuninitialized",
	20	"-Wunused",
	21	"-Wno-unused-parameter",
	22	"-Wwrite-strings",
	23	"-Wchar-subscripts",
	24	"-funsigned-char",
	25	"-fwrapv",
	26	};
	27
	28	const defines: []const struct { name: []const u8, value: []const u8 } = &.{
	29	.{ .name = "_GNU_SOURCE", .value = "1" },
	30	.{ .name = "CONFIG_BIGNUM", .value = "1" },
	31	.{ .name = "CONFIG_VERSION", .value = b.fmt("\"{s}\"", .{version}) },
	32	};
	33
	34	const libquickjs = b.addStaticLibrary(.{
	35	.name = "quickjs",
	36	.target = target,
	37	.optimize = optimize,
	38	});
	39
	40	libquickjs.addCSourceFiles(.{
	41	.root = src.path(""),
	42	.files = &.{
	43	"quickjs.c",
	44	"libregexp.c",
	45	"libbf.c",
	46	"libunicode.c",
	47	"cutils.c",
	48	},
	49	.flags = c_flags,
	50	});
	51	for (defines) \|def\| {
	52	libquickjs.root_module.addCMacro(def.name, def.value);
	53	}
	54
	55	const qjsc = b.addExecutable(.{
	56	.name = "qjsc",
	57	.target = b.graph.host,
	58	.optimize = .Debug,
	59	});
	60	qjsc.addCSourceFiles(.{
	61	.root = src.path(""),
	62	.files = &.{
	63	"qjsc.c",
	64	"quickjs-libc.c",
	65	"quickjs.c",
	66	"libregexp.c",
	67	"libbf.c",
	68	"libunicode.c",
	69	"cutils.c",
	70	},
	71	.flags = c_flags,
	72	});
	73	for (defines) \|def\| {
	74	qjsc.root_module.addCMacro(def.name, def.value);
	75	}
	76
	77	const qjs = b.addExecutable(.{
	78	.name = "qjs",
	79	.target = target,
	80	.optimize = optimize,
	81	});
	82	qjs.addCSourceFiles(.{
	83	.root = src.path(""),
	84	.files = &.{
	85	"qjs.c",
	86	"quickjs-libc.c",
	87	},
	88	.flags = c_flags,
	89	});
	90	qjs.root_module.addCSourceFile(.{
	91	.file = runQjsc(b, qjsc, src.path("repl.js")),
	92	.flags = c_flags,
	93	});
	94	qjs.root_module.addCSourceFile(.{
	95	.file = runQjsc(b, qjsc, src.path("qjscalc.js")),
	96	.flags = c_flags,
	97	});
	98	for (defines) \|def\| {
	99	qjs.root_module.addCMacro(def.name, def.value);
	100	}
	101	qjs.linkLibrary(libquickjs);
	102
	103	b.installArtifact(qjs);
	104	b.installArtifact(libquickjs);
	105
	106	const translate = b.addTranslateC(.{
	107	.root_source_file = src.path("quickjs.h"),
	108	.target = target,
	109	.optimize = optimize,
	110	.link_libc = false,
	111	});
	112	const module = translate.addModule("quickjs");
	113	module.linkLibrary(libquickjs);
	114	}
	115
	116	fn runQjsc(b: Build, qjsc: Build.Step.Compile, in: Build.LazyPath) Build.LazyPath {
	117	const run = b.addRunArtifact(qjsc);
	118	run.addArg("-c");
	119	run.addArg("-fbignum");
	120	run.addArg("-o");
	121	const out = run.addOutputFileArg(b.fmt("{s}.c", .{in.getDisplayName()}));
	122	run.addArg("-m");
	123	run.addFileArg(in);
	124	return out;
	125	}
	126
	127	const std = @import("std");
	128	const Build = std.Build;

src/deps/quickjs/build.zig.zon created+16

...	...	@@ -0,0 +1,16 @@
	1	.{
	2	.name = "quickjs",
	3	.version = "0.0.0",
	4
	5	.dependencies = .{
	6	.quickjs = .{
	7	.url = "https://github.com/bellard/quickjs/archive/6e2e68fd0896957f92eb6c242a2e048c1ef3cae0.tar.gz",
	8	.hash = "1220d49e57fc131eebbf4b09fb100f5d802b909d2f36e15d2d849a72f354cd24489d",
	9	},
	10	},
	11
	12	.paths = .{
	13	"build.zig",
	14	"build.zig.zon",
	15	},
	16	}

src/lsp/lsp.zig deleted-41

...	...	@@ -1,41 +0,0 @@
1		const std = @import("std");
2		const builtin = @import("builtin");
3		const core = @import("core");
4		const tracer = @import("tracer");
5
6		pub const tracer_impl = @field(tracer, @import("build_options").tracing);
7
8		pub fn main() !void {
9		if (builtin.mode == .Debug) {
10		// zig build system doesnt print a newline :/
11		std.debug.print("\r\x1b[K", .{});
12		}
13
14		std.debug.print("All your codebase are belong to us\n", .{});
15
16		const use_gpa = builtin.mode == .Debug;
17		var gpa_state: if (use_gpa) std.heap.GeneralPurposeAllocator(.{
18		.stack_trace_frames = 100,
19		}) else struct {} = .{};
20		defer _ = if (use_gpa) gpa_state.deinit();
21		const gpa = if (use_gpa)
22		gpa_state.allocator()
23		else
24		std.heap.c_allocator;
25
26		// if `parse` returns `error.ParseFail`, this `err` is set.
27		//
28		// at some point later this will change to be a `Report`
29		// so that multiple parse errors may be reported at once.
30		var err: core.Parse.AstError.WithTrace = undefined;
31		var ast = try core.Parse.parse(gpa,
32		\\let x = 1;
33		\\
34		\\pub fn hello() -> float {
35		\\ 42.2
36		\\}
37		, &err);
38		defer ast.deinit(gpa);
39
40		try ast.dumpToWriter(std.io.getStdErr().writer());
41		}

src/manual/generate-man.zig+391-70

...	...	@@ -1,46 +1,73 @@
1	1	//! Language reference compiler. Converts markdown into HTML, using
2	2	//! the Tokenizer to highlight code, and the compiler and node.js to
3	3	//! show code results. Only ever built in Debug mode. We are not
4		//! smart about memory. Written at 3am on a Thursday. Enjoy
	4	//! smart about memory. Initially written at 3am on a Thursday. Enjoy
5	5	pub const std = @import("std");
6	6	pub const core = @import("core");
7	7
	8	const mem = std.mem;
	9	const Allocator = mem.Allocator;
8	10	const ArrayList = std.ArrayList;
	11	const ArrayListUnmanaged = std.ArrayListUnmanaged;
	12	const assert = std.debug.assert;
	13
	14	comptime {
	15	assert(std.debug.runtime_safety);
	16	}
9	17
10	18	pub fn main() !void {
11	19	comptime std.debug.assert(@import("builtin").mode == .Debug);
12		var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
13		const gpa = arena.allocator();
	20	var gpa_state: std.heap.GeneralPurposeAllocator(.{}) = .{};
	21	const gpa = gpa_state.allocator();
	22
	23	const progress = std.Progress.start(.{
	24	.root_name = "Build manual",
	25	.estimated_total_items = 0,
	26	});
14	27
15	28	var args = try std.process.argsWithAllocator(gpa);
16	29	_ = args.next().?;
17	30
18	31	const compiler_path = args.next().?;
	32	const runtime_path = args.next().?;
19	33	const markdown_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
20	34	const template_src = try std.fs.cwd().readFileAlloc(gpa, args.next().?, std.math.maxInt(usize));
21	35	const out_path = args.next().?;
22	36
23		var output = std.ArrayList(u8).init(gpa);
24		defer output.deinit();
	37	assert(args.next() == null);
	38
	39	var contents = ArrayList(u8).init(gpa);
	40	var header: HeaderWriter = .{};
	41
25	42	try processMarkdown(
26	43	gpa,
	44	progress,
27	45	markdown_src,
28	46	compiler_path,
29		output.writer(),
	47	runtime_path,
	48	&header,
	49	contents.writer(),
30	50	);
31	51
32		const applied_template = try std.mem.replaceOwned(
33		u8,
34		gpa,
35		template_src,
36		"{contents}",
37		output.items,
38		);
39		defer gpa.free(applied_template);
	52	try header.end(gpa);
	53
	54	const applied_template = try replaceInTemplate(gpa, template_src, .{
	55	.contents = contents.items,
	56	.navigation = header.data.items,
	57	.nav_h1 = header.h1.?,
	58	.repository = "https://github.com/paperdave/wl",
	59	});
40	60
41	61	const out_file = try std.fs.createFileAbsolute(out_path, .{});
42	62	defer out_file.close();
43	63	try out_file.writeAll(applied_template);
	64
	65	progress.end();
	66
	67	std.debug.lockStdErr();
	68	std.debug.unlockStdErr();
	69
	70	std.process.exit(0);
44	71	}
45	72
46	73	const BlockType = enum {
...	...	@@ -48,6 +75,9 @@ const BlockType = enum {
48	75	h1,
49	76	h2,
50	77	h3,
	78	h4,
	79	h5,
	80	h6,
51	81	code_run,
52	82	code_build,
53	83	code,
...	...	@@ -60,6 +90,7 @@ const BlockType = enum {
60	90	const InlineType = enum {
61	91	bold,
62	92	code,
	93	link,
63	94	};
64	95
65	96	const MarkdownTokenizer = struct {
...	...	@@ -67,7 +98,7 @@ const MarkdownTokenizer = struct {
67	98	str: []const u8,
68	99
69	100	fn skipWhitespace(t: *MarkdownTokenizer) void {
70		t.str = std.mem.trimLeft(u8, t.str, " \r\n");
	101	t.str = mem.trimLeft(u8, t.str, " \r\n");
71	102	}
72	103
73	104	fn advance(t: *MarkdownTokenizer, n: usize) []const u8 {
...	...	@@ -78,11 +109,11 @@ const MarkdownTokenizer = struct {
78	109	}
79	110
80	111	fn advanceLine(t: *MarkdownTokenizer) []const u8 {
81		return std.mem.trim(u8, t.advanceLineRaw(), " \r\n");
	112	return mem.trim(u8, t.advanceLineRaw(), " \r\n");
82	113	}
83	114
84	115	fn advanceLineRaw(t: *MarkdownTokenizer) []const u8 {
85		const n = std.mem.indexOfScalar(u8, t.str, '\n');
	116	const n = mem.indexOfScalar(u8, t.str, '\n');
86	117	const line = t.advance(if (n) \|x\| x + 1 else t.str.len);
87	118	return line;
88	119	}
...	...	@@ -98,6 +129,9 @@ const MarkdownTokenizer = struct {
98	129	if (t.str[1] != '#') return .h1;
99	130	if (t.str[2] != '#') return .h2;
100	131	if (t.str[3] != '#') return .h3;
	132	if (t.str[4] != '#') return .h4;
	133	if (t.str[5] != '#') return .h5;
	134	if (t.str[6] != '#') return .h6;
101	135	@panic("headers nested too deep");
102	136	},
103	137	'>' => {
...	...	@@ -108,13 +142,13 @@ const MarkdownTokenizer = struct {
108	142	return .table;
109	143	},
110	144	'`' => {
111		if (std.mem.startsWith(u8, t.str, "```")) {
	145	if (mem.startsWith(u8, t.str, "```")) {
112	146	defer _ = t.advanceLine();
113		if (std.mem.startsWith(u8, t.str, "```lang:run"))
	147	if (mem.startsWith(u8, t.str, "```lang:run"))
114	148	return .code_run;
115		if (std.mem.startsWith(u8, t.str, "```lang:build"))
	149	if (mem.startsWith(u8, t.str, "```lang:build"))
116	150	return .code_build;
117		if (std.mem.startsWith(u8, t.str, "```lang"))
	151	if (mem.startsWith(u8, t.str, "```lang"))
118	152	return .code;
119	153	return .code_generic;
120	154	}
...	...	@@ -130,7 +164,53 @@ const MarkdownTokenizer = struct {
130	164	}
131	165	};
132	166
133		fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []const u8, writer: anytype) !void {
	167	const HeaderWriter = struct {
	168	data: std.ArrayListUnmanaged(u8) = .{},
	169	level: u8 = 0,
	170	h1: ?[]const u8 = null,
	171
	172	pub fn add(header: *HeaderWriter, alloc: Allocator, name: []const u8, id: []const u8, level: u8) !void {
	173	const w = header.data.writer(alloc);
	174
	175	while (header.level > level) {
	176	header.level -= 1;
	177	try w.writeAll("</ul>");
	178	}
	179
	180	while (header.level < level) {
	181	if (header.level == 0) {
	182	try w.writeAll("<ul id=\"toc\">");
	183	} else {
	184	try w.writeAll("<li><ul>");
	185	}
	186	header.level += 1;
	187	}
	188
	189	try w.print("<li><a class=\"header-link\" href=\"#{s}\">{s}</a></li>", .{ id, name });
	190	}
	191
	192	pub fn end(header: *HeaderWriter, alloc: Allocator) !void {
	193	const w = header.data.writer(alloc);
	194	while (header.level > 1) {
	195	header.level -= 1;
	196	try w.writeAll("</ul></li>");
	197	}
	198	if (header.level == 1) {
	199	header.level -= 1;
	200	try w.writeAll("</ul>");
	201	}
	202	}
	203	};
	204
	205	fn processMarkdown(
	206	gpa: Allocator,
	207	progress: std.Progress.Node,
	208	src: []const u8,
	209	compiler_path: []const u8,
	210	runtime_path: []const u8,
	211	header: *HeaderWriter,
	212	writer: anytype,
	213	) !void {
134	214	var t = MarkdownTokenizer{ .str = src };
135	215
136	216	t.skipWhitespace();
...	...	@@ -157,13 +237,44 @@ fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []con
157	237	.h1,
158	238	.h2,
159	239	.h3,
	240	.h4,
	241	.h5,
	242	.h6,
160	243	=> \|tag\| {
161	244	const html_tag = @tagName(tag);
162	245	const name = t.advanceLine();
163		try writer.print("<{0s}>{1s}</{0s}>\n", .{
164		html_tag,
165		name,
166		});
	246	const id = try nameToHeaderId(gpa, name);
	247
	248	if (tag != .h1) {
	249	try writer.print("<{0s} id=\"{2s}\"><a href=\"#{2s}\">{1s}</a></{0s}>\n", .{
	250	html_tag,
	251	name,
	252	id,
	253	});
	254
	255	if (tag != .h1)
	256	try header.add(gpa, name, id, switch (tag) {
	257	.h2 => 1,
	258	.h3 => 2,
	259	.h4 => 3,
	260	.h5 => 4,
	261	.h6 => 5,
	262	else => unreachable,
	263	});
	264	} else {
	265	assert(header.h1 == null);
	266
	267	header.h1 = try std.fmt.allocPrint(
	268	gpa,
	269	"<a href=\"#top\">{s}</a>",
	270	.{name},
	271	);
	272
	273	try writer.print("<{0s}><a href=\"#top\">{1s}</a></{0s}>\n", .{
	274	html_tag,
	275	name,
	276	});
	277	}
167	278	},
168	279	.code_run,
169	280	.code_build,
...	...	@@ -173,50 +284,154 @@ fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []con
173	284	var code = ArrayList(u8).init(gpa);
174	285
175	286	var next_line = t.advanceLineRaw();
176		while (t.str.len > 0 and !std.mem.startsWith(u8, next_line, "```")) {
	287	while (t.str.len > 0 and !mem.startsWith(u8, next_line, "```")) {
177	288	try code.appendSlice(next_line);
178	289	next_line = t.advanceLineRaw();
179	290	}
180	291
	292	if (tag == .code_run or tag == .code_build or tag == .code) {
	293	try writer.writeAll("<figure><figcaption><cite>hello.wl</cite></figcaption>");
	294	} else {
	295	try writer.writeAll("<figure>");
	296	}
	297
181	298	try writer.writeAll("<pre><code>");
182		try writeTokenizedCode(gpa, code.items, writer);
	299	if (tag == .code_generic) {
	300	try writeHtmlEscapedSlice(code.items, writer, false);
	301	} else {
	302	try writeTokenizedCode(gpa, code.items, writer);
	303	}
183	304	try writer.writeAll("</code></pre>");
184	305
185	306	try code.append('\n');
186	307
187		switch (tag) {
188		.code_run,
189		.code_build,
190		=> {
191		var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
192		build.stdin_behavior = .Pipe;
193		build.stdout_behavior = .Pipe;
194		build.stderr_behavior = .Pipe;
195		var env_map = try std.process.getEnvMap(gpa);
196		defer env_map.deinit();
197		try env_map.put("YES_COLOR", "1");
198		build.env_map = &env_map;
199		try build.spawn();
200		try build.stdin.?.writeAll(code.items);
201		var out = ArrayList(u8).init(gpa);
202		var err = ArrayList(u8).init(gpa);
203		try build.collectOutput(&out, &err, std.math.maxInt(usize));
204		const result = try build.wait();
205		_ = result;
206		if (std.mem.trim(u8, err.items, " \n\r").len > 0) {
207		try writer.writeAll("<pre class='term'><code>");
208		try writeAnsi(gpa, err.items, writer);
209		try writer.writeAll("</code></pre>");
210		}
211		if (std.mem.trim(u8, out.items, " \n\r").len > 0) {
212		try writer.writeAll("<pre><code>");
213		try writer.writeAll("// Compiled JavaScript Code");
214		try writer.writeAll(out.items[std.mem.indexOfScalar(u8, out.items, '\n').?..]);
215		try writer.writeAll("</code></pre>");
	308	if (tag == .code_run or tag == .code_build) {
	309	var prog = progress.start("Code Snippet", if (tag == .code_run) 2 else 1);
	310	defer prog.end();
	311
	312	var prog_step = prog.start("Compile", 1);
	313	var build = std.process.Child.init(&.{ compiler_path, "/dev/stdin" }, gpa);
	314	build.stdin_behavior = .Pipe;
	315	build.stdout_behavior = .Pipe;
	316	build.stderr_behavior = .Pipe;
	317	var env_map = try std.process.getEnvMap(gpa);
	318	defer env_map.deinit();
	319	try env_map.put("YES_COLOR", "1");
	320	build.env_map = &env_map;
	321	try build.spawn();
	322	try build.stdin.?.writeAll(code.items);
	323	var out = ArrayList(u8).init(gpa);
	324	var err = ArrayList(u8).init(gpa);
	325	try build.collectOutput(&out, &err, std.math.maxInt(usize));
	326	const result = try build.wait();
	327	if (result != .Exited or result.Exited != 0) {
	328	try err.writer().print("[Process {s} with {d}]", .{
	329	switch (result) {
	330	.Exited => "exited",
	331	.Stopped => "stopped",
	332	.Signal => "signaled",
	333	.Unknown => "hit unknown state",
	334	},
	335	switch (result) {
	336	inline else => \|d\| @as(u32, d),
	337	},
	338	});
	339	}
	340	if (mem.trim(u8, err.items, " \n\r").len > 0) {
	341	try writer.writeAll("<pre class='term'><code>");
	342	try writeAnsi(gpa, err.items, writer);
	343	try writer.writeAll("</code></pre>");
	344	}
	345	const trimmed_out = mem.trim(u8, out.items, " \n\r");
	346	if (trimmed_out.len > 0) {
	347	try writer.writeAll("<pre><code>");
	348	try writer.writeAll("// Compiled JavaScript Code");
	349	try writer.writeAll(trimmed_out[mem.indexOfScalar(u8, trimmed_out, '\n').?..]);
	350	try writer.writeAll("</code></pre>");
	351	}
	352
	353	prog.completeOne();
	354	prog_step.end();
	355
	356	if (tag == .code_run and runtime_path.len > 0) {
	357	const run_step = prog.start("Execute", 0);
	358	defer run_step.end();
	359
	360	var temp_dir = std.testing.tmpDir(.{});
	361	defer temp_dir.cleanup();
	362
	363	const input_sub_path = "result.js";
	364	try temp_dir.dir.writeFile(.{
	365	.sub_path = input_sub_path,
	366	.data = trimmed_out,
	367	});
	368
	369	var env_map_run = try std.process.getEnvMap(gpa);
	370	defer env_map_run.deinit();
	371	try env_map_run.put("NO_COLOR", "1");
	372	try env_map_run.put("TERM", "dumb");
	373
	374	var run_out = ArrayList(u8).init(gpa);
	375	var run_err = ArrayList(u8).init(gpa);
	376
	377	// Different JavaScript runtimes have different quirks with
	378	// regards to CLI parsing and standard input streams.
	379	const run_sub_command = std.mem.indexOf(u8, runtime_path, "deno") != null or
	380	std.mem.indexOf(u8, runtime_path, "porf") != null;
	381
	382	const cmd: []const []const u8 = if (run_sub_command)
	383	&.{ runtime_path, "run", input_sub_path }
	384	else
	385	&.{ runtime_path, input_sub_path };
	386
	387	var run = std.process.Child.init(cmd, gpa);
	388
	389	run.cwd_dir = temp_dir.dir;
	390	var path_buf: [std.fs.max_path_bytes]u8 = undefined;
	391	if (@import("builtin").os.tag == .windows)
	392	run.cwd = try std.os.getFdPath(temp_dir.dir.fd, &path_buf);
	393
	394	run.stdin_behavior = .Ignore;
	395	run.stdout_behavior = .Pipe;
	396	run.stderr_behavior = .Pipe;
	397
	398	run.env_map = &env_map_run;
	399	try run.spawn();
	400
	401	try run.collectOutput(&run_out, &run_err, std.math.maxInt(usize));
	402	const run_result = run.wait() catch \|e\| {
	403	if (e == error.FileNotFound) {
	404	std.debug.panic(
	405	\\Missing JavaScript evaluator: {s}
	406	\\Specify another one using -Djs_runtime=... or set to "" to skip evaluation.
	407	\\
	408	, .{runtime_path});
	409	}
	410	return e;
	411	};
	412
	413	if (run_result != .Exited or run_result.Exited != 0) {
	414	try run_err.writer().print("[Process {s} with {d}]", .{
	415	switch (run_result) {
	416	.Exited => "exited",
	417	.Stopped => "stopped",
	418	.Signal => "signaled",
	419	.Unknown => "hit unknown state",
	420	},
	421	switch (run_result) {
	422	inline else => \|d\| @as(u32, d),
	423	},
	424	});
216	425	}
217		},
218		else => {},
	426
	427	try writer.writeAll("<pre><code>");
	428	try writer.writeAll(run_out.items);
	429	try writer.writeAll(run_err.items);
	430	try writer.writeAll("</code></pre>");
	431	}
219	432	}
	433
	434	try writer.writeAll("</figure>");
220	435	},
221	436	.ul,
222	437	=> {
...	...	@@ -235,7 +450,7 @@ fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []con
235	450	try writer.writeAll("<table><tbody>");
236	451	{
237	452	try writer.writeAll("<tr>");
238		var it = std.mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
	453	var it = mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
239	454	while (it.next()) \|tok\| {
240	455	try writer.writeAll("<th>");
241	456	try processInlineMarkdown(gpa, tok, writer);
...	...	@@ -247,7 +462,7 @@ fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []con
247	462	t.skipWhitespace();
248	463	while (t.str.len > 0 and t.str[0] == '\|') {
249	464	try writer.writeAll("<tr>");
250		var it = std.mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
	465	var it = mem.tokenizeScalar(u8, t.advanceLine()[1..], '\|');
251	466	while (it.next()) \|tok\| {
252	467	try writer.writeAll("<td>");
253	468	try processInlineMarkdown(gpa, tok, writer);
...	...	@@ -282,7 +497,7 @@ fn processMarkdown(gpa: std.mem.Allocator, src: []const u8, compiler_path: []con
282	497	pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
283	498	var pos: usize = 0;
284	499	while (true) : (pos += 1) {
285		pos = std.mem.indexOfAnyPos(u8, src, pos, "*`") orelse return null;
	500	pos = mem.indexOfAnyPos(u8, src, pos, "*`[") orelse return null;
286	501
287	502	switch (src[pos]) {
288	503	'*' => switch (src[pos + 1]) {
...	...	@@ -290,12 +505,13 @@ pub fn nextInlineToken(src: []const u8) ?struct { InlineType, usize } {
290	505	else => continue,
291	506	},
292	507	'`' => return .{ .code, pos },
	508	'[' => return .{ .link, pos },
293	509	else => unreachable,
294	510	}
295	511	}
296	512	}
297	513
298		fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytype) !void {
	514	fn processInlineMarkdown(gpa: Allocator, src: []const u8, writer: anytype) !void {
299	515	var bold = false;
300	516	var remain = src;
301	517
...	...	@@ -312,13 +528,32 @@ fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytyp
312	528	remain = remain[2..];
313	529	},
314	530	.code => {
315		const end = std.mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
	531	const end = mem.indexOfScalarPos(u8, remain, 1, '`') orelse @panic("invalid code block");
316	532	const code = remain[1..end];
317	533	try writer.writeAll("<code>");
318	534	try writeTokenizedCode(gpa, code, writer);
319	535	try writer.writeAll("</code>");
320	536	remain = remain[end + 1 ..];
321	537	},
	538	.link => {
	539	const label_end = mem.indexOfScalarPos(u8, remain, 1, ']') orelse @panic("invalid link block");
	540	assert(remain[label_end + 1] == '('); // invalid link. '[' syntax is not implemented
	541
	542	const href_end = mem.indexOfScalarPos(u8, remain, label_end + 2, ')') orelse @panic("invalid link block");
	543
	544	const label = remain[1..label_end];
	545	const href = remain[label_end + 2 .. href_end];
	546
	547	try writer.writeAll("<a href=\"");
	548	try writeHtmlEscapedSlice(href, writer, true);
	549	if (!mem.startsWith(u8, href, "#")) {
	550	try writer.writeAll("\" class=\"external\" target=\"_blank\" ref=\"noopener noreferrer");
	551	}
	552	try writer.writeAll("\">");
	553	try processInlineMarkdown(gpa, label, writer);
	554	try writer.writeAll("</a>");
	555	remain = remain[href_end + 1 ..];
	556	},
322	557	}
323	558	}
324	559	try writer.writeAll(remain);
...	...	@@ -330,7 +565,7 @@ fn processInlineMarkdown(gpa: std.mem.Allocator, src: []const u8, writer: anytyp
330	565
331	566	// TODO: Change this to use Tree-sitter, which would also make it very simple
332	567	// to highlight JavaScript code in addition to the language.
333		fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	568	fn writeTokenizedCode(gpa: Allocator, code: []const u8, writer: anytype) !void {
334	569	const code_z = try gpa.dupeZ(u8, code);
335	570	defer gpa.free(code_z);
336	571
...	...	@@ -344,7 +579,7 @@ fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype)
344	579	t.pos += 1;
345	580	continue;
346	581	} orelse break;
347		try writer.writeAll(t.code[last..tok.offset]);
	582	try writeHtmlEscapedSlice(t.code[last..tok.offset], writer, false);
348	583	try writer.print("<span class='tok-{s}{s}'>{s}</span>", .{
349	584	@tagName(tok.tag),
350	585	if (tok.tag.isKeyword()) " kw" else "",
...	...	@@ -352,17 +587,18 @@ fn writeTokenizedCode(gpa: std.mem.Allocator, code: []const u8, writer: anytype)
352	587	});
353	588	last = t.pos;
354	589	}
	590	try writeHtmlEscapedSlice(t.code[last..], writer, false);
355	591	}
356	592
357		fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
	593	fn writeAnsi(gpa: Allocator, code: []const u8, writer: anytype) !void {
358	594	var remain = code;
359	595	var count: usize = 0;
360		while (std.mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
361		try writer.writeAll(remain[0..esc_i]);
	596	while (mem.indexOfScalar(u8, remain, '\x1b')) \|esc_i\| {
	597	try writeHtmlEscapedSlice(remain[0..esc_i], writer, false);
362	598	switch (remain[esc_i + 1]) {
363	599	'[' => switch (remain[esc_i + 2]) {
364	600	'0'...'9' => {
365		const end = std.mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
	601	const end = mem.indexOfScalarPos(u8, remain, esc_i + 2, 'm').?;
366	602	const int = std.fmt.parseInt(u8, remain[esc_i + 2 .. end], 10) catch unreachable;
367	603	switch (int) {
368	604	else => {
...	...	@@ -385,3 +621,88 @@ fn writeAnsi(gpa: std.mem.Allocator, code: []const u8, writer: anytype) !void {
385	621	}
386	622	_ = gpa;
387	623	}
	624
	625	fn nameToHeaderId(alloc: Allocator, input: []const u8) ![]const u8 {
	626	var out = try ArrayList(u8).initCapacity(alloc, input.len);
	627
	628	var is_split = false;
	629	for (input) \|char\| {
	630	switch (char) {
	631	'a'...'z', 'A'...'Z', '0'...'9' => {
	632	if (is_split) {
	633	out.appendAssumeCapacity('-');
	634	is_split = false;
	635	}
	636	out.appendAssumeCapacity(std.ascii.toLower(char));
	637	},
	638	else => {
	639	is_split = true;
	640	},
	641	}
	642	}
	643
	644	return try out.toOwnedSlice();
	645	}
	646
	647	fn replaceInTemplate(alloc: Allocator, template: []const u8, replacements: anytype) ![]const u8 {
	648	const Replacements = @TypeOf(replacements);
	649	const Map = std.StringArrayHashMapUnmanaged([]const u8);
	650
	651	var map: Map = .{};
	652	inline for (@typeInfo(Replacements).Struct.fields) \|field\| {
	653	try map.put(alloc, field.name, @field(replacements, field.name));
	654	}
	655
	656	var out: ArrayListUnmanaged(u8) = .{};
	657	const w = out.writer(alloc);
	658
	659	var i: usize = 0;
	660	while (mem.indexOfScalarPos(u8, template, i, '{')) \|open_at\| {
	661	const end = mem.indexOfScalarPos(u8, template, open_at + 1, '}') orelse break;
	662	const inner = template[open_at + 1 .. end];
	663
	664	if (map.get(inner)) \|replacement\| {
	665	// write previous
	666	try w.writeAll(template[i..open_at]);
	667	// write template
	668	try w.writeAll(replacement);
	669	// move one past the ending '}'
	670	i = end + 1;
	671	} else {
	672	// write up to and including the {
	673	try w.writeAll(template[i .. open_at + 1]);
	674	// move one past it
	675	i = open_at + 1;
	676	}
	677	}
	678
	679	// write ending
	680	try w.writeAll(template[i..]);
	681
	682	return out.items;
	683	}
	684
	685	fn writeHtmlEscapedSlice(text: []const u8, writer: anytype, escape_quotes: bool) !void {
	686	var i: usize = 0;
	687	while (mem.indexOfAnyPos(
	688	u8,
	689	text,
	690	i,
	691	if (escape_quotes)
	692	&.{ '&', '<', '>', '"' }
	693	else
	694	&.{ '&', '<', '>' },
	695	)) \|at\| {
	696	const replace = switch (text[at]) {
	697	'&' => "&",
	698	'<' => "<",
	699	'>' => ">",
	700	'"' => """,
	701	else => unreachable,
	702	};
	703	try writer.writeAll(text[i..at]);
	704	try writer.writeAll(replace);
	705	i = at + 1;
	706	}
	707	try writer.writeAll(text[i..]);
	708	}

test/behavior.zig created+393

...	...	@@ -0,0 +1,393 @@
	1	//! This program executes behavior tests found in ./test/behavior. Additionally, it can be run
	2	//! as a standalone program to compile and execute a single behavior test (for debugging)
	3
	4	/// This file is included for all behavior tests for the JavaScript backend.
	5	const helpers =
	6	\\(function (self) {
	7	\\ function assert(condition) {
	8	\\ if (!condition) {
	9	\\ throw new Error("Assertion failure");
	10	\\ }
	11	\\ }
	12	\\
	13	\\ function assertEqual(actual, expected) {
	14	\\ if (actual !== expected) {
	15	\\ var err;
	16	\\ try {
	17	\\ err = new Error("Expected " + expected + ", but got " + actual);
	18	\\ } catch (e) {
	19	\\ err = new Error("Assertion failure");
	20	\\ }
	21	\\ throw err;
	22	\\ }
	23	\\ }
	24	\\
	25	\\ function print(str) {
	26	\\ console.log(str);
	27	\\ }
	28	\\
	29	\\ self.assert = assert;
	30	\\ self.assertEqual = assertEqual;
	31	\\ self.print = print;
	32	\\})(typeof globalThis !== "undefined" ? globalThis : this);
	33	\\
	34	;
	35
	36	const CaseOptions = struct {
	37	dump_ir: bool = false,
	38	sync: bool = false,
	39	};
	40
	41	const Case = struct {
	42	path: []const u8,
	43	display: []const u8,
	44	success: bool = false,
	45	logs: []const u8 = "",
	46	failure: ?struct {
	47	err: anyerror,
	48	stack: core.StoredTrace,
	49	} = null,
	50	};
	51
	52	fn runTestCase(
	53	backing_allocator: std.mem.Allocator,
	54	file_path: []const u8,
	55	case_options: CaseOptions,
	56	out: std.io.AnyWriter,
	57	) !void {
	58	var gpa = std.heap.GeneralPurposeAllocator(.{
	59	.stack_trace_frames = 100,
	60	}){ .backing_allocator = backing_allocator };
	61	const alloc = gpa.allocator();
	62
	63	var backend = core.Js{};
	64	var comp: core.Comp = .{
	65	.gpa = tracer.tracingAllocator(&alloc),
	66	.options = .{ .dump_ir = case_options.dump_ir },
	67	.backend = .{ .js = &backend },
	68	.out = .{ .writer = out, .tty = .escape_codes },
	69	};
	70	try comp.init();
	71
	72	const main_file = try comp.getFile(file_path);
	73
	74	var fail_report: core.Report = undefined;
	75	const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
	76	error.TransformFail => {
	77	try out.print("Transform failed:\n", .{});
	78	try fail_report.dumpToWriter(&comp, out, .escape_codes);
	79	return error.TransformFail;
	80	},
	81	error.ParseFail => {
	82	// Already printed.
	83	return error.ParseFail;
	84	},
	85	else => return err,
	86	};
	87
	88	if (case_options.dump_ir) {
	89	try teg.dumpToWriter(out, .escape_codes);
	90	}
	91
	92	const entry_point = try comp.semaEntryPoint(main_file);
	93
	94	if (comp.temp_flag_had_errors)
	95	return error.SemaErrors;
	96
	97	if (teg.report.warnings.len > 0)
	98	return error.TegWarnings;
	99
	100	var js_out: std.ArrayListUnmanaged(u8) = .{};
	101	defer js_out.deinit(alloc);
	102
	103	try js_out.appendSlice(alloc, helpers);
	104	try js_out.appendSlice(alloc, "\n\n(function() {\n\n");
	105
	106	const start_i = js_out.items.len;
	107	try backend.link(&comp, entry_point, js_out.writer(alloc).any());
	108
	109	try out.writeAll("--- JS CODE ---\n");
	110	try out.writeAll(js_out.items[start_i..]);
	111	try out.writeAll("--- JS CODE ---\n");
	112
	113	try js_out.appendSlice(alloc, "\n\n})();");
	114	try js_out.append(alloc, 0);
	115
	116	const runtime = qjs.JS_NewRuntime() orelse
	117	return error.UnexpectedError;
	118	defer qjs.JS_FreeRuntime(runtime);
	119	const js_ctx = qjs.JS_NewContext(runtime) orelse
	120	return error.UnexpectedError;
	121	defer qjs.JS_FreeContext(js_ctx);
	122
	123	const result = qjs.JS_Eval(
	124	js_ctx,
	125	js_out.items.ptr,
	126	js_out.items.len - 1,
	127	"test.js",
	128	qjs.JS_EVAL_TYPE_GLOBAL,
	129	);
	130	defer qjs.JS_FreeValue(js_ctx, result);
	131
	132	if (qjs.JS_IsException(result) != 0) {
	133	const exception = qjs.JS_GetException(js_ctx);
	134	defer qjs.JS_FreeValue(js_ctx, exception);
	135
	136	const exception_str = qjs.JS_ToCString(js_ctx, exception);
	137	defer qjs.JS_FreeCString(js_ctx, exception_str);
	138
	139	const slice = std.mem.span(exception_str);
	140
	141	try out.print("Exception: {s}\n", .{slice});
	142
	143	return error.RuntimeFailed;
	144	}
	145	}
	146
	147	const threadlocal_state = struct {
	148	threadlocal var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
	149	};
	150
	151	pub fn runTestCaseWrap(
	152	gpa: std.mem.Allocator,
	153	case: *Case,
	154	case_options: CaseOptions,
	155	progress: std.Progress.Node,
	156	) void {
	157	const node = progress.start(case.display, 0);
	158	defer node.end();
	159
	160	const alloc = threadlocal_state.arena.allocator();
	161	defer _ = threadlocal_state.arena.reset(.{ .retain_with_limit = std.mem.page_size * 8 });
	162
	163	var logs: std.ArrayListUnmanaged(u8) = .{};
	164	defer logs.deinit(alloc);
	165
	166	defer case.logs = gpa.dupe(u8, logs.items) catch
	167	"[logs failed to allocate]";
	168
	169	if (case_options.sync) std.debug.lockStdErr();
	170	defer if (case_options.sync) std.debug.unlockStdErr();
	171
	172	runTestCase(
	173	alloc,
	174	case.path,
	175	case_options,
	176	if (case_options.sync)
	177	std.io.getStdErr().writer().any()
	178	else
	179	logs.writer(alloc).any(),
	180	) catch \|err\| {
	181	const stack = switch (err) {
	182	error.TransformFail,
	183	error.ParseFail,
	184	error.TegWarnings,
	185	error.SemaErrors,
	186	error.AnalysisFail,
	187	=> core.StoredTrace.empty,
	188
	189	else => core.StoredTrace.from(@errorReturnTrace()),
	190	};
	191
	192	case.failure = .{
	193	.err = err,
	194	.stack = stack,
	195	};
	196
	197	return;
	198	};
	199
	200	case.success = true;
	201	}
	202
	203	pub fn main() !void {
	204	// unicode props is asserted to be initialized before core.Tokenizer is used
	205	// this will result in a static page allocation, which is never cleaned up.
	206	try core.unicode_props.init();
	207	var gpa = std.heap.GeneralPurposeAllocator(.{
	208	.stack_trace_frames = 100,
	209	.thread_safe = true,
	210	}){};
	211
	212	defer _ = gpa.deinit();
	213	const alloc = gpa.allocator();
	214
	215	const args = try std.process.argsAlloc(alloc);
	216	defer std.process.argsFree(alloc, args);
	217
	218	var test_root: []const u8 = "test/behavior";
	219	var case_options: CaseOptions = .{};
	220	var all_logs = false;
	221	var filter: ?[]const u8 = null;
	222	defer if (filter) \|f\| alloc.free(f);
	223
	224	var argument_error = false;
	225
	226	for (args[1..]) \|full_arg\| {
	227	if (std.mem.startsWith(u8, full_arg, "--")) {
	228	if (getFlag("--test-root", full_arg)) \|flag\| {
	229	test_root = flag;
	230	} else if (std.mem.eql(u8, "--dump-ir", full_arg)) {
	231	case_options.dump_ir = true;
	232	} else if (std.mem.eql(u8, "--all-logs", full_arg)) {
	233	all_logs = true;
	234	} else if (std.mem.eql(u8, "--sync", full_arg)) {
	235	case_options.sync = true;
	236	} else {
	237	std.debug.print("Unknown argument: {s}\n", .{full_arg});
	238	argument_error = true;
	239	}
	240	} else {
	241	if (filter != null) {
	242	std.debug.print("Unknown positional argument: {s}\n", .{full_arg});
	243	} else {
	244	filter = try std.mem.replaceOwned(
	245	u8,
	246	alloc,
	247	full_arg,
	248	if (std.fs.path.sep == '/') "\\" else '/',
	249	std.fs.path.sep_str,
	250	);
	251	}
	252	}
	253	}
	254
	255	if (argument_error) std.process.exit(1);
	256
	257	const root = try std.fs.cwd().openDir(test_root, .{ .iterate = true });
	258
	259	var test_cases: std.ArrayListUnmanaged(Case) = .{};
	260	defer test_cases.deinit(alloc);
	261	defer for (test_cases.items) \|test_case\| {
	262	alloc.free(test_case.path);
	263	};
	264
	265	var path_buf: std.BoundedArray(u8, std.fs.max_path_bytes) = .{};
	266	path_buf.len = @intCast((try std.os.getFdPath(root.fd, &path_buf.buffer)).len);
	267	try iterateTestCases(alloc, root, path_buf.len, &path_buf, &test_cases, filter);
	268
	269	if (test_cases.items.len == 0) {
	270	if (filter) \|f\| {
	271	std.debug.print("No test cases match filter \"{s}\"\n", .{f});
	272	} else {
	273	std.debug.print("No test cases\n", .{});
	274	}
	275	std.process.exit(1);
	276	}
	277
	278	const progress = std.Progress.start(.{ .root_name = "Behavior Tests" });
	279	defer progress.end();
	280	progress.setEstimatedTotalItems(test_cases.items.len);
	281
	282	var wg: std.Thread.WaitGroup = .{};
	283
	284	var pool: std.Thread.Pool = undefined;
	285	if (!case_options.sync) try pool.init(.{ .allocator = alloc });
	286	defer if (!case_options.sync) pool.deinit();
	287
	288	for (test_cases.items) \|*test_case\| {
	289	const call_args = .{
	290	alloc,
	291	test_case,
	292	case_options,
	293	progress,
	294	};
	295
	296	if (case_options.sync) {
	297	@call(.auto, runTestCaseWrap, call_args);
	298	} else {
	299	pool.spawnWg(&wg, runTestCaseWrap, call_args);
	300	}
	301	}
	302
	303	wg.wait();
	304
	305	for (test_cases.items) \|*case\| {
	306	if (case.success) {
	307	if (all_logs) {
	308	std.debug.print("PASS: {s}\n", .{case.display});
	309	std.debug.print("{s}", .{case.logs});
	310	}
	311	} else if (case.failure) \|*fail\| {
	312	std.debug.print("FAIL: {s}\n", .{case.display});
	313	std.debug.print("{s}", .{case.logs});
	314	if (fail.stack.index > 0) {
	315	std.debug.print("Error return trace:\n", .{});
	316	std.debug.dumpStackTrace(fail.stack.trace());
	317	}
	318	} else {
	319	// success was never set true but failure was left null
	320	// this means runTestCaseWrap was never called
	321	std.debug.print("INVALID STATE: {s}\n", .{case.path});
	322	}
	323	if (case.logs.len > 0)
	324	alloc.free(case.logs);
	325	}
	326	}
	327
	328	fn iterateTestCases(
	329	alloc: std.mem.Allocator,
	330	root: Dir,
	331	root_path_len: usize,
	332	path_buf: *std.BoundedArray(u8, std.fs.max_path_bytes),
	333	test_cases: *std.ArrayListUnmanaged(Case),
	334	filter: ?[]const u8,
	335	) !void {
	336	var it = root.iterate();
	337	while (try it.next()) \|entry\| {
	338	switch (entry.kind) {
	339	.file => {
	340	if (entry.name.len > 0 and entry.name[0] == '.')
	341	continue;
	342
	343	const path = try std.fs.path.join(alloc, &.{
	344	path_buf.slice(),
	345	entry.name,
	346	});
	347
	348	if (filter) \|f\| {
	349	if (std.mem.indexOf(u8, path, f) == null) {
	350	alloc.free(path);
	351	continue;
	352	}
	353	}
	354
	355	try test_cases.append(alloc, .{
	356	.path = path,
	357	.display = path[root_path_len + 1 ..],
	358	});
	359	},
	360	.directory => {
	361	const top = path_buf.len;
	362	defer path_buf.len = top;
	363
	364	try path_buf.append(std.fs.path.sep);
	365	try path_buf.appendSlice(entry.name);
	366
	367	const sub_dir = try root.openDir(entry.name, .{ .iterate = true });
	368	try iterateTestCases(alloc, sub_dir, root_path_len, path_buf, test_cases, filter);
	369	},
	370	else => return error.UnsupportedFileType,
	371	}
	372	}
	373	}
	374
	375	fn getFlag(name: []const u8, full_arg: []const u8) ?[]const u8 {
	376	if (full_arg.len > name.len + 1 and
	377	full_arg[name.len] == '=' and
	378	std.mem.startsWith(u8, name, full_arg))
	379	{
	380	return full_arg[name.len + 1 ..];
	381	}
	382
	383	return null;
	384	}
	385
	386	const std = @import("std");
	387	const Dir = std.fs.Dir;
	388
	389	const core = @import("core");
	390	const qjs = @import("quickjs");
	391
	392	const tracer = @import("tracer");
	393	pub const tracer_impl = @field(tracer, @import("build_options").tracing);

test/behavior/arithmetic created+67

...	...	@@ -0,0 +1,67 @@
	1	fn f(a: float) -> float { a }
	2
	3	fn addFloat(a: float, b: float) -> float {
	4	a + b
	5	}
	6
	7	fn subFloat(a: float, b: float) -> float {
	8	a - b
	9	}
	10
	11	fn mulFloat(a: float, b: float) -> float {
	12	a * b
	13	}
	14
	15	fn divFloat(a: float, b: float) -> float {
	16	a / b
	17	}
	18
	19	// fn modFloat(a: float, b: float) -> float {
	20	// a % b
	21	// }
	22
	23	// fn expFloat(a: float, b: float) -> float {
	24	// a ** b
	25	// }
	26
	27	fn i(a: int) -> int { a }
	28
	29	fn addInt(a: int, b: int) -> int {
	30	a + b
	31	}
	32
	33	fn subInt(a: int, b: int) -> int {
	34	a - b
	35	}
	36
	37	// fn mulInt(a: int, b: int) -> int {
	38	// a * b
	39	// }
	40
	41	// fn divInt(a: int, b: int) -> int {
	42	// a / b
	43	// }
	44
	45	// fn expInt(a: int, b: int) -> int {
	46	// a * b
	47	// }
	48
	49	// fn modFloat(a: float, b: float) -> float {
	50	// a % b
	51	// }
	52
	53	let assertEqual = @js("assertEqual");
	54
	55	assertEqual(addFloat(1, 2), f(3));
	56	assertEqual(addFloat(1.5, 2.2), f(3.7));
	57	assertEqual(subFloat(1, 2), f(-1));
	58	assertEqual(subFloat(1.5, 2.2), f(-0.7000000000000002));
	59	assertEqual(mulFloat(1, 2), f(2));
	60	assertEqual(mulFloat(1.5, 2.2), f(3.3000000000000003));
	61	assertEqual(divFloat(1, 2), f(0.5));
	62	assertEqual(divFloat(1.5, 2.2), f(0.6818181818181818));
	63
	64	assertEqual(addInt(1, 2), i(3));
	65	assertEqual(addInt(-421, 592), i(171));
	66	assertEqual(subInt(1, 2), i(-1));
	67	assertEqual(subInt(528, -43), i(571));