Commit 5d80ae2c0e35a2f617b42d22cf112402ad5ca63b | ix

author	dave caruso <me@paperdave.net> 2024-05-27 20:49:10-07:00
committer	dave caruso <me@paperdave.net> 2024-05-27 20:49:10-07:00
log	5d80ae2c0e35a2f617b42d22cf112402ad5ca63b
tree	d797eb2decd8e1e6a9e4bf8828a23575f12b3a90
parent	d024b032f7ff08f0876ca53d5e4b55b294f419c4
signature	Commit is signed but in an unrecognized format.

move compiler->core, add lsp binary

45 files changed, 11874 insertions(+), 11778 deletions(-)

build.zig+104-62

...	...	@@ -19,47 +19,29 @@ pub fn build(b: *std.Build) void {
19	19	},
20	20	};
21	21
22		const tracing = b.option(enum {
23		none,
24		spall,
25		chrome,
26		log,
27		}, "trace", "Enable tracing") orelse .none;
	22	const tracing = b.option(
	23	TracingMode,
	24	"trace",
	25	"Enable tracing",
	26	) orelse .none;
28	27
29		const dep_opts = .{ .target = target, .optimize = optimize };
30
31		const zg = b.dependency("zg", dep_opts);
32		const tracer = b.dependency("tracer", dep_opts);
33		// const ada = b.dependency("ada", dep_opts);
	28	const main = buildWithTarget(b, target, optimize, tracing);
34	29
35		const opts = b.addOptions();
36		opts.addOption([]const u8, "tracing", @tagName(tracing));
37		const opts_mod = opts.createModule();
	30	inline for (.{
	31	.{ main.compiler, "compiler" },
	32	.{ main.lsp, "lsp" },
	33	}) \|entry\| {
	34	const artifact, const name = entry;
38	35
39		const exe = b.addExecutable(.{
40		.name = "compiler",
41		.root_source_file = b.path("src/compiler/main.zig"),
42		.target = target,
43		.optimize = optimize,
44		.link_libc = true,
45		});
46		b.installArtifact(exe);
47
48		exe.root_module.addImport("build_options", opts_mod);
49		exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
50		exe.root_module.addImport("tracer", tracer.module("tracer"));
51		// if (!target.result.isWasm()) {
52		// exe.root_module.addImport("ada", ada.module("ada"));
53		// }
54
55		{
56		const run_cmd = b.addRunArtifact(exe);
57		run_cmd.step.dependOn(b.getInstallStep());
58		if (b.args) \|args\| {
59		run_cmd.addArgs(args);
	36	{
	37	const run_cmd = b.addRunArtifact(artifact);
	38	run_cmd.step.dependOn(b.getInstallStep());
	39	if (b.args) \|args\| {
	40	run_cmd.addArgs(args);
	41	}
	42	const run_step = b.step(name, "Run " ++ name ++ " binary");
	43	run_step.dependOn(&run_cmd.step);
60	44	}
61		const run_step = b.step("compiler", "Run compiler CLI");
62		run_step.dependOn(&run_cmd.step);
63	45	}
64	46
65	47	// const wasm_api = b.addExecutable(.{
...	...	@@ -102,30 +84,90 @@ pub fn build(b: *std.Build) void {
102	84	},
103	85	) catch @panic("OOM");
104	86
105		// std.log.info("{s}\n", .{cross_target.result.zigTriple(b.allocator) catch @panic("OOM")});
106
107		const cross_exe = b.addExecutable(.{
108		.name = "compiler",
109		.root_source_file = b.path("src/compiler/main.zig"),
110		.target = cross_target,
111		.optimize = .ReleaseSafe,
112		.strip = true,
113		.link_libc = true,
114		});
115		cross_exe.root_module.addImport("build_options", opts_mod);
116		cross_exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
117		cross_exe.root_module.addImport("tracer", tracer.module("tracer"));
118
119		release.dependOn(&b.addInstallFile(
120		cross_exe.getEmittedBin(),
121		b.pathJoin(&.{
122		"release",
123		name,
124		std.mem.concat(b.allocator, u8, &.{
125		"compiler",
126		cross_target.result.exeFileExt(),
127		}) catch @panic("OOM"),
128		}),
129		).step);
	87	const artifacts = buildWithTarget(b, cross_target, .ReleaseSafe, .none);
	88
	89	inline for (.{
	90	.{ artifacts.compiler, "compiler" },
	91	.{ artifacts.lsp, "lsp" },
	92	}) \|entry\| {
	93	const artifact, const bin_name = entry;
	94	release.dependOn(&b.addInstallFile(
	95	artifact.getEmittedBin(),
	96	b.pathJoin(&.{
	97	"release",
	98	name,
	99	std.mem.concat(b.allocator, u8, &.{
	100	bin_name,
	101	cross_target.result.exeFileExt(),
	102	}) catch @panic("OOM"),
	103	}),
	104	).step);
	105	}
130	106	}
131	107	}
	108
	109	const TracingMode = enum { none, spall, chrome, log };
	110
	111	const Artifacts = struct {
	112	core: *Build.Module,
	113	compiler: *Build.Step.Compile,
	114	lsp: *Build.Step.Compile,
	115	};
	116
	117	fn buildWithTarget(
	118	b: *Build,
	119	target: Build.ResolvedTarget,
	120	optimize: std.builtin.OptimizeMode,
	121	tracing: TracingMode,
	122	) Artifacts {
	123	const dep_opts = .{ .target = target, .optimize = optimize };
	124
	125	// Dependencies
	126	const zg = b.dependency("zg", dep_opts);
	127	const tracer = b.dependency("tracer", dep_opts);
	128
	129	// Options
	130	const opts = b.addOptions();
	131	opts.addOption([]const u8, "tracing", @tagName(tracing));
	132	const opts_mod = opts.createModule();
	133
	134	// Core
	135	const core = b.addModule("compiler", .{
	136	.root_source_file = b.path("src/core/mod.zig"),
	137	.optimize = optimize,
	138	.target = target,
	139	});
	140	core.addImport("build_options", opts_mod);
	141	core.addImport("tracer", tracer.module("tracer"));
	142	core.addImport("zg:PropsData", zg.module("PropsData"));
	143
	144	// Compiler Testing Exe
	145	const compiler_exe = b.addExecutable(.{
	146	.name = "compiler",
	147	.root_source_file = b.path("src/core/main.zig"),
	148	.target = target,
	149	.optimize = optimize,
	150	.link_libc = true,
	151	});
	152	compiler_exe.root_module.addImport("build_options", opts_mod);
	153	compiler_exe.root_module.addImport("core", core);
	154	compiler_exe.root_module.addImport("tracer", tracer.module("tracer"));
	155
	156	// Language Server
	157	const lsp_exe = b.addExecutable(.{
	158	.name = "lsp",
	159	.root_source_file = b.path("src/lsp/lsp.zig"),
	160	.target = target,
	161	.optimize = optimize,
	162	.link_libc = true,
	163	});
	164	lsp_exe.root_module.addImport("build_options", opts_mod);
	165	lsp_exe.root_module.addImport("core", core);
	166	lsp_exe.root_module.addImport("tracer", tracer.module("tracer"));
	167
	168	return .{
	169	.core = core,
	170	.compiler = compiler_exe,
	171	.lsp = lsp_exe,
	172	};
	173	}

src/compiler/Air.Dump.zig deleted-316

...	...	@@ -1,316 +0,0 @@
1		//! This implements a debug formatter for Teg
2		air: Air,
3		ip: *const InternPool,
4
5		pub const State = struct {
6		arena: std.mem.Allocator,
7		air: *const Air,
8		ip: *const InternPool,
9		indent: usize = 0,
10		tty: std.io.tty.Config,
11		visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{},
12
13		fn newline(state: State, writer: anytype) !void {
14		try writer.writeByte('\n');
15		if (state.indent > 0)
16		try writer.writeByteNTimes(' ', state.indent * 2);
17		}
18
19		fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
20		const air = s.air;
21		const tty = s.tty;
22		const tag = s.air.instTag(inst);
23
24		try tty.setColor(w, switch (s.air.instFlags(inst).uses) {
25		.none => .bright_black,
26		.multi => .red,
27		.single => .magenta,
28		});
29		try w.writeByte(switch (s.air.instFlags(inst).uses) {
30		.none => 'X',
31		.multi => 'M',
32		.single => 'S',
33		});
34		try tty.setColor(w, .cyan);
35
36		try w.print(" #{} = ", .{
37		@intFromEnum(inst),
38		});
39
40		try tty.setColor(w, .cyan);
41		try w.print("{s}", .{@tagName(tag)});
42		try tty.setColor(w, .reset);
43
44		if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
45		try tty.setColor(w, .red);
46		try w.writeAll(" [error: recursive tree]");
47		try tty.setColor(w, .reset);
48		return;
49		}
50
51		switch (tag) {
52		.block => {
53		const index_list: Air.IndexList = air.instData(.block, inst);
54		const insts = index_list.slice(air);
55		try s.dumpInstList(w, insts, '{', '}');
56		},
57		.js_global_ref => {
58		const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst);
59		try tty.setColor(w, .magenta);
60		try w.print(" {s}", .{name.toSlice(s.ip)});
61		try tty.setColor(w, .reset);
62		},
63		.arg => {
64		const arg: Air.Arg = air.instData(.arg, inst);
65		try w.writeAll(" ");
66		try tty.setColor(w, .yellow);
67		try w.print("{d}", .{arg.arg_number});
68		try tty.setColor(w, .reset);
69		try w.writeAll(": ");
70		try s.dumpValue(w, arg.type.value());
71		},
72		.@"break" => {
73		const brk: Air.Break = air.instData(.@"break", inst);
74		try w.writeAll("(");
75		try s.dumpInstRef(w, brk.block_ref);
76		try w.writeAll(", ");
77		try s.dumpRef(w, brk.value);
78		try w.writeAll(")");
79		},
80		.store => {
81		const store: Air.Store = air.instData(.store, inst);
82		try w.writeAll("(");
83		try s.dumpInstRef(w, store.memory);
84		try w.writeAll(", ");
85		try s.dumpRef(w, store.operand);
86		try w.writeAll(")");
87		},
88		inline .add,
89		.mul,
90		.sub,
91		.div,
92		.lt,
93		.gt,
94		.eq,
95		=> \|comptime_tag\| {
96		const bin: Air.Bin = air.instData(comptime_tag, inst);
97		try w.writeAll("(");
98		try s.dumpRef(w, bin.lhs);
99		try w.writeAll(", ");
100		try s.dumpRef(w, bin.rhs);
101		try w.writeAll(")");
102		},
103		inline .negate,
104		.bool_not,
105		=> \|comptime_tag\| {
106		const bin: Air.Un = air.instData(comptime_tag, inst);
107		try w.writeAll("(");
108		try s.dumpRef(w, bin.operand);
109		try w.writeAll(")");
110		},
111		.memory => {
112		const memory: Air.Memory = air.instData(.memory, inst);
113		try w.writeAll("(");
114		try s.dumpRef(w, memory.init);
115		try w.writeAll(", type: ");
116		try s.dumpValue(w, memory.ty.value());
117		try w.writeAll(")");
118		},
119		.field => {
120		const field: Air.Field = air.instData(.field, inst);
121		try w.writeAll("(");
122		try s.dumpRef(w, field.base);
123		try w.writeAll(", ");
124		try tty.setColor(w, .magenta);
125		try w.print("{s}", .{field.field.toSlice(s.ip)});
126		try tty.setColor(w, .reset);
127		try w.writeAll(")");
128		},
129		.call => {
130		const call: Air.Call = air.instData(.call, inst);
131		const call_data = call.call_data.get(air);
132		try w.writeAll("(");
133		try s.dumpRef(w, call.base);
134		try w.writeAll(", ");
135		try s.dumpRefList(w, call_data.args.slice(air), '[', ']');
136		try w.writeAll(")");
137		},
138		.@"if" => {
139		const if_inst: Air.If = air.instData(.@"if", inst);
140		const if_data: Air.If.Data = if_inst.data.get(air);
141		try w.writeAll("(");
142		try s.dumpRef(w, if_inst.cond);
143		try w.writeAll(", {");
144		s.indent += 1;
145		try s.newline(w);
146		try w.writeAll("yes: ");
147		try s.dumpRefRecurse(w, if_data.yes);
148		try s.newline(w);
149		try w.writeAll("no: ");
150		try s.dumpRefRecurse(w, if_data.no);
151		s.indent -= 1;
152		try s.newline(w);
153		try w.writeAll("}");
154		},
155		}
156		}
157
158		fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void {
159		switch (ref.decode()) {
160		.index => \|i\| try s.dumpInstRef(w, i),
161		.value => \|v\| try s.dumpValue(w, v),
162		}
163		}
164
165		fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void {
166		switch (ref.decode()) {
167		.index => \|i\| try s.dumpInst(w, i),
168		.value => \|v\| try s.dumpValue(w, v),
169		}
170		}
171
172		fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
173		try s.tty.setColor(w, .red);
174		try w.print("#{d}", .{@intFromEnum(inst)});
175		try s.tty.setColor(w, .reset);
176		}
177
178		pub fn dumpValue(s: *State, w: anytype, val: Value) !void {
179		const tty = s.tty;
180		switch (val.toKey(s.ip)) {
181		.simple_type => \|t\| {
182		try tty.setColor(w, .blue);
183		try w.print("{s}", .{@tagName(t)});
184		try tty.setColor(w, .reset);
185		},
186		.simple_value => \|t\| {
187		try tty.setColor(w, .yellow);
188		try w.print("{s}", .{@tagName(t)});
189		try tty.setColor(w, .reset);
190		},
191		.type_function => \|ty\| {
192		try tty.setColor(w, .blue);
193		try w.writeAll("fn(");
194		try tty.setColor(w, .reset);
195		for (ty.param_types, 0..) \|param, i\| {
196		if (i != 0) {
197		try tty.setColor(w, .blue);
198		try w.writeAll(",");
199		try tty.setColor(w, .reset);
200		}
201		try s.dumpValue(w, param.value());
202		}
203		try tty.setColor(w, .blue);
204		try w.writeAll(") -> ");
205		try tty.setColor(w, .reset);
206		try s.dumpValue(w, ty.return_type.value());
207		},
208		inline .float, .int32, .comptime_float => \|data, tag\| {
209		try tty.setColor(w, .bright_black);
210		try w.writeAll("<");
211		try tty.setColor(w, .blue);
212		try w.writeAll(@tagName(tag));
213		try tty.setColor(w, .reset);
214		try tty.setColor(w, .bright_black);
215		try w.writeAll(", ");
216		try tty.setColor(w, .yellow);
217		try w.print("{d}", .{data});
218		try tty.setColor(w, .bright_black);
219		try w.writeAll(">");
220		try tty.setColor(w, .reset);
221		},
222		.string => \|data\| {
223		try w.writeAll("<");
224		try tty.setColor(w, .blue);
225		try w.writeAll("string");
226		try tty.setColor(w, .reset);
227		try w.writeAll(", ");
228		try tty.setColor(w, .green);
229		try w.writeAll("\"");
230		try w.writeAll(data);
231		try w.writeAll("\"");
232		try tty.setColor(w, .reset);
233		try w.writeAll(">");
234		},
235		.func => \|func\| {
236		try tty.setColor(w, .bright_black);
237		try w.writeAll("<");
238		try s.dumpValue(w, func.ty.value());
239		try tty.setColor(w, .bright_black);
240		try w.writeAll(", ");
241		const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip);
242		try tty.setColor(w, .bright_green);
243		try w.print("decl '{s}'", .{decl_name});
244		try tty.setColor(w, .bright_black);
245		try w.writeAll(">");
246		try tty.setColor(w, .reset);
247		},
248		}
249		}
250
251		fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void {
252		try writer.print(" {c}", .{l});
253		if (list.len > 0) {
254		state.indent += 1;
255
256		for (list) \|inst\| {
257		try state.newline(writer);
258		try state.dumpInst(writer, inst);
259		}
260
261		state.indent -= 1;
262		try state.newline(writer);
263		}
264		try writer.print("{c}", .{r});
265		}
266
267		fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void {
268		try writer.print(" {c}", .{l});
269		if (list.len > 0) {
270		state.indent += 1;
271
272		for (list) \|inst\| {
273		try state.newline(writer);
274		try state.dumpRef(writer, inst);
275		}
276
277		state.indent -= 1;
278		try state.newline(writer);
279		}
280		try writer.print("{c}", .{r});
281		}
282		};
283
284		pub fn format(
285		dump: @This(),
286		comptime unused_fmt: []const u8,
287		_: std.fmt.FormatOptions,
288		writer: anytype,
289		) !void {
290		comptime assert(unused_fmt.len == 0);
291
292		var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
293		defer arena.deinit();
294
295		const air = dump.air;
296		var state = State{
297		.air = &air,
298		.ip = dump.ip,
299		.tty = detectTTYFromWriter(writer),
300		.arena = arena.allocator(),
301		};
302		try state.dumpInst(writer, .main_block);
303		try writer.writeAll("\n");
304		assert(state.indent == 0);
305		}
306
307		const std = @import("std");
308		const assert = std.debug.assert;
309
310		const Air = @import("Air.zig");
311		const Inst = Air.Inst;
312
313		const InternPool = @import("InternPool.zig");
314		const Value = InternPool.Value;
315
316		const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Air.zig deleted-320

...	...	@@ -1,320 +0,0 @@
1		insts: Inst.List.Slice,
2		extra: ExtraStore = .{},
3
4		pub const ExtraStore = dod.ExtraStore(.air, .@"const");
5		pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
6		pub const ExtraIndex = ExtraStore.Index;
7
8		pub const Inst = struct {
9		tag: Tag,
10		data: EncodedData,
11		flags: Flags,
12
13		pub const List = std.MultiArrayList(Inst);
14
15		pub const Tag = enum {
16		block,
17		js_global_ref,
18		call,
19		@"break",
20		@"if",
21		arg,
22		field,
23
24		memory,
25		store,
26
27		// bin
28		add,
29		sub,
30		mul,
31		div,
32
33		eq,
34
35		lt,
36		gt,
37
38		// un
39		negate,
40		bool_not,
41
42		pub fn Data(comptime tag: Tag) type {
43		return switch (tag) {
44		.block => IndexList,
45		.js_global_ref => InternPool.String.WithLen,
46		.@"break" => Break,
47		.@"if" => If,
48		.call => Call,
49		.arg => Arg,
50		.field => Field,
51		.memory => Memory,
52		.store => Store,
53		.add,
54		.sub,
55		.mul,
56		.div,
57		.lt,
58		.gt,
59		.eq,
60		=> Bin,
61		.negate, .bool_not => Un,
62		};
63		}
64		};
65
66		/// reference to either a value or another instruction in this air
67		pub const Ref = packed struct(u32) {
68		kind: Kind,
69		data: u31,
70
71		pub const Kind = enum(u1) { value, index };
72
73		pub const Decoded = union(enum) {
74		value: Value,
75		index: Index,
76		};
77
78		pub fn decode(ref: Ref) Decoded {
79		return switch (ref.kind) {
80		.value => .{ .value = @enumFromInt(ref.data) },
81		.index => .{ .index = @enumFromInt(ref.data) },
82		};
83		}
84
85		pub fn toOptional(ref: ?Ref) Optional {
86		return if (ref) \|r\| @enumFromInt(@as(u32, @bitCast(r))) else .none;
87		}
88
89		pub const Optional = enum(u32) {
90		none = std.math.maxInt(u32),
91		_,
92
93		pub fn unwrap(opt: Optional) ?Ref {
94		return if (opt != .none) @bitCast(@intFromEnum(opt)) else null;
95		}
96		};
97
98		pub fn isVoid(ref: Ref) bool {
99		return ref.kind == .value and
100		ref.decode().value == .void;
101		}
102
103		pub fn eql(ref: Ref, other: Ref) bool {
104		return @as(u32, @bitCast(ref)) == @as(u32, @bitCast(other));
105		}
106		};
107
108		/// reference to another instruction in this air
109		/// note that these cannot point to code in other functions, such makes no sense
110		pub const Index = enum(u32) {
111		main_block,
112		_,
113
114		pub fn toRef(index: Index) Ref {
115		return .{
116		.kind = .index,
117		.data = @intCast(@intFromEnum(index)),
118		};
119		}
120
121		pub fn toOptional(index: Index) Optional {
122		return @enumFromInt(@intFromEnum(index));
123		}
124
125		pub const Optional = enum(u32) {
126		none = std.math.maxInt(u32),
127		_,
128
129		pub fn unwrap(oi: Optional) ?Index {
130		return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null;
131		}
132		};
133		};
134		};
135
136		pub const Flags = packed struct {
137		uses: Uses = .none,
138
139		pub const Uses = enum(u2) {
140		none,
141		single,
142		multi,
143
144		pub fn increment(uses: Uses) Uses {
145		return switch (uses) {
146		.none => .single,
147		.single, .multi => .multi,
148		};
149		}
150		};
151		};
152
153		pub const IndexList = struct {
154		start: u32,
155		len: u32,
156
157		pub fn slice(self: IndexList, air: *const Air) []const Inst.Index {
158		return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
159		}
160		};
161
162		pub const RefList = struct {
163		start: u32,
164		len: u32,
165
166		pub fn slice(self: RefList, air: *const Air) []const Inst.Ref {
167		return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
168		}
169		};
170
171		pub const Break = struct {
172		block_ref: Inst.Index,
173		value: Inst.Ref,
174		};
175
176		pub const Bin = struct {
177		lhs: Inst.Ref,
178		rhs: Inst.Ref,
179		};
180
181		pub const Store = struct {
182		memory: Inst.Index,
183		operand: Inst.Ref,
184		};
185
186		pub const Un = struct {
187		operand: Inst.Ref,
188		unused: Unused = .zero,
189		};
190
191		pub const Field = struct {
192		base: Inst.Ref,
193		field: InternPool.NullTerminatedString,
194		};
195
196		pub const Call = struct {
197		base: Inst.Ref,
198		call_data: ExtraIndex(Data),
199
200		pub const Data = struct {
201		args: RefList,
202		};
203		};
204
205		pub const Arg = struct {
206		type: Type,
207		arg_number: u32,
208		};
209
210		pub const If = struct {
211		cond: Inst.Ref,
212		data: ExtraIndex(Data),
213
214		pub const Data = struct {
215		yes: Inst.Ref,
216		no: Inst.Ref,
217		union_ty: Type,
218		};
219		};
220
221		pub const Memory = struct {
222		init: Inst.Ref,
223		ty: Type,
224		};
225
226		pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
227		defer air.* = undefined;
228		air.insts.deinit(gpa);
229		air.extra.deinit(gpa);
230		}
231
232		pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag {
233		return air.insts.items(.tag)[@intFromEnum(index)];
234		}
235
236		pub fn instFlags(air: *const Air, index: Inst.Index) Flags {
237		return air.insts.items(.flags)[@intFromEnum(index)];
238		}
239
240		pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData {
241		return air.insts.items(.data)[@intFromEnum(index)];
242		}
243
244		pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
245		return air.instEncodedData(index).decode(tag, air.instTag(index));
246		}
247
248		pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
249		return air.instData(tag, index).extra_index.get(air);
250		}
251
252		pub fn typeOf(air: const Air, ref: Inst.Ref, ip: InternPool) InternPool.Type {
253		return switch (ref.decode()) {
254		.value => \|v\| v.typeOf(ip),
255		.index => \|i\| air.typeOfIndex(i, ip),
256		};
257		}
258
259		pub fn typeOfIndex(air: const Air, index: Inst.Index, ip: InternPool) InternPool.Type {
260		return switch (air.instTag(index)) {
261		.js_global_ref => .@"opaque",
262
263		.block => {
264		const items = air.instData(.block, index).slice(air);
265		return air.typeOfIndex(items[items.len - 1], ip);
266		},
267
268		inline .add, .sub, .mul, .div => \|t\| air.typeOf(air.instData(t, index).lhs, ip),
269
270		.lt, .gt, .eq, .bool_not => .bool,
271
272		.arg => air.instData(.arg, index).type,
273
274		.negate => air.typeOf(air.instData(.negate, index).operand, ip),
275
276		.@"break" => .noreturn,
277
278		// todo:
279		.field => .@"opaque",
280
281		.call => {
282		const base_type = air.typeOf(air.instData(.call, index).base, ip);
283		if (base_type == .@"opaque") return .@"opaque";
284		return base_type.value().toKey(ip).type_function.return_type;
285		},
286
287		.memory => air.instData(.memory, index).ty,
288
289		.store => air.instData(.memory, air.instData(.store, index).memory).ty,
290
291		.@"if" => air.instData(.@"if", index).data.get(air).union_ty,
292		};
293		}
294
295		pub fn paramCount(air: Air) u32 {
296		for (air.insts.items(.tag)[1..], 0..) \|tag, i\| {
297		if (tag != .arg) return @intCast(i);
298		}
299		return @intCast(air.insts.len);
300		}
301
302		pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
303		try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
304		}
305
306		pub const Dump = @import("Air.Dump.zig");
307
308		const Air = @This();
309		const std = @import("std");
310		const mem = std.mem;
311		const assert = std.debug.assert;
312
313		const InternPool = @import("InternPool.zig");
314		const Value = InternPool.Value;
315		const Type = InternPool.Type;
316
317		const Teg = @import("Teg.zig");
318		const dod = @import("array.zig");
319
320		const Unused = enum(u32) { zero = 0 };

src/compiler/Ast.Dump.zig deleted-300

...	...	@@ -1,300 +0,0 @@
1		//! A flat array is not very useful for debugging, so this structure implements
2		//! a debug formatter for Ast.
3		ast: Ast,
4
5		const State = struct {
6		ast: *const Ast,
7		indent: usize = 0,
8		node_tags: []const Node.Tag,
9		node_data: []const Node.Data,
10		node_tokens: []const TokenIndex,
11		tty: std.io.tty.Config,
12
13		fn newline(state: State, writer: anytype) !void {
14		try writer.writeByte('\n');
15		if (state.indent > 0)
16		try writer.writeByteNTimes(' ', state.indent * 2);
17		}
18		};
19
20		pub fn format(
21		dump: Dump,
22		comptime unused_fmt: []const u8,
23		_: std.fmt.FormatOptions,
24		writer: anytype,
25		) !void {
26		comptime assert(unused_fmt.len == 0);
27		const ast = dump.ast;
28		var state = State{
29		.ast = &ast,
30		.node_tags = ast.nodes.items(.tag),
31		.node_data = ast.nodes.items(.data),
32		.node_tokens = ast.nodes.items(.main_token),
33		.tty = detectTTYFromWriter(writer),
34		};
35		try dumpNode(writer, &state, 0);
36		try writer.writeAll("\n");
37		assert(state.indent == 0);
38		}
39
40		fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void {
41		const tag = state.node_tags[node];
42		const data = state.node_data[node];
43		const data_tag = Node.data_tags[@intFromEnum(tag)];
44		const main_token = state.node_tokens[node];
45		const tty = state.tty;
46
47		try state.tty.setColor(writer, .bright_black);
48		try writer.print("{d}", .{node});
49		try state.tty.setColor(writer, .reset);
50		try writer.writeAll(" ");
51
52		try tty.setColor(writer, .cyan);
53		try writer.print("{s}", .{@tagName(tag)});
54		try tty.setColor(writer, .reset);
55
56		switch (tag) {
57		.builtin_call => {
58		try tty.setColor(writer, .red);
59		try writer.print(" {s}", .{state.ast.tokenSlice(main_token)});
60		try tty.setColor(writer, .reset);
61		},
62		else => {},
63		}
64
65		switch (data_tag) {
66		.none => {
67		switch (tag) {
68		.id => try tty.setColor(writer, .red),
69		.string => try tty.setColor(writer, .green),
70		.number_int, .number_float => try tty.setColor(writer, .yellow),
71		else => {},
72		}
73		switch (tag) {
74		.id,
75		.string,
76		.number_int,
77		.number_float,
78		=> try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}),
79
80		else => unreachable,
81		}
82		try tty.setColor(writer, .reset);
83		},
84		.fn_decl => {
85		try writer.writeAll(" ");
86		const fn_decl = state.ast.fullFnDecl(node);
87		if (fn_decl.public) {
88		try writer.writeAll("[pub] ");
89		}
90		try tty.setColor(writer, .magenta);
91		try writer.writeAll(fn_decl.id);
92		try tty.setColor(writer, .reset);
93		try writer.writeAll(" {");
94		{
95		state.indent += 1;
96		defer state.indent -= 1;
97		try state.newline(writer);
98		try writer.writeAll("params:");
99		try dumpNodeList(writer, state, fn_decl.params, '[', ']');
100		if (fn_decl.return_type) \|return_type\| {
101		try state.newline(writer);
102		try writer.writeAll("return_type: ");
103		try dumpNode(writer, state, return_type);
104		}
105		try state.newline(writer);
106		try writer.writeAll("body: ");
107		try dumpNode(writer, state, fn_decl.body);
108		}
109		try state.newline(writer);
110		try writer.writeAll("}");
111		},
112		.list => {
113		const list = data.list.slice(state.ast);
114		switch (tag) {
115		.root,
116		.block_void,
117		.block_implicit_return,
118		=> try dumpNodeList(writer, state, list, '{', '}'),
119		.builtin_call,
120		=> try dumpNodeList(writer, state, list, '(', ')'),
121		else => unreachable,
122		}
123		},
124		.var_decl => {
125		try writer.writeAll(" ");
126		const var_decl = state.ast.fullVarDecl(node);
127		if (var_decl.public) {
128		try writer.writeAll("[pub] ");
129		}
130		try writer.writeAll(switch (var_decl.is_const) {
131		true => "constant",
132		false => "mutable",
133		});
134		try tty.setColor(writer, .magenta);
135		try writer.print(" {s}", .{var_decl.id});
136		try tty.setColor(writer, .reset);
137		var indent = false;
138		if (var_decl.type) \|type_node\| {
139		state.indent += 1;
140		indent = true;
141		try state.newline(writer);
142		try writer.writeAll(": ");
143		try dumpNode(writer, state, type_node);
144		try writer.writeAll(", ");
145		}
146		try writer.writeAll(" = ");
147		try dumpNode(writer, state, var_decl.init);
148		if (indent) {
149		state.indent -= 1;
150		}
151		},
152		.param_decl => {
153		try writer.writeAll(" ");
154		const param_decl = state.ast.fullParamDecl(node);
155		try tty.setColor(writer, .magenta);
156		try writer.writeAll(param_decl.id);
157		try tty.setColor(writer, .reset);
158		try writer.writeAll(": ");
159		try dumpNode(writer, state, param_decl.type);
160		},
161		.bin => {
162		try writer.writeAll(" (");
163		{
164		state.indent += 1;
165		defer state.indent -= 1;
166		try state.newline(writer);
167		try dumpNode(writer, state, data.bin.lhs);
168		try state.newline(writer);
169		try dumpNode(writer, state, data.bin.rhs);
170		}
171		try state.newline(writer);
172		try writer.writeAll(")");
173		},
174		.un => {
175		try writer.writeAll(" (");
176		{
177		state.indent += 1;
178		defer state.indent -= 1;
179		try state.newline(writer);
180		try dumpNode(writer, state, data.un.expr);
181		}
182		try state.newline(writer);
183		try writer.writeAll(")");
184		},
185		.label => {
186		const label_text = state.ast.tokenSlice(main_token);
187		try writer.print(" {s}:", .{label_text});
188
189		try writer.writeAll(" (");
190		{
191		state.indent += 1;
192		defer state.indent -= 1;
193		try state.newline(writer);
194		try dumpNode(writer, state, data.label.expr);
195		}
196		try state.newline(writer);
197		try writer.writeAll(")");
198		},
199		.field => {
200		try writer.writeAll(" (");
201		{
202		state.indent += 1;
203		defer state.indent -= 1;
204		try state.newline(writer);
205		try dumpNode(writer, state, data.field.base);
206		}
207		try state.newline(writer);
208		try writer.writeAll(")");
209		},
210		.@"return" => {
211		if (data.@"return".expr.unwrap()) \|expr\| {
212		try writer.writeAll(" (");
213		try dumpNode(writer, state, expr);
214		try writer.writeAll(")");
215		}
216		},
217		.@"break" => {
218		const label = data.@"break".label;
219		const value = data.@"break".value;
220		if (label != .none or value != .none) {
221		// try writer.writeAll("(");
222		// try writer.writeAll(")");
223		if (label.unwrap()) \|l\| {
224		const label_text = state.ast.tokenSlice(l);
225		try writer.print(" :{s}", .{label_text});
226		}
227		if (value.unwrap()) \|expr\| {
228		try writer.writeAll(" (");
229		try dumpNode(writer, state, expr);
230		try writer.writeAll(")");
231		}
232		}
233		},
234		.call => {
235		const base = data.call.base;
236		const extra = state.ast.extraData(data.call.extra, Call);
237		const arg_list = extra.args.slice(state.ast);
238
239		try writer.writeAll("(");
240		try writer.writeAll("base: ");
241		try dumpNode(writer, state, base);
242		try writer.writeAll(", ");
243		try writer.writeAll("arg:");
244		try dumpNodeList(writer, state, arg_list, '[', ']');
245		try writer.writeAll(")");
246		},
247		.@"if" => {
248		const branch = data.@"if";
249		const if_data = state.ast.extraData(branch.extra, Ast.If);
250
251		try writer.writeAll(" (");
252		try dumpNode(writer, state, branch.cond);
253		try writer.writeAll(") {");
254		{
255		state.indent += 1;
256		defer state.indent -= 1;
257		try state.newline(writer);
258		try writer.writeAll("yes: ");
259		try dumpNode(writer, state, if_data.yes);
260		try state.newline(writer);
261		if (if_data.no.unwrap()) \|no\| {
262		try writer.writeAll("no: ");
263		try dumpNode(writer, state, no);
264		} else {
265		try writer.writeAll("no: [none]");
266		}
267		}
268		try state.newline(writer);
269		try writer.writeAll("}");
270		},
271		}
272		}
273
274		fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void {
275		try writer.print(" {c}", .{l});
276		{
277		state.indent += 1;
278		defer state.indent -= 1;
279
280		for (list) \|node\| {
281		try state.newline(writer);
282		try dumpNode(writer, state, node);
283		}
284		}
285		try state.newline(writer);
286		try writer.print("{c}", .{r});
287		}
288
289		const Dump = @This();
290
291		const std = @import("std");
292		const assert = std.debug.assert;
293
294		const Ast = @import("Ast.zig");
295		const NodeIndex = Ast.NodeIndex;
296		const Node = Ast.Node;
297		const TokenIndex = Ast.TokenIndex;
298		const Call = Ast.Call;
299
300		const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Ast.zig deleted-678

...	...	@@ -1,678 +0,0 @@
1		//! The Abstract Syntax Tree preserves most details of the source code,
2		//! as it is used for formatting. Before semantic analysis, this
3		//! structure is normalized into `Teg`, which removes constructs like
4		//! grouping, unneded let bindings, and other transforms.
5		//!
6		//! `ast.nodes` contains nodes stored as three parts:
7		//! - tag: the type of the node
8		//! - main_token: index to it's primary token, depends on the tag
9		//! - data: two u32 values, depends on the tag
10		//!
11		//! Many nodes encode all their metadata with just `node.data`, using
12		//! facts about the language grammar to be able to infer data without
13		//! storing it.
14		//!
15		//! For example, `var_decl` has the following encoding:
16		//! - `main_token` is either a `kw_var` or `kw_let`
17		//! - the token after `main_token` is always the identifier
18		//! - `data.var_decl` contains `NodeIndex`es pointing at the type
19		//! and the initializer, which are both expressions.
20		//! - If the keyword before `main_token` is `kw_pub` it is a public variable.
21		//!
22		//! To make decoding these easier, many nodes contain a "full" variant,
23		//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`.
24		//!
25		//! For nodes that must store more than 64 bits of payload data, an "extra"
26		//! struct is stored out-of-line in the `extra` field. These can be read
27		//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use
28		//! the `full` structures with their convenience functions.
29		//!
30		//! Documentation comments are not stored anywhere, but you can find them
31		//! by iterating backwards from a declaration and seeing if a `.doc_comment`
32		//! is present. This is provided by `ast.declDocComment(id)`.
33		//!
34		//! The `Ast` owns token and node data, but not the source code.
35
36		/// Externally owned source code.
37		utf8_source: [:0]const u8,
38
39		tokens: Token.List.Slice,
40		nodes: Node.List.Slice,
41		extra: []const u32,
42
43		pub const Node = struct {
44		tag: Tag,
45		main_token: TokenIndex,
46		data: Data,
47
48		pub const List = std.MultiArrayList(Node);
49
50		pub const Tag = enum {
51		/// Read with `ast.rootNodes()`
52		/// data is .list (containing stmt slice)
53		/// TODO: rename to `module`
54		root,
55		/// main_token is the {
56		/// data is .list (containing stmt slice)
57		///
58		/// This guarantees that the block either empty, or the last statement
59		/// ends with a semicolon.
60		block_void,
61		/// main_token is the {
62		/// data is .list (containing stmt slice)
63		///
64		/// This guarantees that the block is not empty and that the last
65		/// statement is an expression statement without a trailing semicolon.
66		block_implicit_return,
67		/// Variable or constant declaration
68		/// `var x: type = init;`
69		/// `let y = x;`
70		/// main_token is the keyword
71		/// data is .var_decl
72		var_decl,
73		/// Function declaration
74		/// `fn foo(x: i32) -> i32 { rhs }`
75		/// Params and return type are stored in extra data
76		fn_decl,
77		/// 'name: type' declaration
78		/// Main token is the identifier
79		param_decl,
80		/// An identifier. main_token is the identifier.
81		id,
82		/// Main token is the data. May not fit in i32 range,
83		/// but will contain no dots or e's.
84		number_int,
85		/// Main token is data. Will contain '.' and/or 'e'.
86		number_float,
87		/// A string literal
88		string,
89		/// A multi-line string literal
90		/// data is .token_list of the tokens, which is guaranteed to be contiguous
91		multiline_string,
92		/// Call to a function
93		/// `foo(x, y);`
94		/// main_token is the `(`
95		/// data is .call
96		call,
97		/// Call to a builtin function
98		/// `@raw("hello");`
99		/// data is .list (containing arg slice)
100		builtin_call,
101		/// `if cond { ... } else ...`
102		/// main_token is the `if`
103		/// data is .@"if"
104		@"if",
105		/// `if let id = ... { ... } else ...`
106		/// main_token is the `if`
107		/// data is .@"if"
108		// if_let_nonvoid,
109		/// `id: ...`
110		/// data is un
111		label,
112		/// `expr.field`
113		/// main_token is the `.`
114		/// data is .field
115		field,
116		/// `lhs[rhs]`
117		/// main_token is the `[`
118		/// data is .bin
119		index,
120		/// `return expr;`
121		/// main_token is the `return`
122		/// data is .@"return"
123		@"return",
124		/// `break :brk expr;`
125		/// main_token is the `break`
126		/// data is .@"break"
127		@"break",
128		/// `(expr)`
129		/// main_token is the `(`
130		/// data is .un
131		group,
132		/// `lhs + rhs`. main_token is the operator.
133		add,
134		/// `lhs - rhs`. main_token is the operator.
135		sub,
136		/// `lhs * rhs`. main_token is the operator.
137		mul,
138		/// `lhs / rhs`. main_token is the operator.
139		div,
140		/// `lhs % rhs`. main_token is the operator.
141		rem,
142		/// `lhs ** rhs`. main_token is the operator.
143		pow,
144		/// `lhs && rhs`. main_token is the operator.
145		@"and",
146		/// `lhs \|\| rhs`. main_token is the operator.
147		@"or",
148		/// `lhs & rhs`. main_token is the operator.
149		bit_and,
150		/// `lhs \| rhs`. main_token is the operator.
151		bit_or,
152		/// `lhs ^ rhs`. main_token is the operator.
153		bit_xor,
154		/// `lhs << rhs`. main_token is the operator.
155		shl,
156		/// `lhs >> rhs`. main_token is the operator.
157		shr,
158		/// `lhs == rhs`. main_token is the operator.
159		eq,
160		/// `lhs != rhs`. main_token is the operator.
161		ne,
162		/// `lhs < rhs`. main_token is the operator.
163		lt,
164		/// `lhs <= rhs`. main_token is the operator.
165		le,
166		/// `lhs > rhs`. main_token is the operator.
167		gt,
168		/// `lhs >= rhs`. main_token is the operator.
169		ge,
170		/// `lhs = rhs`. main_token is the operator.
171		assign,
172		/// `lhs += rhs`. main_token is the operator.
173		add_assign,
174		/// `lhs -= rhs`. main_token is the operator.
175		sub_assign,
176		/// `lhs *= rhs`. main_token is the operator.
177		mul_assign,
178		/// `lhs /= rhs`. main_token is the operator.
179		div_assign,
180		/// `lhs %= rhs`. main_token is the operator.
181		rem_assign,
182		/// `lhs **= rhs`. main_token is the operator.
183		pow_assign,
184		/// `lhs &&= rhs`. main_token is the operator.
185		and_assign,
186		/// `lhs \|\|= rhs`. main_token is the operator.
187		or_assign,
188		/// `lhs &= rhs`. main_token is the operator.
189		bit_and_assign,
190		/// `lhs \|= rhs`. main_token is the operator.
191		bit_or_assign,
192		/// `lhs ^= rhs`. main_token is the operator.
193		bit_xor_assign,
194		/// `lhs <<= rhs`. main_token is the operator.
195		shl_assign,
196		/// `lhs >>= rhs`. main_token is the operator.
197		shr_assign,
198		/// `!expr`. main_token is the operator.
199		bool_not,
200		/// `-expr`. main_token is the operator.
201		negate,
202		/// `~expr`. main_token is the operator.
203		bit_not,
204		/// `comptime expr`. main_token is the operator.
205		@"comptime",
206		/// `pure expr`. main_token is the operator.
207		// pure,
208		/// `assert expr`. main_token is the operator.
209		// assert,
210
211		pub fn isAssignBin(op: Tag) bool {
212		return switch (op) {
213		.assign,
214		.add_assign,
215		.sub_assign,
216		.mul_assign,
217		.div_assign,
218		.rem_assign,
219		.pow_assign,
220		.and_assign,
221		.or_assign,
222		.bit_and_assign,
223		.bit_or_assign,
224		.bit_xor_assign,
225		.shl_assign,
226		.shr_assign,
227		=> true,
228		.add,
229		.sub,
230		.mul,
231		.div,
232		.rem,
233		.pow,
234		.@"and",
235		.@"or",
236		.bit_and,
237		.bit_or,
238		.bit_xor,
239		.shl,
240		.shr,
241		.eq,
242		.ne,
243		.lt,
244		.le,
245		.gt,
246		.ge,
247		=> false,
248		else => unreachable, // not a binary operator
249		};
250		}
251
252		pub fn isBasic(op: Tag) bool {
253		return switch (op) {
254		.id,
255		.string,
256		.number_int,
257		.number_float,
258		.builtin_call,
259		=> true,
260		else => false,
261		};
262		}
263		};
264
265		pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{
266		.root = .list,
267		.block_void = .list,
268		.block_implicit_return = .list,
269		.var_decl = .var_decl,
270		.fn_decl = .fn_decl,
271		.param_decl = .param_decl,
272		.id = .none,
273		.number_int = .none,
274		.number_float = .none,
275		.string = .none,
276		.multiline_string = .list,
277		.call = .call,
278		.builtin_call = .list,
279		.@"if" = .@"if",
280		.field = .field,
281		.index = .bin,
282		.@"return" = .@"return",
283		.@"break" = .@"break",
284		.group = .un,
285		.add = .bin,
286		.sub = .bin,
287		.mul = .bin,
288		.div = .bin,
289		.rem = .bin,
290		.pow = .bin,
291		.@"and" = .bin,
292		.@"or" = .bin,
293		.bit_and = .bin,
294		.bit_or = .bin,
295		.bit_xor = .bin,
296		.shl = .bin,
297		.shr = .bin,
298		.eq = .bin,
299		.ne = .bin,
300		.lt = .bin,
301		.le = .bin,
302		.gt = .bin,
303		.ge = .bin,
304		.assign = .bin,
305		.add_assign = .bin,
306		.sub_assign = .bin,
307		.mul_assign = .bin,
308		.div_assign = .bin,
309		.rem_assign = .bin,
310		.pow_assign = .bin,
311		.and_assign = .bin,
312		.or_assign = .bin,
313		.bit_and_assign = .bin,
314		.bit_or_assign = .bin,
315		.bit_xor_assign = .bin,
316		.shl_assign = .bin,
317		.shr_assign = .bin,
318		.bool_not = .un,
319		.negate = .un,
320		.bit_not = .un,
321		.@"comptime" = .un,
322		.label = .label,
323		});
324
325		pub const Data = union {
326		none: struct { Unused = .zero, Unused = .zero },
327		fn_decl: struct {
328		// A `FnDecl`
329		extra: ExtraIndex,
330		body: NodeIndex,
331		},
332		/// either is a node list or a token list, depending on the node's tag
333		list: SubList,
334		var_decl: struct {
335		type: OptionalNodeIndex,
336		init: NodeIndex,
337		},
338		param_decl: struct {
339		type: NodeIndex,
340		unused: Unused = .zero,
341		},
342		bin: struct {
343		lhs: NodeIndex,
344		rhs: NodeIndex,
345		},
346		field: struct {
347		base: NodeIndex,
348		field_token: TokenIndex,
349		},
350		un: struct {
351		expr: NodeIndex,
352		unused: Unused = .zero,
353		},
354		label: struct {
355		expr: NodeIndex,
356		unused: Unused = .zero,
357		},
358		@"return": struct {
359		expr: OptionalNodeIndex,
360		unused: Unused = .zero,
361		},
362		@"break": struct {
363		label: OptionalTokenIndex,
364		value: OptionalNodeIndex,
365		},
366		call: struct {
367		base: NodeIndex,
368		/// A `Call`
369		extra: ExtraIndex,
370		},
371		@"if": struct {
372		cond: NodeIndex,
373		/// A `If`
374		extra: ExtraIndex,
375		},
376
377		const FieldEnum = std.meta.FieldEnum(Data);
378		};
379		};
380
381		pub const Error = error{ OutOfMemory, ParseFail };
382
383		pub const Location = struct {
384		line: Ordinal(u32),
385		column: Ordinal(u32),
386		// Byte offset of the start of the line
387		line_start: ByteOffset,
388		// Byte offset of the end of the line
389		line_end: ByteOffset,
390		};
391
392		pub const Span = struct {
393		start: u32,
394		end: u32,
395		main: u32,
396		};
397
398		// This can be removed when the following issue is implemented:
399		// See: https://github.com/ziglang/zig/issues/104
400		// See: https://github.com/ziglang/zig/issues/3806
401		pub const OptionalNodeIndex = enum(u32) {
402		none = 0,
403		_,
404
405		pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
406		assert(idx != 0);
407		return @enumFromInt(idx orelse return .none);
408		}
409
410		pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
411		return switch (idx) {
412		.none => null,
413		_ => @intFromEnum(idx),
414		};
415		}
416		};
417
418		pub const OptionalTokenIndex = enum(u32) {
419		none = 0,
420		_,
421
422		pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
423		assert(idx != 0);
424		return @enumFromInt(idx orelse return .none);
425		}
426
427		pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
428		return switch (idx) {
429		.none => null,
430		_ => @intFromEnum(idx),
431		};
432		}
433		};
434
435		pub const OptionalExtraIndex = enum(u32) {
436		none = std.math.maxInt(u32),
437		_,
438
439		pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
440		assert(idx != std.math.maxInt(u32));
441		return @enumFromInt(idx orelse return .none);
442		}
443
444		pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
445		return switch (idx) {
446		.none => null,
447		_ => @intFromEnum(idx),
448		};
449		}
450		};
451
452		pub const SubList = struct {
453		/// Either NodeIndex or TokenIndex, depending on where this came from.
454		start: u32,
455		len: u32,
456
457		pub fn slice(self: SubList, ast: *const Ast) []const u32 {
458		return ast.extra[self.start..][0..self.len];
459		}
460		};
461
462		pub const FnDecl = struct {
463		params: SubList,
464		return_type: OptionalNodeIndex,
465		};
466
467		pub const Call = struct {
468		args: SubList,
469		};
470
471		pub const If = struct {
472		/// either .block_void or .block_implicit_return
473		yes: NodeIndex,
474		/// if it exists, it is either
475		/// .block_void, .block_implicit_return, or .@"if"
476		no: OptionalNodeIndex,
477		};
478
479		pub const full = struct {
480		pub const FnDecl = struct {
481		start_token: TokenIndex,
482		public: bool,
483		id: []const u8,
484		params: []const NodeIndex,
485		return_type: ?NodeIndex,
486		body: NodeIndex,
487		};
488
489		pub const VarDecl = struct {
490		start_token: TokenIndex,
491		public: bool,
492		is_const: bool,
493		id: []const u8,
494		type: ?NodeIndex,
495		init: NodeIndex,
496		};
497
498		pub const ParamDecl = struct {
499		start_token: TokenIndex,
500		id: []const u8,
501		type: NodeIndex,
502		};
503		};
504
505		pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void {
506		ast.tokens.deinit(gpa);
507		ast.nodes.deinit(gpa);
508		gpa.free(ast.extra);
509		ast.* = undefined;
510		}
511
512		pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag {
513		return ast.nodes.items(.tag)[node];
514		}
515
516		pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data {
517		return ast.nodes.items(.data)[node];
518		}
519
520		pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex {
521		return ast.nodes.items(.main_token)[node];
522		}
523
524		pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl {
525		const tok_tags = ast.tokens.items(.tag);
526
527		const main_token = ast.nodes.items(.main_token)[node];
528		const public = tok_tags[main_token -\| 1] == .kw_pub;
529		assert(tok_tags[main_token] == .kw_fn);
530		assert(tok_tags[main_token + 1] == .id);
531		const id = ast.tokenSlice(main_token + 1);
532
533		const data = ast.nodes.items(.data)[node].fn_decl;
534		const extra = ast.extraData(data.extra, FnDecl);
535
536		return .{
537		.start_token = if (public) main_token - 1 else main_token,
538		.public = public,
539		.id = id,
540		.body = data.body,
541		.params = extra.params.slice(&ast),
542		.return_type = extra.return_type.unwrap(),
543		};
544		}
545
546		pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl {
547		const tok_tags = ast.tokens.items(.tag);
548
549		const main_token = ast.nodes.items(.main_token)[node];
550		const public = tok_tags[main_token -\| 1] == .kw_pub;
551		const is_const = switch (tok_tags[main_token]) {
552		.kw_let => true,
553		.kw_var => false,
554		else => \|tag\| {
555		if (std.debug.runtime_safety) {
556		std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)});
557		}
558		unreachable;
559		},
560		};
561		assert(tok_tags[main_token + 1] == .id);
562		const id = ast.tokenSlice(main_token + 1);
563		const data = ast.nodes.items(.data)[node].var_decl;
564		return .{
565		.start_token = if (public) main_token - 1 else main_token,
566		.public = public,
567		.is_const = is_const,
568		.id = id,
569		.type = data.type.unwrap(),
570		.init = data.init,
571		};
572		}
573
574		pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
575		const tok_tags = ast.tokens.items(.tag);
576
577		const main_token = ast.nodes.items(.main_token)[node];
578		assert(tok_tags[main_token] == .id);
579		const id = ast.tokenSlice(main_token);
580		const data = ast.nodes.items(.data)[node].param_decl;
581		return .{
582		.start_token = main_token,
583		.id = id,
584		.type = data.type,
585		};
586		}
587
588		pub fn idToken(ast: Ast, node: NodeIndex) TokenIndex {
589		return switch (ast.nodeTag(node)) {
590		.param_decl,
591		.id,
592		.label,
593		=> ast.nodeToken(node),
594		.fn_decl,
595		.var_decl,
596		=> ast.nodeToken(node) + 1,
597		else => unreachable, // does not have an id
598		};
599		}
600
601		pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
602		const data = ast.nodes.items(.data)[node].list;
603		return data.slice(ast);
604		}
605
606		pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
607		const token_start = self.tokens.items(.offset)[token_index];
608		return Tokenizer.tokenLocation(self.utf8_source, start_offset, token_start);
609		}
610
611		pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 {
612		return ast.tokens.get(token_index).slice(ast.utf8_source);
613		}
614
615		pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
616		assert(ast.nodes.items(.tag)[node].isBasic());
617		const token = ast.nodes.items(.main_token)[node];
618		return ast.tokenSlice(token);
619		}
620
621		pub fn fnDeclIdToken(tree: Ast, node: NodeIndex) TokenIndex {
622		assert(tree.nodeTag(node) == .fn_decl);
623		return tree.nodeToken(node) + 1;
624		}
625
626		pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
627		const fields = std.meta.fields(T);
628		var result: T = undefined;
629		var i: usize = 0;
630		inline for (fields) \|field\| {
631		@field(result, field.name) = switch (field.type) {
632		NodeIndex => tree.extra[index + i],
633		SubList => SubList{
634		.start = tree.extra[index + i],
635		.len = tree.extra[index + i + 1],
636		},
637		OptionalNodeIndex => @enumFromInt(tree.extra[index + i]),
638		OptionalTokenIndex => @enumFromInt(tree.extra[index + i]),
639		OptionalExtraIndex => @enumFromInt(tree.extra[index + i]),
640		else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)),
641		};
642		i += switch (field.type) {
643		NodeIndex,
644		OptionalNodeIndex,
645		OptionalTokenIndex,
646		OptionalExtraIndex,
647		=> 1,
648		SubList => 2,
649		else => comptime unreachable,
650		};
651		}
652		return result;
653		}
654
655		pub fn dumpToWriter(ast: Ast, writer: anytype) !void {
656		try writer.print("{}", .{Dump{ .ast = ast }});
657		}
658
659		pub const Dump = @import("Ast.Dump.zig");
660
661		pub const Unused = enum(u32) { zero = 0 };
662		pub const TokenIndex = u32;
663		pub const ExtraIndex = u32;
664		pub const NodeIndex = u32;
665
666		const Ast = @This();
667		const std = @import("std");
668		const assert = std.debug.assert;
669
670		const Tokenizer = @import("Tokenizer.zig");
671		const Token = Tokenizer.Token;
672		const TokenList = Tokenizer.Token.List;
673		const ByteOffset = Tokenizer.ByteOffset;
674
675		const Report = @import("Report.zig");
676
677		const util = @import("util.zig");
678		const Ordinal = util.Ordinal;

src/compiler/Cache.zig deleted-9

...	...	@@ -1,9 +0,0 @@
1		const std = @import("std");
2
3		/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
4		pub const bin_digest_len = 16;
5		pub const hex_digest_len = bin_digest_len * 2;
6		pub const BinDigest = [bin_digest_len]u8;
7		pub const HexDigest = [hex_digest_len]u8;
8
9		pub const HashHelper = std.Build.Cache.HashHelper;

src/compiler/Comp.zig deleted-443

...	...	@@ -1,443 +0,0 @@
1		//! Compilation
2
3		/// General purpose allocator
4		gpa: Allocator,
5		/// Stores all Type and Value objects.
6		intern_pool: InternPool = .{},
7		/// All files in the compilation
8		files: std.StringArrayHashMapUnmanaged(*File) = .{},
9
10		/// # of sema instructions in this compilation cycle
11		analyzed_instructions: usize = 0,
12
13		options: Options = .{},
14
15		// TODO: consider computing this dynamicly per-link
16		current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
17
18		pub fn init(comp: *Comp) !void {
19		try comp.intern_pool.init(comp.gpa);
20		}
21
22		pub fn deinit(comp: *Comp) void {
23		comp.intern_pool.deinit(comp.gpa);
24		for (comp.files.values()) \|file\| {
25		file.deinit(comp.gpa);
26		}
27		comp.files.deinit(comp.gpa);
28		comp.current_compile_decls.deinit(comp.gpa);
29		}
30
31		pub fn getFile(comp: Comp, absolute_file_path: []const u8) !File {
32		assert(std.fs.path.isAbsolute(absolute_file_path));
33		const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path);
34		if (gop.found_existing) {
35		return gop.value_ptr.*;
36		}
37		const file = try comp.gpa.create(File);
38		file.* = .{
39		.file_path = absolute_file_path,
40		.path_digest = null,
41		.root_module = .none,
42		.status = .never_loader,
43		.source = null,
44		.ast = null,
45		.teg = null,
46		};
47		gop.value_ptr.* = file;
48		return file;
49		}
50
51		pub const Source = struct {
52		code: [:0]const u8,
53		stat: Stat,
54		};
55
56		pub const File = struct {
57		/// TODO: change this to a URL
58		file_path: []const u8,
59
60		root_module: Decl.Index.Optional,
61		status: enum {
62		never_loader,
63		/// A failure occurred during loading the file. Retrying might succeed.
64		retryable_failure,
65		/// Errors during parse
66		parse_failure,
67		/// Errors during transform
68		transform_failure,
69		/// Teg is valid
70		success,
71		},
72
73		source: ?Source,
74		ast: ?Ast,
75		teg: ?Teg,
76
77		path_digest: ?Cache.BinDigest,
78
79		const GetSourceError =
80		std.fs.File.OpenError \|\|
81		std.fs.File.ReadError \|\|
82		std.fs.File.StatError \|\|
83		Allocator.Error \|\| error{
84		FileTooBig,
85		UnexpectedEndOfFile,
86		};
87
88		pub fn getSource(file: *File, gpa: Allocator) GetSourceError!Source {
89		if (file.source) \|source\| {
90		return source;
91		}
92
93		// Keep track of inode, file size, mtime, hash so we can detect which files
94		// have been modified when an incremental update is requested.
95		var f = try std.fs.openFileAbsolute(file.file_path, .{});
96		defer f.close();
97
98		const stat = try f.stat();
99
100		if (stat.size > std.math.maxInt(u32))
101		return error.FileTooBig;
102
103		const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
104		errdefer gpa.free(source);
105		const amt = try f.readAll(source);
106		if (amt != stat.size)
107		return error.UnexpectedEndOfFile;
108
109		// Here we do not modify stat fields because this function is the one
110		// used for error reporting. We need to keep the stat fields stale so that
111		// transformFile can know to regenerate Teg.
112
113		const loaded_source = Source{
114		.code = source,
115		.stat = .{
116		.size = stat.size,
117		.inode = stat.inode,
118		.mtime = stat.mtime,
119		},
120		};
121		file.source = loaded_source;
122		return loaded_source;
123		}
124
125		pub const GetAstError = GetSourceError \|\| Parse.Error;
126
127		pub fn getAst(file: File, comp: Comp) GetAstError!Ast {
128		if (file.ast) \|ast\| {
129		return ast;
130		}
131
132		var err_data: Parse.AstError.WithTrace = undefined;
133		const ast = Parse.parse(
134		comp.gpa,
135		(try file.getSource(comp.gpa)).code,
136		&err_data,
137		) catch \|err\| switch (err) {
138		error.OutOfMemory => return err,
139		error.ParseFail => {
140		// TODO: let this merge with other report
141		var report = try err_data.toReport(comp, file);
142		report.dumpToStdErr(comp) catch {};
143		report.deinit(comp.gpa);
144		return err;
145		},
146		};
147
148		if (comp.options.dump_ir) {
149		ast.dumpToWriter(std.io.getStdErr().writer()) catch {
150		// Simplify the error set.
151		};
152		}
153
154		file.ast = ast;
155		return ast;
156		}
157
158		pub const GetTegError = GetAstError \|\| Transform.Error;
159
160		pub fn getTeg(file: File, comp: Comp, fail_report: *Report) GetTegError!Teg {
161		if (file.teg) \|teg\| {
162		return teg;
163		}
164
165		const bin_digest = hash: {
166		var path_hash: Cache.HashHelper = .{};
167		path_hash.addBytes("0.1.0");
168		path_hash.addBytes(file.file_path);
169		var bin: Cache.BinDigest = undefined;
170		path_hash.hasher.final(&bin);
171		break :hash bin;
172		};
173		file.path_digest = bin_digest;
174
175		const teg = try Transform.transform(
176		try file.getAst(comp),
177		comp.gpa,
178		file,
179		fail_report,
180		);
181
182		file.teg = teg;
183		return teg;
184		}
185
186		pub fn unloadSource(file: *File, gpa: Allocator) void {
187		if (file.source) \|source\| {
188		gpa.free(source.code);
189		file.source = null;
190		}
191		}
192
193		pub fn unloadAst(file: *File, gpa: Allocator) void {
194		if (file.ast) \|*ast\| {
195		ast.deinit(gpa);
196		file.ast = null;
197		}
198		}
199
200		pub fn unloadTeg(file: *File, gpa: Allocator) void {
201		if (file.teg) \|*teg\| {
202		teg.deinit(gpa);
203		file.teg = null;
204		}
205		}
206
207		pub fn deinit(file: *File, gpa: Allocator) void {
208		file.unloadTeg(gpa);
209		file.unloadAst(gpa);
210		file.unloadSource(gpa);
211		gpa.destroy(file);
212		}
213		};
214
215		pub const Decl = struct {
216		name: InternPool.NullTerminatedString,
217		val: InternPool.Value,
218		/// The direct parent namespace of the Decl. In the case of the Decl
219		/// corresponding to a file, this is the namespace of the struct, since
220		/// there is no parent.
221		src_namespace: Namespace.Index,
222
223		/// The AST node index of this declaration.
224		/// Must be recomputed when the corresponding source file is modified.
225		// src_node: Ast.NodeIndex,
226		/// Line number corresponding to `src_node`. Stored separately so that source files
227		/// do not need to be loaded into memory in order to compute debug line numbers.
228		/// This value is absolute.
229		// src_line: u32,
230		/// Index of the ZIR `declaration` instruction from which this `Decl` was created.
231		/// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`.
232		teg_decl_index: InternPool.TrackedInst.Index.Optional,
233
234		/// Represents the "shallow" analysis status. For example, for decls that are functions,
235		/// the function type is analyzed with this set to `in_progress`, however, the semantic
236		/// analysis of the function body is performed with this value set to `success`. Functions
237		/// have their own analysis status field.
238		analysis: enum {
239		/// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
240		/// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
241		unreferenced,
242		/// Semantic analysis for this Decl is running right now.
243		/// This state detects dependency loops.
244		in_progress,
245		/// The file corresponding to this Decl had a parse error or ZIR error.
246		/// There will be a corresponding ErrorMsg in Zcu.failed_files.
247		file_failure,
248		/// This Decl might be OK but it depends on another one which did not
249		/// successfully complete semantic analysis.
250		dependency_failure,
251		/// Semantic analysis failure.
252		/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
253		sema_failure,
254		/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
255		codegen_failure,
256		/// Sematic analysis and constant value codegen of this Decl has
257		/// succeeded. However, the Decl may be outdated due to an in-progress
258		/// update. Note that for a function, this does not mean codegen of the
259		/// function body succeded: that state is indicated by the function's
260		/// `analysis` field.
261		complete,
262		},
263
264		codegen: ?JsTree = null,
265
266		is_pub: bool,
267
268		/// What kind of a declaration is this.
269		kind: Kind,
270
271		pub fn deinit(decl: *Decl, gpa: Allocator) void {
272		defer decl.* = undefined;
273		if (decl.codegen) \|*cg\| cg.deinit(gpa);
274		}
275
276		pub const Kind = enum {
277		named,
278		};
279
280		pub const Index = InternPool.DeclIndex;
281		};
282
283		pub const Namespace = struct {
284		parent: Index.Optional,
285		file_scope: *File,
286		/// Direct children of the namespace.
287		/// Declaration order is preserved via entry order.
288		/// These are only declarations named directly by the AST; anonymous
289		/// declarations are not stored here.
290		decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{},
291
292		pub const DeclContext = struct {
293		comp: *Comp,
294
295		pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 {
296		const decl = ctx.comp.intern_pool.declPtr(decl_index);
297		return std.hash.uint32(@intFromEnum(decl.name));
298		}
299
300		pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool {
301		_ = b_index;
302		const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index);
303		const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index);
304		return a_decl.name == b_decl.name;
305		}
306		};
307
308		pub const DeclNameAdapter = struct {
309		comp: *Comp,
310
311		pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 {
312		_ = self;
313		return std.hash.uint32(@intFromEnum(s));
314		}
315
316		pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool {
317		_ = b_index;
318		return a == self.comp.intern_pool.declPtr(b_decl_index).name;
319		}
320		};
321
322		pub fn lookupDirect(ns: Namespace, comp: Comp, name: InternPool.NullTerminatedString) ?Decl.Index {
323		return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp });
324		}
325
326		pub const Index = InternPool.NamespaceIndex;
327		};
328
329		/// Index a module in a Teg.
330		/// When a File's Teg is loaded, we call this on the root module.
331		/// submodules are analyzed lazily.
332		pub fn indexTegModule(comp: Comp, file: File, module_index: Teg.Inst.Index) !Namespace.Index {
333		const teg = file.teg.?;
334
335		const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
336		.parent = .none,
337		.file_scope = file,
338		});
339		const ns_ptr = comp.intern_pool.namespacePtr(ns);
340		errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns);
341
342		for (teg.moduleDeclarations(module_index)) \|decl_index\| {
343		const name = try comp.intern_pool.getOrPutString(
344		comp.gpa,
345		teg.declName(decl_index),
346		.no_embedded_nulls,
347		);
348		const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index);
349		const decl = try comp.intern_pool.createDecl(comp.gpa, .{
350		.name = name,
351		.val = undefined,
352		.src_namespace = ns,
353		.teg_decl_index = tracked_inst.toOptional(),
354		.analysis = .unreferenced,
355		.is_pub = false, // todo:
356		.kind = .named,
357		});
358		try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp });
359		}
360
361		return ns;
362		}
363
364		pub fn semaEntryPoint(comp: Comp, file: File) !void {
365		var report: Report = undefined;
366		const teg = try file.getTeg(comp, &report);
367
368		// TODO, move this elsewhere. we should print all errors and warnings at once
369		if (teg.report.warnings.len > 0) {
370		try teg.report.dumpToStdErr(comp);
371		}
372
373		const ns = try comp.indexTegModule(file, .main_module_inst);
374		const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
375		@panic("no top level code?");
376		const decl = comp.intern_pool.declPtr(decl_index);
377
378		assert(comp.current_compile_decls.items.len == 0);
379
380		var arena = std.heap.ArenaAllocator.init(comp.gpa);
381		defer arena.deinit();
382
383		var sema: Sema = .{
384		.comp = comp,
385		.gpa = comp.gpa,
386		.code = teg,
387		.arena = arena.allocator(),
388		.ip = &comp.intern_pool,
389		.decl = decl,
390		.decl_index = decl_index,
391		.namespace_index = ns,
392		};
393		try sema.init();
394		defer sema.deinit();
395		_ = try sema.analyzeDeclaration();
396		try sema.finalize();
397		}
398
399		pub fn codeGenDecl(c: Comp, decl: Decl, air: *const Air) !void {
400		const t = tracer.trace(@src());
401		defer t.end();
402
403		var arena_state = std.heap.ArenaAllocator.init(c.gpa);
404		defer arena_state.deinit();
405		const arena = arena_state.allocator();
406
407		var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air);
408		errdefer cg.deinit();
409		try cg.build();
410		decl.codegen = try cg.finalize();
411		}
412
413		pub fn link(c: *Comp, writer: anytype) !void {
414		try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
415		}
416
417		const Options = struct {
418		dump_ir: bool = false,
419		};
420
421		const Comp = @This();
422
423		const std = @import("std");
424		const Allocator = std.mem.Allocator;
425		const assert = std.debug.assert;
426		const Stat = std.Build.Cache.File.Stat;
427		const log = std.log.scoped(.comp);
428
429		const InternPool = @import("InternPool.zig");
430
431		const Ast = @import("Ast.zig");
432		const Air = @import("Air.zig");
433		const Teg = @import("Teg.zig");
434		const Cache = @import("Cache.zig");
435		const Sema = @import("Sema.zig");
436		const Transform = @import("Transform.zig");
437		const JsNodeGen = @import("JsNodeGen.zig");
438		const JsTree = @import("JsTree.zig");
439		const JsLink = @import("JsLink.zig");
440		const Report = @import("Report.zig");
441		const Parse = @import("Parse.zig");
442
443		const tracer = @import("tracer");

src/compiler/InternPool.zig deleted-1196

...	...	@@ -1,1196 +0,0 @@
1		//! Large parts of this file are based on the Zig compiler's InternPool.
2		//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig
3
4		/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
5		/// constructed lazily.
6		map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
7		items: std.MultiArrayList(Item) = .{},
8		extra: ExtraStore = .{},
9
10		/// In order to store references to strings in fewer bytes, we copy all
11		/// string bytes into here. String bytes can be null. It is up to whomever
12		/// is referencing the data here whether they want to store both index and length,
13		/// thus allowing null bytes, or store only index, and use null-termination. The
14		/// `string_bytes` array is agnostic to either usage.
15		string_bytes: std.ArrayListUnmanaged(u8) = .{},
16
17		/// Used for finding the index inside `string_bytes`.
18		string_table: std.HashMapUnmanaged(
19		u32,
20		void,
21		std.hash_map.StringIndexContext,
22		std.hash_map.default_max_load_percentage,
23		) = .{},
24
25		/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
26		/// persists across incremental updates.
27		tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
28
29		/// Rather than allocating Decl objects with an Allocator, we instead allocate
30		/// them with this SegmentedList. This provides four advantages:
31		/// * Stable memory so that one thread can access a Decl object while another
32		/// thread allocates additional Decl objects from this list.
33		/// * It allows us to use u32 indexes to reference Decl objects rather than
34		/// pointers, saving memory in Type, Value, and dependency sets.
35		/// * Using integers to reference Decl objects rather than pointers makes
36		/// serialization trivial.
37		/// * It provides a unique integer to be used for anonymous symbol names, avoiding
38		/// multi-threaded contention on an atomic counter.
39		allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{},
40		/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack.
41		decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{},
42
43		/// Same pattern as with `allocated_decls`.
44		allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{},
45		/// Same pattern as with `decls_free_list`.
46		namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{},
47
48		/// How many items in the InternPool are statically known.
49		/// This is specified with an integer literal and a corresponding comptime
50		/// assert below to break an unfortunate and arguably incorrect dependency loop
51		/// when compiling.
52		pub const static_len = Teg.Inst.Index.static_len;
53		comptime {
54		// @compileLog(static_len, static_keys.len);
55		assert(static_len == static_keys.len);
56		}
57
58		pub const Item = struct {
59		tag: Tag,
60		data: EncodedData,
61
62		pub fn encode(comptime tag: Tag, data: tag.Data()) Item {
63		return .{ .tag = tag, .data = EncodedData.encode(tag, data) };
64		}
65		};
66
67		pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var");
68		pub const EncodedData = dod.EncodedData(Tag, u32);
69		pub const ExtraIndex = ExtraStore.Index;
70
71		pub const Tag = enum(u8) {
72		/// This special tag represents a value which was removed from this pool via
73		/// `InternPool.remove`. The item remains allocated to preserve indices, but
74		/// lookups will consider it not equal to any other item, and all queries
75		/// assert not this tag. `data` is unused.
76		removed,
77
78		/// A function type
79		/// data is an extra index into a `FuncType`
80		type_function,
81		/// A type that can be represented with only an enum tag.
82		/// data is SimpleType enum value.
83		simple_type,
84
85		// --- before this are types ---
86		// --- past this are values ---
87
88		/// A value that can be represented with only an enum tag.
89		/// data is SimpleValue enum value.
90		simple_value,
91		/// A numeric literal that can be represented as a float.
92		/// Integers greater than 2^53 known at compile time are currently not supported.
93		/// data is an extra index into a `Float`
94		comptime_float,
95		/// An int32 value
96		/// data is i32
97		int32,
98		/// A float value
99		/// data is an extra index into a `Float`
100		float,
101		/// A function
102		/// data is an extra index into a `FuncDecl`
103		func,
104		/// A string
105		/// data is an extra index into a `String`
106		string,
107
108		pub fn Data(comptime tag: Tag) type {
109		return switch (tag) {
110		.removed,
111		=> unreachable, // does not have a data
112
113		.simple_type => SimpleType,
114		.simple_value => SimpleValue,
115		.int32 => i32,
116
117		else => ExtraIndex(tag.Payload()),
118		};
119		}
120
121		/// When `data` is an extra index, this is the type of the payload.
122		pub fn Payload(comptime tag: Tag) type {
123		return switch (tag) {
124		.removed,
125		.simple_type,
126		.simple_value,
127		.int32,
128		=> unreachable, // does not have a payload
129		.float, .comptime_float => Float,
130		.func => FuncDecl,
131		.type_function => TypeFunction,
132		.string => String.WithLen,
133		};
134		}
135
136		/// Trailing: N/A
137		pub const FuncDecl = struct {
138		analysis: FuncAnalysis,
139		owner_decl: DeclIndex,
140		ty: Type,
141		teg_body_inst: TrackedInst.Index,
142		};
143
144		/// Trailing:
145		/// 0. param_type: Index for each params_len
146		pub const TypeFunction = struct {
147		params_len: u32,
148		return_type: Type,
149		};
150		};
151
152		pub const Value = enum(u32) {
153		pub const first_type: Value = .opaque_type;
154		pub const last_type: Value = .noreturn_type;
155		pub const first_value: Value = .void;
156		pub const last_value: Value = .empty_string;
157
158		opaque_type,
159		bool_type,
160		int32_type,
161		float_type,
162		string_type,
163		noreturn_type,
164		type_type,
165		numeric_literal_type,
166
167		void,
168		bool_true,
169		bool_false,
170		int32_zero,
171		float_zero,
172		nan,
173		empty_string,
174		standard_library,
175
176		none = std.math.maxInt(u32),
177		_,
178
179		/// An array of `Index` existing within the `extra` array.
180		/// This type exists to provide a struct with lifetime that is
181		/// not invalidated when items are added to the `InternPool`.
182		pub const Slice = struct {
183		start: u32,
184		len: u32,
185
186		pub fn get(slice: Slice, ip: *const InternPool) []Value {
187		return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
188		}
189		};
190
191		/// Used for a map of `Index` values to the index within a list of `Index` values.
192		const Adapter = struct {
193		indexes: []const Value,
194
195		pub fn eql(ctx: @This(), a: Value, b_void: void, b_map_index: usize) bool {
196		_ = b_void;
197		return a == ctx.indexes[b_map_index];
198		}
199
200		pub fn hash(ctx: @This(), a: Value) u32 {
201		_ = ctx;
202		return std.hash.uint32(@intFromEnum(a));
203		}
204		};
205
206		pub fn valueTag(value: Value, ip: *const InternPool) Tag {
207		return ip.items.items(.tag)[@intFromEnum(value)];
208		}
209
210		pub fn valueData(value: Value, ip: *const InternPool, comptime tag: Tag) tag.Data() {
211		return ip.items.items(.data)[@intFromEnum(value)].decode(tag, value.valueTag(ip));
212		}
213
214		pub fn toAir(value: Value) Air.Inst.Ref {
215		return .{ .kind = .value, .data = @intCast(@intFromEnum(value)) };
216		}
217
218		pub fn toKey(value: Value, ip: *const InternPool) Key {
219		return ip.indexToKey(value);
220		}
221
222		pub fn dump(value: Value, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) {
223		return .{ .data = .{ .val = value, .ip = ip } };
224		}
225
226		fn formatValue(
227		ctx: struct { val: Value, ip: *const InternPool },
228		comptime _: []const u8,
229		_: std.fmt.FormatOptions,
230		writer: anytype,
231		) !void {
232		// TODO: move the formatter out of Air.Dump
233		var state: Air.Dump.State = undefined;
234		state.ip = ctx.ip;
235		state.tty = @import("util.zig").detectTTYFromWriter(writer);
236		try Air.Dump.State.dumpValue(&state, writer, ctx.val);
237		}
238
239		pub fn typeOf(value: Value, ip: *const InternPool) Type {
240		return switch (value.valueTag(ip)) {
241		.removed => unreachable, // UAF
242
243		.type_function => .type,
244		.simple_type => .type,
245
246		.simple_value => switch (value.valueData(ip, .simple_value)) {
247		.true => .bool,
248		.false => .bool,
249		.void => .void,
250		},
251
252		.comptime_float => .numeric_literal,
253		.int32 => .int32,
254		.float => .float,
255		.string => .string,
256		.func => value.toKey(ip).func.ty,
257		};
258		}
259
260		/// Try to re-interpret this `Value` as a `Type`.
261		/// Returns `null` if the value is not a type.
262		pub fn toType(value: Value, ip: *const InternPool) ?Type {
263		return switch (value.valueTag(ip)) {
264		.removed => unreachable, // UAF
265
266		.type_function,
267		.simple_type,
268		=> @enumFromInt(@intFromEnum(value)),
269
270		// exception: void is a value AND a type
271		.simple_value => if (value == .void) .void else null,
272
273		else => null,
274		};
275		}
276
277		pub fn toTypeUnsafe(value: Value, proof_ip: *const InternPool) Type {
278		if (std.debug.runtime_safety) {
279		return value.toType(proof_ip).?;
280		}
281		return @enumFromInt(@intFromEnum(value));
282		}
283
284		pub fn asString(val: Value, ip: *const InternPool) InternPool.String.WithLen {
285		return val.valueData(ip, .string).get(ip);
286		}
287
288		pub fn toFloat(val: Value, ip: *const InternPool) ?f64 {
289		return switch (val.valueTag(ip)) {
290		inline .comptime_float, .float => \|tag\| val.valueData(ip, tag).get(ip).decode(),
291		else => null,
292		};
293		}
294		};
295
296		pub const Type = enum(u32) {
297		@"opaque" = @intFromEnum(Value.opaque_type),
298		bool = @intFromEnum(Value.bool_type),
299		int32 = @intFromEnum(Value.int32_type),
300		float = @intFromEnum(Value.float_type),
301		string = @intFromEnum(Value.string_type),
302		noreturn = @intFromEnum(Value.noreturn_type),
303		type = @intFromEnum(Value.type_type),
304		numeric_literal = @intFromEnum(Value.numeric_literal_type),
305		void = @intFromEnum(Value.void),
306		_,
307
308		// Restricted version of `InternPool.Tag`
309		pub const Tag = enum(u32) {
310		type_function = @intFromEnum(InternPool.Tag.type_function),
311		simple_type = @intFromEnum(InternPool.Tag.simple_type),
312		};
313
314		/// An array of `Index` existing within the `extra` array.
315		/// This type exists to provide a struct with lifetime that is
316		/// not invalidated when items are added to the `InternPool`.
317		pub const Slice = struct {
318		start: u32,
319		len: u32,
320
321		pub fn get(slice: Slice, ip: *const InternPool) []Type {
322		return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
323		}
324		};
325
326		pub fn value(ty: Type) Value {
327		return @enumFromInt(@intFromEnum(ty));
328		}
329
330		pub fn tag(ty: Type, ip: *const InternPool) Type.Tag {
331		return @enumFromInt(@intFromEnum(ty.value().valueTag(ip)));
332		}
333
334		pub fn toFunctionPrototype(ty: Type, ip: *const InternPool) ?Key.TypeFunction {
335		if (ty.tag(ip) != .type_function) return null;
336		return ip.indexToKey(ty.value()).type_function;
337		}
338
339		pub fn displayName(ty: Type, ip: *const InternPool) std.fmt.Formatter(formatDisplayName) {
340		return .{ .data = .{ .type = ty, .ip = ip } };
341		}
342
343		pub fn dump(ty: Type, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) {
344		return ty.value().dump(ip);
345		}
346
347		fn formatDisplayName(
348		ctx: struct { type: Type, ip: *const InternPool },
349		comptime _: []const u8,
350		_: std.fmt.FormatOptions,
351		writer: anytype,
352		) !void {
353		const ty = ctx.type;
354		const ip = ctx.ip;
355
356		switch (ty) {
357		.@"opaque" => return try writer.writeAll("opaque"),
358		.bool => return try writer.writeAll("bool"),
359		.int32 => return try writer.writeAll("int"),
360		.float => return try writer.writeAll("float"),
361		.string => return try writer.writeAll("string"),
362		.noreturn => return try writer.writeAll("noreturn"),
363		.type => return try writer.writeAll("type"),
364		.numeric_literal => return try writer.writeAll("numeric literal"),
365		.void => return try writer.writeAll("void"),
366		else => {},
367		}
368
369		switch (ty.tag(ip)) {
370		.simple_type => unreachable, // all simple types are already printed
371		.type_function => {
372		const type_function = ty.toFunctionPrototype(ip).?;
373		try writer.writeAll("fn(");
374		for (type_function.param_types, 0..) \|param, i\| {
375		if (i != 0) try writer.writeAll(", ");
376		try writer.print("{}", .{param.displayName(ip)});
377		}
378		try writer.writeAll(") -> ");
379		try writer.print("{}", .{type_function.return_type.displayName(ip)});
380		},
381		}
382		}
383		};
384
385		pub const static_keys = [_]Key{
386		SimpleType.@"opaque".key(),
387		SimpleType.bool.key(),
388		SimpleType.int32.key(),
389		SimpleType.float.key(),
390		SimpleType.string.key(),
391		SimpleType.noreturn.key(),
392		SimpleType.type.key(),
393		SimpleType.numeric_literal.key(),
394
395		SimpleValue.void.key(),
396		SimpleValue.true.key(),
397		SimpleValue.false.key(),
398		.{ .int32 = 0 },
399		.{ .float = 0.0 },
400		.{ .float = Float.pure_nan.decode() },
401		.{ .string = "" },
402		};
403
404		/// State that is mutable during semantic analysis. This data is not used for
405		/// equality or hashing.
406		pub const FuncAnalysis = packed struct(u32) {
407		state: State,
408
409		_: u24 = 0,
410
411		pub const State = enum(u8) {
412		/// This function has not yet undergone analysis, because we have not
413		/// seen a potential runtime call. It may be analyzed in future.
414		none,
415		/// Analysis for this function has been queued, but not yet completed.
416		queued,
417		/// This function intentionally only has ZIR generated because it is marked
418		/// inline, which means no runtime version of the function will be generated.
419		inline_only,
420		in_progress,
421		/// There will be a corresponding ErrorMsg in Module.failed_decls
422		sema_failure,
423		/// This function might be OK but it depends on another Decl which did not
424		/// successfully complete semantic analysis.
425		dependency_failure,
426		/// There will be a corresponding ErrorMsg in Module.failed_decls.
427		/// Indicates that semantic analysis succeeded, but code generation for
428		/// this function failed.
429		codegen_failure,
430		/// Semantic analysis and code generation of this function succeeded.
431		success,
432		};
433		};
434
435		/// An index into `string_bytes`.
436		pub const String = enum(u32) {
437		/// An empty string.
438		empty = 0,
439		_,
440
441		pub const WithLen = struct {
442		string: String,
443		len: u32,
444
445		pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 {
446		return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)];
447		}
448
449		pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString {
450		return string.string.tryToNullTerminatedString(string.len, ip, gpa);
451		}
452		};
453
454		pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 {
455		return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)];
456		}
457
458		pub fn withLen(string: String, len: u64) WithLen {
459		return .{ .string = string, .len = @intCast(len) };
460		}
461
462		pub fn at(string: String, index: u64, ip: *const InternPool) u8 {
463		return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)];
464		}
465
466		pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString {
467		assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null);
468		assert(string.at(len, ip) == 0);
469		return @enumFromInt(@intFromEnum(string));
470		}
471
472		pub fn tryToNullTerminatedString(
473		string: String,
474		len: u64,
475		ip: *InternPool,
476		gpa: Allocator,
477		) !NullTerminatedString {
478		const slice = string.toSlice(len, ip);
479		if (std.mem.indexOfScalar(u8, slice, 0)) \|_\| {
480		return error.HasNulls;
481		}
482		if (string.at(len, ip) == 0) {
483		return @enumFromInt(@intFromEnum(string));
484		}
485		return ip.getOrPutString(gpa, slice, .no_embedded_nulls);
486		}
487		};
488
489		pub const NullTerminatedString = enum(u32) {
490		empty = 0,
491		_,
492
493		pub fn toString(self: NullTerminatedString) String {
494		return @enumFromInt(@intFromEnum(self));
495		}
496
497		// pub fn toOptional(self: NullTerminatedString) Optional{
498		// return @enumFromInt(@intFromEnum(self));
499		// }
500
501		pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
502		const slice = ip.string_bytes.items[@intFromEnum(string)..];
503		return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
504		}
505
506		pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
507		return @intCast(string.toSlice(ip).len);
508		}
509
510		pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
511		return std.mem.eql(u8, string.toSlice(ip), slice);
512		}
513
514		const Adapter = struct {
515		strings: []const NullTerminatedString,
516
517		pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool {
518		_ = b_void;
519		return a == ctx.strings[b_map_index];
520		}
521
522		pub fn hash(ctx: @This(), a: NullTerminatedString) u32 {
523		_ = ctx;
524		return std.hash.uint32(@intFromEnum(a));
525		}
526		};
527
528		const FormatData = struct {
529		string: NullTerminatedString,
530		ip: *const InternPool,
531		};
532		fn format(
533		data: FormatData,
534		comptime specifier: []const u8,
535		_: std.fmt.FormatOptions,
536		writer: anytype,
537		) @TypeOf(writer).Error!void {
538		const slice = data.string.toSlice(data.ip);
539		if (comptime std.mem.eql(u8, specifier, "")) {
540		try writer.writeAll(slice);
541		} else if (comptime std.mem.eql(u8, specifier, "i")) {
542		try writer.print("{p}", .{std.zig.fmtId(slice)});
543		} else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'");
544		}
545
546		pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) {
547		return .{ .data = .{ .string = self, .ip = ip } };
548		}
549		};
550
551		pub const TrackedInst = extern struct {
552		path_digest: Cache.BinDigest,
553		inst: Teg.Inst.Index,
554
555		comptime {
556		// The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
557		assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index));
558		}
559
560		pub const Index = enum(u32) {
561		_,
562		pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index {
563		return ip.tracked_insts.keys()[@intFromEnum(i)].inst;
564		}
565		pub fn toOptional(i: TrackedInst.Index) Optional {
566		return @enumFromInt(@intFromEnum(i));
567		}
568		pub const Optional = enum(u32) {
569		none = std.math.maxInt(u32),
570		_,
571		pub fn unwrap(opt: Optional) ?TrackedInst.Index {
572		return switch (opt) {
573		.none => null,
574		_ => @enumFromInt(@intFromEnum(opt)),
575		};
576		}
577		};
578		};
579		};
580
581		pub const DeclIndex = enum(u32) {
582		_,
583
584		pub fn toOptional(i: DeclIndex) Optional {
585		return @enumFromInt(@intFromEnum(i));
586		}
587
588		pub const Optional = enum(u32) {
589		none = std.math.maxInt(u32),
590		_,
591
592		pub fn init(oi: ?DeclIndex) Optional {
593		return @enumFromInt(@intFromEnum(oi orelse return .none));
594		}
595
596		pub fn unwrap(oi: Optional) ?DeclIndex {
597		if (oi == .none) return null;
598		return @enumFromInt(@intFromEnum(oi));
599		}
600		};
601		};
602
603		pub const NamespaceIndex = enum(u32) {
604		_,
605
606		pub fn toOptional(i: DeclIndex) Optional {
607		return @enumFromInt(@intFromEnum(i));
608		}
609
610		pub const Optional = enum(u32) {
611		none = std.math.maxInt(u32),
612		_,
613
614		pub fn init(oi: ?NamespaceIndex) Optional {
615		return @enumFromInt(@intFromEnum(oi orelse return .none));
616		}
617
618		pub fn unwrap(oi: Optional) ?NamespaceIndex {
619		if (oi == .none) return null;
620		return @enumFromInt(@intFromEnum(oi));
621		}
622		};
623		};
624
625		/// A single value captured in the closure of a namespace type. This is not a plain
626		/// `Index` because we must differentiate between the following cases:
627		/// * runtime-known value (where we store the type)
628		/// * comptime-known value (where we store the value)
629		/// * decl val (so that we can analyze the value lazily)
630		/// * decl ref (so that we can analyze the reference lazily)
631		pub const Capture = packed struct(u32) {
632		tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref },
633		idx: u30,
634
635		pub fn wrap(val: Unwrapped) Capture {
636		return switch (val) {
637		.@"comptime" => \|i\| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) },
638		.runtime => \|i\| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) },
639		.decl_val => \|i\| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) },
640		.decl_ref => \|i\| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) },
641		};
642		}
643		pub fn unwrap(val: Capture) Unwrapped {
644		return switch (val.tag) {
645		.@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) },
646		.runtime => .{ .runtime = @enumFromInt(val.idx) },
647		.decl_val => .{ .decl_val = @enumFromInt(val.idx) },
648		.decl_ref => .{ .decl_ref = @enumFromInt(val.idx) },
649		};
650		}
651
652		pub const Unwrapped = union(enum) {
653		/// Index refers to the value.
654		@"comptime": Value,
655		/// Index refers to the type.
656		runtime: Value,
657		decl_val: DeclIndex,
658		decl_ref: DeclIndex,
659		};
660
661		pub const Slice = struct {
662		start: u32,
663		len: u32,
664		pub fn get(slice: Slice, ip: *const InternPool) []Capture {
665		return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
666		}
667		};
668		};
669
670		const SimpleValue = enum {
671		true,
672		false,
673		void,
674
675		pub fn key(self: SimpleValue) Key {
676		return .{ .simple_value = self };
677		}
678		};
679
680		const SimpleType = enum {
681		@"opaque",
682		int32,
683		float,
684		string,
685		bool,
686		noreturn,
687		type,
688		numeric_literal,
689
690		pub fn key(self: SimpleType) Key {
691		return .{ .simple_type = self };
692		}
693		};
694
695		pub const Key = union(enum) {
696		comptime_float: f64,
697		int32: i32,
698		float: f64,
699		string: []const u8,
700		simple_value: SimpleValue,
701		simple_type: SimpleType,
702		type_function: TypeFunction,
703		func: Func,
704
705		const Adapter = struct {
706		intern_pool: *const InternPool,
707
708		pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
709		_ = b_void;
710		if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
711		return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
712		}
713
714		pub fn hash(ctx: @This(), a: Key) u32 {
715		return a.hash32(ctx.intern_pool);
716		}
717		};
718
719		pub const TypeFunction = struct {
720		param_types: []Type,
721		return_type: Type,
722
723		pub fn hash(self: TypeFunction, hasher: *Hash) void {
724		const t = tracer.trace(@src());
725		defer t.end();
726
727		for (self.param_types) \|param_type\| {
728		std.hash.autoHash(hasher, param_type);
729		}
730		std.hash.autoHash(hasher, self.return_type);
731		}
732
733		pub fn eql(a: TypeFunction, b: TypeFunction) bool {
734		return std.mem.eql(Type, a.param_types, b.param_types) and
735		a.return_type == b.return_type;
736		}
737		};
738
739		pub const Func = struct {
740		ty: Type,
741		owner_decl: DeclIndex,
742		teg_body_inst: TrackedInst.Index,
743
744		pub fn hash(self: Func, hasher: *Hash) void {
745		std.hash.autoHash(hasher, self.ty);
746		std.hash.autoHash(hasher, self.owner_decl);
747		std.hash.autoHash(hasher, self.teg_body_inst);
748		}
749		};
750
751		pub fn hash32(key: Key, ip: *const InternPool) u32 {
752		return @truncate(key.hash64(ip));
753		}
754
755		pub fn hash64(key: Key, ip: *const InternPool) u64 {
756		const t = tracer.trace(@src());
757		defer t.end();
758
759		_ = ip; // autofix
760		const asBytes = std.mem.asBytes;
761		const KeyTag = @typeInfo(Key).Union.tag_type.?;
762		const seed = @intFromEnum(@as(KeyTag, key));
763
764		return switch (key) {
765		inline .int32,
766		.simple_value,
767		.simple_type,
768		=> \|x\| Hash.hash(seed, asBytes(&x)),
769		.string,
770		=> \|x\| Hash.hash(seed, x),
771		.float, .comptime_float => \|x\| Hash.hash(seed, asBytes(&Float.assertPure(x))),
772		.type_function => \|x\| {
773		var hasher = Hash.init(seed);
774		x.hash(&hasher);
775		return hasher.final();
776		},
777		.func => \|x\| {
778		var hasher = Hash.init(seed);
779		x.hash(&hasher);
780		return hasher.final();
781		},
782		};
783		}
784
785		pub fn eql(a: Key, b: Key, ip: *const InternPool) bool {
786		const t = tracer.trace(@src());
787		defer t.end();
788
789		_ = ip; // autofix
790		const KeyTag = @typeInfo(Key).Union.tag_type.?;
791		const a_tag: KeyTag = a;
792		const b_tag: KeyTag = b;
793		if (a_tag != b_tag) return false;
794		return switch (a) {
795		inline .int32,
796		.simple_value,
797		.simple_type,
798		=> \|x, tag\| x == @field(b, @tagName(tag)),
799		.string,
800		=> \|x\| std.mem.eql(u8, x, b.string),
801		.float => \|x\| Float.assertPure(x) == Float.assertPure(b.float),
802		.comptime_float => \|x\| Float.assertPure(x) == Float.assertPure(b.comptime_float),
803		.type_function => \|x\| x.eql(b.type_function),
804		.func => \|a_info\| {
805		const b_info = b.func;
806		if (a_info.owner_decl != b_info.owner_decl)
807		return false;
808		if (a_info.ty == b_info.ty)
809		return true;
810
811		// TODO: zig has something extra here for inferred error sets
812		return false;
813		},
814		};
815		}
816		};
817
818		const EmbeddedNulls = enum {
819		no_embedded_nulls,
820		maybe_embedded_nulls,
821
822		const OptionalNullTerminatedString = @compileError("TODO");
823		const OptionalString = @compileError("TODO");
824
825		fn StringType(comptime embedded_nulls: EmbeddedNulls) type {
826		return switch (embedded_nulls) {
827		.no_embedded_nulls => NullTerminatedString,
828		.maybe_embedded_nulls => String,
829		};
830		}
831
832		fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type {
833		return switch (embedded_nulls) {
834		.no_embedded_nulls => OptionalNullTerminatedString,
835		.maybe_embedded_nulls => OptionalString,
836		};
837		}
838		};
839
840		// implementations:
841
842		/// To construct, instantiate `.{ .gpa = gpa }` then call this.
843		pub fn init(ip: *InternPool, gpa: Allocator) !void {
844		const t = tracer.trace(@src());
845		defer t.end();
846
847		assert(ip.items.len == 0);
848
849		// Reserve string index 0 for an empty string.
850		assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty);
851
852		// So that we can use `catch unreachable` below.
853		try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
854		try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
855		try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len);
856
857		// This inserts all the statically-known values into the intern pool in the
858		// order expected.
859		for (static_keys) \|key\| {
860		_ = ip.get(gpa, key) catch unreachable;
861		}
862
863		if (std.debug.runtime_safety) {
864		// Sanity check.
865		assert(ip.indexToKey(.bool_true).simple_value == .true);
866		assert(ip.indexToKey(.bool_false).simple_value == .false);
867		}
868
869		assert(ip.items.len == static_keys.len);
870		}
871
872		pub fn deinit(ip: *InternPool, gpa: Allocator) void {
873		const t = tracer.trace(@src());
874		defer t.end();
875
876		ip.map.deinit(gpa);
877		ip.items.deinit(gpa);
878		ip.extra.deinit(gpa);
879		// ip.limbs.deinit(gpa);
880		ip.string_bytes.deinit(gpa);
881
882		var decls = ip.allocated_decls.iterator(0);
883		while (decls.next()) \|decl\| {
884		decl.deinit(gpa);
885		}
886
887		ip.allocated_decls.deinit(gpa);
888		ip.decls_free_list.deinit(gpa);
889
890		var namespaces = ip.allocated_namespaces.iterator(0);
891		while (namespaces.next()) \|ns\| {
892		ns.decls.deinit(gpa);
893		}
894
895		ip.namespaces_free_list.deinit(gpa);
896		ip.allocated_namespaces.deinit(gpa);
897
898		// for (ip.maps.items) \|*map\| map.deinit(gpa);
899		// ip.maps.deinit(gpa);
900
901		ip.string_table.deinit(gpa);
902
903		ip.tracked_insts.deinit(gpa);
904
905		// ip.src_hash_deps.deinit(gpa);
906		// ip.decl_val_deps.deinit(gpa);
907		// ip.func_ies_deps.deinit(gpa);
908		// ip.namespace_deps.deinit(gpa);
909		// ip.namespace_name_deps.deinit(gpa);
910
911		// ip.first_dependency.deinit(gpa);
912
913		// ip.dep_entries.deinit(gpa);
914		// ip.free_dep_entries.deinit(gpa);
915
916		ip.* = undefined;
917		}
918
919		pub fn indexToKey(ip: *const InternPool, index: Value) Key {
920		assert(index != .none);
921		const item: Item = ip.items.get(@intFromEnum(index));
922		const data = item.data;
923		return switch (item.tag) {
924		.removed => unreachable,
925
926		.type_function => .{
927		.type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)),
928		},
929
930		.simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) },
931		.simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) },
932		.int32 => .{ .int32 = data.decode(.int32, item.tag) },
933		.float => .{ .float = data.decode(.float, item.tag).get(ip).decode() },
934		.comptime_float => .{ .comptime_float = data.decode(.comptime_float, item.tag).get(ip).decode() },
935		.string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) },
936		.func => {
937		const func = data.decode(.func, item.tag).get(ip);
938		return .{ .func = .{
939		.ty = func.ty,
940		.owner_decl = func.owner_decl,
941		.teg_body_inst = func.teg_body_inst,
942		} };
943		},
944		};
945		}
946
947		pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value {
948		const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)});
949		defer t.end();
950
951		const adapter: Key.Adapter = .{ .intern_pool = ip };
952		const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
953		if (gop.found_existing) return @enumFromInt(gop.index);
954		if (ip.items.len >= std.math.maxInt(u31) - 1) {
955		return error.TooManyValues;
956		}
957
958		const t2 = tracer.traceFmt(@src(), "(insert)", .{});
959		defer t2.end();
960
961		try ip.items.ensureUnusedCapacity(gpa, 1);
962		const item: Item = switch (key) {
963		inline .int32,
964		.simple_type,
965		.simple_value,
966		=> \|s, tag\| Item.encode(@field(Tag, @tagName(tag)), s),
967		.string => \|s\| Item.encode(
968		.string,
969		try ip.extra.append(
970		gpa,
971		String.WithLen,
972		(try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len),
973		),
974		),
975		.float => \|f\| Item.encode(
976		.float,
977		try ip.extra.append(gpa, Float, Float.encode(f)),
978		),
979		.comptime_float => \|f\| Item.encode(
980		.comptime_float,
981		try ip.extra.append(gpa, Float, Float.encode(f)),
982		),
983		.func => \|f\| brk: {
984		const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{
985		.analysis = .{
986		.state = .none,
987		},
988		.owner_decl = f.owner_decl,
989		.ty = f.ty,
990		.teg_body_inst = f.teg_body_inst,
991		});
992
993		break :brk Item.encode(.func, extra);
994		},
995		.type_function => \|f\| brk: {
996		const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
997		.params_len = @intCast(f.param_types.len),
998		.return_type = f.return_type,
999		}, f.param_types.len);
1000		defer trail.end();
1001		trail.writeSlice(f.param_types);
1002		break :brk Item.encode(.type_function, extra);
1003		},
1004		};
1005		ip.items.appendAssumeCapacity(item);
1006		const index: Value = @enumFromInt(ip.items.len - 1);
1007		if (std.debug.runtime_safety) {
1008		assert(ip.indexToKey(index).eql(key, ip));
1009		}
1010		return index;
1011		}
1012
1013		fn extraTypeFunction(
1014		ip: *const InternPool,
1015		extra_index: ExtraIndex(Tag.TypeFunction),
1016		) Key.TypeFunction {
1017		const type_function, var trail = extra_index.getTrail(ip);
1018		const param_types = trail.readTypeSlice(type_function.params_len);
1019		const result: Key.TypeFunction = .{
1020		.return_type = type_function.return_type,
1021		.param_types = param_types.get(ip),
1022		};
1023		return result;
1024		}
1025
1026		pub fn getOrPutString(
1027		ip: *InternPool,
1028		gpa: Allocator,
1029		slice: []const u8,
1030		comptime embedded_nulls: EmbeddedNulls,
1031		) Allocator.Error!embedded_nulls.StringType() {
1032		const t = tracer.trace(@src());
1033		defer t.end();
1034
1035		try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1);
1036		ip.string_bytes.appendSliceAssumeCapacity(slice);
1037		ip.string_bytes.appendAssumeCapacity(0);
1038		return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls);
1039		}
1040
1041		fn getOrPutTrailingString(
1042		ip: *InternPool,
1043		gpa: Allocator,
1044		len: usize,
1045		comptime embedded_nulls: EmbeddedNulls,
1046		) Allocator.Error!embedded_nulls.StringType() {
1047		const t = tracer.trace(@src());
1048		defer t.end();
1049
1050		const string_bytes = &ip.string_bytes;
1051		const str_index: u32 = @intCast(string_bytes.items.len - len);
1052		if (len > 0 and string_bytes.getLast() == 0) {
1053		_ = string_bytes.pop();
1054		} else {
1055		try string_bytes.ensureUnusedCapacity(gpa, 1);
1056		}
1057		const key: []const u8 = string_bytes.items[str_index..];
1058		const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
1059		switch (embedded_nulls) {
1060		.no_embedded_nulls => assert(!has_embedded_null),
1061		.maybe_embedded_nulls => if (has_embedded_null) {
1062		string_bytes.appendAssumeCapacity(0);
1063		return @enumFromInt(str_index);
1064		},
1065		}
1066		const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{
1067		.bytes = string_bytes,
1068		}, std.hash_map.StringIndexContext{
1069		.bytes = string_bytes,
1070		});
1071		if (gop.found_existing) {
1072		string_bytes.shrinkRetainingCapacity(str_index);
1073		return @enumFromInt(gop.key_ptr.*);
1074		} else {
1075		gop.key_ptr.* = str_index;
1076		string_bytes.appendAssumeCapacity(0);
1077		return @enumFromInt(str_index);
1078		}
1079		}
1080
1081		pub fn trackInst(ip: InternPool, gpa: Allocator, file: Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index {
1082		const t = tracer.trace(@src());
1083		defer t.end();
1084
1085		const key: TrackedInst = .{
1086		.path_digest = file.path_digest.?,
1087		.inst = inst,
1088		};
1089		const gop = try ip.tracked_insts.getOrPut(gpa, key);
1090		return @enumFromInt(gop.index);
1091		}
1092
1093		pub fn declPtr(ip: InternPool, index: DeclIndex) Comp.Decl {
1094		return ip.allocated_decls.at(@intFromEnum(index));
1095		}
1096
1097		pub fn declPtrConst(ip: const InternPool, index: DeclIndex) const Comp.Decl {
1098		return ip.allocated_decls.at(@intFromEnum(index));
1099		}
1100
1101		pub fn namespacePtr(ip: InternPool, index: NamespaceIndex) Comp.Namespace {
1102		return ip.allocated_namespaces.at(@intFromEnum(index));
1103		}
1104
1105		pub fn createDecl(
1106		ip: *InternPool,
1107		gpa: Allocator,
1108		initialization: Comp.Decl,
1109		) Allocator.Error!DeclIndex {
1110		const t = tracer.trace(@src());
1111		defer t.end();
1112
1113		if (ip.decls_free_list.popOrNull()) \|index\| {
1114		ip.allocated_decls.at(@intFromEnum(index)).* = initialization;
1115		return index;
1116		}
1117		const ptr = try ip.allocated_decls.addOne(gpa);
1118		ptr.* = initialization;
1119		return @enumFromInt(ip.allocated_decls.len - 1);
1120		}
1121
1122		pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void {
1123		const t = tracer.trace(@src());
1124		defer t.end();
1125
1126		ip.declPtr(index).* = undefined;
1127		ip.decls_free_list.append(gpa, index) catch {
1128		// In order to keep `destroyDecl` a non-fallible function, we ignore memory
1129		// allocation failures here, instead leaking the Decl until garbage collection.
1130		};
1131		}
1132
1133		pub fn createNamespace(
1134		ip: *InternPool,
1135		gpa: Allocator,
1136		initialization: Comp.Namespace,
1137		) Allocator.Error!NamespaceIndex {
1138		const t = tracer.trace(@src());
1139		defer t.end();
1140
1141		if (ip.namespaces_free_list.popOrNull()) \|index\| {
1142		ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization;
1143		return index;
1144		}
1145		const ptr = try ip.allocated_namespaces.addOne(gpa);
1146		ptr.* = initialization;
1147		return @enumFromInt(ip.allocated_namespaces.len - 1);
1148		}
1149
1150		pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void {
1151		const t = tracer.trace(@src());
1152		defer t.end();
1153
1154		ip.namespacePtr(index).* = undefined;
1155		ip.namespaces_free_list.append(gpa, index) catch {
1156		// In order to keep `destroyNamespace` a non-fallible function, we ignore memory
1157		// allocation failures here, instead leaking the Namespace until garbage collection.
1158		};
1159		}
1160
1161		const InternPool = @This();
1162
1163		const std = @import("std");
1164		const Allocator = std.mem.Allocator;
1165		const Hash = std.hash.Wyhash;
1166		const assert = std.debug.assert;
1167
1168		const Comp = @import("Comp.zig");
1169		const Cache = @import("Cache.zig");
1170
1171		const Teg = @import("Teg.zig");
1172		const Air = @import("Air.zig");
1173
1174		const dod = @import("array.zig");
1175		const Float = dod.Float;
1176
1177		const tracer = @import("tracer");
1178
1179		test InternPool {
1180		const testing = std.testing;
1181		std.testing.refAllDeclsRecursive(@This());
1182
1183		var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
1184		const ip = &ip_struct;
1185		defer ip.deinit();
1186
1187		const str = try ip.getOrPutString("hello", .no_embedded_nulls);
1188		try testing.expectEqualStrings("hello", str.toSlice(ip));
1189		const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
1190		try testing.expectEqual(str, str2);
1191
1192		const int1 = try ip.get(.{ .int32 = 21483 });
1193		try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
1194		const int2 = try ip.get(.{ .int32 = 21483 });
1195		try testing.expectEqual(int1, int2);
1196		}

src/compiler/JsLink.zig deleted-519

...	...	@@ -1,519 +0,0 @@
1		/// This is the JavaScript backend's linker.
2		/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript
3		/// bundle out of them.
4		comp: *Comp,
5
6		/// Value is an index into `link_strings`
7		decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{},
8		link_strings: std.ArrayHashMapUnmanaged(
9		LinkString,
10		GlobalOwner,
11		LinkString.Context,
12		true,
13		) = .{},
14		string_bytes: std.ArrayListUnmanaged(u8) = .{},
15
16		/// The first declaration is the function containing top-level code.
17		pub fn linkJsCode(
18		comp: *Comp,
19		decls: []const Decl.Index,
20		writer: anytype,
21		) !void {
22		var t = tracer.trace(@src());
23		defer t.end();
24
25		// Arena allocator is used for the main flow
26		var link: JsLink = .{ .comp = comp };
27		defer link.deinit();
28
29		// Temp allocator is used for extremely temporary string work.
30		// var temp_sfa = std.heap.stackFallback(4096, comp.gpa);
31		// const temp_alloc = temp_sfa.get();
32
33		// 1. Resolve all global identifiers. Function names must not shadow external
34		// names. We also need this list prepared so that local identifier names do not
35		// shadow functions or externals.
36		const root_decl = comp.intern_pool.declPtrConst(decls[0]);
37		assert(root_decl.kind == .named);
38		assert(root_decl.name == .empty);
39
40		for (root_decl.codegen.?.extern_symbols) \|extern_symbol\| {
41		try link.reserveExternSymbol(extern_symbol);
42		}
43
44		for (decls[1..]) \|decl_index\| {
45		assert(decl_index != decls[0]);
46
47		const decl = comp.intern_pool.declPtrConst(decl_index);
48
49		// todo: add all externs first
50		for (decl.codegen.?.extern_symbols) \|extern_symbol\| {
51		try link.reserveExternSymbol(extern_symbol);
52		}
53
54		const name = decl.name.toSlice(&comp.intern_pool);
55		assert(name.len > 0);
56
57		const gop = try link.link_strings.getOrPutContextAdapted(
58		comp.gpa,
59		name,
60		LinkString.SliceContext{ .link = &link },
61		.{ .link = &link },
62		);
63		if (gop.found_existing) {
64		@panic("TODO: handle this shadowing case.");
65		}
66		const name_str = try link.appendLinkString(name);
67		gop.key_ptr.* = name_str;
68		gop.value_ptr.* = .decl;
69		try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
70		}
71
72		try writer.print("// Language Compiler v0.1.2. Alpha quality software.\n", .{});
73
74		// 2. Print every function decl
75		for (decls[1..]) \|decl_index\| {
76		try link.printDecl(decl_index, writer);
77		}
78
79		// 3. Print the top-level code
80		if (root_decl.codegen.?.nodes.len > 1) {
81		try link.printDeclContents(&root_decl.codegen.?, writer);
82		} else {
83		try writer.print("// (program has no observable effects, thus an empty file)\n", .{});
84		}
85		}
86
87		fn deinit(link: *JsLink) void {
88		var t = tracer.trace(@src());
89		defer t.end();
90
91		defer link.* = undefined;
92		link.decls.deinit(link.comp.gpa);
93		link.link_strings.deinit(link.comp.gpa);
94		link.string_bytes.deinit(link.comp.gpa);
95		}
96
97		fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
98		var t = tracer.trace(@src());
99		defer t.end();
100
101		const name = extern_symbol.toSlice(&link.comp.intern_pool);
102		const gop = try link.link_strings.getOrPutContextAdapted(
103		link.comp.gpa,
104		name,
105		LinkString.SliceContext{ .link = link },
106		.{ .link = link },
107		);
108
109		if (gop.found_existing) {
110		if (gop.value_ptr.* != .external) {
111		@panic("TODO: Rename Decl");
112		}
113		}
114		gop.key_ptr.* = try link.appendLinkString(name);
115		gop.value_ptr.* = .external;
116		}
117
118		fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
119		var t = tracer.trace(@src());
120		defer t.end();
121
122		const start = link.string_bytes.items.len;
123		try link.string_bytes.appendSlice(link.comp.gpa, string);
124		return .{ .start = @intCast(start), .len = @intCast(string.len) };
125		}
126
127		fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 {
128		var t = tracer.trace(@src());
129		defer t.end();
130
131		return (link.decls.get(decl) orelse if (std.debug.runtime_safety) {
132		std.debug.panic("Decl {d} ({}) is not present in link decl map", .{
133		@intFromEnum(decl),
134		link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool),
135		});
136		} else unreachable).toString(link).slice(link);
137		}
138
139		fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void {
140		var t = tracer.trace(@src());
141		defer t.end();
142
143		try writer.writeAll("let ");
144		try writer.writeAll(link.declIdentifier(decl_index));
145		try writer.writeAll(" = ");
146
147		const decl = link.comp.intern_pool.declPtrConst(decl_index);
148		const tree = &decl.codegen.?;
149
150		try writer.writeAll("(");
151		for (0..tree.param_count) \|id\| {
152		try link.printLocal(@intCast(id), writer);
153		if (id != tree.param_count - 1) try writer.writeAll(", ");
154		}
155		try writer.writeAll(") => {\n");
156		try link.printDeclContents(tree, writer);
157		try writer.writeAll("};\n");
158		}
159
160		fn printDeclContents(link: JsLink, tree: const JsTree, writer: anytype) !void {
161		var t = tracer.trace(@src());
162		defer t.end();
163
164		for (tree.mainStatements()) \|stmt\| {
165		try link.printExpr(tree, stmt, writer);
166		try writer.writeAll(";\n");
167		}
168		}
169
170		fn printBlockContents(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
171		var t = tracer.trace(@src());
172		defer t.end();
173
174		assert(list.slice(tree).len > 0);
175		for (list.slice(tree)) \|stmt\| {
176		try link.printExpr(tree, stmt, writer);
177		switch (tree.nodeTag(stmt)) {
178		else => try writer.writeAll(";"),
179		.stmt_block,
180		.stmt_if,
181		=> {},
182		}
183		try writer.writeAll("\n");
184		}
185		}
186
187		fn printBlock(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
188		var t = tracer.trace(@src());
189		defer t.end();
190
191		if (list.len != 1) {
192		try writer.writeAll("{");
193		try link.printBlockContents(tree, list, writer);
194		try writer.writeAll("}");
195		} else {
196		try link.printBlockContents(tree, list, writer);
197		}
198		}
199
200		fn printExpr(link: JsLink, tree: const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
201		var t = tracer.trace(@src());
202		defer t.end();
203
204		switch (tree.nodeTag(node)) {
205		.root => unreachable,
206
207		.plain_value => {
208		const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node);
209		switch (plain_value.value.toKey(&link.comp.intern_pool)) {
210		// TODO: minification
211		.int32 => \|val\| try writer.print("{d}", .{val}),
212		// TODO: correct printing of many value types
213		.float => \|val\| try writer.print("{d}", .{val}),
214		.simple_value => \|val\| try writer.writeAll(switch (val) {
215		.true => "true",
216		.false => "false",
217		// TODO: make codegen decide from null and undefined
218		// instead of emitting void nodes.
219		.void => "undefined",
220		}),
221		// TODO: escapes
222		.string => \|val\| try writer.print("\"{s}\"", .{val}),
223
224		.comptime_float => unreachable, // was not lowered to float
225
226		else => unreachable, // this value is not a plain_value
227		}
228		},
229
230		.local => {
231		const ref: JsTree.LocalRef = tree.nodeData(.local, node);
232		try link.printLocal(ref.id, writer);
233		},
234
235		.global => {
236		const ref: JsTree.GlobalRef = tree.nodeData(.global, node);
237		try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool));
238		},
239
240		.decl => {
241		const ref: JsTree.DeclRef = tree.nodeData(.decl, node);
242		try writer.writeAll(link.declIdentifier(ref.index));
243		},
244
245		.stmt_break => {
246		const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node);
247		try writer.writeAll("break");
248		if (true) {
249		try writer.writeAll(" ");
250		try link.printLabelIdentifier(stmt.label, writer);
251		}
252		},
253
254		.stmt_return => {
255		const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node);
256		try writer.writeAll("return");
257		if (stmt.operand.unwrap()) \|operand\| {
258		try writer.writeAll(" ");
259		try link.printExpr(tree, operand, writer);
260		}
261		},
262
263		.stmt_declare_local => {
264		const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node);
265		try writer.writeAll("let ");
266		try link.printLocal(stmt.identifier_index, writer);
267		if (stmt.init.unwrap()) \|init\| {
268		try writer.writeAll(" = ");
269		try link.printExpr(tree, init, writer);
270		}
271		},
272
273		.stmt_if => {
274		const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node);
275		const data = stmt.data.get(tree);
276		try writer.writeAll("if (");
277		try link.printExpr(tree, stmt.cond, writer);
278		try writer.writeAll(") ");
279		if (data.flags.yes_label) {
280		try link.printLabelIdentifier(data.flags.block_label, writer);
281		try writer.writeAll(":");
282		}
283		try link.printBlock(tree, data.yes, writer);
284		if (data.no.len > 0) {
285		try writer.writeAll(" else ");
286		if (data.flags.no_label) {
287		try link.printLabelIdentifier(data.flags.block_label, writer);
288		try writer.writeAll(":");
289		}
290		try link.printBlock(tree, data.no, writer);
291		}
292		},
293
294		.stmt_block => {
295		const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node);
296		try link.printLabelIdentifier(stmt.label, writer);
297		try writer.writeAll(": {\n");
298		try link.printBlockContents(tree, stmt.content.get(tree), writer);
299		try writer.writeAll("}");
300		},
301
302		inline .assign,
303		.add,
304		.sub,
305		.mul,
306		.div,
307		.eq,
308		.lt,
309		.gt,
310		.assign_add,
311		.assign_sub,
312		.assign_mul,
313		.assign_div,
314		=> \|tag\| {
315		const bin: JsTree.Bin = tree.nodeData(tag, node);
316		const op: Operator = @field(Operator, @tagName(tag));
317		try link.printExpr(tree, bin.lhs, writer);
318		try writer.writeAll(" ");
319		try writer.writeAll(comptime op.symbol());
320		try writer.writeAll(" ");
321		try link.printExpr(tree, bin.rhs, writer);
322		},
323
324		inline .negate,
325		.bool_not,
326		.pre_dec,
327		.pre_inc,
328		=> \|tag\| {
329		const un: JsTree.Un = tree.nodeData(tag, node);
330		const op: Operator = @field(Operator, @tagName(tag));
331		try writer.writeAll(comptime op.symbol());
332		try link.printExpr(tree, un.operand, writer);
333		},
334
335		.ternary => {
336		const ternary: JsTree.Ternary = tree.nodeData(.ternary, node);
337		const data = ternary.data.get(tree);
338		try writer.writeAll("(");
339		try link.printExpr(tree, ternary.cond, writer);
340		try writer.writeAll(" ? ");
341		try link.printExpr(tree, data.yes, writer);
342		try writer.writeAll(" : ");
343		try link.printExpr(tree, data.no, writer);
344		try writer.writeAll(")");
345		},
346
347		.call => {
348		const call: JsTree.Call = tree.nodeData(.call, node);
349		const args = call.args.get(tree);
350
351		try link.printExpr(tree, call.base, writer);
352		try writer.writeAll("(");
353
354		for (args.slice(tree), 0..) \|arg, i\| {
355		if (i != 0) try writer.writeAll(", ");
356		try link.printExpr(tree, arg, writer);
357		}
358
359		try writer.writeAll(")");
360		},
361
362		.field => {
363		const field: JsTree.Field = tree.nodeData(.field, node);
364
365		try link.printExpr(tree, field.base, writer);
366		try writer.writeAll(".");
367		try writer.writeAll(field.field.toSlice(&link.comp.intern_pool));
368		},
369		}
370		}
371
372		fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
373		var t = tracer.trace(@src());
374		defer t.end();
375
376		_ = link; // todo: be smarter about this
377		try writer.print("v{d}", .{local});
378		}
379
380		fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
381		var t = tracer.trace(@src());
382		defer t.end();
383
384		_ = link; // todo: be smarter about this
385		try writer.print("label{d}", .{local});
386		}
387
388		const GlobalOwner = enum {
389		external,
390		decl,
391		};
392
393		// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence
394		const Level = enum {
395		comma,
396		assignment,
397		conditional,
398		logical_or,
399		logical_and,
400		bitwise_or,
401		bitwise_xor,
402		bitwise_and,
403		equality,
404		relational,
405		bitshift,
406		additive,
407		multiplicative,
408		exponentiation,
409		prefix,
410		postfix,
411		new,
412		access_and_call,
413		member,
414		};
415
416		const Operator = enum {
417		assign,
418		assign_add,
419		assign_sub,
420		assign_mul,
421		assign_div,
422		add,
423		sub,
424		mul,
425		div,
426		eq,
427		lt,
428		gt,
429		negate,
430		bool_not,
431		pre_inc,
432		pre_dec,
433
434		pub fn symbol(op: Operator) []const u8 {
435		return switch (op) {
436		.assign => "=",
437		.assign_add => "+=",
438		.assign_sub => "-=",
439		.assign_mul => "*=",
440		.assign_div => "/=",
441		.add => "+",
442		.sub => "-",
443		.mul => "*",
444		.div => "/",
445		.eq => "===",
446		.lt => "<",
447		.gt => ">",
448
449		.pre_inc => "++",
450		.pre_dec => "++",
451
452		.negate => "-",
453		.bool_not => "!",
454		};
455		}
456		};
457
458		const LinkString = struct {
459		start: u32,
460		len: u32,
461
462		fn slice(str: LinkString, link: *const JsLink) []const u8 {
463		return link.string_bytes.items[str.start..][0..str.len];
464		}
465
466		const Index = enum(u32) {
467		_,
468
469		pub fn toString(index: Index, link: *const JsLink) LinkString {
470		return link.link_strings.keys()[@intFromEnum(index)];
471		}
472		};
473
474		const Context = struct {
475		link: *const JsLink,
476
477		pub fn hash(ctx: Context, str: LinkString) u32 {
478		// assert(str.start + str.len < ctx.link.string_bytes.items.len);
479
480		return @truncate(Wyhash.hash(0, str.slice(ctx.link)));
481		}
482
483		pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool {
484		assert(a.start + a.len < ctx.link.string_bytes.items.len);
485		assert(b.start + b.len < ctx.link.string_bytes.items.len);
486		return a.start == b.start;
487		}
488		};
489
490		const SliceContext = struct {
491		link: *const JsLink,
492
493		pub fn hash(_: SliceContext, str: []const u8) u32 {
494		return @truncate(Wyhash.hash(0, str));
495		}
496
497		pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool {
498		// since the given string is not in string_bytes we do comparisons
499		return std.mem.eql(u8, a, b.slice(ctx.link));
500		}
501		};
502		};
503
504		const JsLink = @This();
505		const std = @import("std");
506		const assert = std.debug.assert;
507		const Wyhash = std.hash.Wyhash;
508		const log = std.log.scoped(.js_link);
509
510		const Comp = @import("Comp.zig");
511		const Decl = Comp.Decl;
512
513		const JsTree = @import("JsTree.zig");
514		const Node = JsTree.Node;
515
516		const InternPool = @import("InternPool.zig");
517		const String = InternPool.String;
518
519		const tracer = @import("tracer");

src/compiler/JsNodeGen.zig deleted-700

...	...	@@ -1,700 +0,0 @@
1		//! Air -> JsTree
2		gpa: Allocator,
3		arena: Allocator,
4		air: *const Air,
5		ip: *const InternPool,
6
7		nodes: Node.List,
8		extra: dod.ExtraStore(.tfr, .@"var"),
9		extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void),
10		decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void),
11
12		param_count: u32,
13		air_node_map: []MapEntry,
14		scratch: std.ArrayListUnmanaged(u32),
15		next_id: u32,
16		next_label: u32,
17
18		const Error = error{
19		OutOfMemory,
20		};
21
22		const MapEntry = union {
23		none: void,
24		node: Node.Index,
25		block: packed struct {
26		used: bool = false,
27		index: u31,
28		},
29		};
30
31		pub fn init(gpa: Allocator, arena: Allocator, ip: const InternPool, air: const Air) !JsNodeGen {
32		const t = tracer.trace(@src());
33		defer t.end();
34
35		const air_node_map = try arena.alloc(MapEntry, air.insts.len);
36		if (std.debug.runtime_safety) {
37		@memset(air_node_map, .{ .none = {} });
38		}
39		const param_count = air.paramCount();
40		return .{
41		.gpa = gpa,
42		.arena = arena,
43		.air = air,
44		.ip = ip,
45		.param_count = param_count,
46		.nodes = .{},
47		.extern_symbols = .{},
48		.decl_deps = .{},
49		.air_node_map = air_node_map,
50		.extra = .{},
51		.scratch = .{},
52		.next_id = param_count,
53		.next_label = 0,
54		};
55		}
56
57		pub fn deinit(c: *JsNodeGen) void {
58		const t = tracer.trace(@src());
59		defer t.end();
60
61		defer c.* = undefined;
62		c.nodes.deinit(c.gpa);
63		c.extra.deinit(c.gpa);
64		c.extern_symbols.deinit(c.gpa);
65		c.decl_deps.deinit(c.gpa);
66
67		// arena can deinit the rest
68		}
69
70		pub fn finalize(c: *JsNodeGen) !JsTree {
71		const t = tracer.trace(@src());
72		defer t.end();
73
74		// todo: errdefer will probably leave memory in a fully invalid state
75		try c.nodes.setCapacity(c.gpa, c.nodes.len);
76
77		var extra = try c.extra.toOwnedStore(c.gpa);
78		errdefer extra.deinit(c.gpa);
79
80		// todo: i dont like this dupe
81		const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys());
82		errdefer c.gpa.free(extern_symbols);
83
84		const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys());
85		errdefer c.gpa.free(decl_deps);
86
87		defer c.deinit();
88
89		return .{
90		.param_count = c.param_count,
91		.nodes = c.nodes.toOwnedSlice(),
92		.extern_symbols = extern_symbols,
93		.decl_deps = decl_deps,
94		.extra = extra,
95		};
96		}
97
98		pub fn build(c: *JsNodeGen) !void {
99		const t = tracer.trace(@src());
100		defer t.end();
101
102		assert(c.air.instTag(.main_block) == .block);
103		const statements = c.air.instData(.block, .main_block).slice(c.air);
104
105		const scratch_top = c.scratch.items.len;
106		defer c.scratch.shrinkRetainingCapacity(scratch_top);
107
108		_ = try c.add(.root, undefined);
109
110		if (statements.len > 0) {
111		for (statements[0..c.param_count], 1..) \|stmt, expected_id\| {
112		assert(@intFromEnum(stmt) == expected_id);
113		assert(c.air.instTag(stmt) == .arg);
114		c.air_node_map[@intFromEnum(stmt)] = .{
115		.node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }),
116		};
117		}
118
119		for (statements[c.param_count..]) \|stmt\| {
120		try c.codeGenInst(stmt);
121		}
122		}
123
124		const items = c.scratch.items[scratch_top..];
125		const top = c.extra.list.items.len;
126		try c.extra.list.appendSlice(c.gpa, items);
127		c.nodes.items(.data)[0] = EncodedData.encode(.root, .{
128		.start = @intCast(top),
129		.len = @intCast(items.len),
130		});
131		}
132
133		fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
134		const t = tracer.traceFmt(@src(), "#{d} {s}", .{ @intFromEnum(inst), @tagName(c.air.instTag(inst)) });
135		defer t.end();
136
137		switch (c.air.instTag(inst)) {
138		.arg => unreachable, // codegen must skip over these instructions.
139
140		.block => try c.codeGenBlock(inst),
141
142		.js_global_ref => {
143		const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst);
144		try c.extern_symbols.put(c.gpa, data, {});
145		c.air_node_map[@intFromEnum(inst)] = .{
146		.node = try c.add(.global, .{ .name = data }),
147		};
148		},
149
150		.memory => {
151		const i = @intFromEnum(inst);
152		const data: Air.Memory = c.air.instData(.memory, inst);
153		const value: JsTree.Node.Index.Optional = if (data.init.isVoid())
154		.none
155		else
156		(try c.fetchExpression(data.init)).toOptional();
157
158		try c.addStmt(try c.add(.stmt_declare_local, .{
159		.identifier_index = c.next_id,
160		.init = value,
161		}));
162		const id = try c.add(.local, .{ .id = c.next_id });
163		c.next_id += 1;
164		c.air_node_map[i] = .{ .node = id };
165		},
166
167		inline .gt,
168		.lt,
169		.eq,
170		.add,
171		.sub,
172		.mul,
173		.div,
174		=> \|tag\| {
175		const data: Air.Bin = c.air.instData(tag, inst);
176		const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
177		.lhs = try c.fetchExpression(data.lhs),
178		.rhs = try c.fetchExpression(data.rhs),
179		});
180		try c.markExpression(inst, node);
181		},
182
183		.store => {
184		const data: Air.Store = c.air.instData(.store, inst);
185
186		const target = try c.fetchInstExpression(data.memory);
187
188		if (data.operand.kind == .index) {
189		const index = data.operand.decode().index;
190		switch (c.air.instTag(index)) {
191		inline .add,
192		.sub,
193		.mul,
194		.div,
195		=> \|tag\| {
196		const inner_data: Air.Bin = c.air.instData(tag, index);
197		if (inner_data.lhs.eql(data.memory.toRef())) {
198		if ((tag == .add or tag == .sub) and
199		inner_data.rhs.kind == .value)
200		{
201		// if (inner_data.rhs.decode().value.toFloat(c.ip)) \|f\| {
202		// if (f == 1) {
203		// // convert this into ++ or --
204		// const node = try c.add(
205		// if (tag == .add)
206		// .pre_inc
207		// else
208		// .pre_dec,
209		// .{
210		// .operand = target,
211		// },
212		// );
213		// try c.markExpression(inst, node);
214
215		// return;
216		// }
217		// }
218		}
219
220		// convert into a compound assignment
221		const node = try c.add(
222		@field(JsTree.Tag, "assign_" ++ @tagName(tag)),
223		.{
224		.lhs = target,
225		.rhs = try c.fetchExpression(inner_data.rhs),
226		},
227		);
228		try c.markExpression(inst, node);
229
230		return;
231		}
232		},
233		else => {},
234		}
235		}
236
237		const node = try c.add(.assign, .{
238		.lhs = target,
239		.rhs = try c.fetchExpression(data.operand),
240		});
241		try c.markExpression(inst, node);
242		},
243
244		inline .negate,
245		.bool_not,
246		=> \|tag\| {
247		const data: Air.Un = c.air.instData(tag, inst);
248		const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
249		.operand = try c.fetchExpression(data.operand),
250		});
251		try c.markExpression(inst, node);
252		},
253
254		.field => {
255		const data: Air.Field = c.air.instData(.field, inst);
256		const node = try c.add(.field, .{
257		.base = try c.fetchExpression(data.base),
258		.field = data.field,
259		});
260		try c.markExpression(inst, node);
261		},
262
263		.call => {
264		const data: Air.Call = c.air.instData(.call, inst);
265		const call_data = data.call_data.get(c.air);
266
267		try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len);
268		const start = c.extra.list.items.len;
269		c.extra.list.items.len += call_data.args.len;
270
271		const items = c.extra.list.items[start..][0..call_data.args.len];
272
273		for (call_data.args.slice(c.air), items) \|arg, *dest\| {
274		dest.* = @intFromEnum(try c.fetchExpression(arg));
275		}
276
277		const node = try c.add(.call, .{
278		.base = try c.fetchExpression(data.base),
279		.args = try c.extra.append(c.gpa, JsTree.IndexList, .{
280		.start = @intCast(start),
281		.len = call_data.args.len,
282		}),
283		});
284		try c.markExpression(inst, node);
285		},
286
287		.@"if" => {
288		const data: Air.If = c.air.instData(.@"if", inst);
289		const extra = data.data.get(c.air);
290		var cond = try c.fetchExpression(data.cond);
291
292		if (c.canLowerToExpression(extra.yes) and
293		c.canLowerToExpression(extra.no))
294		{
295		const yes = try c.lowerToExpression(extra.yes);
296		const no = try c.lowerToExpression(extra.yes);
297
298		const ternary = try c.add(.ternary, .{
299		.cond = cond,
300		.data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{
301		.yes = yes,
302		.no = no,
303		}),
304		});
305
306		try c.markExpression(inst, ternary);
307		} else {
308		const flags = c.air.instFlags(inst);
309		const uses = flags.uses;
310
311		const target = if (uses != .none)
312		try c.addTemporary()
313		else
314		null;
315
316		var yes = try c.codeGenBranch(extra.yes, target);
317		var no = try c.codeGenBranch(extra.no, target);
318		assert(!((yes == null) and (no == null))); // no "void else void" branches
319
320		// when `yes` is void, swap the condition so that the `else` is omitted
321		if (yes == null) {
322		const temp_yes = yes;
323		yes = no;
324		no = temp_yes;
325		cond = try c.add(.bool_not, .{
326		.operand = cond,
327		});
328		}
329
330		const main_expr = try c.add(.stmt_if, .{
331		.cond = cond,
332		.data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{
333		.yes = yes.?.list,
334		.no = if (no) \|branch\|
335		branch.list
336		else
337		.{ .start = 0, .len = 0 },
338		.flags = .{
339		.block_label = @intCast(c.next_label + 1),
340		.yes_label = yes.?.requires_label,
341		.no_label = if (no) \|branch\| branch.requires_label else false,
342		},
343		}),
344		});
345
346		try c.addStmt(main_expr);
347
348		if (target) \|node\| {
349		try c.markExpression(inst, node);
350		}
351		}
352		},
353
354		.@"break" => {
355		const brk: Air.Break = c.air.instData(.@"break", inst);
356
357		const operand = try c.fetchExpression(brk.value);
358
359		// break #0 -> return
360		if (brk.block_ref == .main_block) {
361		try c.addStmt(try c.add(.stmt_return, .{
362		.operand = operand.toOptional(),
363		}));
364		} else {
365		const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block;
366
367		const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air);
368		if (block_insts[block_insts.len - 1] != inst) {
369		block.used = true;
370		}
371
372		const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]);
373		if (assignment.unwrap()) \|target\| {
374		try c.addStmt(try c.add(.assign, .{
375		.lhs = target,
376		.rhs = operand,
377		}));
378		}
379
380		try c.addStmt(try c.add(.stmt_break, .{
381		.label = c.scratch.items[block.index + 1],
382		}));
383		}
384		},
385		}
386		}
387
388		fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void {
389		const t = tracer.trace(@src());
390		defer t.end();
391
392		const flags: Air.Flags = c.air.instFlags(inst);
393		const i = @intFromEnum(inst);
394		switch (flags.uses) {
395		.none => {
396		try c.addStmt(node);
397		},
398		.single => {
399		// TODO: this is not aware of side effect fences
400		c.air_node_map[i] = .{ .node = node };
401		},
402		.multi => {
403		const decl = try c.add(.stmt_declare_local, .{
404		.identifier_index = c.next_id,
405		.init = node.toOptional(),
406		});
407		try c.addStmt(decl);
408		const id = try c.add(.local, .{ .id = c.next_id });
409		c.next_id += 1;
410		c.air_node_map[i] = .{ .node = id };
411		},
412		}
413		}
414
415		fn fetchExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
416		const t = tracer.trace(@src());
417		defer t.end();
418
419		return switch (ref.decode()) {
420		.value => \|v\| c.valueToExpression(v),
421		.index => \|index\| fetchInstExpression(c, index),
422		};
423		}
424
425		fn fetchInstExpression(c: *JsNodeGen, index: Air.Inst.Index) !Node.Index {
426		const t = tracer.trace(@src());
427		defer t.end();
428
429		const data = c.air_node_map[@intFromEnum(index)];
430		if (std.debug.runtime_safety) {
431		const flags: Air.Flags = c.air.instFlags(index);
432		switch (flags.uses) {
433		.none => unreachable, // incorrectly marked instruction as unused,
434		.single => {},
435		.multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
436		}
437		}
438		return data.node;
439		}
440
441		fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index {
442		const t = tracer.trace(@src());
443		defer t.end();
444
445		const intern_tag = v.valueTag(c.ip);
446		switch (intern_tag) {
447		.removed => unreachable, // UAF
448
449		.type_function,
450		.simple_type,
451		=> unreachable, // Type values are not allowed here
452
453		.int32,
454		.float,
455		.comptime_float,
456		.string,
457		.simple_value,
458		=> return try c.add(.plain_value, .{ .value = v }),
459
460		// funcs always point to decls
461		.func => {
462		const func = v.toKey(c.ip).func;
463		try c.decl_deps.put(c.gpa, func.owner_decl, {});
464		return try c.add(.decl, .{ .index = func.owner_decl });
465		},
466		}
467		}
468
469		fn addTemporary(c: *JsNodeGen) !Node.Index {
470		const t = tracer.trace(@src());
471		defer t.end();
472
473		try c.addStmt(try c.add(.stmt_declare_local, .{
474		.identifier_index = c.next_id,
475		.init = .none,
476		}));
477		const id = try c.add(.local, .{ .id = c.next_id });
478		c.next_id += 1;
479		return id;
480		}
481
482		const Branch = struct {
483		requires_label: bool,
484		list: JsTree.IndexList,
485		};
486
487		fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch {
488		const t = tracer.trace(@src());
489		defer t.end();
490
491		const scratch_top = c.scratch.items.len;
492		defer c.scratch.shrinkRetainingCapacity(scratch_top);
493
494		c.next_label += 1;
495		defer c.next_label -= 1;
496		const this_label = c.next_label;
497
498		var requires_label = false;
499		var items = switch (inst.decode()) {
500		.value => \|v\| items: {
501		if (v == .void)
502		return null;
503		if (temporary) \|temp\| {
504		const node = try c.valueToExpression(v);
505		try c.addStmt(try c.add(.assign, .{
506		.lhs = temp,
507		.rhs = node,
508		}));
509		}
510		break :items c.scratch.items[scratch_top..];
511		},
512		.index => \|i\| items: {
513		assert(c.air.instTag(i) == .block);
514		c.air_node_map[@intFromEnum(i)] = .{
515		.block = .{ .index = @intCast(scratch_top) },
516		};
517
518		try c.scratch.appendSlice(c.arena, &.{
519		@intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
520		this_label,
521		});
522
523		const block: Air.IndexList = c.air.instData(.block, i);
524		for (block.slice(c.air)) \|item\| {
525		try c.codeGenInst(item);
526		}
527
528		requires_label = c.air_node_map[@intFromEnum(i)].block.used;
529
530		break :items c.scratch.items[scratch_top + 2 ..];
531		},
532		};
533
534		const last_node = c.scratch.items[c.scratch.items.len - 1];
535		if (c.nodes.items(.tag)[last_node] == .stmt_break and
536		c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label)
537		{
538		items.len -= 1;
539		c.nodes.len -= 1;
540		}
541
542		const top = c.extra.list.items.len;
543		try c.extra.list.appendSlice(c.gpa, items);
544		return .{
545		.requires_label = requires_label,
546		.list = .{
547		.start = @intCast(top),
548		.len = @intCast(items.len),
549		},
550		};
551		}
552
553		fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void {
554		const t = tracer.trace(@src());
555		defer t.end();
556
557		const scratch_top = c.scratch.items.len;
558		errdefer c.scratch.shrinkRetainingCapacity(scratch_top);
559
560		c.next_label += 1;
561		defer c.next_label -= 1;
562
563		const next_label = c.next_label;
564
565		try c.scratch.appendSlice(c.arena, &.{
566		// @intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
567		@intFromEnum(Node.Index.Optional.none),
568		next_label,
569		});
570
571		c.air_node_map[@intFromEnum(inst)] = .{
572		.block = .{ .index = @intCast(scratch_top) },
573		};
574		const block: Air.IndexList = c.air.instData(.block, inst);
575
576		for (block.slice(c.air)) \|item\| {
577		try c.codeGenInst(item);
578		}
579
580		const items = c.scratch.items[scratch_top + 2 ..];
581		const top = c.extra.list.items.len;
582		try c.extra.list.appendSlice(c.gpa, items);
583		c.scratch.shrinkRetainingCapacity(scratch_top);
584		try c.addStmt(try c.add(.stmt_block, .{
585		.label = c.next_label,
586		.content = try c.extra.append(c.gpa, JsTree.IndexList, .{
587		.start = @intCast(top),
588		.len = @intCast(items.len),
589		}),
590		}));
591		}
592
593		// Ideally, we convert everything to an expression. This allows for much
594		// more compact code. When that isnt possible (`return`, `while`, etc),
595		// we detect that here and ensure to lower `if` to actual if statements,
596		// and so on.
597		fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool {
598		const t = tracer.trace(@src());
599		defer t.end();
600
601		return switch (inst.decode()) {
602		.value => true,
603		.index => \|index\| switch (c.air.instTag(index)) {
604		.js_global_ref,
605		.call,
606		.arg,
607		.field,
608		.add,
609		.sub,
610		.mul,
611		.div,
612		.eq,
613		.lt,
614		.gt,
615		.negate,
616		.bool_not,
617		.store,
618		=> true,
619
620		// JavaScript variable declarations require statements.
621		// Later, this could be made true if we can hoist temporaries up.
622		.memory => false,
623
624		// break/return must be a js statement
625		.@"break",
626		=> false,
627
628		.block => {
629		// TODO: blocks that are
630		// - all statements except last are lowerable
631		// - the last statement is lowerable OR a break of this block
632		return false;
633		},
634
635		.@"if" => {
636		const call = c.air.instData(.@"if", index);
637		const data = call.data.get(c.air);
638
639		return c.canLowerToExpression(call.cond) and
640		c.canLowerToExpression(data.yes) and
641		c.canLowerToExpression(data.no);
642		},
643		},
644		};
645		}
646
647		fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
648		const t = tracer.trace(@src());
649		defer t.end();
650
651		assert(c.canLowerToExpression(ref));
652		return switch (ref.decode()) {
653		.value => \|v\| try c.valueToExpression(v),
654		.index => \|index\| {
655		try c.codeGenInst(index);
656		return c.fetchExpression(ref);
657		},
658		};
659		}
660
661		fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index {
662		const t = tracer.trace(@src());
663		defer t.end();
664
665		try c.nodes.append(c.gpa, .{
666		.tag = tag,
667		.data = EncodedData.encode(tag, data),
668		});
669		return @enumFromInt(c.nodes.len - 1);
670		}
671
672		fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void {
673		const t = tracer.trace(@src());
674		defer t.end();
675
676		try c.scratch.append(c.arena, @intFromEnum(stmt));
677		}
678
679		const JsNodeGen = @This();
680
681		const std = @import("std");
682		const assert = std.debug.assert;
683		const Allocator = std.mem.Allocator;
684
685		const InternPool = @import("InternPool.zig");
686		const Value = InternPool.Value;
687		const NullTerminatedString = InternPool.NullTerminatedString;
688
689		const Comp = @import("Comp.zig");
690		const Decl = Comp.Decl;
691
692		const JsTree = @import("JsTree.zig");
693		const EncodedData = JsTree.EncodedData;
694		const Node = JsTree.Node;
695		const LocalIndex = JsTree.LocalIndex;
696
697		const Air = @import("Air.zig");
698
699		const dod = @import("array.zig");
700		const tracer = @import("tracer");

src/compiler/JsTree.zig deleted-257

...	...	@@ -1,257 +0,0 @@
1		//! JavaScript Function Representation is the last representation before javascript
2		//! source bytes are created. One notable thing here is that all local identifiers
3		//! are numbered, instead of being named. The first `param_count` local identifiers
4		//! refer to parameters, and the rest refer to the respective block that defined them.
5
6		param_count: u32,
7		nodes: Node.List.Slice,
8		extern_symbols: []String.WithLen,
9		decl_deps: []Decl.Index,
10		extra: ExtraStore,
11
12		pub const LocalIndex = u32;
13		pub const ExtraStore = dod.ExtraStore(.tfr, .@"const");
14		pub const EncodedData = dod.EncodedData(Tag, u64);
15		pub const ExtraIndex = ExtraStore.Index;
16
17		pub const Tag = enum {
18		root,
19		/// Only allow simple value types: int, float, string, bool
20		plain_value,
21		/// Reference to a local identifier (give an VarId)
22		local,
23		/// Reference to a global identifier (give a NullTerminatedString)
24		global,
25		/// Reference to a declaration (contains a Decl.Index)
26		decl,
27		// bin ops
28		assign,
29		assign_add,
30		assign_sub,
31		assign_mul,
32		assign_div,
33		add,
34		sub,
35		mul,
36		div,
37		eq,
38		lt,
39		gt,
40		// exprs
41		call,
42		field,
43		ternary,
44		// prefix ops
45		negate,
46		bool_not,
47		pre_inc,
48		pre_dec,
49		// stmts
50		/// `let ... = ...`
51		stmt_declare_local,
52		/// `if (...) ... else ...`
53		stmt_if,
54		/// `break ... ...`
55		stmt_break,
56		/// `return ...` (break from block #0)
57		stmt_return,
58		/// `...: { ... }`
59		stmt_block,
60
61		pub fn Data(comptime tag: Tag) type {
62		return switch (tag) {
63		.root => IndexList,
64		.plain_value => PlainValue,
65		.local => LocalRef,
66		.global => GlobalRef,
67		.decl => DeclRef,
68		.assign,
69		.assign_add,
70		.assign_sub,
71		.assign_mul,
72		.assign_div,
73		.add,
74		.sub,
75		.mul,
76		.div,
77		.eq,
78		.lt,
79		.gt,
80		=> Bin,
81		.ternary => Ternary,
82		.negate,
83		.bool_not,
84		.pre_inc,
85		.pre_dec,
86		=> Un,
87		.call => Call,
88		.field => Field,
89		.stmt_if => StmtIf,
90		.stmt_break => StmtBreak,
91		.stmt_return => StmtReturn,
92		.stmt_declare_local => StmtDeclareLocal,
93		.stmt_block => StmtBlock,
94		};
95		}
96		};
97
98		pub const Node = struct {
99		tag: Tag,
100		data: EncodedData,
101
102		pub const List = std.MultiArrayList(Node);
103
104		pub const Index = enum(u32) {
105		root = 0,
106		_,
107
108		pub fn toOptional(i: Index) Optional {
109		return @enumFromInt(@intFromEnum(i));
110		}
111
112		pub const Optional = enum(u32) {
113		none = std.math.maxInt(u32),
114		_,
115
116		pub fn unwrap(oi: Optional) ?Index {
117		return if (oi == .none)
118		null
119		else
120		@enumFromInt(@intFromEnum(oi));
121		}
122		};
123		};
124		};
125
126		pub const PlainValue = struct {
127		value: Value,
128		unused: Unused = .zero,
129		};
130
131		pub const LocalRef = struct {
132		id: LocalIndex,
133		unused: Unused = .zero,
134		};
135
136		pub const GlobalRef = struct {
137		name: InternPool.String.WithLen,
138		};
139
140		pub const DeclRef = struct {
141		index: Decl.Index,
142		unused: Unused = .zero,
143		};
144
145		pub const StmtDeclareLocal = struct {
146		identifier_index: u32,
147		init: Node.Index.Optional,
148		};
149
150		pub const Bin = struct {
151		lhs: Node.Index,
152		rhs: Node.Index,
153		};
154
155		pub const Un = struct {
156		operand: Node.Index,
157		unused: Unused = .zero,
158		};
159
160		pub const Call = struct {
161		base: Node.Index,
162		args: ExtraIndex(IndexList),
163		};
164
165		pub const Field = struct {
166		base: Node.Index,
167		field: NullTerminatedString,
168		};
169
170		pub const IndexList = struct {
171		start: u32,
172		len: u32,
173
174		pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index {
175		return @ptrCast(tree.extra.list.items[list.start..][0..list.len]);
176		}
177		};
178
179		pub const StmtIf = struct {
180		cond: Node.Index,
181		data: ExtraIndex(Data),
182
183		pub const Data = struct {
184		yes: IndexList,
185		no: IndexList,
186		flags: Flags,
187		};
188
189		pub const Flags = packed struct(u32) {
190		block_label: u16,
191		yes_label: bool,
192		no_label: bool,
193		_: u14 = 0,
194		};
195		};
196
197		pub const Ternary = struct {
198		cond: Node.Index,
199		data: ExtraIndex(Data),
200
201		pub const Data = struct {
202		yes: Node.Index,
203		no: Node.Index,
204		};
205		};
206
207		pub const StmtBreak = struct {
208		label: u32,
209		unused: Unused = .zero,
210		};
211
212		pub const StmtReturn = struct {
213		operand: Node.Index.Optional,
214		unused: Unused = .zero,
215		};
216
217		pub const StmtBlock = struct {
218		label: u32,
219		content: ExtraIndex(IndexList),
220		};
221
222		pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void {
223		tree.nodes.deinit(gpa);
224		gpa.free(tree.decl_deps);
225		gpa.free(tree.extern_symbols);
226		tree.extra.deinit(gpa);
227		}
228
229		pub fn nodeTag(tree: JsTree, index: Node.Index) Tag {
230		return tree.nodes.items(.tag)[@intFromEnum(index)];
231		}
232
233		pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() {
234		return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index));
235		}
236
237		pub fn mainStatements(tree: *const JsTree) []const Node.Index {
238		return tree.nodeData(.root, .root).slice(tree);
239		}
240
241		const JsTree = @This();
242
243		const std = @import("std");
244		const assert = std.debug.assert;
245
246		const InternPool = @import("InternPool.zig");
247		const Value = InternPool.Value;
248
249		const NullTerminatedString = InternPool.NullTerminatedString;
250		const String = InternPool.String;
251
252		const Comp = @import("Comp.zig");
253		const Decl = Comp.Decl;
254
255		const dod = @import("array.zig");
256
257		const Unused = @import("Ast.zig").Unused;

src/compiler/Parse.zig deleted-1146

...	...	@@ -1,1146 +0,0 @@
1		//! Parser takes a Token list and creates a Node list. It does not own the source code or
2		//! token data, however transferring into an `Ast` does move token list ownership.
3		gpa: Allocator,
4		source: [:0]const u8,
5		token_tags: []const Token.Tag,
6		token_starts: []const ByteOffset,
7
8		pos: TokenIndex = 0,
9		nodes: Node.List = .{},
10		extra: ArrayListUnmanaged(u32) = .{},
11		scratch: ArrayListUnmanaged(NodeIndex) = .{},
12
13		ast_error: ?AstError.WithTrace = null,
14
15		depth: u32 = 0,
16
17		/// Instead of implementing a non-recursive parser, the stack size is limited by this constant.
18		const depth_limit = 4096;
19
20		pub fn parse(gpa: std.mem.Allocator, source: [:0]const u8, ast_error: *AstError.WithTrace) Parse.Error!Ast {
21		var err_byte: ByteOffset = 0;
22		var tokens = Tokenizer.tokenize(
23		source,
24		gpa,
25		&err_byte,
26		) catch \|err\| switch (err) {
27		error.OutOfMemory => return error.OutOfMemory,
28		else => {
29		ast_error.* = .{
30		.err = .{ .tokenization = .{
31		.err = err,
32		.byte = err_byte,
33		} },
34		.trace = Report.DebugStoredTrace.from(@errorReturnTrace()),
35		};
36		return error.ParseFail;
37		},
38		};
39		errdefer tokens.deinit(gpa);
40
41		var parser = Parse.init(gpa, source, tokens);
42		errdefer parser.deinit();
43
44		try parser.nodes.ensureUnusedCapacity(
45		gpa,
46		@max(1, @divFloor(tokens.len, 4)),
47		);
48
49		parser.parseRoot() catch \|err\| switch (err) {
50		error.OutOfMemory => return error.OutOfMemory,
51		error.ParseFail => {
52		ast_error.* = parser.ast_error.?;
53		return error.ParseFail;
54		},
55		};
56
57		return try parser.finalize(tokens);
58		}
59
60		fn init(gpa: Allocator, source: [:0]const u8, tokens: Token.List.Slice) Parse {
61		return Parse{
62		.gpa = gpa,
63		.source = source,
64		.token_tags = tokens.items(.tag),
65		.token_starts = tokens.items(.offset),
66		};
67		}
68
69		/// This transfers the parse result into an Ast
70		fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
71		const extra = try p.extra.toOwnedSlice(p.gpa);
72		defer p.* = undefined;
73		assert(token_slice.items(.tag).ptr == p.token_tags.ptr);
74		assert(token_slice.items(.offset).ptr == p.token_starts.ptr);
75		p.scratch.deinit(p.gpa);
76
77		const ast: Ast = .{
78		.utf8_source = p.source,
79		.tokens = token_slice,
80		.nodes = p.nodes.toOwnedSlice(),
81		.extra = extra,
82		};
83
84		return ast;
85		}
86
87		/// This discards all parser work
88		fn deinit(p: *Parse) void {
89		defer p.* = undefined;
90		p.nodes.deinit(p.gpa);
91		p.extra.deinit(p.gpa);
92		p.scratch.deinit(p.gpa);
93		}
94
95		pub const Error = error{
96		OutOfMemory,
97		ParseFail,
98		};
99
100		pub const ExpectedContext = enum {
101		param,
102		argument,
103		statement,
104		expression,
105		else_clause,
106
107		pub fn description(tag: ExpectedContext) []const u8 {
108		return switch (tag) {
109		.param => "a parameter",
110		.argument => "an argument",
111		.statement => "a statement",
112		.expression => "an expression",
113		.else_clause => "an else clause",
114		};
115		}
116		};
117
118		/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
119		/// When printing these, token list and Ast is not actually available, so full tokens
120		/// are stored in the errors.
121		pub const AstError = union(enum) {
122		/// Failure during tokenization
123		tokenization: struct {
124		err: Tokenizer.TokenizeError,
125		byte: ByteOffset,
126		},
127		/// Unexpected data, expected a token.
128		unexpected_token: struct {
129		expected: Token.Tag,
130		got: Token,
131		},
132		/// Unexpected data, expected <ExpectedContext>
133		unexpected_context: struct {
134		expected: ExpectedContext,
135		got: Token,
136		},
137		/// Went past depth maximum
138		depth_limit: Token,
139
140		pub fn toReport(err: AstError, comp: Comp, file: Comp.File, trace: Report.DebugStoredTrace) !Report {
141		return (Report.Err{ .ast = err }).toReport(comp, file, trace);
142		}
143
144		pub fn build(err: AstError, ctx: *Report.MsgContext) !void {
145		switch (err) {
146		.tokenization => \|data\| {
147		// TODO: consider making tokenizer emit AstError more directly,
148		// so that it can just annotate start and ends
149		try ctx.msg(switch (data.err) {
150		error.UnterminatedString => .{ .codepoint_eol = data.byte },
151		else => .{ .codepoint = data.byte },
152		}, "{s}", .{switch (data.err) {
153		error.OutOfMemory => unreachable,
154		error.InvalidSyntax => "unexpected token",
155		error.TrailingNumericSeparator => "trailing numberic separator",
156		error.UnterminatedString => "unterminated string literal",
157		error.InvalidEscapeSequence => "invalid escape sequence",
158		error.InvalidEscapeCodepoint => "invalid escape codepoint",
159		error.InvalidUtf8 => "file is not valid utf8",
160		error.GitConflict => "git merge conflict encountered",
161		}});
162		},
163		.unexpected_token => \|data\| {
164		try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{
165		data.expected.description(),
166		data.got.tag.description(),
167		});
168		},
169		.unexpected_context => \|data\| {
170		try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{
171		data.expected.description(),
172		data.got.tag.description(),
173		});
174		},
175		.depth_limit => \|data\| {
176		try ctx.msg(.{ .token_stored = data }, "syntax tree is nested too deeply (limit: {d})", .{depth_limit});
177		},
178		}
179		}
180
181		pub const WithTrace = struct {
182		err: AstError,
183		trace: Report.DebugStoredTrace,
184
185		pub fn toReport(err: WithTrace, comp: Comp, file: Comp.File) !Report {
186		return err.err.toReport(comp, file, err.trace);
187		}
188		};
189		};
190
191		pub fn parseRoot(p: *Parse) Error!void {
192		const trace = t.trace(@src());
193		defer trace.end();
194
195		assert(p.nodes.len == 0);
196		assert(p.extra.items.len == 0);
197		assert(p.pos == 0);
198		assert(p.nodes.capacity > 0);
199
200		p.nodes.appendAssumeCapacity(.{
201		.tag = .root,
202		.main_token = 0,
203		.data = undefined,
204		});
205
206		const list = try p.parseModuleMembers();
207		p.nodes.items(.data)[0] = .{
208		.list = list,
209		};
210		}
211
212		fn parseModuleMembers(p: *Parse) Error!SubList {
213		p.depth += 1;
214		defer p.depth -= 1;
215		if (p.depth > depth_limit) {
216		return p.fail(.{ .depth_limit = p.fullToken(p.pos) });
217		}
218
219		const trace = t.trace(@src());
220		defer trace.end();
221
222		const scratch_top = p.scratch.items.len;
223		defer p.scratch.shrinkRetainingCapacity(scratch_top);
224
225		// skip module doc comment
226		while (p.eatToken(.module_doc_comment)) \|_\| {}
227
228		while (true) switch (p.currentTag()) {
229		.kw_let,
230		.kw_var,
231		.kw_fn,
232		.kw_pub,
233		=> try p.scratch.append(p.gpa, try p.parseModuleDecl()),
234
235		.kw_struct,
236		.kw_enum,
237		.kw_trait,
238		.kw_impl,
239		=> \|tag\| std.debug.panic("TODO: parse {s}", .{@tagName(tag)}),
240
241		.eof => break,
242
243		else => {
244		if (!p.isExpressionStart()) {
245		return p.failUnexpectedToken(.statement);
246		}
247
248		try p.scratch.append(p.gpa, try p.parseExpression(true));
249		_ = try p.expect(.semi);
250		},
251		};
252
253		return p.scratchToSubList(scratch_top);
254		}
255
256		fn parseModuleDecl(p: *Parse) Error!NodeIndex {
257		const trace = t.trace(@src());
258		defer trace.end();
259
260		_ = p.eatToken(.kw_pub);
261
262		return switch (p.currentTag()) {
263		.kw_var, .kw_let => try p.parseVarDecl(),
264
265		.kw_fn => try p.parseFnDecl(),
266
267		else => unreachable,
268		};
269		}
270
271		fn parseFnDecl(p: *Parse) Error!NodeIndex {
272		const trace = t.trace(@src());
273		defer trace.end();
274
275		const main_token = p.assertToken(.kw_fn);
276
277		_ = try p.expect(.id);
278
279		_ = try p.expect(.start_paren);
280
281		const params = try p.parseParamList();
282
283		const return_type = if (p.eatToken(.minus_lt)) \|_\|
284		if (!p.isExpressionStart())
285		return p.failUnexpectedToken(.expression)
286		else
287		try p.parseExpression(false)
288		else
289		null;
290
291		try p.expectWithoutAdvance(.start_brace);
292		const body = try p.parseBlock();
293
294		const extra = try p.addExtra(Ast.FnDecl, .{
295		.params = params,
296		.return_type = optionalNodeIndex(return_type),
297		});
298
299		return try p.addNode(.{
300		.tag = .fn_decl,
301		.main_token = main_token,
302		.data = .{ .fn_decl = .{
303		.body = body,
304		.extra = extra,
305		} },
306		});
307		}
308
309		fn parseParamList(p: *Parse) Error!SubList {
310		const trace = t.trace(@src());
311		defer trace.end();
312
313		const scratch_top = p.scratch.items.len;
314		defer p.scratch.shrinkRetainingCapacity(scratch_top);
315
316		while (true) {
317		const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top});
318		defer trace_param.end();
319
320		if (p.currentTag() == .end_paren) break;
321
322		try p.scratch.append(p.gpa, try p.parseParamDecl());
323
324		switch (p.currentTag()) {
325		.comma => {
326		_ = p.nextToken();
327		},
328		.end_paren => break,
329		else => return p.failUnexpectedToken(.param),
330		}
331		}
332
333		_ = p.assertToken(.end_paren);
334
335		return p.scratchToSubList(scratch_top);
336		}
337
338		fn parseParamDecl(p: *Parse) Error!NodeIndex {
339		const trace = t.trace(@src());
340		defer trace.end();
341
342		const main_token = p.assertToken(.id);
343
344		_ = try p.expect(.colon);
345
346		const ty = try p.parseExpression(false);
347
348		return try p.addNode(.{
349		.tag = .param_decl,
350		.main_token = main_token,
351		.data = .{ .param_decl = .{
352		.type = ty,
353		} },
354		});
355		}
356
357		fn parseBlock(p: *Parse) Error!NodeIndex {
358		const trace = t.trace(@src());
359		defer trace.end();
360
361		const main_token = p.assertToken(.start_brace);
362
363		const tag, const list = try p.parseBlockContents();
364		assert(p.lastTag() == .end_brace);
365
366		return try p.addNode(.{
367		.tag = tag,
368		.main_token = main_token,
369		.data = .{
370		.list = list,
371		},
372		});
373		}
374
375		fn parseLabelExpression(p: *Parse) Error!NodeIndex {
376		const trace = t.trace(@src());
377		defer trace.end();
378
379		const main_token = p.assertToken(.id);
380		_ = try p.expect(.colon);
381		const expr = try p.parseExpression(false);
382		return try p.addNode(.{
383		.tag = .label,
384		.main_token = main_token,
385		.data = .{ .label = .{
386		.expr = expr,
387		} },
388		});
389		}
390
391		fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } {
392		const trace = t.trace(@src());
393		defer trace.end();
394
395		assert(p.lastTag() == .start_brace);
396		const scratch_top = p.scratch.items.len;
397		defer p.scratch.shrinkRetainingCapacity(scratch_top);
398
399		const tag: Node.Tag = while (true) switch (p.currentTag()) {
400		.kw_let,
401		.kw_var,
402		=> try p.scratch.append(p.gpa, try p.parseVarDecl()),
403
404		.end_brace => break .block_void,
405
406		else => {
407		if (!p.isExpressionStart()) {
408		return p.failUnexpectedToken(.statement);
409		}
410
411		const first_tag = p.currentTag();
412
413		try p.scratch.append(p.gpa, try p.parseExpression(true));
414
415		switch (first_tag) {
416		// some control flow and blocks are allowed to not have a
417		// trailing semicolons. this is a stylistic choice.
418		.kw_if,
419		.start_brace,
420		=> {
421		// ending a block here should also count as an implicit return,
422		// even though voiding it with a semicolon is not allowed.
423		if (p.currentTag() == .end_brace) break .block_implicit_return;
424		},
425
426		// TODO: make kw_break and kw_return REQUIRE a semicolon, as
427		// they are noreturn statements, doing implicit break on
428		// them makes no sense
429
430		else => {
431		if (p.eatToken(.semi) == null) {
432		// this must be last statement in block if it isnt,
433		// the p.expect(.end_brace) later will fail.
434		break .block_implicit_return;
435		}
436		},
437		}
438		},
439		};
440
441		_ = try p.expect(.end_brace);
442
443		return .{ tag, try p.scratchToSubList(scratch_top) };
444		}
445
446		/// VarDecl <- (kw_pub)? (kw_var \| kw_let) id (colon expr)? equal expr semi
447		fn parseVarDecl(p: *Parse) Error!NodeIndex {
448		const trace = t.trace(@src());
449		defer trace.end();
450
451		const kind = p.currentTag();
452		assert(kind == .kw_var or kind == .kw_let);
453
454		const main_token = p.nextToken();
455
456		_ = try p.expect(.id);
457
458		const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) \|_\|
459		optionalNodeIndex(try p.parseExpression(false))
460		else
461		.none;
462
463		_ = try p.expect(.equal);
464
465		const data_init = try p.parseExpression(true);
466
467		_ = try p.expect(.semi);
468
469		return try p.addNode(.{
470		.tag = .var_decl,
471		.main_token = main_token,
472		.data = .{ .var_decl = .{
473		.type = data_type,
474		.init = data_init,
475		} },
476		});
477		}
478
479		fn isExpressionStart(p: *Parse) bool {
480		return switch (p.currentTag()) {
481		.id,
482		.number_int,
483		.number_float,
484		.string,
485		.builtin_id,
486		.kw_return,
487		.kw_break,
488		.kw_if,
489		.start_paren,
490		.start_brace,
491		.bang,
492		.tilde,
493		.minus,
494		=> true,
495		else => false,
496		};
497		}
498
499		const Operator = enum(u8) {
500		add = @intFromEnum(Token.Tag.plus),
501		sub = @intFromEnum(Token.Tag.minus),
502		mul = @intFromEnum(Token.Tag.star),
503		div = @intFromEnum(Token.Tag.slash),
504		rem = @intFromEnum(Token.Tag.percent),
505		pow = @intFromEnum(Token.Tag.star_star),
506		@"and" = @intFromEnum(Token.Tag.and_and),
507		@"or" = @intFromEnum(Token.Tag.pipe_pipe),
508		bit_and = @intFromEnum(Token.Tag.@"and"),
509		bit_or = @intFromEnum(Token.Tag.pipe),
510		bit_xor = @intFromEnum(Token.Tag.caret),
511		shl = @intFromEnum(Token.Tag.lt_lt),
512		shr = @intFromEnum(Token.Tag.gt_gt),
513		eq = @intFromEnum(Token.Tag.equal_equal),
514		ne = @intFromEnum(Token.Tag.bang_equal),
515		lt = @intFromEnum(Token.Tag.lt),
516		le = @intFromEnum(Token.Tag.lt_equal),
517		gt = @intFromEnum(Token.Tag.gt),
518		ge = @intFromEnum(Token.Tag.gt_equal),
519		// Following operators are assignment operators
520		assign = @intFromEnum(Token.Tag.equal),
521		add_assign = @intFromEnum(Token.Tag.plus_equal),
522		sub_assign = @intFromEnum(Token.Tag.minus_equal),
523		mul_assign = @intFromEnum(Token.Tag.star_equal),
524		div_assign = @intFromEnum(Token.Tag.slash_equal),
525		rem_assign = @intFromEnum(Token.Tag.percent_equal),
526		pow_assign = @intFromEnum(Token.Tag.star_star_equal),
527		and_assign = @intFromEnum(Token.Tag.and_and_equal),
528		or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal),
529		bit_and_assign = @intFromEnum(Token.Tag.and_equal),
530		bit_or_assign = @intFromEnum(Token.Tag.pipe_equal),
531		bit_xor_assign = @intFromEnum(Token.Tag.caret_equal),
532		shl_assign = @intFromEnum(Token.Tag.lt_lt_equal),
533		shr_assign = @intFromEnum(Token.Tag.gt_gt_equal),
534
535		const Direction = enum { ltr, rtl };
536
537		/// Only the binary operator levels are actually used,
538		/// but all are listed for reference.
539		const Level = enum {
540		/// e. e[] e()
541		accessor,
542		/// new
543		new,
544		/// ++e, --e, !e, ~e, +e, -e, typeof, void, delete
545		prefix,
546		/// **
547		exponentiation,
548		/// *, /, %
549		multiplicative,
550		/// +, -
551		additive,
552		/// <<, >>, >>>
553		bitwise_shift,
554		/// <, <=, >, >=
555		relational,
556		/// ==, !=
557		equality,
558		/// &
559		bit_and,
560		/// ^
561		bit_xor,
562		/// \|
563		bit_or,
564		/// &&
565		logical_and,
566		/// \|\|, ??
567		logical_or,
568		/// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
569		assign,
570		};
571
572		pub fn direction(op: Operator) Direction {
573		return switch (op) {
574		.pow => .rtl,
575
576		.assign,
577		.add_assign,
578		.sub_assign,
579		.mul_assign,
580		.div_assign,
581		.rem_assign,
582		.pow_assign,
583		.and_assign,
584		.or_assign,
585		.bit_and_assign,
586		.bit_or_assign,
587		.bit_xor_assign,
588		.shl_assign,
589		.shr_assign,
590		=> .rtl,
591
592		else => .ltr,
593		};
594		}
595
596		pub fn level(op: Operator) Level {
597		return switch (op) {
598		.pow => .exponentiation,
599		.mul, .div, .rem => .multiplicative,
600		.add, .sub => .additive,
601		.shl, .shr => .bitwise_shift,
602		.lt, .le, .gt, .ge => .relational,
603		.eq, .ne => .equality,
604		.@"and" => .logical_and,
605		.@"or" => .logical_or,
606		.bit_and => .bit_and,
607		.bit_xor => .bit_xor,
608		.bit_or => .bit_or,
609		.assign,
610		.add_assign,
611		.sub_assign,
612		.mul_assign,
613		.div_assign,
614		.rem_assign,
615		.pow_assign,
616		.and_assign,
617		.or_assign,
618		.bit_and_assign,
619		.bit_or_assign,
620		.bit_xor_assign,
621		.shl_assign,
622		.shr_assign,
623		=> .assign,
624		};
625		}
626
627		const lut = brk: {
628		const token_tags = std.enums.values(Token.Tag);
629		const op_tags = std.enums.values(Operator);
630
631		var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len;
632		for (op_tags) \|op\| {
633		is_operator_lut[@intFromEnum(op)] = true;
634		}
635
636		var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len;
637		for (op_tags) \|op\| {
638		ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op));
639		}
640
641		break :brk struct {
642		const is_operator = is_operator_lut;
643		const ast_tag = ast_tag_lut;
644		};
645		};
646
647		pub fn fromToken(tag: Token.Tag) ?Operator {
648		return if (lut.is_operator[@intFromEnum(tag)])
649		@enumFromInt(@intFromEnum(tag))
650		else
651		null;
652		}
653
654		pub fn astTag(op: Operator) Node.Tag {
655		const tag = lut.ast_tag[@intFromEnum(op)];
656		assert(tag != .root); // invalid operator or lut construction
657		return tag;
658		}
659
660		pub fn isAssign(op: Operator) bool {
661		return switch (op) {
662		.assign,
663		.add_assign,
664		.sub_assign,
665		.mul_assign,
666		.div_assign,
667		.rem_assign,
668		.pow_assign,
669		.and_assign,
670		.or_assign,
671		.bit_and_assign,
672		.bit_or_assign,
673		.bit_xor_assign,
674		.shl_assign,
675		.shr_assign,
676		=> true,
677		else => false,
678		};
679		}
680		};
681
682		fn parseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex {
683		const trace = t.trace(@src());
684		defer trace.end();
685
686		p.depth += 1;
687		defer p.depth -= 1;
688		if (p.depth > depth_limit) {
689		return p.fail(.{ .depth_limit = p.fullToken(p.pos) });
690		}
691
692		assert(p.isExpressionStart());
693
694		// These unary operators are greedy, so handle them first.
695		switch (p.currentTag()) {
696		.kw_return => return p.parseReturn(),
697		.kw_break => return p.parseBreak(),
698		else => {},
699		}
700
701		var expr = try p.parseExpressionPart();
702
703		while (Operator.fromToken(p.currentTag())) \|op\| {
704		if (!allow_assign_operators and op.isAssign()) break;
705		expr = try p.parseBinarySuffix(expr, op, allow_assign_operators);
706		}
707
708		return expr;
709		}
710
711		fn parseExpressionPart(p: *Parse) Error!NodeIndex {
712		const trace = t.trace(@src());
713		defer trace.end();
714
715		var expr = switch (p.currentTag()) {
716		// Literals
717		.id => if (p.nextTag() == .colon)
718		try p.parseLabelExpression()
719		else
720		try p.blankNode(.id),
721		.number_int => try p.blankNode(.number_int),
722		.number_float => try p.blankNode(.number_float),
723		.string => try p.blankNode(.string),
724
725		// Complex expressions
726		.builtin_id => try p.parseBuiltinCall(),
727		.kw_return => try p.parseReturn(),
728		.kw_break => try p.parseBreak(),
729		.kw_if => try p.parseIf(),
730		.start_paren => try p.parseGroup(),
731		.start_brace => try p.parseBlock(),
732
733		// Unary operators
734		inline .bang, .tilde, .minus => \|tag\| unary: {
735		const main_token = p.nextToken();
736		const expr = try p.parseExpressionPart();
737
738		break :unary try p.addNode(.{
739		.tag = switch (tag) {
740		.bang => .bool_not,
741		.tilde => .bit_not,
742		.minus => .negate,
743		else => comptime unreachable,
744		},
745		.main_token = main_token,
746		.data = .{ .un = .{
747		.expr = expr,
748		} },
749		});
750		},
751
752		else => return p.failUnexpectedToken(.expression),
753		};
754
755		// parse field, calls, and index expressions
756		while (true) {
757		const trace2 = t.traceFmt(@src(), "(field/call)", .{});
758		defer trace2.end();
759
760		switch (p.currentTag()) {
761		.dot => {
762		const main_token = p.nextToken();
763		const field = try p.expect(.id);
764
765		expr = try p.addNode(.{
766		.tag = .field,
767		.main_token = main_token,
768		.data = .{ .field = .{
769		.base = expr,
770		.field_token = field,
771		} },
772		});
773		},
774		.start_paren => {
775		const main_token = p.nextToken();
776		const args = try p.parseCallArgs();
777
778		const extra = try p.addExtra(Ast.Call, .{
779		.args = args,
780		});
781
782		expr = try p.addNode(.{
783		.tag = .call,
784		.main_token = main_token,
785		.data = .{ .call = .{
786		.base = expr,
787		.extra = extra,
788		} },
789		});
790		},
791		.start_bracket => {
792		const main_token = p.nextToken();
793		const index = try p.parseExpression(false);
794
795		expr = try p.addNode(.{
796		.tag = .index,
797		.main_token = main_token,
798		.data = .{ .bin = .{
799		.lhs = expr,
800		.rhs = index,
801		} },
802		});
803		},
804
805		else => break,
806		}
807		}
808
809		return expr;
810		}
811
812		fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator, allow_assign_operators: bool) Error!NodeIndex {
813		const trace = t.trace(@src());
814		defer trace.end();
815
816		const main_token = p.nextToken();
817		const level = op.level();
818		var rhs = try p.parseExpressionPart();
819
820		while (Operator.fromToken(p.currentTag())) \|next_op\| {
821		if (!allow_assign_operators and next_op.isAssign()) break;
822
823		const new_level = next_op.level();
824		const use = switch (op.direction()) {
825		.ltr => @intFromEnum(new_level) < @intFromEnum(level),
826		.rtl => @intFromEnum(new_level) <= @intFromEnum(level),
827		};
828		if (!use) break;
829
830		rhs = try p.parseBinarySuffix(rhs, next_op, allow_assign_operators);
831		}
832
833		return try p.addNode(.{
834		.tag = op.astTag(),
835		.main_token = main_token,
836		.data = .{ .bin = .{
837		.lhs = lhs,
838		.rhs = rhs,
839		} },
840		});
841		}
842
843		fn parseReturn(p: *Parse) Error!NodeIndex {
844		const trace = t.trace(@src());
845		defer trace.end();
846
847		const main_token = p.assertToken(.kw_return);
848
849		const expr = if (p.isExpressionStart())
850		try p.parseExpression(false)
851		else
852		null;
853
854		return try p.addNode(.{
855		.tag = .@"return",
856		.main_token = main_token,
857		.data = .{ .@"return" = .{
858		.expr = optionalNodeIndex(expr),
859		} },
860		});
861		}
862
863		fn parseBreak(p: *Parse) Error!NodeIndex {
864		const trace = t.trace(@src());
865		defer trace.end();
866
867		const main_token = p.assertToken(.kw_break);
868
869		const label = if (p.eatToken(.colon)) \|_\|
870		try p.expect(.id)
871		else
872		null;
873
874		const expr = if (p.isExpressionStart())
875		try p.parseExpression(false)
876		else
877		null;
878
879		return try p.addNode(.{
880		.tag = .@"break",
881		.main_token = main_token,
882		.data = .{ .@"break" = .{
883		.label = optionalTokenIndex(label),
884		.value = optionalNodeIndex(expr),
885		} },
886		});
887		}
888
889		fn parseBuiltinCall(p: *Parse) Error!NodeIndex {
890		const trace = t.trace(@src());
891		defer trace.end();
892
893		const main_token = p.assertToken(.builtin_id);
894
895		_ = try p.expect(.start_paren);
896
897		const args = try p.parseCallArgs();
898
899		return try p.addNode(.{
900		.tag = .builtin_call,
901		.main_token = main_token,
902		.data = .{ .list = args },
903		});
904		}
905
906		fn parseCallArgs(p: *Parse) Error!SubList {
907		const trace = t.trace(@src());
908		defer trace.end();
909
910		const scratch_top = p.scratch.items.len;
911		defer p.scratch.shrinkRetainingCapacity(scratch_top);
912
913		while (p.isExpressionStart()) {
914		try p.scratch.append(p.gpa, try p.parseExpression(false));
915
916		switch (p.currentTag()) {
917		.end_paren => {
918		break;
919		},
920		.comma => {
921		_ = p.nextToken();
922		if (p.currentTag() == .end_paren)
923		break;
924		},
925		else => return p.failUnexpectedToken(.argument),
926		}
927		}
928
929		_ = try p.expect(.end_paren);
930
931		return p.scratchToSubList(scratch_top);
932		}
933
934		fn parseIf(p: *Parse) Error!NodeIndex {
935		const trace = t.trace(@src());
936		defer trace.end();
937
938		const main_token = p.assertToken(.kw_if);
939
940		const cond = try p.parseExpression(false);
941
942		_ = try p.expectWithoutAdvance(.start_brace);
943		const yes = try p.parseBlock();
944
945		const no = if (p.eatToken(.kw_else)) \|_\|
946		try p.parseElseValue()
947		else
948		null;
949
950		const extra = try p.addExtra(Ast.If, .{
951		.yes = yes,
952		.no = optionalNodeIndex(no),
953		});
954
955		return try p.addNode(.{
956		.tag = .@"if",
957		.main_token = main_token,
958		.data = .{ .@"if" = .{
959		.cond = cond,
960		.extra = extra,
961		} },
962		});
963		}
964
965		fn parseElseValue(p: *Parse) Error!NodeIndex {
966		const trace = t.trace(@src());
967		defer trace.end();
968
969		assert(p.lastTag() == .kw_else);
970
971		return switch (p.currentTag()) {
972		.start_brace => p.parseBlock(),
973		.kw_if => p.parseIf(),
974		.id => p.parseLabelExpression(),
975
976		else => return p.failUnexpectedToken(.else_clause),
977		};
978		}
979
980		fn parseGroup(p: *Parse) Error!NodeIndex {
981		const trace = t.trace(@src());
982		defer trace.end();
983
984		const main_token = p.assertToken(.start_paren);
985
986		const expr = try p.parseExpression(false);
987
988		_ = try p.expect(.end_paren);
989
990		return try p.addNode(.{
991		.tag = .group,
992		.main_token = main_token,
993		.data = .{ .un = .{
994		.expr = expr,
995		} },
996		});
997		}
998
999		// helpers
1000
1001		/// Append a node and return it's index.
1002		fn addNode(p: *Parse, node: Node) !NodeIndex {
1003		const trace = t.trace(@src());
1004		defer trace.end();
1005
1006		try p.nodes.append(p.gpa, node);
1007		// @intCast is always safe because 2^32 tokens alwayss produce less than
1008		// 2^32 nodes, and there is a 2^32 limit on source file bytes.
1009		return @intCast(p.nodes.len - 1);
1010		}
1011
1012		fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
1013		const trace = t.trace(@src());
1014		defer trace.end();
1015
1016		const extra_top = p.extra.items.len;
1017		const bytes = std.mem.asBytes(&extra);
1018		try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes));
1019		return @intCast(extra_top);
1020		}
1021
1022		fn fail(p: *Parse, ast_error: AstError) Error {
1023		p.ast_error = .{
1024		.err = ast_error,
1025		.trace = Report.DebugStoredTrace.capture(@returnAddress()),
1026		};
1027		return Error.ParseFail;
1028		}
1029
1030		fn failUnexpectedToken(p: *Parse, kind: ExpectedContext) Error {
1031		return p.fail(.{ .unexpected_context = .{
1032		.got = p.fullToken(p.pos),
1033		.expected = kind,
1034		} });
1035		}
1036
1037		fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
1038		const main_token = p.assertToken(tag);
1039		return try p.addNode(.{
1040		.tag = @field(Node.Tag, @tagName(tag)),
1041		.main_token = main_token,
1042		.data = .{ .none = .{} },
1043		});
1044		}
1045
1046		fn currentTag(p: *Parse) Token.Tag {
1047		return p.token_tags[p.pos];
1048		}
1049
1050		fn lastTag(p: *Parse) Token.Tag {
1051		return p.token_tags[p.pos - 1];
1052		}
1053
1054		fn nextTag(p: *Parse) Token.Tag {
1055		return p.token_tags[p.pos + 1];
1056		}
1057
1058		fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex {
1059		return if (p.token_tags[p.pos] == tag) p.nextToken() else null;
1060		}
1061
1062		fn expect(p: *Parse, tag: Token.Tag) !TokenIndex {
1063		if (p.currentTag() != tag) {
1064		return p.fail(.{ .unexpected_token = .{
1065		.got = p.fullToken(p.pos),
1066		.expected = tag,
1067		} });
1068		}
1069		return p.nextToken();
1070		}
1071
1072		fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void {
1073		if (p.currentTag() != tag) {
1074		return p.fail(.{ .unexpected_token = .{
1075		.got = p.fullToken(p.pos),
1076		.expected = tag,
1077		} });
1078		}
1079		}
1080
1081		fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex {
1082		const token = p.nextToken();
1083		assert(p.token_tags[token] == tag);
1084		return token;
1085		}
1086
1087		fn nextToken(p: *Parse) TokenIndex {
1088		const result = p.pos;
1089		p.pos += 1;
1090		return result;
1091		}
1092
1093		fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool {
1094		return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
1095		}
1096
1097		fn fullToken(p: *Parse, token: TokenIndex) Token {
1098		return Token{
1099		.tag = p.token_tags[token],
1100		.offset = p.token_starts[token],
1101		};
1102		}
1103
1104		fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList {
1105		const trace = t.trace(@src());
1106		defer trace.end();
1107
1108		const items = p.scratch.items[scratch_top..];
1109		const extra_top = p.extra.items.len;
1110		try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]);
1111		return .{
1112		.start = @intCast(extra_top),
1113		.len = @intCast(items.len),
1114		};
1115		}
1116
1117		const Parse = @This();
1118
1119		const std = @import("std");
1120		const assert = std.debug.assert;
1121		const mem = std.mem;
1122		const Allocator = mem.Allocator;
1123		const ArrayListUnmanaged = std.ArrayListUnmanaged;
1124
1125		const Tokenizer = @import("Tokenizer.zig");
1126		const Token = Tokenizer.Token;
1127		const ByteOffset = Tokenizer.ByteOffset;
1128
1129		const Ast = @import("Ast.zig");
1130		const Node = Ast.Node;
1131		const Data = Node.Data;
1132		const SubList = Ast.SubList;
1133		const NodeIndex = Ast.NodeIndex;
1134		const TokenIndex = Ast.TokenIndex;
1135		const ExtraIndex = Ast.ExtraIndex;
1136		const OptionalNodeIndex = Ast.OptionalNodeIndex;
1137		const OptionalTokenIndex = Ast.OptionalTokenIndex;
1138		const OptionalExtraIndex = Ast.OptionalExtraIndex;
1139		const optionalTokenIndex = OptionalTokenIndex.init;
1140		const optionalNodeIndex = OptionalNodeIndex.init;
1141		const optionalExtraIndex = OptionalExtraIndex.init;
1142
1143		const t = @import("tracer");
1144
1145		const Report = @import("Report.zig");
1146		const Comp = @import("Comp.zig");

src/compiler/Report.zig deleted-864

...	...	@@ -1,864 +0,0 @@
1		//! A Report is a list of errors and warnings for a part of a compilation.
2		//!
3		//! `Err/Warn` structs are unions of every possible message, the logic
4		//! for printing them is all done here.
5		//!
6		//! Across the compiler, `Report.Builder`s' `.err` and `.warn` methods
7		//! can add messages by ID.
8		errors: []Item = &.{},
9		warnings: []Item = &.{},
10		string_bytes: []u8 = &.{},
11
12		pub var debug_show_report_trace: if (builtin.mode == .Debug) bool else void =
13		if (builtin.mode == .Debug) false;
14
15		pub const Item = struct {
16		/// What is this diagnostic
17		data: union { @"error": Err, warn: Warn },
18		/// What file was it in
19		url: String,
20
21		debug_stack_trace: DebugStoredTrace,
22
23		/// Type is never stored, inferred from which list it is in.
24		pub const Type = enum { @"error", warn };
25		};
26
27		pub const Err = union(enum) {
28		pub const FieldEnum = std.meta.FieldEnum(Err);
29
30		/// Constructs a `Report` with just a single error.
31		pub fn toReport(err: Err, comp: Comp, file: File, trace: DebugStoredTrace) !Report {
32		const string_bytes = try comp.gpa.dupe(u8, file.file_path);
33		errdefer comp.gpa.free(string_bytes);
34
35		const errors = try comp.gpa.dupe(Item, &.{.{
36		.data = .{ .@"error" = err },
37		.url = .{ .start = 0, .len = @intCast(string_bytes.len) },
38		.debug_stack_trace = trace,
39		}});
40		errdefer comp.gpa.free(errors);
41
42		return .{ .errors = errors, .string_bytes = string_bytes };
43		}
44
45		ast: @import("Parse.zig").AstError,
46
47		unknown_identifier: struct {
48		node: NodeIndex,
49
50		fn build(data: @This(), ctx: *MsgContext) !void {
51		try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{
52		ctx.ast.basicNodeSlice(data.node),
53		});
54		}
55		},
56		invalid_builtin: struct {
57		node: NodeIndex,
58
59		fn build(data: @This(), ctx: *MsgContext) !void {
60		try ctx.msgToken(
61		ctx.ast.nodeToken(data.node),
62		"unknown built-in function \"{s}\"",
63		.{ctx.ast.basicNodeSlice(data.node)},
64		);
65		}
66		},
67		duplicate_function: struct {
68		node: NodeIndex,
69		prev: NodeIndex,
70
71		fn build(data: @This(), ctx: *MsgContext) !void {
72		try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
73		ctx.ast.fullFnDecl(data.node).id,
74		});
75		try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{});
76		// TODO: be able to store a full list of other declarations
77		// and add them here as "and here" and so on
78		}
79		},
80		duplicate_param: struct {
81		node: NodeIndex,
82		prev: NodeIndex,
83
84		fn build(data: @This(), ctx: *MsgContext) !void {
85		try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
86		ctx.ast.fullParamDecl(data.node).id,
87		});
88		try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{});
89		// TODO: be able to store a full list of other declarations
90		// and add them here as "and here" and so on
91		}
92		},
93		invalid_return: struct {
94		node: NodeIndex,
95
96		fn build(data: @This(), ctx: *MsgContext) !void {
97		try ctx.msgNode(
98		data.node,
99		"return cannot be used outside of a function body",
100		.{},
101		);
102		}
103		},
104		unresolved_label: struct {
105		node: NodeIndex,
106
107		fn build(data: @This(), ctx: *MsgContext) !void {
108		try ctx.msg(
109		.{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 },
110		"unknown block label \"{s}\"",
111		.{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)},
112		);
113		}
114		},
115		shadow_builtin_variable: struct {
116		node: NodeIndex,
117
118		fn build(data: @This(), ctx: *MsgContext) !void {
119		try ctx.msgToken(
120		ctx.ast.nodeToken(data.node) + 1,
121		"declaration shadows reserved built-in \"{s}\"",
122		.{ctx.ast.fullVarDecl(data.node).id},
123		);
124		}
125		},
126		comptime_float_to_int: struct {
127		val: f64,
128		src: LazyNode,
129
130		fn build(data: @This(), ctx: *MsgContext) !void {
131		const node = try data.src.get(ctx);
132		try ctx.msgNode(node, "numeric literal \"{d}\" cannot be coerced to \"int\"", .{data.val});
133		}
134		},
135		coersion: struct {
136		from: Type,
137		to: Type,
138		src: LazyNode,
139
140		fn build(data: @This(), ctx: *MsgContext) !void {
141		const node = try data.src.get(ctx);
142		try ctx.msgNode(node, "expected \"{s}\", got \"{s}\"", .{
143		data.to.displayName(&ctx.comp.intern_pool),
144		data.from.displayName(&ctx.comp.intern_pool),
145		});
146		}
147		},
148		call_non_function: struct {
149		call: LazyNode,
150		ty: Type,
151
152		fn build(data: @This(), ctx: *MsgContext) !void {
153		const node = try data.call.get(ctx);
154		try ctx.msgNode(node, "\"{s}\" is not callable", .{
155		data.ty.displayName(&ctx.comp.intern_pool),
156		});
157		}
158		},
159		call_mismatch_args: struct {
160		call: LazyNode,
161		actual: u32,
162		expected: u32,
163
164		fn build(data: @This(), ctx: *MsgContext) !void {
165		const node = try data.call.get(ctx);
166		try ctx.msgNode(node, "function expected {d} arguments, recieved {d}", .{
167		data.expected,
168		data.actual,
169		});
170		}
171		},
172		assign_builtin: struct {
173		node: NodeIndex,
174
175		fn build(data: @This(), ctx: *MsgContext) !void {
176		try ctx.msgNode(
177		data.node,
178		"cannot re-assign built-in \"{s}\"",
179		.{ctx.ast.basicNodeSlice(data.node)},
180		);
181		}
182		},
183		invalid_assignment_symbol: struct {
184		node: NodeIndex,
185		decl: NodeIndex,
186		kind: UnusedKind,
187
188		fn build(data: @This(), ctx: *MsgContext) !void {
189		const id = switch (ctx.ast.nodeTag(data.decl)) {
190		.param_decl => ctx.ast.fullParamDecl(data.decl).id,
191		.fn_decl => ctx.ast.fullFnDecl(data.decl).id,
192		.var_decl => ctx.ast.fullVarDecl(data.decl).id,
193		else => unreachable, // not a valid symbol node
194		};
195
196		try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{
197		@tagName(data.kind),
198		id,
199		});
200		try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{});
201		}
202		},
203		invalid_assignment_target: struct {
204		node: NodeIndex,
205
206		fn build(data: @This(), ctx: *MsgContext) !void {
207		try ctx.msgNode(
208		data.node,
209		"invalid assignment target",
210		.{},
211		);
212		}
213		},
214		};
215
216		pub const Warn = union(enum) {
217		pub const FieldEnum = std.meta.FieldEnum(Warn);
218
219		unused: struct {
220		node: NodeIndex,
221		kind: UnusedKind,
222
223		fn build(data: @This(), ctx: *MsgContext) !void {
224		const id = switch (ctx.ast.nodeTag(data.node)) {
225		.param_decl => ctx.ast.fullParamDecl(data.node).id,
226		.fn_decl => ctx.ast.fullFnDecl(data.node).id,
227		.var_decl => ctx.ast.fullVarDecl(data.node).id,
228		else => unreachable, // not a valid symbol node
229		};
230
231		try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{
232		@tagName(data.kind),
233		id,
234		});
235
236		try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id});
237		}
238		},
239		not_mutated_var: struct {
240		node: NodeIndex,
241
242		fn build(data: @This(), ctx: *MsgContext) !void {
243		const id = switch (ctx.ast.nodeTag(data.node)) {
244		.var_decl => ctx.ast.fullVarDecl(data.node).id,
245		else => unreachable, // not a valid variable symbol node
246		};
247
248		try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{
249		id,
250		});
251		}
252		},
253		invalid_discard: struct {
254		node: NodeIndex,
255
256		fn build(data: @This(), ctx: *MsgContext) !void {
257		try ctx.msgToken(ctx.ast.nodeToken(data.node), "\"let\" is not required for a discard", .{});
258		}
259		},
260		discard_builtin: struct {
261		node: NodeIndex,
262
263		fn build(data: @This(), ctx: *MsgContext) !void {
264		try ctx.msgToken(ctx.ast.nodeToken(data.node), "discarding a built-in symbol is meaningless", .{});
265		}
266		},
267		unnecessary_discard: struct {
268		node: NodeIndex,
269
270		fn build(data: @This(), ctx: *MsgContext) !void {
271		try ctx.msgToken(ctx.ast.nodeToken(data.node), "useless discard", .{});
272		}
273		},
274		dangling_identifier: struct {
275		node: NodeIndex,
276
277		fn build(data: @This(), ctx: *MsgContext) !void {
278		try ctx.msgToken(ctx.ast.nodeToken(data.node), "dangling identifier does nothing", .{});
279		}
280		},
281		numeric_int_var_ambiguity: struct {
282		node: LazyNode,
283
284		fn build(data: @This(), ctx: *MsgContext) !void {
285		const node = try data.node.get(ctx);
286		try ctx.msgNode(node, "constant integer in a \"var\" has an unknown type, assuming \"float\"", .{});
287		try ctx.msgAlone("add \": float\" or \": int\" after the variable name to set the type", .{});
288		}
289		},
290		};
291
292		/// A variant of std.builtin.StackTrace that stores it's data within itself
293		/// instead of being a pointer. This allows capturing the stack trace and
294		/// saving it for later printing.
295		pub const StoredTrace = struct {
296		data: [32]usize,
297		index: usize,
298
299		pub const empty = .{
300		.data = .{0} ** 32,
301		.index = 0,
302		};
303
304		fn trace(stored: *StoredTrace) std.builtin.StackTrace {
305		return .{
306		.index = stored.index,
307		.instruction_addresses = &stored.data,
308		};
309		}
310
311		pub fn capture(begin: ?usize) StoredTrace {
312		var stored: StoredTrace = undefined;
313		var frame = stored.trace();
314		std.debug.captureStackTrace(begin orelse @returnAddress(), &frame);
315		stored.index = frame.index;
316		return stored;
317		}
318
319		pub fn from(stack_trace: ?*std.builtin.StackTrace) StoredTrace {
320		if (stack_trace) \|stack\| {
321		var data: [32]usize = undefined;
322		@memset(&data, 0);
323		const items = @min(stack.instruction_addresses.len, 32);
324		@memcpy(data[0..items], stack.instruction_addresses[0..items]);
325		return .{
326		.data = data,
327		.index = @min(items, stack.index),
328		};
329		} else {
330		return empty;
331		}
332		}
333		};
334
335		pub const DebugStoredTrace = if (builtin.mode == .Debug)
336		StoredTrace
337		else
338		struct {
339		pub fn capture(_: ?usize) @This() {
340		return .{};
341		}
342
343		pub fn from(_: ?*std.builtin.StackTrace) @This() {
344		return .{};
345		}
346		};
347
348		pub const UnusedKind = enum {
349		param,
350		constant,
351		variable,
352		function,
353		};
354
355		pub const String = struct {
356		start: u32,
357		len: u32,
358
359		pub fn slice(str: String, report: Report) []const u8 {
360		return report.string_bytes[str.start..][0..str.len];
361		}
362		};
363
364		const Prefix = enum {
365		@"error",
366		warn,
367		note,
368
369		fn color(prefix: Prefix) std.io.tty.Color {
370		return switch (prefix) {
371		.@"error" => .red,
372		.warn => .yellow,
373		.note => .blue,
374		};
375		}
376		};
377
378		pub fn deinit(report: Report, gpa: Allocator) void {
379		gpa.free(report.errors);
380		gpa.free(report.warnings);
381		gpa.free(report.string_bytes);
382		}
383
384		pub fn dumpToWriter(report: Report, comp: *Comp, writer: anytype, tty: std.io.tty.Config) !void {
385		// try writer.print("{d} errors, {d} warnings\n", .{ report.errors.len, report.warnings.len });
386		for (report.errors) \|item\| {
387		try report.formatItem(item, .@"error", comp, writer, tty);
388		}
389		for (report.warnings) \|item\| {
390		try report.formatItem(item, .warn, comp, writer, tty);
391		}
392		}
393
394		pub fn formatItem(
395		report: Report,
396		item: Item,
397		item_type: Item.Type,
398		comp: *Comp,
399		writer: anytype,
400		tty: std.io.tty.Config,
401		) !void {
402		var arena = std.heap.ArenaAllocator.init(comp.gpa);
403		defer arena.deinit();
404		var sfa = std.heap.stackFallback(16384, arena.allocator());
405		const alloc = sfa.get();
406
407		const file = try comp.getFile(item.url.slice(report));
408
409		const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast);
410		const ast_data = if (need_load_ast) try file.getAst(comp) else undefined;
411
412		var ctx: MsgContext = .{
413		.arena = alloc,
414		.comp = comp,
415		.file = file,
416		.ast = &ast_data,
417		.utf8_source = (try file.getSource(comp.gpa)).code,
418		.components = ArrayListUnmanaged(Component).initCapacity(alloc, 16) catch
419		unreachable, // stack-fallback allocator has enough space.
420		};
421
422		switch (item_type) {
423		inline else => \|tag\| switch (@field(item.data, @tagName(tag))) {
424		inline else => \|payload\| try payload.build(&ctx),
425		},
426		}
427
428		assert(ctx.components.items.len > 0);
429
430		for (ctx.components.items, 1..) \|component, i\| {
431		const prefix: Prefix = if (i == 1)
432		switch (item_type) {
433		inline else => \|tag\| @field(Prefix, @tagName(tag)),
434		}
435		else
436		.note;
437
438		const is_last = i == ctx.components.items.len;
439
440		if (i != 1) {
441		try tty.setColor(writer, .dim);
442		try writer.writeAll(" │ ");
443		try tty.setColor(writer, .reset);
444		}
445
446		try tty.setColor(writer, prefix.color());
447		try tty.setColor(writer, .bold);
448		try writer.writeAll(@tagName(prefix));
449		try tty.setColor(writer, .reset);
450		try tty.setColor(writer, .dim);
451		try writer.writeAll(": ");
452		try tty.setColor(writer, .reset);
453		try writer.writeAll(component.message_utf8);
454		try writer.writeAll("\n");
455
456		const utf8_source = ctx.utf8_source;
457
458		const at = component.location orelse {
459		if (is_last) {
460		try tty.setColor(writer, .dim);
461		try writer.print(" {s}", .{"┴"});
462		try writer.writeAll("\n");
463		try tty.setColor(writer, .reset);
464		}
465		continue;
466		};
467
468		const line_before = utf8_source[at.loc.line_start..at.byte];
469		const trimmed_before = std.mem.trimLeft(u8, line_before, " \t\r");
470		const line_after = utf8_source[at.byte + at.slice.len .. at.loc.line_end + 1];
471		const width = std.fmt.count("{d}", .{at.loc.line.oneBased()});
472
473		try tty.setColor(writer, .dim);
474		try writer.writeByteNTimes(' ', width);
475		try writer.print("{s}{s}:{d}:{d}\n", .{
476		if (i == 1) "┬ " else "├ ",
477		file.file_path,
478		at.loc.line.oneBased(),
479		at.loc.column.oneBased(),
480		});
481		try writer.print("{d}", .{at.loc.line.oneBased()});
482		try writer.print("{s} ", .{"│"});
483		try tty.setColor(writer, .reset);
484		try writer.writeAll(trimmed_before);
485		try tty.setColor(writer, prefix.color());
486		try tty.setColor(writer, .bold);
487		try writer.writeAll(at.slice);
488		try tty.setColor(writer, .reset);
489		try writer.writeAll(line_after);
490		try tty.setColor(writer, .dim);
491		try writer.writeAll(if (is_last) " ╰" else " ├");
492		try writer.writeBytesNTimes(
493		"─",
494		width +
495		(at.byte - at.loc.line_start) -
496		(line_before.len - trimmed_before.len),
497		);
498		try tty.setColor(writer, .reset);
499		try tty.setColor(writer, prefix.color());
500		try tty.setColor(writer, .bold);
501		try writer.writeBytesNTimes("━", at.slice.len);
502		try tty.setColor(writer, .reset);
503		try writer.writeAll("\n");
504		}
505
506		if (builtin.mode == .Debug and debug_show_report_trace) {
507		var trace = item.debug_stack_trace;
508		std.debug.dumpStackTrace(trace.trace());
509		}
510		}
511
512		pub fn dumpToStdErr(report: Report, comp: *Comp) !void {
513		const stderr = std.io.getStdErr();
514		const tty = std.io.tty.detectConfig(stderr);
515
516		if (false or
517		// Windows API uses it's own API calls, so a buffered writer
518		// cannot be used :/
519		(builtin.os.tag == .windows and tty == .windows_api) or
520		// Debug stack trace printer writes directly to standard err,
521		// so we need to be unbuffered for that.
522		(builtin.mode == .Debug and debug_show_report_trace))
523		{
524		try dumpToWriter(report, comp, stderr.writer(), tty);
525		} else {
526		var buf = std.io.bufferedWriter(stderr.writer());
527		const writer = buf.writer();
528		try dumpToWriter(report, comp, writer, tty);
529		try buf.flush();
530		}
531		}
532
533		pub const Builder = struct {
534		gpa: Allocator,
535		current_file: *Comp.File,
536		errors: ArrayListUnmanaged(Item) = .{},
537		warnings: ArrayListUnmanaged(Item) = .{},
538		string_bytes: ArrayListUnmanaged(u8) = .{},
539		current_file_str: ?String = null,
540
541		pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void {
542		try b.errors.append(b.gpa, .{
543		.data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) },
544		.url = try b.currentFileString(),
545		.debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{},
546		});
547		}
548
549		pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void {
550		try b.warnings.append(b.gpa, .{
551		.data = .{ .warn = @unionInit(Warn, @tagName(kind), data) },
552		.url = try b.currentFileString(),
553		.debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{},
554		});
555		}
556
557		pub fn addString(b: *Builder, str: []const u8) !String {
558		const top = b.string_bytes.items.len;
559		try b.string_bytes.appendSlice(b.gpa, str);
560		return .{ .start = @intCast(top), .len = @intCast(str.len) };
561		}
562
563		pub fn currentFileString(b: *Builder) !String {
564		return b.current_file_str orelse {
565		b.current_file_str = try b.addString(b.current_file.file_path);
566		return b.current_file_str.?;
567		};
568		}
569
570		pub fn deinit(b: *Builder) void {
571		b.errors.deinit(b.gpa);
572		b.warnings.deinit(b.gpa);
573		b.string_bytes.deinit(b.gpa);
574		}
575
576		pub fn finalize(b: *Builder) !Report {
577		defer b.* = undefined;
578		errdefer b.deinit();
579
580		const errors = try b.errors.toOwnedSlice(b.gpa);
581		errdefer b.gpa.free(errors);
582
583		const warnings = try b.warnings.toOwnedSlice(b.gpa);
584		errdefer b.gpa.free(warnings);
585
586		const string_bytes = try b.string_bytes.toOwnedSlice(b.gpa);
587		errdefer b.gpa.free(string_bytes);
588
589		return .{
590		.errors = errors,
591		.warnings = warnings,
592		.string_bytes = string_bytes,
593		};
594		}
595		};
596
597		pub const SliceLoc = struct {
598		loc: Location,
599		byte: ByteOffset,
600		slice: []const u8,
601		};
602
603		const Component = struct {
604		message_utf8: []const u8,
605		location: ?SliceLoc,
606		};
607
608		pub const Highlight = union(enum) {
609		// No source view
610		alone,
611		// Highlight one token
612		token: TokenIndex,
613		// Highlight one token without referring to the Ast
614		token_stored: Tokenizer.Token,
615		// Highlight two tokens
616		two_tokens: TokenIndex,
617		// Highlight a Node
618		node: NodeIndex,
619		// Highlight a codepoint
620		codepoint: ByteOffset,
621		// Highlight from the start of a codepoint to '\n'
622		codepoint_eol: ByteOffset,
623
624		pub fn loc(highlight: Highlight, ast: *const Ast) ?SliceLoc {
625		return switch (highlight) {
626		.alone => null,
627		inline .node,
628		.token,
629		.two_tokens,
630		=> \|payload, tag\| loc: {
631		const token_index: TokenIndex = switch (tag) {
632		.node => ast.nodeToken(payload),
633		.token, .two_tokens => payload,
634		else => comptime unreachable,
635		};
636
637		const location = ast.tokenLocation(0, token_index);
638		const token = ast.tokens.get(token_index);
639
640		break :loc .{
641		.loc = location,
642		.slice = token.slice(ast.utf8_source),
643		.byte = token.offset,
644		};
645		},
646
647		.token_stored => \|token\| loc: {
648		const location = Tokenizer.tokenLocation(ast.utf8_source, 0, token.offset);
649
650		break :loc .{
651		.loc = location,
652		.slice = token.slice(ast.utf8_source),
653		.byte = token.offset,
654		};
655		},
656		inline .codepoint,
657		.codepoint_eol,
658		=> \|b, tag\| loc: {
659		const len = switch (tag) {
660		.codepoint => std.unicode.utf8ByteSequenceLength(ast.utf8_source[b]) catch 1,
661		.codepoint_eol => if (std.mem.indexOfScalar(u8, ast.utf8_source[b..], '\n')) \|n\|
662		n
663		else
664		(ast.utf8_source.len - b),
665		else => comptime unreachable,
666		};
667		break :loc .{
668		.byte = b,
669		.slice = ast.utf8_source[b..][0..len],
670		.loc = .{
671		// TODO:
672		.column = .start,
673		.line = .start,
674		.line_start = 0,
675		.line_end = @intCast(ast.utf8_source.len),
676		},
677		};
678		},
679		};
680		}
681		};
682
683		/// This serves as a small API to make creating messages easier.
684		pub const MsgContext = struct {
685		file: *File,
686		comp: *Comp,
687		ast: *const Ast,
688		utf8_source: [:0]const u8,
689
690		/// This arena is is a stack fallback with 16kb, then
691		/// falling back to an arena allocator. This should
692		/// make the cost of arbitrarily creating `Component`
693		/// nodes extremely lightweight.
694		arena: Allocator,
695
696		/// First component is "error" or "warn"
697		/// Trailing components are "note"
698		components: ArrayListUnmanaged(Component),
699
700		pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void {
701		const text = try std.fmt.allocPrint(m.arena, fmt, args);
702		try m.components.append(m.arena, .{
703		.message_utf8 = text,
704		.location = highlight.loc(m.ast),
705		});
706		}
707
708		pub fn msgNode(m: *MsgContext, node: NodeIndex, comptime fmt: []const u8, args: anytype) !void {
709		return m.msg(.{ .node = node }, fmt, args);
710		}
711
712		pub fn msgToken(m: *MsgContext, token: TokenIndex, comptime fmt: []const u8, args: anytype) !void {
713		return m.msg(.{ .token = token }, fmt, args);
714		}
715
716		pub fn msgAlone(m: *MsgContext, comptime fmt: []const u8, args: anytype) !void {
717		return m.msg(.alone, fmt, args);
718		}
719		};
720
721		pub const LazyNode = struct {
722		decl: Comp.Decl.Index,
723		inst: Teg.Inst.Index,
724		operand: Operand,
725
726		pub fn get(lazy: LazyNode, ctx: *MsgContext) !NodeIndex {
727		const decl = ctx.comp.intern_pool.declPtrConst(lazy.decl);
728		const ns = ctx.comp.intern_pool.namespacePtr(decl.src_namespace);
729
730		// TODO: This might change if we have errors like
731		// - function arguments not matching at <callsite>
732		// - function declared <definition>
733		// And they are both in different files.
734		// To resolve this, `Component` needs to store which file it is in.
735		assert(ns.file_scope == ctx.file);
736
737		if (ns.parent.unwrap() != null) {
738		@panic("TODO: LazyNode.get when the decl is in a nested namespace");
739		}
740
741		const teg = &(ctx.file.teg orelse
742		// this will be required once incremental compilation hits
743		@panic("TODO: teg was unloaded"));
744
745		const decl_inst = decl.teg_decl_index.unwrap() orelse
746		unreachable; // file was not analyzed, yet teg loaded?
747
748		const teg_decl = teg.instData(.declaration, decl_inst.resolve(&ctx.comp.intern_pool));
749
750		const node_offset = teg.instNodeOffset(lazy.inst);
751
752		const base_node: NodeIndex = @intCast(teg_decl.src_node + node_offset);
753		if (lazy.operand == .main_node)
754		return base_node;
755
756		const data = ctx.ast.nodeData(base_node);
757
758		const node_tag = ctx.ast.nodeTag(base_node);
759
760		switch (teg.instTag(lazy.inst)) {
761		.as => switch (node_tag) {
762		.var_decl => {
763		switch (lazy.operand) {
764		.rhs => return data.var_decl.init,
765		else => {},
766		}
767		},
768		else => {},
769		},
770		.call => switch (node_tag) {
771		.call => switch (lazy.operand) {
772		.operand => return data.call.base,
773		else => \|op\| return ctx.ast.extraData(data.call.extra, Ast.Call)
774		.args.slice(ctx.ast)[op.getArgId()],
775		},
776		else => {},
777		},
778		.@"break" => switch (node_tag) {
779		.@"return" => return data.@"return".expr.unwrap().?,
780		.@"break" => return data.@"break".value.unwrap().?,
781		.block_implicit_return => {
782		const statements = ctx.ast.nodeStatements(base_node);
783		return statements[statements.len - 1];
784		},
785		else => return base_node,
786		},
787		.memory => switch (node_tag) {
788		.var_decl => switch (lazy.operand) {
789		.operand => return data.var_decl.init,
790		else => {},
791		},
792		else => {},
793		},
794		.store => {
795		assert(node_tag.isAssignBin());
796		switch (lazy.operand) {
797		.lhs => return data.bin.lhs,
798		.rhs => return data.bin.rhs,
799		else => unreachable,
800		}
801		},
802		else => {},
803		}
804
805		if (!std.debug.runtime_safety) unreachable;
806
807		std.debug.panic("TODO: operand lookup: .{s} + .{s} + {}", .{
808		@tagName(teg.instTag(lazy.inst)),
809		@tagName(ctx.ast.nodeTag(base_node)),
810		lazy.operand,
811		});
812		}
813		};
814
815		pub const Operand = enum(u32) {
816		/// Point to the main node of an instruction
817		main_node = 0,
818		/// Base of a call, field, or operand to a unary instruction
819		operand = 1,
820		/// Return type of a function declaration or lambda node
821		return_type = 2,
822		/// Left hand side of a binary instruction
823		lhs = 4,
824		/// Right hand side of a binary instruction
825		rhs = 5,
826
827		/// One of:
828		/// - An argument of a .call node
829		/// - A parameter of a .fn_decl node
830		pub fn argument(id: u32) Operand {
831		return @enumFromInt(id + 5);
832		}
833
834		fn getArgId(op: Operand) u32 {
835		return @intFromEnum(op) - 5;
836		}
837		};
838
839		const Report = @This();
840		const std = @import("std");
841		const ArrayListUnmanaged = std.ArrayListUnmanaged;
842		const Allocator = std.mem.Allocator;
843		const assert = std.debug.assert;
844
845		const builtin = @import("builtin");
846
847		const Tokenizer = @import("Tokenizer.zig");
848		const ByteOffset = Tokenizer.ByteOffset;
849
850		const Ast = @import("Ast.zig");
851		const TokenIndex = Ast.TokenIndex;
852		const NodeIndex = Ast.NodeIndex;
853		const Location = Ast.Location;
854
855		const Teg = @import("Teg.zig");
856
857		const Comp = @import("Comp.zig");
858		const File = Comp.File;
859
860		const InternPool = @import("InternPool.zig");
861		const Value = InternPool.Value;
862		const Type = InternPool.Type;
863
864		const util = @import("util.zig");

src/compiler/Sema.zig deleted-999

...	...	@@ -1,999 +0,0 @@
1		/// absolutely stealing zig on this one haha
2		/// This struct is created per declaration and chills
3
4		// inputs:
5		comp: *Comp,
6		gpa: Allocator,
7		arena: Allocator,
8		code: Teg,
9		ip: *InternPool,
10		decl: *Decl,
11		decl_index: Decl.Index,
12		namespace_index: Namespace.Index,
13
14		// temp data:
15		inst_map: InstMap = undefined,
16		/// scratch memory. can be used in any place you might temporarily allocate an arraylist
17		/// this is most notably used to store block contents before the block is finalized
18		scratch: std.ArrayListUnmanaged(u32) = .{},
19		params: []Type = undefined,
20
21		// outputs
22		report: Report.Builder = undefined,
23		air_instructions: std.MultiArrayList(Air.Inst) = .{},
24		air_extra: dod.ExtraStore(.air, .@"var") = .{},
25
26		temp_result_map: std.AutoArrayHashMapUnmanaged(Teg.Inst.Index, Type) = .{},
27
28		pub const Error = error{
29		/// Analysis has failed, see error list
30		AnalysisFail,
31		// This analysis is waiting on other dependencies to finish
32		// Interrupt,
33
34		// TODO: add handling of this
35		TooManyValues,
36		HasNulls,
37
38		OutOfMemory,
39		} \|\| Comp.File.GetTegError;
40
41		/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything.
42		/// This is fine because all instructions are going to be adacent in a single declaration.
43		const InstMap = struct {
44		items: []Air.Inst.Ref.Optional,
45		start: Teg.Inst.Index,
46
47		pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
48		const decl = code.declData(inst_start);
49		const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
50		@memset(insts, .none);
51		return .{
52		.items = insts,
53		.start = inst_start,
54		};
55		}
56
57		pub fn deinit(map: *InstMap, allocator: Allocator) void {
58		allocator.free(map.items);
59		map.* = undefined;
60		}
61
62		pub fn get(map: *const InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref {
63		const source_i = @intFromEnum(teg_code);
64		const start_i = @intFromEnum(map.start);
65		assert(source_i >= start_i and source_i <= (start_i + map.items.len));
66		return map.items[source_i - start_i].unwrap();
67		}
68
69		pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void {
70		if (assertWithMessage(map.get(from) == null)) {
71		std.debug.panic("%{d} is already mapped (to #{d})", .{
72		@intFromEnum(from),
73		@intFromEnum(map.get(from).?.decode().index),
74		});
75		}
76		const source_i = @intFromEnum(from);
77		const start_i = @intFromEnum(map.start);
78		map.items[source_i - start_i] = to.toOptional();
79		}
80		};
81
82		pub fn init(s: *Sema) !void {
83		const t = tracer.trace(@src());
84		defer t.end();
85
86		s.report = .{
87		.gpa = s.gpa,
88		.current_file = s.ip.namespacePtr(s.namespace_index).file_scope,
89		};
90
91		try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
92
93		s.inst_map = try InstMap.init(
94		s.arena,
95		&s.code,
96		s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip),
97		);
98		}
99
100		pub fn deinit(s: *Sema) void {
101		const t = tracer.trace(@src());
102		defer t.end();
103
104		const report = s.report.finalize() catch @panic("OOM");
105		report.dumpToStdErr(s.comp) catch @panic("Error");
106		report.deinit(s.gpa);
107
108		defer s.* = undefined;
109		s.inst_map.deinit(s.arena);
110		s.scratch.deinit(s.arena);
111		s.air_extra.deinit(s.gpa);
112		s.air_instructions.deinit(s.gpa);
113		s.temp_result_map.deinit(s.arena);
114		}
115
116		pub fn finalize(s: *Sema) !void {
117		const air: Air = s.tempAir();
118
119		if (s.comp.options.dump_ir) {
120		std.debug.print("=" ** 20, .{});
121		std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)});
122		air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
123		}
124
125		try s.comp.codeGenDecl(s.decl, &air);
126		}
127
128		// --- analyze functions ---
129
130		/// This is the entrypoint to semantic analysis: It all starts at a declaration.
131		pub fn analyzeDeclaration(s: *Sema) Error!Value {
132		const t = tracer.trace(@src());
133		defer t.end();
134
135		// TODO: turn this entire function into a nonrecursive state machine, so
136		// that analysis can pause and continue at any point. for example, when a
137		// dependency on another declaration is required, but that declaration has
138		// not been resolved yet. or, we are waiting on a file read to happen
139
140		const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start);
141		assert(decl.kind == .function);
142
143		s.params = try s.gpa.alloc(Type, fn_decl.params.len);
144		defer s.gpa.free(s.params);
145
146		for (fn_decl.params.slice(&s.code), 0..) \|param, i\| {
147		s.params[i] = try s.resolveType(
148		s.inst_map.start.op(Report.Operand.argument(@intCast(i))),
149		param,
150		);
151		}
152
153		const return_type = try s.resolveType(
154		s.inst_map.start.op(.return_type),
155		fn_decl.return_type,
156		);
157
158		const root_block = fn_decl.body;
159		switch (root_block.decode()) {
160		.value => \|value\| {
161		const coerced = try s.coerceValue(
162		s.inst_map.start.op(.operand),
163		value,
164		return_type,
165		);
166		const block = try s.addBlock();
167		_ = try s.addInstRef(.@"break", .{
168		.block_ref = block.block_index,
169		.value = coerced.toAir(),
170		});
171		},
172		.index => \|index\| {
173		try s.temp_result_map.putNoClobber(s.arena, index, return_type);
174		_ = try s.analyzeBlock(index);
175		},
176		}
177
178		const func_prototype = (try s.ip.get(s.gpa, .{
179		.type_function = .{
180		.param_types = s.params,
181		.return_type = return_type,
182		},
183		})).toTypeUnsafe(s.ip);
184		const value = try s.ip.get(s.gpa, .{
185		.func = .{
186		.ty = func_prototype,
187		.owner_decl = s.decl_index,
188		.teg_body_inst = s.decl.teg_decl_index.unwrap().?,
189		},
190		});
191		s.decl.val = value;
192		return value;
193		}
194
195		pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref {
196		const t = tracer.trace(@src());
197		defer t.end();
198
199		const scratch_top = s.scratch.items.len;
200		defer s.scratch.shrinkRetainingCapacity(scratch_top);
201
202		const block = try s.addBlock();
203		try s.inst_map.add(in, block.block_index.toRef());
204
205		var last: Air.Inst.Ref = undefined;
206		const instructions = s.code.blockInstructions(in);
207		assert(instructions.len != 0);
208		for (instructions) \|inst\| {
209		last = try s.analyzeCode(inst);
210		}
211
212		if (scratch_top + 1 == s.scratch.items.len) {
213		const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1];
214		if (single_instruction == .@"break") {
215		const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1]
216		.decode(.@"break", single_instruction);
217		if (data.block_ref == block.block_index) {
218		// This is a block that resolved to just a single value ref with
219		// no side effects. This means we can just lower this to the ref
220		// itself, and give codegen an easier time.
221
222		// Delete the block and the single instruction
223		// Break does not use any extra data, so this call removes both items
224		s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index));
225
226		return data.value;
227		}
228		}
229		}
230
231		return (try block.finish(s, scratch_top)).toRef();
232		}
233
234		fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
235		const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
236		defer t.end();
237
238		const out = try s.analyzeCodeInner(in);
239
240		// Block refs are added automatically
241		if (s.code.instTag(in) == .block) {
242		assert(s.inst_map.get(in) != null);
243		} else {
244		try s.inst_map.add(in, out);
245		}
246
247		return out;
248		}
249
250		fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref {
251		switch (s.code.instTag(in)) {
252		.string => {
253		const string: Teg.SlicedString = s.code.instPayload(.string, in);
254		return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) });
255		},
256		.number => {
257		const number: dod.Float = s.code.instPayload(.number, in);
258		return s.makeValueAir(.{ .comptime_float = number.decode() });
259		},
260		.js => {
261		const operand_ref = s.code.instData(.js, in).operand;
262		const operand = try s.resolveRefComptimeType(in.op(.operand), operand_ref, .string);
263		return s.addInstRef(.js_global_ref, operand.asString(s.ip));
264		},
265		.field => {
266		const field: Teg.Inst.Bin = s.code.instPayload(.field, in);
267		const base = try s.resolveRefRuntime(field.lhs);
268
269		const rhs = try s.resolveRefComptimeType(in.op(.rhs), field.rhs, .string);
270		const string = rhs.asString(s.ip);
271
272		const base_type = s.typeOfRef(base);
273		switch (base_type) {
274		.@"opaque" => {
275		return s.addInstRef(.field, .{
276		.base = base,
277		.field = try string.tryToNullTerminatedString(s.ip, s.gpa),
278		});
279		},
280		else => \|tag\| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}),
281		}
282		},
283		.load_decl => {
284		const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in);
285		const decl_name = try s.ip.getOrPutString(
286		s.gpa,
287		request.string.slice(s.code.string_bytes),
288		.no_embedded_nulls,
289		);
290		const decl_index = s.lookupDecl(decl_name) orelse {
291		std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)});
292		return error.AnalysisFail;
293		};
294		const resolved = try s.resolveDecl(decl_index);
295		return resolved.toAir();
296		},
297		.param => {
298		const param: Teg.Inst.DataNode = s.code.instData(.param, in);
299		return s.addInstRef(.arg, .{
300		.type = s.params[param.data],
301		.arg_number = param.data,
302		});
303		},
304		inline .add,
305		.sub,
306		.mul,
307		.div,
308		.lt,
309		.gt,
310		.eq,
311		=> \|tag\| {
312		const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
313
314		const lhs = try s.resolveRefRuntime(bin.lhs);
315		const rhs = try s.resolveRefRuntime(bin.rhs);
316
317		if (lhs.kind == .value and rhs.kind == .value) {
318		return (try s.comptimeBin(
319		@field(BinaryOp, @tagName(tag)),
320		lhs.decode().value,
321		rhs.decode().value,
322		));
323		}
324
325		const left_type = s.typeOfRef(lhs);
326		_ = left_type; // autofix
327		const right_type = s.typeOfRef(rhs);
328		_ = right_type; // autofix
329
330		// TODO: validate type for operation
331
332		return s.addInstRef(tag.airTag(), .{
333		.lhs = lhs,
334		.rhs = rhs,
335		});
336		},
337		inline .negate, .bool_not => \|tag\| {
338		const un: Teg.Inst.RefNode = s.code.instData(tag, in);
339
340		const operand = try s.resolveRefRuntime(un.operand);
341
342		const operand_type = s.typeOfRef(operand);
343		_ = operand_type; // autofix
344
345		// TODO: validate type for operation
346
347		return s.addInstRef(tag.airTag(), .{
348		.operand = operand,
349		});
350		},
351		.@"break" => {
352		const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in);
353		const operand = try s.resolveRefRuntime(brk.operand);
354		const block_ref = s.inst_map.get(brk.block).?.decode().index; // block ref is resolved by teg
355
356		const coerced = if (s.temp_result_map.get(brk.block)) \|ty\|
357		try s.coerceRef(in.op(.operand), operand, ty)
358		else
359		operand;
360
361		return s.addInstRef(.@"break", .{
362		.block_ref = block_ref,
363		.value = coerced,
364		});
365		},
366		.call => {
367		const call: Teg.Inst.Call = s.code.instPayload(.call, in);
368		const base = try s.resolveRefRuntime(call.base);
369		const base_type = s.typeOfRef(base);
370
371		const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len);
372		defer s.arena.free(scratch);
373		if (base_type != .@"opaque") {
374		const proto = base_type.toFunctionPrototype(s.ip) orelse {
375		return s.fail(.call_non_function, .{
376		.call = s.nodeLoc(in, .operand),
377		.ty = base_type,
378		});
379		};
380
381		if (proto.param_types.len != call.args.len) {
382		// TODO: also attempt to coerce some of the params to reveal more errors.
383		return s.fail(.call_mismatch_args, .{
384		.call = s.nodeLoc(in, .operand),
385		.actual = @intCast(call.args.len),
386		.expected = @intCast(proto.param_types.len),
387		});
388		}
389
390		// validate function args
391		for (proto.param_types, call.args.slice(&s.code), 0..) \|param, arg, i\| {
392		const ref = try s.resolveRefRuntime(arg);
393		const coerced = try s.coerceRef(
394		in.op(Report.Operand.argument(@intCast(i))),
395		ref,
396		param,
397		);
398		scratch[i] = coerced;
399		}
400		} else {
401		// just copy the args in without checking
402		for (call.args.slice(&s.code), 0..) \|arg, i\| {
403		const ref = try s.resolveRefRuntime(arg);
404		scratch[i] = ref;
405		}
406		}
407
408		const start = s.air_extra.list.items.len;
409		try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch));
410
411		return s.addInstRef(.call, .{
412		.base = base,
413		.call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{
414		.args = .{
415		.start = @intCast(start),
416		.len = @intCast(scratch.len),
417		},
418		}),
419		});
420		},
421		.ensure_void => {
422		const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false);
423		const t = s.typeOfRef(operand);
424		_ = t;
425		// if (!t.isEnsureVoid(s.ip)) {
426		// std.debug.print("oh no, we are discarding {} on accident\n", .{
427		// t.fmt(s.ip),
428		// });
429		// }
430		return InternPool.Value.void.toAir();
431		},
432		.discard => {
433		const operand = try s.resolveRefRuntimeExtra(s.code.instData(.discard, in).operand, false);
434		const t = s.typeOfRef(operand);
435		if (t == .void) {
436		std.debug.print("oh no, we have a useless _ = void;\n", .{});
437		}
438		return InternPool.Value.void.toAir();
439		},
440		.@"if" => {
441		const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code);
442
443		const cond = try s.resolveRefRuntime(data.cond);
444
445		// try s.validateType(s.typeOfRef(cond), .bool_type);
446
447		// If known at comptime, only analyze the taken path
448		if (cond.kind == .value) {
449		@panic("TODO: comptime IF");
450		}
451
452		// Emit a runtime branch
453		const yes_block = try s.analyzeRefToBlock(data.yes);
454		const no_block = try s.analyzeRefToBlock(data.no);
455
456		return try s.addInstRef(.@"if", .{
457		.cond = cond,
458		.data = try s.air_extra.append(s.gpa, Air.If.Data, .{
459		.yes = yes_block,
460		.no = no_block,
461		.union_ty = s.unionType(s.typeOfRef(yes_block), s.typeOfRef(no_block)),
462		}),
463		});
464		},
465		.block => {
466		const block = try s.analyzeBlock(in);
467		switch (block.decode()) {
468		.index => \|t\| if (s.tempAir().instTag(t) == .block) {
469		try s.scratch.append(s.gpa, @intFromEnum(t));
470		},
471		else => {},
472		}
473		return block;
474		},
475		.memory => {
476		const data = s.code.instData(.memory, in);
477
478		var initial_value = try s.resolveRefRuntime(data.operand);
479
480		switch (initial_value.decode()) {
481		.value => \|v\| switch (v.toKey(s.ip)) {
482		.comptime_float => \|val\| {
483		try s.report.warn(.numeric_int_var_ambiguity, .{
484		.node = s.nodeLoc(in, .operand),
485		});
486		initial_value = (try s.makeValue(.{ .float = val })).toAir();
487		},
488		else => {},
489		},
490		else => {},
491		}
492
493		return try s.addInstRef(.memory, .{
494		.init = initial_value,
495		.ty = s.typeOfRef(initial_value),
496		});
497		},
498		.store => {
499		const data: Teg.Inst.Store = s.code.instPayload(.store, in);
500		assert(s.code.instTag(data.memory) == .memory); // corrupt store instruction
501
502		const location = s.mapTeg(data.memory.toRef()).decode().index;
503		s.markInst(location);
504		const operand = try s.resolveRefRuntime(data.operand);
505		const operand_coerced = try s.coerceRef(
506		in.op(.rhs),
507		operand,
508		s.typeOfRef(location.toRef()),
509		);
510		return try s.addInstRef(.store, .{
511		.memory = location,
512		.operand = operand_coerced,
513		});
514		},
515		.as => {
516		const as: Teg.Inst.As = s.code.instPayload(.as, in);
517		const coerce_to = try s.resolveRefComptimeType(in.op(.lhs), as.type, .type);
518		const operand = try s.resolveRefRuntime(as.operand);
519		return try s.coerceRef(in.op(.rhs), operand, coerce_to.toType(s.ip).?);
520		},
521		else => \|tag\| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}),
522		}
523		}
524
525		fn markInst(s: *Sema, inst: Air.Inst.Index) void {
526		const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
527		flags.uses = flags.uses.increment();
528		}
529
530		fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) Error!Air.Inst.Ref {
531		const t = tracer.trace(@src());
532		defer t.end();
533
534		const scratch_top = s.scratch.items.len;
535		defer s.scratch.shrinkRetainingCapacity(scratch_top);
536
537		const visited = try s.resolveRefRuntime(in);
538
539		if (scratch_top == s.scratch.items.len and visited.kind == .value) {
540		return visited;
541		}
542
543		if (visited.kind == .index and
544		s.air_instructions.items(.tag)[@intFromEnum(visited.decode().index)] == .block)
545		{
546		return visited;
547		}
548
549		const block = try s.addBlock();
550		_ = try s.addInstRef(.@"break", .{
551		.block_ref = block.block_index,
552		.value = visited,
553		});
554		return (try block.finish(s, scratch_top)).toRef();
555		}
556
557		fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) Error!Air.Inst.Ref {
558		const index = ref.toIndex() orelse {
559		return ref.toValue().?.toAir();
560		};
561
562		if (s.inst_map.get(index)) \|air\| {
563		if (mark) {
564		switch (air.decode()) {
565		.index => \|i\| s.markInst(i),
566		else => {},
567		}
568		}
569		return air;
570		}
571
572		return s.analyzeCode(index);
573		}
574
575		fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) Error!Air.Inst.Ref {
576		return resolveRefRuntimeExtra(s, ref, true);
577		}
578
579		fn resolveRefComptimeAny(s: *Sema, ref: Teg.Inst.Ref) !Value {
580		const index = switch (ref.decode()) {
581		.value => \|value\| return value,
582		.index => \|index\| index,
583		};
584
585		if (s.inst_map.get(index)) \|air\| {
586		switch (air.decode()) {
587		.value => \|value\| return value,
588		else => {},
589		}
590		}
591
592		switch (s.code.instTag(index)) {
593		.string => {
594		const string: Teg.SlicedString = s.code.instPayload(.string, index);
595		return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
596		},
597
598		else => \|tag\| std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
599		}
600		}
601
602		fn resolveRefComptimeType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref, ty: Type) !Value {
603		const any = try s.resolveRefComptimeAny(ref);
604		return try s.coerceValue(src, any, ty);
605		}
606
607		fn resolveType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref) !Type {
608		return (try s.resolveRefComptimeType(src, ref, .type)).toTypeUnsafe(s.ip);
609		}
610
611		/// Coerce `value` runtime ref to `coerce_type`, and emit an error at `teg_ref`'s location
612		fn coerceRef(s: *Sema, src: InstOp, value: Air.Inst.Ref, to: Type) !Air.Inst.Ref {
613		const from = s.typeOfRef(value);
614		if (from == to) {
615		return value;
616		}
617		const index = switch (value.decode()) {
618		.value => \|v\| return (try s.coerceValue(src, v, to)).toAir(),
619		.index => \|index\| index,
620		};
621		switch (to) {
622		.@"opaque" => {
623		//
624		},
625		.bool => {
626		//
627		},
628		.int32 => switch (from) {
629		.int32 => unreachable, // same type handled
630		.numeric_literal => unreachable, // must should comptime known
631		else => {},
632		},
633		.float => switch (from) {
634		.float => unreachable, // same type handled
635		.int32 => return index.toRef(),
636		else => {},
637		},
638		.string => {
639		//
640		},
641		.noreturn => {
642		//
643		},
644		.type => {
645		//
646		},
647		.numeric_literal => {
648		@panic("TODO: error message for not known at compile time");
649		},
650		.void => {
651		//
652		},
653		else => {},
654		}
655		return s.coersionError(src, from, to);
656		}
657
658		fn coerceValue(s: *Sema, src: InstOp, value: Value, to: Type) !Value {
659		const from = value.typeOf(s.ip);
660		if (from == to) {
661		return value;
662		}
663		switch (to) {
664		.@"opaque" => {
665		//
666		},
667		.bool => {
668		//
669		},
670		.int32 => switch (from) {
671		.int32 => unreachable, // same type handled
672		.numeric_literal, .float => switch (value.toKey(s.ip)) {
673		.comptime_float, .float => \|val\| {
674		if (!math.isFinite(val) or
675		val < math.minInt(i32) or
676		val > math.maxInt(i32) or
677		(@rem(val, 1) != 0))
678		{
679		return s.fail(.comptime_float_to_int, .{
680		.val = val,
681		.src = s.lazyNode(src),
682		});
683		}
684
685		return try s.makeValue(.{
686		.int32 = @intFromFloat(val),
687		});
688		},
689		else => unreachable, // not a numeric literal
690		},
691		else => return s.coersionError(src, from, to),
692		},
693		.float => switch (from) {
694		.float => unreachable, // same type handled
695		.numeric_literal => if (value.toFloat(s.ip)) \|val\|
696		return try s.makeValue(.{
697		.float = val,
698		}),
699		else => {},
700		},
701		.string => {
702		//
703		},
704		.noreturn => {
705		//
706		},
707		.type => {
708		if (value == .void) return .void;
709		},
710		.numeric_literal => {},
711		.void => {
712		if (value == .void) return .void;
713		},
714		else => {},
715		}
716		return s.coersionError(src, from, to);
717		}
718
719		fn resolveDecl(s: *Sema, decl_index: Decl.Index) Error!Value {
720		const decl = s.ip.declPtr(decl_index);
721		switch (decl.analysis) {
722		.unreferenced => {
723		decl.analysis = .in_progress;
724
725		// TODO: make this short-circuit the current analysis, move this analyze
726		// job into a "deferred" list, and then make this thread analyze something else
727
728		var arena = std.heap.ArenaAllocator.init(s.gpa);
729		defer arena.deinit();
730
731		var todo: Report = undefined;
732		var sema: Sema = .{
733		.comp = s.comp,
734		.gpa = s.gpa,
735		.ip = s.ip,
736		.arena = arena.allocator(),
737		.code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.comp, &todo),
738		.decl = decl,
739		.decl_index = decl_index,
740		.namespace_index = decl.src_namespace,
741		};
742
743		try sema.init();
744		defer sema.deinit();
745		const value = try sema.analyzeDeclaration();
746		try sema.finalize();
747
748		decl.analysis = .complete;
749
750		return value;
751		},
752		else => {
753		return decl.val;
754		},
755		}
756		}
757
758		fn unionType(s: *Sema, a: Type, b: Type) Type {
759		if (a == b) {
760		return a;
761		}
762
763		if (a == .noreturn)
764		return b;
765
766		if (b == .noreturn)
767		return a;
768
769		std.debug.panic(
770		"TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}",
771		.{
772		a.dump(s.ip),
773		b.dump(s.ip),
774		},
775		);
776		// return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } });
777		}
778
779		const BinaryOp = enum {
780		add,
781		sub,
782		mul,
783		div,
784		lt,
785		gt,
786		eq,
787		};
788
789		fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref {
790		switch (op) {
791		.add => {
792		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
793		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
794		return (try s.ip.get(s.gpa, .{
795		.float = l + r,
796		})).toAir();
797		},
798		inline .sub, .mul, .div => \|tag\| {
799		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
800		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
801
802		const result = switch (tag) {
803		.sub => l - r,
804		.mul => l * r,
805		.div => l / r,
806		else => comptime unreachable,
807		};
808
809		return (try s.ip.get(s.gpa, .{
810		.float = result,
811		})).toAir();
812		},
813		inline .lt,
814		.gt,
815		=> \|tag\| {
816		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
817		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
818		const result = switch (tag) {
819		.lt => l < r,
820		.gt => l > r,
821		else => comptime unreachable,
822		};
823		return @as(
824		InternPool.Value,
825		if (result) .bool_true else .bool_false,
826		).toAir();
827		},
828		.eq,
829		=> {
830		const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
831		const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
832		const result = l == r;
833		return @as(
834		InternPool.Value,
835		if (result) .bool_true else .bool_false,
836		).toAir();
837		},
838		}
839		}
840
841		/// --- helper ---
842		const IncompleteBlock = struct {
843		block_index: Air.Inst.Index,
844
845		pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index {
846		s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top));
847		return block.block_index;
848		}
849		};
850
851		fn addBlock(s: *Sema) !IncompleteBlock {
852		try s.air_instructions.append(s.gpa, .{
853		.tag = .block,
854		.data = undefined,
855		.flags = .{},
856		});
857		return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) };
858		}
859
860		fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
861		try s.air_instructions.append(s.gpa, .{
862		.tag = tag,
863		.data = Air.EncodedData.encode(tag, data),
864		.flags = .{},
865		});
866		const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1);
867		try s.scratch.append(s.arena, @intFromEnum(index));
868		return index.toRef();
869		}
870
871		fn setInstData(
872		s: *Sema,
873		comptime tag: Air.Inst.Tag,
874		index: Air.Inst.Index,
875		data: tag.Data(),
876		) void {
877		const i = @intFromEnum(index);
878		assert(s.air_instructions.items(.tag)[i] == tag);
879		s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data);
880		}
881
882		fn makeValue(s: *Sema, key: InternPool.Key) !Value {
883		return try s.ip.get(s.gpa, key);
884		}
885
886		fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref {
887		return (try s.makeValue(key)).toAir();
888		}
889
890		fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
891		if (inst.toValue()) \|interned\| {
892		return interned.toAir();
893		}
894		const index = inst.toIndex().?;
895		return s.inst_map.get(index) orelse {
896		if (std.debug.runtime_safety) {
897		std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)});
898		}
899		unreachable;
900		};
901		}
902
903		fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList {
904		const items = s.scratch.items[scratch_top..];
905		const extra_top = s.air_extra.list.items.len;
906		try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]);
907		return .{
908		.start = @intCast(extra_top),
909		.len = @intCast(items.len),
910		};
911		}
912
913		fn tempAir(s: *const Sema) Air {
914		return .{
915		.insts = s.air_instructions.slice(),
916		.extra = s.air_extra.constant(),
917		};
918		}
919
920		fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Type {
921		return s.tempAir().typeOf(ref, s.ip);
922		}
923
924		fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex {
925		var nsi = s.namespace_index;
926		while (true) {
927		const ns = s.ip.namespacePtr(nsi);
928		if (ns.lookupDirect(s.comp, name)) \|decl_id\| {
929		return decl_id;
930		}
931		nsi = ns.parent.unwrap() orelse
932		unreachable; // transform ensures all identifiers get resolved
933		}
934		}
935
936		fn nodeLoc(s: *const Sema, in: Teg.Inst.Index, op: Report.Operand) Report.LazyNode {
937		return s.lazyNode(in.op(op));
938		}
939
940		fn addWarn(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void {
941		try s.report.warn(kind, data);
942		}
943
944		fn addError(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void {
945		try s.report.err(kind, data);
946		}
947
948		fn fail(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) Error {
949		try s.report.err(kind, data);
950		return Error.AnalysisFail;
951		}
952
953		fn coersionError(s: *Sema, src: InstOp, from: Type, to: Type) Error {
954		return s.fail(.coersion, .{
955		.from = from,
956		.to = to,
957		.src = s.lazyNode(src),
958		});
959		}
960
961		fn lazyNode(s: *const Sema, src: InstOp) Report.LazyNode {
962		return src.lazyNode(s.decl_index);
963		}
964
965		const Sema = @This();
966
967		const std = @import("std");
968		const mem = std.mem;
969		const math = std.math;
970		const Allocator = std.mem.Allocator;
971		const assert = std.debug.assert;
972		const log = std.log.scoped(.sema);
973
974		inline fn assertWithMessage(cond: bool) bool {
975		if (std.debug.runtime_safety) {
976		return !cond;
977		} else if (!cond) {
978		unreachable; // assertion failure
979		}
980		}
981
982		const Comp = @import("Comp.zig");
983		const File = Comp.File;
984		const Decl = Comp.Decl;
985		const Namespace = Comp.Namespace;
986
987		const Teg = @import("Teg.zig");
988		const InstOp = Teg.InstOp;
989
990		const Air = @import("Air.zig");
991		const Report = @import("Report.zig");
992		const Ast = @import("Ast.zig");
993		const dod = @import("array.zig");
994		const tracer = @import("tracer");
995
996		const InternPool = @import("InternPool.zig");
997		const Value = InternPool.Value;
998		const Type = InternPool.Type;
999		const NullTerminatedString = InternPool.NullTerminatedString;

src/compiler/Teg.Dump.zig deleted-314

...	...	@@ -1,314 +0,0 @@
1		//! This implements a debug formatter for Teg
2		teg: Teg,
3
4		const State = struct {
5		arena: std.mem.Allocator,
6		teg: *const Teg,
7		indent: usize = 0,
8		tty: std.io.tty.Config,
9		visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{},
10
11		fn newline(state: State, writer: anytype) !void {
12		try writer.writeByte('\n');
13		if (state.indent > 0)
14		try writer.writeByteNTimes(' ', state.indent * 2);
15		}
16
17		fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void {
18		const teg = s.teg;
19		const tty = s.tty;
20
21		try writer.print("%{} = ", .{@intFromEnum(inst)});
22
23		const tag = s.teg.instTag(inst);
24
25		try tty.setColor(writer, .bright_black);
26		const node_offset = teg.instNodeOffset(inst);
27		try writer.print("{c}{d} ", .{
28		@as(u8, if (node_offset >= 0) '+' else '-'),
29		@abs(node_offset),
30		});
31		try tty.setColor(writer, .reset);
32
33		try tty.setColor(writer, .cyan);
34		try writer.print("{s}", .{@tagName(tag)});
35		try tty.setColor(writer, .reset);
36
37		if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
38		try tty.setColor(writer, .red);
39		try writer.writeAll(" [error: recursive tree]");
40		try tty.setColor(writer, .reset);
41		return;
42		}
43
44		switch (tag) {
45		.module => {
46		try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}');
47		},
48		.declaration => {
49		const decl_info = teg.declData(inst);
50		try writer.print(" ({s}) ", .{@tagName(decl_info.kind)});
51
52		const name = decl_info.name.slice(teg.string_bytes);
53		if (name.len == 0) {
54		try tty.setColor(writer, .bold);
55		try writer.writeAll("<top level code>");
56		} else {
57		try tty.setColor(writer, .magenta);
58		try writer.writeAll(name);
59		}
60		try tty.setColor(writer, .reset);
61
62		try writer.writeAll(" {");
63		s.indent += 1;
64		try s.newline(writer);
65
66		switch (decl_info.kind) {
67		.function => {
68		const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1";
69		try writer.writeAll("params: ");
70		try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']');
71		try s.newline(writer);
72		try writer.writeAll("return_type: ");
73		try s.dumpRef(writer, fn_data.return_type);
74		try s.newline(writer);
75		try writer.writeAll("body: ");
76		try s.dumpRef(writer, fn_data.body);
77		},
78		}
79		s.indent -= 1;
80		try s.newline(writer);
81		try writer.writeAll("}");
82		},
83		.param => {
84		const data = teg.instData(.param, inst);
85		try writer.writeAll("(");
86		try tty.setColor(writer, .yellow);
87		try writer.print("{d}", .{data.data});
88		try tty.setColor(writer, .reset);
89		try writer.writeAll(")");
90		},
91		.block => {
92		const block = teg.instData(.block, inst);
93		const index_list = block.extra_index.get(teg);
94		try s.dumpInstList(writer, index_list.slice(teg), '[', ']');
95		},
96		.number => {
97		const number = teg.instData(.number, inst);
98		const float = number.extra_index.get(teg).decode();
99		try tty.setColor(writer, .yellow);
100		try writer.print(" {d}", .{float});
101		try tty.setColor(writer, .reset);
102		},
103		inline .field,
104		.add,
105		.sub,
106		.mul,
107		.div,
108		.rem,
109		.pow,
110		.@"and",
111		.@"or",
112		.bit_and,
113		.bit_or,
114		.bit_xor,
115		.shl,
116		.shr,
117		.eq,
118		.ne,
119		.lt,
120		.le,
121		.gt,
122		.ge,
123		=> \|comptime_tag\| {
124		const data = teg.instData(comptime_tag, inst).extra_index.get(teg);
125		try writer.writeAll("(");
126		try s.dumpRefMarker(writer, data.lhs);
127		try writer.writeAll(", ");
128		try s.dumpRefMarker(writer, data.rhs);
129		try writer.writeAll(")");
130		},
131		.as => {
132		const data = teg.instData(.as, inst).extra_index.get(teg);
133		try writer.writeAll("(");
134		try s.dumpRefMarker(writer, data.type);
135		try writer.writeAll(", ");
136		try s.dumpRefMarker(writer, data.operand);
137		try writer.writeAll(")");
138		},
139		.@"break" => {
140		const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg);
141		try writer.writeAll("(");
142		try s.dumpRefMarker(writer, data.block.toRef());
143		try writer.writeAll(", ");
144		try s.dumpRefMarker(writer, data.operand);
145		try writer.writeAll(")");
146		},
147		.store => {
148		const data: Teg.Inst.Store = teg.instData(.store, inst).extra_index.get(teg);
149		try writer.writeAll("(");
150		try s.dumpRefMarker(writer, data.memory.toRef());
151		try writer.writeAll(", ");
152		try s.dumpRefMarker(writer, data.operand);
153		try writer.writeAll(")");
154		},
155		.string => {
156		const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg);
157		try tty.setColor(writer, .green);
158		try writer.writeAll(" \"");
159		try writer.writeAll(data.slice(teg.string_bytes));
160		try writer.writeAll("\"");
161		try tty.setColor(writer, .reset);
162		},
163		inline .memory,
164		.discard,
165		.ensure_void,
166		.ensure_bool,
167		.negate,
168		.bool_not,
169		.bit_not,
170		.js,
171		.import,
172		.fetch,
173		.compile_error,
174		=> \|comptime_tag\| {
175		const data = teg.instData(comptime_tag, inst);
176		try writer.writeAll("(");
177		try s.dumpRefMarker(writer, data.operand);
178		try writer.writeAll(")");
179		},
180		.call => {
181		const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg);
182		try writer.writeAll("(");
183		try s.dumpRefMarker(writer, call.base);
184		try writer.writeAll(", ");
185		try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']');
186		try writer.writeAll(")");
187		},
188		.load_decl => {
189		const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst);
190
191		try tty.setColor(writer, .red);
192		try writer.print(" {s}", .{call.string.slice(teg.string_bytes)});
193		try tty.setColor(writer, .reset);
194		},
195		.@"if" => {
196		const data: Teg.Inst.If = teg.instPayload(.@"if", inst);
197		try writer.writeAll("(");
198		try s.dumpRefMarker(writer, data.cond);
199		try writer.writeAll(") {");
200		s.indent += 1;
201		try s.newline(writer);
202		try writer.writeAll("yes: ");
203		try s.dumpRef(writer, data.yes);
204		try s.newline(writer);
205		try writer.writeAll("no: ");
206		try s.dumpRef(writer, data.no);
207		s.indent -= 1;
208		try s.newline(writer);
209		try writer.writeAll("}");
210		},
211		}
212		}
213
214		fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
215		if (ref.toIndex()) \|index\| {
216		try s.dumpInst(writer, index);
217		} else {
218		try s.tty.setColor(writer, .blue);
219		try writer.print("%{s}", .{@tagName(ref)});
220		try s.tty.setColor(writer, .reset);
221		}
222		}
223
224		fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
225		if (ref.toIndex()) \|index\| {
226		try s.tty.setColor(writer, .blue);
227		try writer.print("%{d}", .{@intFromEnum(index)});
228		try s.tty.setColor(writer, .reset);
229		} else {
230		try s.tty.setColor(writer, .blue);
231		try writer.print("%{s}", .{@tagName(ref)});
232		try s.tty.setColor(writer, .reset);
233		}
234		}
235
236		fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void {
237		try writer.print(" {c}", .{l});
238		if (list.len > 0) {
239		state.indent += 1;
240
241		for (list) \|inst\| {
242		try state.newline(writer);
243		try state.dumpInst(writer, inst);
244		}
245
246		state.indent -= 1;
247		try state.newline(writer);
248		}
249		try writer.print("{c}", .{r});
250		}
251
252		fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
253		try writer.print(" {c}", .{l});
254		if (list.len > 0) {
255		state.indent += 1;
256
257		for (list) \|ref\| {
258		try state.newline(writer);
259		try state.dumpRef(writer, ref);
260		}
261
262		state.indent -= 1;
263
264		try state.newline(writer);
265		}
266		try writer.print("{c}", .{r});
267		}
268
269		fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
270		try writer.print(" {c}", .{l});
271		if (list.len > 0) {
272		state.indent += 1;
273
274		for (list) \|ref\| {
275		try state.newline(writer);
276		try state.dumpRefMarker(writer, ref);
277		}
278
279		state.indent -= 1;
280
281		try state.newline(writer);
282		}
283		try writer.print("{c}", .{r});
284		}
285		};
286
287		pub fn format(
288		dump: @This(),
289		comptime unused_fmt: []const u8,
290		_: std.fmt.FormatOptions,
291		writer: anytype,
292		) !void {
293		comptime assert(unused_fmt.len == 0);
294
295		var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
296		defer arena.deinit();
297
298		const teg = dump.teg;
299		var state = State{
300		.teg = &teg,
301		.tty = detectTTYFromWriter(writer),
302		.arena = arena.allocator(),
303		};
304		try state.dumpInst(writer, .main_module_inst);
305		try writer.writeAll("\n");
306		assert(state.indent == 0);
307		}
308
309		const std = @import("std");
310		const assert = std.debug.assert;
311
312		const Teg = @import("Teg.zig");
313
314		const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Teg.zig deleted-486

...	...	@@ -1,486 +0,0 @@
1		//! TODO: rename this to something else. Teg is not a graph
2
3		insts: Inst.List.Slice,
4		string_bytes: []const u8,
5		extra: ExtraStore = .{},
6		report: Report,
7
8		pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
9		pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
10		pub const ExtraIndex = ExtraStore.Index;
11		pub const Dump = @import("Teg.Dump.zig");
12
13		pub const Inst = struct {
14		pub const List = std.MultiArrayList(Inst);
15
16		tag: Tag,
17		data: EncodedData,
18
19		// todo: fully document every thing
20		pub const Tag = enum {
21		module,
22		declaration,
23
24		block,
25		/// Creates a memory cell in the current scope.
26		memory,
27		/// Performs a type coersion, such as in variable declarations.
28		/// Emits a sema error if the type conversion fails.
29		/// `let x: i32 = 1`
30		as,
31		load_decl,
32		/// Load a parameter
33		param,
34		store,
35		discard,
36		ensure_void,
37		ensure_bool,
38		/// lowered from `break` or `return`
39		/// ends a block with a resulting value
40		@"break",
41		number,
42		string,
43
44		@"if",
45
46		add,
47		sub,
48		mul,
49		div,
50		rem,
51		pow,
52		@"and",
53		@"or",
54		bit_and,
55		bit_or,
56		bit_xor,
57		shl,
58		shr,
59		eq,
60		ne,
61		lt,
62		le,
63		gt,
64		ge,
65
66		negate,
67		bool_not,
68		bit_not,
69		field,
70		call,
71
72		/// Implements the `@js` builtin
73		js,
74		/// Implements the `@import` builtin
75		import,
76		/// Implements the `@fetch` builtin
77		fetch,
78		/// Implements the `@compileError` builtin
79		compile_error,
80
81		pub fn Data(comptime tag: Tag) type {
82		return switch (tag) {
83		.module => PlNode(IndexList),
84		.declaration => PlNode(Decl),
85		.block => PlNode(IndexList),
86		.as => PlNode(As),
87		.@"break" => PlNode(Break),
88		.number => PlNode(Float),
89		.string => PlNode(SlicedString),
90		.call => PlNode(Call),
91		.@"if" => PlNode(If),
92		.load_decl => StrNode,
93		.param => DataNode,
94		.store => PlNode(Store),
95
96		// Single instruction operand
97		.memory,
98		.discard,
99		.ensure_void,
100		.ensure_bool,
101		.negate,
102		.bool_not,
103		.bit_not,
104		.js,
105		.import,
106		.fetch,
107		.compile_error,
108		=> RefNode,
109
110		// Two instruction operands
111		.field,
112		.add,
113		.sub,
114		.mul,
115		.div,
116		.rem,
117		.pow,
118		.@"and",
119		.@"or",
120		.bit_and,
121		.bit_or,
122		.bit_xor,
123		.shl,
124		.shr,
125		.eq,
126		.ne,
127		.lt,
128		.le,
129		.gt,
130		.ge,
131		=> PlNode(Bin),
132		};
133		}
134
135		pub fn airTag(comptime tag: Tag) Air.Inst.Tag {
136		return @field(Air.Inst.Tag, @tagName(tag));
137		}
138		};
139
140		/// TODO: make enum? idk
141		const NodeOffset = i32;
142
143		pub fn PlNode(comptime Payload: type) type {
144		return struct {
145		pub const T = Payload;
146
147		src_node: NodeOffset,
148		extra_index: ExtraIndex(Payload),
149		};
150		}
151
152		pub const RefNode = struct {
153		src_node: NodeOffset,
154		operand: Inst.Ref,
155		};
156
157		pub const StrNode = struct {
158		src_node: NodeOffset,
159		string: NullTerminatedString,
160		};
161
162		pub const DataNode = struct {
163		src_node: NodeOffset,
164		data: u32,
165		};
166
167		/// The position of a Teg instruction within the `Teg` instructions array.
168		pub const Index = enum(u32) {
169		/// Teg is structured so that the outermost "main" struct of any file
170		/// is always at index 0.
171		main_module_inst = 0,
172		_,
173
174		pub const static_len = @intFromEnum(Ref.empty_string) + 1;
175
176		pub fn toRef(i: Index) Inst.Ref {
177		return @enumFromInt(static_len + @intFromEnum(i));
178		}
179
180		pub fn toOptional(i: Index) OptionalIndex {
181		return @enumFromInt(@intFromEnum(i));
182		}
183
184		pub fn op(i: Index, operand: Report.Operand) InstOp {
185		return .{ .index = i, .op = operand };
186		}
187		};
188
189		pub const OptionalIndex = enum(u32) {
190		/// Teg is structured so that the outermost "main" module of any file
191		/// is always at index 0.
192		main_module_inst = 0,
193		ref_start_index = Index.static_len,
194		none = std.math.maxInt(u32),
195		_,
196
197		pub fn unwrap(oi: OptionalIndex) ?Index {
198		return if (oi == .none) null else @enumFromInt(@intFromEnum(oi));
199		}
200		};
201
202		/// A reference to instruction, or to an InternPool index, or neither.
203		///
204		/// If the integer tag value is < InternPool.static_len, then it
205		/// corresponds to an InternPool index. Otherwise, this refers to a ZIR
206		/// instruction.
207		///
208		/// The tag type is specified so that it is safe to bitcast between `[]u32`
209		/// and `[]Ref`.
210		pub const Ref = enum(u32) {
211		opaque_type,
212		bool_type,
213		int32_type,
214		float_type,
215		string_type,
216		noreturn_type,
217		type_type,
218		numeric_literal_type,
219
220		void,
221		bool_true,
222		bool_false,
223		int32_zero,
224		float_zero,
225		nan,
226		empty_string,
227		standard_library,
228
229		/// This Ref does not correspond to any instruction or constant
230		/// value and may instead be used as a sentinel to indicate null.
231		none = std.math.maxInt(u32),
232
233		_,
234
235		pub const Decoded = union(enum) {
236		value: InternPool.Value,
237		index: Teg.Inst.Index,
238		};
239
240		pub fn decode(ref: Ref) Decoded {
241		const ref_int = @intFromEnum(ref);
242		return if (ref_int >= Index.static_len)
243		.{ .index = @enumFromInt(ref_int - Index.static_len) }
244		else
245		.{ .value = @enumFromInt(ref_int) };
246		}
247
248		pub fn toIndex(ref: Ref) ?Index {
249		assert(ref != .none);
250		const ref_int = @intFromEnum(ref);
251		if (ref_int >= Index.static_len) {
252		return @enumFromInt(ref_int - Index.static_len);
253		} else {
254		return null;
255		}
256		}
257
258		pub fn toIndexAllowNone(inst: Ref) ?Index {
259		if (inst == .none) return null;
260		return toIndex(inst);
261		}
262
263		pub fn toValue(inst: Ref) ?InternPool.Value {
264		const ref_int = @intFromEnum(inst);
265
266		// todo: is this > or >=
267		if (ref_int > Index.static_len)
268		return null;
269
270		return @enumFromInt(ref_int);
271		}
272		};
273
274		pub const Bin = struct {
275		lhs: Inst.Ref,
276		rhs: Inst.Ref,
277		};
278
279		pub const Store = struct {
280		memory: Inst.Index,
281		operand: Inst.Ref,
282		};
283
284		pub const As = struct {
285		type: Inst.Ref,
286		operand: Inst.Ref,
287		};
288
289		pub const Break = struct {
290		block: Inst.Index,
291		operand: Inst.Ref,
292		};
293
294		pub const Call = struct {
295		base: Inst.Ref,
296		args: RefList,
297		};
298
299		pub const If = struct {
300		cond: Inst.Ref,
301		yes: Inst.Ref,
302		no: Inst.Ref,
303		};
304
305		/// This will contain trailing data depending on the .kind
306		pub const Decl = struct {
307		name: NullTerminatedString,
308		hash: Cache.BinDigest,
309		kind: Kind,
310		instruction_count: u32,
311
312		pub const Kind = enum {
313		function,
314		// future: struct, let, var, etc
315		};
316		};
317
318		/// Trailng data for function declarations
319		pub const DeclFn = struct {
320		params: RefList,
321		return_type: Ref,
322		body: Ref,
323		};
324		};
325
326		pub const InstOp = struct {
327		index: Inst.Index,
328		op: Report.Operand,
329
330		pub fn lazyNode(inst: InstOp, decl: Comp.Decl.Index) LazyNode {
331		return .{
332		.inst = inst.index,
333		.operand = inst.op,
334		.decl = decl,
335		};
336		}
337		};
338
339		pub const NullTerminatedString = enum(u32) {
340		empty = 0,
341		_,
342
343		pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 {
344		if (str == .empty) return "";
345		const sub = string_bytes[@intFromEnum(str)..];
346		const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable;
347
348		return sub[0..i :0];
349		}
350		};
351
352		pub const SlicedString = struct {
353		start: u32,
354		len: u32,
355
356		pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 {
357		return string_bytes[str.start..][0..str.len];
358		}
359		};
360
361		pub const RefList = struct {
362		start: u32,
363		len: u32,
364
365		pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref {
366		return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
367		}
368		};
369
370		pub const IndexList = struct {
371		start: u32,
372		len: u32,
373
374		pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index {
375		return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
376		}
377		};
378
379		pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
380		gpa.free(teg.string_bytes);
381		teg.extra.deinit(gpa);
382		teg.insts.deinit(gpa);
383		teg.report.deinit(gpa);
384		teg.* = undefined;
385		}
386
387		pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag {
388		return teg.insts.items(.tag)[@intFromEnum(index)];
389		}
390
391		pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData {
392		return teg.insts.items(.data)[@intFromEnum(index)];
393		}
394
395		pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
396		return teg.instEncodedData(index).decode(tag, teg.instTag(index));
397		}
398
399		pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
400		return teg.instData(tag, index).extra_index.get(teg);
401		}
402
403		pub fn instNodeOffset(teg: *const Teg, index: Inst.Index) i32 {
404		switch (teg.instTag(index)) {
405		inline else => \|tag\| {
406		return teg.instData(tag, index).src_node;
407		},
408		}
409		}
410
411		pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index {
412		return teg.instData(.block, index).extra_index.get(teg).slice(teg);
413		}
414
415		pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } {
416		const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg);
417		const decl_fn = trail.read(&teg.extra, Inst.DeclFn);
418		return .{ decl, decl_fn };
419		}
420
421		pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl {
422		return teg.instData(.declaration, index).extra_index.get(teg);
423		}
424
425		pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index {
426		const module = teg.instData(.module, module_index);
427		const module_data: IndexList = module.extra_index.get(teg);
428		const slice = module_data.slice(teg);
429		if (std.debug.runtime_safety) {
430		for (slice) \|item\| {
431		if (teg.instTag(item) != .declaration) {
432		std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))});
433		unreachable;
434		}
435		}
436		}
437		return slice;
438		}
439
440		pub fn rootFunction(teg: *const Teg) Inst.Index {
441		const decls = teg.moduleDeclarations(.main_module_inst);
442		const root_fn = decls[decls.len - 1];
443		assert(teg.declName(root_fn).len == 0);
444		return root_fn;
445		}
446
447		pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
448		const decl = teg.instData(.declaration, decl_index);
449		const decl_data: Inst.Decl = decl.extra_index.get(teg);
450		return decl_data.name.slice(teg.string_bytes);
451		}
452
453		pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
454		try writer.print("{}", .{Dump{ .teg = teg }});
455		}
456
457		const Teg = @This();
458
459		const std = @import("std");
460		const mem = std.mem;
461		const assert = std.debug.assert;
462
463		const Cache = @import("Cache.zig");
464
465		const Ast = @import("Ast.zig");
466		const Unused = Ast.Unused;
467		const NodeIndex = Ast.NodeIndex;
468		const TokenIndex = Ast.TokenIndex;
469		const OptionalNodeIndex = Ast.OptionalNodeIndex;
470		const OptionalTokenIndex = Ast.OptionalTokenIndex;
471		const OptionalExtraIndex = Ast.OptionalExtraIndex;
472		const optionalTokenIndex = OptionalTokenIndex.init;
473		const optionalNodeIndex = OptionalNodeIndex.init;
474		const optionalExtraIndex = OptionalExtraIndex.init;
475
476		const dod = @import("array.zig");
477		const Float = dod.Float;
478
479		const InternPool = @import("InternPool.zig");
480
481		const Air = @import("Air.zig");
482
483		const Comp = @import("Comp.zig");
484
485		const Report = @import("Report.zig");
486		const LazyNode = Report.LazyNode;

src/compiler/Tokenizer.zig deleted-995

...	...	@@ -1,995 +0,0 @@
1		//! The tokenizer is extremely simple structure that, given a view of source
2		//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit
3		//! offset to where that token is located.
4		//!
5		//! Retriving payloads like string/numeric values, identifier names, etc is
6		//! done separately, either in Parse or Transform, when the respective data
7		//! is actually needed. Tokenization does guarante that such data is valid.
8		code: [:0]const u8,
9		pos: ByteOffset = 0,
10
11		/// Tokenize source code into a list of tokens.
12		/// When an error occurs, the position of the error is returned in `error_pos`.
13		/// Caller owns returned slice.
14		pub fn tokenize(
15		source_code: [:0]const u8,
16		gpa: mem.Allocator,
17		error_pos: *ByteOffset,
18		) TokenizeError!Token.List.Slice {
19		const trace = T.trace(@src());
20		defer trace.end();
21
22		var tokens = Token.List{};
23		errdefer tokens.deinit(gpa);
24
25		// pre-allocate some space based on a very rough guess
26		try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8));
27
28		var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 };
29		errdefer error_pos.* = tokenizer.pos;
30
31		tokenizer.skipWhitespace();
32		while (try tokenizer.next()) \|token\| {
33		try tokens.append(gpa, token);
34		tokenizer.skipWhitespace();
35		}
36
37		try tokens.append(gpa, .{
38		.tag = .eof,
39		.offset = @intCast(source_code.len),
40		});
41
42		return tokens.toOwnedSlice();
43		}
44
45		pub const ByteOffset = u32;
46
47		pub const Token = struct {
48		tag: Tag,
49		offset: ByteOffset,
50
51		pub const List = std.MultiArrayList(Token);
52
53		pub const Tag = enum {
54		id,
55		builtin_id,
56		number_int,
57		number_float,
58		string,
59		multiline_string,
60		doc_comment,
61		module_doc_comment,
62
63		// sort-lines: start
64		kw_assert,
65		kw_break,
66		kw_comptime,
67		kw_defer,
68		kw_else,
69		kw_enum,
70		kw_fn,
71		kw_infer,
72		kw_for,
73		kw_while,
74		kw_if,
75		kw_impl,
76		kw_json,
77		kw_let,
78		kw_match,
79		kw_module,
80		kw_pub,
81		kw_pure,
82		kw_return,
83		kw_struct,
84		kw_trait,
85		kw_var,
86		// sort-lines: end
87
88		// sort-lines: start
89		@"and",
90		and_and,
91		and_and_equal,
92		and_equal,
93		bang,
94		bang_equal,
95		caret,
96		caret_equal,
97		colon,
98		comma,
99		dot,
100		end_brace,
101		end_bracket,
102		end_paren,
103		equal,
104		equal_equal,
105		equal_lt,
106		gt,
107		gt_equal,
108		gt_gt,
109		gt_gt_equal,
110		gt_gt_gt,
111		gt_gt_gt_equal,
112		lt,
113		lt_equal,
114		lt_lt,
115		lt_lt_equal,
116		minus,
117		minus_equal,
118		minus_lt,
119		percent,
120		percent_equal,
121		pipe,
122		pipe_equal,
123		pipe_pipe,
124		pipe_pipe_equal,
125		plus,
126		plus_equal,
127		semi,
128		slash,
129		slash_equal,
130		star,
131		star_equal,
132		star_star,
133		star_star_equal,
134		start_brace,
135		start_bracket,
136		start_paren,
137		tilde,
138		tilde_equal,
139		// sort-lines: end
140
141		/// Represents a tokenization error. Instead of reporting an error during
142		/// tokenization, parsing will begin, and an error will be reported when
143		/// this token is hit. That way, the error can tell you what it expected.
144		invalid,
145		/// End of file
146		/// Used so that parser does not need to do out of bounds checks
147		eof,
148
149		/// Most tokens have statically known lexeme, such as "struct" or "+="
150		pub fn lexeme(tag: Tag) ?[]const u8 {
151		return switch (tag) {
152		.invalid => unreachable,
153
154		// do not have a static lexeme
155		.eof,
156		.id,
157		.builtin_id,
158		.number_int,
159		.number_float,
160		.string,
161		.multiline_string,
162		.doc_comment,
163		.module_doc_comment,
164		=> null,
165
166		.@"and" => "&",
167		.and_and => "&&",
168		.bang => "!",
169		.bang_equal => "!=",
170		.equal_lt => "=<",
171		.caret => "^",
172		.end_brace => "}",
173		.end_bracket => "]",
174		.end_paren => ")",
175		.colon => ":",
176		.comma => ",",
177		.dot => ".",
178		.equal => "=",
179		.equal_equal => "==",
180		.gt => ">",
181		.gt_equal => ">=",
182		.gt_gt => ">>",
183		.gt_gt_gt => ">>>",
184		.lt => "<",
185		.lt_equal => "<=",
186		.lt_lt => "<<",
187		.minus => "-",
188		.start_brace => "{",
189		.start_bracket => "[",
190		.start_paren => "(",
191		.pipe => "\|",
192		.pipe_pipe => "\|\|",
193		.percent => "%",
194		.plus => "+",
195		.minus_lt => "->",
196		.semi => ";",
197		.slash => "/",
198		.star => "*",
199		.star_star => "**",
200		.tilde => "~",
201		.slash_equal => "/=",
202		.percent_equal => "%=",
203		.tilde_equal => "~=",
204		.caret_equal => "^=",
205		.and_equal => "&=",
206		.pipe_equal => "\|=",
207		.lt_lt_equal => "<<=",
208		.gt_gt_equal => ">>=",
209		.gt_gt_gt_equal => ">>>=",
210		.plus_equal => "+=",
211		.minus_equal => "-=",
212		.star_equal => "*=",
213		.star_star_equal => "**=",
214		.pipe_pipe_equal => "\|\|=",
215		.and_and_equal => "&&=",
216
217		inline else => \|keyword\| comptime len: {
218		if (!mem.startsWith(u8, @tagName(keyword), "kw_")) {
219		@compileError("Token tag is not a keyword: " ++ @tagName(keyword));
220		}
221		break :len @tagName(keyword)[3..];
222		},
223		};
224		}
225
226		pub fn description(tag: Tag) []const u8 {
227		return switch (tag) {
228		.invalid => unreachable,
229		.eof => "end of file",
230		.id => "an identifier",
231		.builtin_id => "a builtin identifier",
232		.number_int => "a numeric literal",
233		.number_float => "a numeric literal",
234		.string => "a string literal",
235		.multiline_string => "a multiline string literal",
236		.doc_comment => "a documentation comment",
237		.module_doc_comment => "a module documentation comment",
238		inline else => \|t\| comptime "\"" ++ t.lexeme().? ++ "\"",
239		};
240		}
241
242		/// Most tokens have a static length, such as "struct" or "+="
243		pub fn staticLen(tag: Tag) ?u32 {
244		return if (tag.lexeme()) \|str\| @intCast(str.len) else null;
245		}
246		};
247
248		/// Instead of storing the length of a token, we can recompute this
249		/// information on demand, as it is not often needed.
250		///
251		/// If you are computing the length of something that is knowable
252		/// without the source code, use token.tag.staticLen() instead.
253		pub fn len(token: Token, utf8_source: [:0]const u8) u32 {
254		return token.tag.staticLen() orelse {
255		var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
256		assert(
257		(tokenizer.next() catch
258		unreachable // data corruption
259		orelse unreachable // data corruption
260		).tag == token.tag,
261		);
262		return tokenizer.pos - token.offset;
263		};
264		}
265
266		pub fn slice(token: Token, utf8_source: [:0]const u8) []const u8 {
267		return token.tag.lexeme() orelse
268		utf8_source[token.offset..][0..token.len(utf8_source)];
269		}
270
271		const keywords = std.ComptimeStringMap(Tag, keywords: {
272		const fields = std.enums.values(Tag);
273		const KV = struct { []const u8, Tag };
274		var kws: [fields.len]KV = undefined;
275		var i = 0;
276		for (fields) \|field\| {
277		if (mem.startsWith(u8, @tagName(field), "kw_")) {
278		kws[i] = .{ @tagName(field)[3..], field };
279		i += 1;
280		}
281		}
282		break :keywords kws[0..i];
283		});
284		};
285
286		/// When any of these error occurs, the tokenizer's position is set to
287		/// the start of the offending token.
288		pub const NextTokenError = error{
289		/// Generic token syntax error
290		InvalidSyntax,
291		/// A trailing '_' was seen in a numeric literal.
292		TrailingNumericSeparator,
293		/// Newline or EOF encountered while parsing a string literal.
294		UnterminatedString,
295		/// A string escape sequence was invalid.
296		InvalidEscapeSequence,
297		/// The codepoint represented by a unicode escape sequence
298		/// is not a valid unicode codepoint.
299		InvalidEscapeCodepoint,
300		/// There are unresolved git conflicts in the source code.
301		GitConflict,
302		/// The source code is not valid UTF-8.
303		InvalidUtf8,
304		};
305
306		pub const TokenizeError = mem.Allocator.Error \|\| NextTokenError;
307
308		/// Next assumes the position is already at the start of a token.
309		/// You must call `skipWhitespace` before calling this.
310		pub fn next(t: *Tokenizer) !?Token {
311		while (true) {
312		const trace = T.trace(@src());
313		defer trace.end();
314
315		switch (t.byte()) {
316		' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
317
318		'a'...'z', 'A'...'Z', '_', '$' => {
319		const start = t.pos;
320		t.pos += 1;
321		return try t.tokenizeIdentifier(start);
322		},
323
324		'@' => switch (t.nextByte()) {
325		'"' => {
326		const start = t.pos;
327		t.pos += 1;
328		try t.consumeString();
329		return .{ .tag = .id, .offset = start };
330		},
331		else => {
332		const start = t.pos;
333		t.pos += 1;
334		return try t.tokenizeIdentifier(start);
335		},
336		},
337
338		'0'...'9' => return try t.tokenizeNumber(),
339
340		'.' => switch (t.nextByte()) {
341		'0'...'9' => return try t.tokenizeNumber(),
342		else => return t.simpleToken(.dot),
343		},
344
345		'/' => switch (t.nextByte()) {
346		'=' => return t.simpleToken(.slash_equal),
347		'/' => {
348		if ((t.byteAt(2) orelse 0 == '/') and
349		(t.byteAt(3) orelse 0 != '/'))
350		{
351		const start = t.pos;
352		t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len);
353		return .{ .tag = .doc_comment, .offset = start };
354		}
355
356		if (t.byteAt(2) orelse 0 == '!') {
357		const start = t.pos;
358		t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
359		return .{ .tag = .module_doc_comment, .offset = start };
360		}
361
362		// skip the comment
363		t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
364		t.skipWhitespace();
365		continue;
366		},
367		else => return t.simpleToken(.slash),
368		},
369
370		'\\' => switch (t.nextByte()) {
371		'\\' => {
372		const start = t.pos;
373		t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
374		return .{ .tag = .multiline_string, .offset = start };
375		},
376		else => return t.maybeUTF8Error(error.InvalidSyntax),
377		},
378
379		'"' => return try t.tokenizeString(),
380
381		'&' => switch (t.nextByte()) {
382		'&' => return t.simpleToken(.and_and),
383		'=' => return t.simpleToken(.and_equal),
384		else => return t.simpleToken(.@"and"),
385		},
386		'\|' => switch (t.nextByte()) {
387		'\|' => return t.simpleToken(.pipe_pipe),
388		'=' => return t.simpleToken(.pipe_equal),
389		else => return t.simpleToken(.pipe),
390		},
391		'^' => switch (t.nextByte()) {
392		'^' => return t.simpleToken(.caret),
393		'=' => return t.simpleToken(.caret_equal),
394		else => return t.simpleToken(.caret),
395		},
396		'!' => switch (t.nextByte()) {
397		'=' => return t.simpleToken(.equal_equal),
398		else => return t.simpleToken(.bang_equal),
399		},
400		'=' => switch (t.nextByte()) {
401		'=' => return t.simpleToken(.equal_equal),
402		else => return t.simpleToken(.equal),
403		},
404		'>' => switch (t.nextByte()) {
405		'=' => return t.simpleToken(.gt_equal),
406		'>' => switch (t.byteAt(2) orelse 0) {
407		'>' => return t.simpleToken(.gt_gt_gt),
408		else => return t.simpleToken(.gt_gt),
409		},
410		else => return t.simpleToken(.gt),
411		},
412		'<' => switch (t.nextByte()) {
413		'=' => return t.simpleToken(.lt_equal),
414		'<' => switch (t.byteAt(2) orelse 0) {
415		'=' => return t.simpleToken(.lt_lt_equal),
416		else => {
417		// Git merge conflicts are <<<<<<<
418		const start = t.pos;
419		t.pos += 2;
420		var remain: u3 = 4;
421		while (t.byte() == '<') {
422		t.pos += 1;
423		remain -= 1;
424		if (remain == 0) {
425		return error.GitConflict;
426		}
427		}
428		t.pos = start;
429		return t.simpleToken(.lt_lt);
430		},
431		},
432		else => return t.simpleToken(.lt),
433		},
434		'+' => switch (t.nextByte()) {
435		'=' => return t.simpleToken(.plus_equal),
436		else => return t.simpleToken(.plus),
437		},
438		'-' => switch (t.nextByte()) {
439		'=' => return t.simpleToken(.minus_equal),
440		'>' => return t.simpleToken(.minus_lt),
441		else => return t.simpleToken(.minus),
442		},
443		'*' => switch (t.nextByte()) {
444		'*' => return t.simpleToken(.star_star),
445		'=' => return t.simpleToken(.star_equal),
446		else => return t.simpleToken(.star),
447		},
448		'%' => switch (t.nextByte()) {
449		'=' => return t.simpleToken(.percent_equal),
450		else => return t.simpleToken(.percent),
451		},
452		'~' => switch (t.nextByte()) {
453		'=' => return t.simpleToken(.tilde_equal),
454		else => return t.simpleToken(.tilde),
455		},
456
457		';' => return t.simpleToken(.semi),
458		':' => return t.simpleToken(.colon),
459		'(' => return t.simpleToken(.start_paren),
460		')' => return t.simpleToken(.end_paren),
461		'[' => return t.simpleToken(.start_bracket),
462		']' => return t.simpleToken(.end_bracket),
463		'{' => return t.simpleToken(.start_brace),
464		'}' => return t.simpleToken(.end_brace),
465		',' => return t.simpleToken(.comma),
466
467		0x80...0xFF => {
468		// need to decode utf-8
469		const byte_len, const code_point = try t.codePoint();
470
471		if (unicode_props.isXidStart(code_point)) {
472		const start = t.pos;
473		t.pos += byte_len;
474		return try t.tokenizeIdentifier(start);
475		}
476
477		return error.InvalidSyntax;
478		},
479
480		else => return error.InvalidSyntax,
481
482		0 => {
483		if (t.pos == t.code.len) return null;
484		return error.InvalidSyntax;
485		},
486		}
487		}
488		}
489
490		/// Source code is null terminated, so accessing this is safe
491		fn byte(t: Tokenizer) u8 {
492		return t.code[t.pos];
493		}
494
495		/// Source code is null terminated.
496		fn nextByte(t: Tokenizer) u8 {
497		assert(t.byte() < 0x80);
498		return t.code[t.pos + 1];
499		}
500
501		/// Remember that using this only makes sense if you know for certain that all
502		/// previous bytes are ASCII, otherwise you may hit UB
503		fn byteAt(t: Tokenizer, comptime offset: u32) ?u8 {
504		if (offset == 1) @compileError("use nextByte()");
505
506		// assert that the previous bytes are ASCII
507		// current byte is allowed to be a continuation byte
508		for (t.code[t.pos .. t.pos + offset]) \|b\| {
509		assert(b < 0x80);
510		}
511		return t.code[t.pos + offset];
512		}
513
514		fn byteAtRt(t: Tokenizer, offset: u32) ?u8 {
515		// assert that the previous bytes are ASCII
516		// current byte is allowed to be a continuation byte
517		for (t.code[t.pos .. t.pos + offset]) \|b\| {
518		assert(b < 0x80);
519		}
520		return t.code[t.pos + offset];
521		}
522
523		fn codePoint(t: Tokenizer) !struct { u3, u21 } {
524		const b = t.byte(); // must check the first byte manually
525		assert(b >= 0x80); // must be a continuation byte
526
527		const byte_len = unicode.utf8ByteSequenceLength(b) catch
528		return error.InvalidUtf8;
529		assert(byte_len > 1);
530
531		const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch
532		return error.InvalidUtf8;
533
534		return .{ byte_len, code_point };
535		}
536
537		fn isAtIdentifierStart(t: *Tokenizer) !bool {
538		switch (t.byte()) {
539		'a'...'z', 'A'...'Z', '_', '$' => return true,
540		else => return false,
541		0x80...0xFF => {
542		_, const code_point = try t.codePoint();
543		return unicode_props.isXidStart(code_point);
544		},
545		}
546		}
547
548		fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
549		const trace = T.trace(@src());
550		defer trace.end();
551
552		while (true) {
553		switch (t.byte()) {
554		'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
555		t.pos += 1;
556		},
557		0x80...0xFF => {
558		const byte_len, const code_point = try t.codePoint();
559
560		t.pos += byte_len;
561
562		if (!unicode_props.isXidContinue(code_point)) {
563		break;
564		}
565		},
566		else => break,
567		}
568		}
569
570		return .{
571		.tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse
572		if (t.code[start_byte] == '@')
573		.builtin_id
574		else
575		.id,
576		.offset = start_byte,
577		};
578		}
579
580		/// Numbers have the following rules:
581		/// - May start with a `.`, but cannot end with `.`
582		/// - May contain leading zeros, but transform pass will warn.
583		/// - May contain a `_` separator, cannot have two in a row.
584		/// - May contain `e` or `E` for exponents.
585		/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers.
586		/// The token is tagged as `number_int` or `number_float` depending
587		/// on if it contained a `.` This is to help later re-parses.
588		//
589		// TODO: is it a zig compiler bug that `?` is required here?
590		fn tokenizeNumber(t: *Tokenizer) !?Token {
591		const trace = T.trace(@src());
592		defer trace.end();
593
594		// Caller ensures the following invariant:
595		switch (t.byte()) {
596		'.' => switch (t.nextByte()) {
597		'0'...'9' => {},
598		else => unreachable,
599		},
600		'0'...'9' => {},
601		else => unreachable,
602		}
603
604		// Parse Hex, Binary, Octal numbers
605		if (t.byte() == '0') {
606		switch (t.nextByte()) {
607		'x', 'X' => {
608		t.pos += 2;
609		try t.consumeIntegerRadix(.hex);
610		},
611		'b', 'B' => {
612		t.pos += 2;
613		try t.consumeIntegerRadix(.binary);
614		},
615		'o', 'O' => {
616		t.pos += 2;
617		try t.consumeIntegerRadix(.octal);
618		},
619		else => {},
620		}
621		}
622
623		const start = t.pos;
624		var tag: Token.Tag = .number_int;
625
626		if (t.byte() == '.') {
627		t.pos += 1;
628		tag = .number_float;
629		try t.consumeInteger();
630		} else {
631		try t.consumeInteger();
632
633		// decimal point notation
634		switch (t.byte()) {
635		'.' => switch (t.nextByte()) {
636		'0'...'9' => {
637		t.pos += 1;
638		tag = .number_float;
639		try t.consumeInteger();
640		},
641		else => {
642		// we have not advanced the tokenizer past the `.`
643		assert(tag == .number_int);
644		return .{ .tag = tag, .offset = start };
645		},
646		},
647		else => {},
648		}
649		}
650
651		switch (t.byte()) {
652		'e', 'E' => {
653		tag = .number_float;
654		t.pos += 1;
655
656		switch (t.byte()) {
657		'+', '-' => {
658		t.pos += 1;
659		switch (t.byte()) {
660		'0'...'9' => {},
661		else => return t.maybeUTF8Error(error.InvalidSyntax),
662		}
663		},
664		'0'...'9' => {},
665		else => return t.maybeUTF8Error(error.InvalidSyntax),
666		}
667
668		try t.consumeInteger();
669
670		// decimal point notation
671		switch (t.byte()) {
672		'.' => switch (t.nextByte()) {
673		'0'...'9' => {
674		t.pos += 1;
675		tag = .number_float;
676		try t.consumeInteger();
677		},
678		else => {},
679		},
680		else => {},
681		}
682		},
683		else => {},
684		}
685
686		// after numbers must be a valid word break the above returns are right
687		// before parsing `.` so those can be unchecked.
688		if (try t.isAtIdentifierStart()) {
689		return error.InvalidSyntax;
690		}
691
692		return .{
693		.tag = tag,
694		.offset = start,
695		};
696		}
697
698		fn maybeConsumeInteger(t: *Tokenizer) !?void {
699		const trace = T.trace(@src());
700		defer trace.end();
701
702		switch (t.byte()) {
703		'0'...'9' => {
704		t.pos += 1;
705		},
706		else => return null,
707		}
708		t.consumeInteger();
709		}
710
711		/// Requires the first byte to be a digit
712		fn consumeInteger(t: *Tokenizer) !void {
713		const trace = T.trace(@src());
714		defer trace.end();
715
716		switch (t.byte()) {
717		'0'...'9' => {
718		t.pos += 1;
719		},
720		else => unreachable,
721		}
722
723		while (true) {
724		switch (t.byte()) {
725		'0'...'9' => {
726		t.pos += 1;
727		},
728		'_' => {
729		t.pos += 1;
730		switch (t.byte()) {
731		'0'...'9' => {
732		t.pos += 1;
733		},
734		else => return t.maybeUTF8Error(error.TrailingNumericSeparator),
735		}
736		},
737		else => break,
738		}
739		}
740		}
741
742		const Radix = enum { hex, binary, octal };
743
744		fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
745		const trace = T.trace(@src());
746		defer trace.end();
747
748		const digits = switch (radix) {
749		.hex => "0123456789abcdefABCDEF",
750		.binary => "01",
751		.octal => "01234567",
752		};
753
754		first: {
755		const b = t.byte();
756		inline for (digits) \|digit\| {
757		if (b == digit) {
758		break :first;
759		}
760		}
761		return t.maybeUTF8Error(error.InvalidSyntax);
762		}
763
764		t.pos += 1;
765
766		while (true) {
767		brk: {
768		const b = t.byte();
769		inline for (digits) \|digit\| {
770		if (b == digit) {
771		break :brk;
772		}
773		}
774		return;
775		}
776		t.pos += 1;
777		}
778		}
779
780		fn tokenizeString(t: *Tokenizer) !Token {
781		assert(t.byte() == '"');
782		const start = t.pos;
783		try t.consumeString();
784		return .{ .tag = .string, .offset = start };
785		}
786
787		fn consumeString(t: *Tokenizer) !void {
788		const trace = T.trace(@src());
789		defer trace.end();
790
791		const start = t.pos;
792		assert(t.byte() == '"');
793
794		t.pos += 1;
795
796		while (true) {
797		switch (t.byte()) {
798		'"' => {
799		t.pos += 1;
800		break;
801		},
802		'\\' => {
803		switch (t.nextByte()) {
804		'\n' => {
805		t.pos = start;
806		return error.UnterminatedString;
807		},
808		'\\' => t.pos += 2,
809		'"' => t.pos += 2,
810		'n' => t.pos += 2,
811		'r' => t.pos += 2,
812		't' => t.pos += 2,
813		'0' => switch (t.byteAt(2) orelse {
814		t.pos = start;
815		return error.UnterminatedString;
816		}) {
817		// '\0xFF' hex escape
818		'x' => {},
819		// '\0' is special cased
820		else => t.pos += 2,
821		},
822		'u' => {
823		const esc_start = t.pos;
824		t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
825		error.InvalidEscapeSequence => {
826		if (t.byte() == '\n')
827		return error.UnterminatedString;
828		t.pos = esc_start;
829		return err;
830		},
831		error.InvalidEscapeCodepoint => {
832		t.pos = esc_start;
833		return err;
834		},
835		// error.UnterminatedString => return err,
836		};
837		},
838		else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
839		}
840		},
841		'\n' => {
842		t.pos = start;
843		return error.UnterminatedString;
844		},
845		else => {
846		t.pos += 1;
847		},
848		0x80...0xFF => {
849		const byte_len, _ = try t.codePoint();
850		t.pos += byte_len;
851		},
852		}
853		}
854		}
855
856		fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
857		const trace = T.trace(@src());
858		defer trace.end();
859
860		assert(t.byte() == '\\');
861		assert(t.nextByte() == 'u');
862		t.pos += 2;
863
864		if (t.byte() != '{')
865		return error.InvalidEscapeSequence;
866		t.pos += 1;
867
868		if (t.byte() == '0') {
869		if (t.nextByte() == 'x') {
870		t.pos += 2;
871		}
872		}
873
874		const start = t.pos;
875		t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence;
876
877		const slice = t.code[start..t.pos];
878
879		if (t.byte() != '}')
880		return error.InvalidEscapeSequence;
881
882		const code_point = std.fmt.parseInt(u21, slice, 16) catch \|err\| switch (err) {
883		error.Overflow => return error.InvalidEscapeCodepoint,
884		error.InvalidCharacter => unreachable,
885		};
886		_ = std.unicode.utf8CodepointSequenceLength(code_point) catch \|err\| switch (err) {
887		error.CodepointTooLarge => return error.InvalidEscapeCodepoint,
888		};
889
890		t.pos += 1;
891		}
892
893		fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token {
894		const start = t.pos;
895		t.pos += comptime (tag.staticLen() orelse unreachable);
896		return .{ .tag = tag, .offset = start };
897		}
898
899		/// We are about to return an error, such as error.TrailingNumericSeparator.
900		/// However, the character we ran into might actually be invalid UTF-8.
901		/// This utility just checks for that to make sure we dont mask error.InvalidUtf8
902		fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
903		.ErrorSet = &[_]std.builtin.Type.Error{
904		.{ .name = "InvalidUtf8" },
905		.{ .name = @errorName(real_err) },
906		},
907		}) {
908		switch (t.byte()) {
909		0x80...0xFF => {
910		_ = try t.codePoint();
911		return real_err;
912		},
913		else => return real_err,
914		}
915		}
916
917		fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
918		.ErrorSet = &[_]std.builtin.Type.Error{
919		.{ .name = "InvalidUtf8" },
920		.{ .name = @errorName(real_err) },
921		},
922		}) {
923		switch (t.byteAtRt(at) orelse return real_err) {
924		0x80...0xFF => {
925		_ = try t.codePoint();
926		return real_err;
927		},
928		else => return real_err,
929		}
930		}
931
932		pub fn skipWhitespace(t: *Tokenizer) void {
933		const trace = T.trace(@src());
934		defer trace.end();
935
936		while (true) {
937		switch (t.byte()) {
938		' ', '\t', '\n', '\r' => {
939		t.pos += 1;
940		},
941		else => break,
942		}
943		}
944		}
945
946		pub fn tokenLocation(
947		utf8_source: []const u8,
948		start_offset: ByteOffset,
949		token_start: ByteOffset,
950		) Location {
951		var loc: Location = .{
952		.line = .start,
953		.column = .start,
954		.line_start = start_offset,
955		.line_end = @intCast(utf8_source.len),
956		};
957
958		// Scan to by line until we go past the token start
959		while (std.mem.indexOfScalarPos(u8, utf8_source, loc.line_start, '\n')) \|i\| {
960		if (i >= token_start) {
961		break; // Went past
962		}
963		loc.line = loc.line.add(1);
964		loc.line_start = @intCast(i + 1);
965		}
966
967		const offset = loc.line_start;
968		for (utf8_source[offset..], 0..) \|c, i\| {
969		if (i + offset == token_start) {
970		loc.line_end = @intCast(i + offset);
971		while (loc.line_end < utf8_source.len and utf8_source[loc.line_end] != '\n') {
972		loc.line_end += 1;
973		}
974		return loc;
975		}
976		if (c == '\n') {
977		loc.line = loc.line.add(1);
978		loc.column = .start;
979		loc.line_start = @intCast(i + 1);
980		} else {
981		loc.column = loc.column.add(1);
982		}
983		}
984
985		return loc;
986		}
987
988		const Tokenizer = @This();
989		const std = @import("std");
990		const unicode = std.unicode;
991		const assert = std.debug.assert;
992		const mem = std.mem;
993		const unicode_props = @import("unicode_props.zig");
994		const T = @import("tracer");
995		const Location = @import("Ast.zig").Location;

src/compiler/Transform.zig deleted-1479

...	...	@@ -1,1479 +0,0 @@
1		//! Transform pass
2		gpa: Allocator,
3		ast: Ast,
4		insts: Inst.List = .{},
5		string_bytes: std.ArrayListUnmanaged(u8) = .{},
6		extra: ExtraStore = .{},
7
8		scratch: std.ArrayListUnmanaged(u32) = .{},
9
10		// TODO: refactor this away
11		// The new home for this memory should be pre-allocated on scratch,
12		// but then remove the elements, moving scratch items on top of this
13		// backwards a few.
14		call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
15		scopes: std.ArrayListUnmanaged(Scope) = .{},
16
17		report: Report.Builder,
18
19		pub const Error = error{
20		OutOfMemory,
21		TransformFail,
22		};
23
24		pub fn transform(
25		ast: Ast,
26		gpa: Allocator,
27		file: *Comp.File,
28		fail_report: *Report,
29		) Error!Teg {
30		const trace = tracer.trace(@src());
31		defer trace.end();
32
33		var t: Transform = .{
34		.ast = ast,
35		.gpa = gpa,
36		.report = .{ .gpa = gpa, .current_file = file },
37		};
38		errdefer t.deinit();
39
40		try t.string_bytes.append(t.gpa, 0);
41
42		const module = t.transformModule(0) catch \|err\| switch (err) {
43		error.OutOfMemory => return err,
44		};
45
46		const report = try t.report.finalize();
47		if (report.errors.len > 0) {
48		fail_report.* = report;
49		return error.TransformFail;
50		}
51
52		assert(@intFromEnum(module) == 0);
53
54		try t.insts.setCapacity(t.gpa, t.insts.len);
55
56		defer t.deinit();
57		return Teg{
58		.insts = t.insts.toOwnedSlice(),
59		.string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
60		.extra = try t.extra.toOwnedStore(t.gpa),
61		.report = report,
62		};
63		}
64
65		pub fn deinit(t: *Transform) void {
66		t.scratch.deinit(t.gpa);
67		t.call_args.deinit(t.gpa);
68		assert(t.scopes.items.len == 0);
69		t.scopes.deinit(t.gpa);
70		t.string_bytes.deinit(t.gpa);
71		t.insts.deinit(t.gpa);
72		t.extra.deinit(t.gpa);
73		t.* = undefined;
74		}
75
76		const Scope = struct {
77		symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
78
79		const Symbol = struct {
80		node: NodeIndex,
81		data: Symbol.Data,
82		// TODO: optimize this structure
83		used: bool = false,
84		mutated: bool = false,
85		discarded: ?NodeIndex = null,
86
87		const Data = union(enum) {
88		declaration,
89		param,
90		mutable: Inst.Index,
91		immutable: Inst.Ref,
92		};
93
94		pub fn addWarnings(symbol: Symbol, t: *Transform) !void {
95		if (!symbol.mutated and symbol.data == .mutable) {
96		try t.report.warn(.not_mutated_var, .{
97		.node = symbol.node,
98		});
99		}
100		if (!symbol.used) {
101		try t.report.warn(.unused, .{
102		.node = symbol.node,
103		.kind = switch (symbol.data) {
104		.declaration => .function,
105		.param => .param,
106		.mutable => .variable,
107		.immutable => switch (t.ast.nodeTag(symbol.node)) {
108		.param_decl => .param,
109		else => .constant,
110		},
111		},
112		});
113		} else if (symbol.discarded) \|discard\| {
114		try t.report.warn(.unnecessary_discard, .{
115		.node = discard,
116		});
117		}
118		}
119		};
120
121		fn addWarnings(s: Scope, t: *Transform) !void {
122		for (s.symbols.values()) \|value\| {
123		try value.addWarnings(t);
124		}
125		}
126
127		fn deinit(s: *Scope, gpa: Allocator) void {
128		s.symbols.deinit(gpa);
129		s.* = undefined;
130		}
131		};
132
133		const ScratchType = enum {
134		ref,
135		index,
136		};
137
138		const BlockContext = struct {
139		node_base: i32,
140		return_target: ?*?Inst.Index = null,
141		break_target: ?*BreakTarget = null,
142
143		/// This struct is always stored on the stack; the idea is to pass
144		/// mutable pointers to it. When `break :brk` is encountered
145		const BreakTarget = struct {
146		label: ?[]const u8,
147		/// This allows `continue` as well as implicit break targetting.
148		is_loop: bool,
149		/// If `null`, this label was unused.
150		block_index: ?Inst.Index = null,
151		parent: ?*BreakTarget = null,
152
153		fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget {
154		return .{
155		.parent = parent,
156		.label = label,
157		.is_loop = is_loop,
158		};
159		}
160
161		fn getInst(target: BreakTarget, t: Transform) !Inst.Index {
162		return target.block_index orelse {
163		const inst = try t.addRootInst(.{
164		.tag = .block,
165		.data = undefined,
166		});
167		target.block_index = inst;
168		return inst;
169		};
170		}
171		};
172
173		fn init(root_node: NodeIndex) BlockContext {
174		return .{
175		.node_base = @intCast(root_node),
176		};
177		}
178
179		fn nodeOffset(self: BlockContext, node: NodeIndex) i32 {
180		return @as(i32, @intCast(node)) - self.node_base;
181		}
182
183		fn declOffset(self: BlockContext, node: NodeIndex) BlockContext {
184		return .{
185		.node_base = self.nodeOffset(node),
186		};
187		}
188
189		fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext {
190		return .{
191		.node_base = self.node_base,
192		.return_target = return_target,
193		};
194		}
195
196		fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext {
197		return .{
198		.node_base = self.node_base,
199		.return_target = self.return_target,
200		.break_target = break_target,
201		};
202		}
203
204		fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index {
205		var break_target = self.break_target;
206		while (break_target) \|target\| : (break_target = target.parent) {
207		if (target.label) \|target_label\| {
208		if (mem.eql(u8, label, target_label)) {
209		return try target.getInst(t);
210		}
211		}
212		}
213		return null;
214		}
215		};
216
217		const Liveness = enum { alive, dead };
218
219		const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
220		.{ "void", .void },
221		.{ "true", .bool_true },
222		.{ "false", .bool_false },
223		.{ "int", .int32_type },
224		.{ "float", .float_type },
225		.{ "string", .string_type },
226		// .{ "bigint", .bigint_type },
227		.{ "bool", .bool_type },
228		.{ "noreturn", .noreturn_type },
229		.{ "type", .type_type },
230		.{ "std", .standard_library },
231		});
232
233		fn transformModule(t: *Transform, module_node: NodeIndex) Allocator.Error!Inst.Index {
234		const trace = tracer.trace(@src());
235		defer trace.end();
236
237		const module = try t.addRootInst(.{
238		.tag = .module,
239		.data = undefined,
240		});
241
242		const decl = BlockContext.init(module_node);
243
244		try t.pushScope();
245		defer t.popScope();
246
247		const statements = t.ast.nodeStatements(module_node);
248		for (statements) \|statement\| {
249		try t.previsitModuleStatement(statement);
250		}
251
252		const s = t.scope();
253
254		const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1);
255		defer t.gpa.free(temp);
256
257		{
258		// prevent concurrent access
259		s.symbols.lockPointers();
260		defer s.symbols.unlockPointers();
261		for (s.symbols.values(), 0..) \|value, i\| {
262		temp[i] = try t.transformFunction(decl, value.node);
263		}
264		}
265
266		const top_level_fn = try t.addRootInst(.{
267		.tag = .declaration,
268		.data = undefined,
269		});
270
271		temp[temp.len - 1] = top_level_fn;
272
273		const declarations: IndexList = .{
274		.start = @intCast(t.extra.list.items.len),
275		.len = @intCast(temp.len),
276		};
277		try t.extra.list.appendSlice(t.gpa, @ptrCast(temp));
278
279		const block = try t.addRootInst(.{
280		.tag = .block,
281		.data = undefined,
282		});
283
284		{
285		const scratch_top = t.scratch.items.len;
286		defer t.scratch.shrinkRetainingCapacity(scratch_top);
287
288		for (statements) \|statement\| {
289		try t.transformModuleStatement(decl, statement);
290		}
291
292		t.getInstData(block).* = EncodedData.encode(.block, .{
293		.src_node = decl.nodeOffset(module_node),
294		.extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)),
295		});
296		}
297
298		const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{
299		.kind = .function,
300		.name = NullTerminatedString.empty,
301		.hash = .{0} ** 16,
302		.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn),
303		}, dod.extraSizeOf(Inst.DeclFn));
304
305		trail.writeStruct(Inst.DeclFn, .{
306		.params = .{ .start = 0, .len = 0 },
307		.return_type = Inst.Ref.void,
308		.body = block.toRef(),
309		});
310
311		t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{
312		.src_node = decl.nodeOffset(module_node),
313		.extra_index = decl_extra,
314		});
315
316		t.getInstData(module).* = EncodedData.encode(.module, .{
317		.src_node = 0,
318		.extra_index = try t.extra.append(t.gpa, IndexList, declarations),
319		});
320
321		try t.scope().addWarnings(t);
322
323		return module;
324		}
325
326		fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void {
327		const trace = tracer.trace(@src());
328		defer trace.end();
329
330		const tag = t.ast.nodes.items(.tag)[node];
331		switch (tag) {
332		.fn_decl => {
333		const fn_decl = t.ast.fullFnDecl(node);
334		const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
335		if (gop.found_existing) {
336		try t.report.err(.duplicate_function, .{
337		.node = node,
338		.prev = gop.value_ptr.node,
339		});
340		return;
341		}
342		gop.value_ptr.* = .{
343		.node = node,
344		.data = .declaration,
345		};
346		},
347		else => {},
348		}
349		}
350
351		fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void {
352		const trace = tracer.trace(@src());
353		defer trace.end();
354
355		const tag = t.ast.nodes.items(.tag)[node];
356		switch (tag) {
357		.root,
358		.param_decl,
359		=> unreachable,
360
361		.fn_decl => {}, // functions analyzed separately
362
363		else => _ = try t.transformBlockStatement(decl, node, false),
364		}
365		}
366
367		fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) Allocator.Error!Inst.Index {
368		const trace = tracer.trace(@src());
369		defer trace.end();
370
371		const decl = outer_decl.declOffset(node);
372		const fn_decl = t.ast.fullFnDecl(node);
373
374		const func = try t.addRootInst(.{
375		.tag = .declaration,
376		.data = undefined,
377		});
378
379		try t.pushScope();
380		defer t.popScope();
381
382		const param_list = brk: {
383		const scratch_top = t.scratch.items.len;
384		defer t.scratch.shrinkRetainingCapacity(scratch_top);
385
386		for (fn_decl.params) \|param\| {
387		try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param));
388		}
389
390		break :brk try t.scratchToRefList(scratch_top);
391		};
392
393		const return_type = if (fn_decl.return_type) \|return_type_node\|
394		try t.transformExpressionToBlock(decl, return_type_node, false)
395		else
396		Inst.Ref.void;
397
398		const body = try t.transformExpressionToBlock(decl, fn_decl.body, true);
399
400		const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{
401		.kind = .function,
402		.name = try t.addNullTerminatedString(fn_decl.id),
403		.hash = .{0} ** 16,
404		.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func),
405		});
406
407		_ = try t.extra.append(t.gpa, Inst.DeclFn, .{
408		.params = param_list,
409		.return_type = return_type,
410		.body = body,
411		});
412
413		t.getInstData(func).* = EncodedData.encode(.declaration, .{
414		.src_node = outer_decl.nodeOffset(node),
415		.extra_index = decl_extra,
416		});
417
418		try t.scope().addWarnings(t);
419
420		return func;
421		}
422
423		const TransformFunctionCtx = struct {
424		param_list: RefList,
425		};
426
427		/// Converts expressions into blocks with a break. This is important
428		/// for places where a block is not allowed, such as a parameter list.
429		/// This is also used for the function body, even though it is a block.
430		/// Simple refs turn into just the ref
431		fn transformExpressionToBlock(
432		t: *Transform,
433		decl: BlockContext,
434		node: NodeIndex,
435		comptime is_function: bool,
436		) Allocator.Error!Inst.Ref {
437		const trace = tracer.trace(@src());
438		defer trace.end();
439
440		const node_tag = t.ast.nodes.items(.tag)[node];
441		switch (node_tag) {
442		.id => {
443		// Lower builtin ids without using a block
444		const id = t.ast.basicNodeSlice(node);
445		if (BuiltInIds.get(id)) \|ref\| {
446		return ref;
447		}
448		},
449
450		else => {},
451		}
452
453		var block: ?Inst.Index = null;
454
455		const scratch_top = t.scratch.items.len;
456		defer t.scratch.shrinkRetainingCapacity(scratch_top);
457
458		if (is_function) {
459		const s = t.scopes.getLast();
460		for (s.symbols.values(), 0..) \|*param, i\| {
461		assert(param.data == .param);
462		const ref = try t.addInstRef(.{
463		.tag = .param,
464		.data = EncodedData.encode(.param, .{
465		.src_node = decl.nodeOffset(node),
466		.data = @intCast(i),
467		}),
468		});
469		param.data = .{ .immutable = ref };
470		}
471		}
472
473		const expr = try t.transformExpression(
474		if (is_function)
475		decl.withReturn(&block)
476		else
477		decl,
478		node,
479		);
480
481		// if there are no new instructions, we dont have to create a block context
482		if (scratch_top == t.scratch.items.len) {
483		const result = expr orelse unreachable; // null means we added break or return
484
485		// TODO: evaluate if `if` blocks would be fine in sema with instruction refs
486		// with this extra if check here, we filter out non-blocks.
487		if (result.toIndex() == null) {
488		return result;
489		}
490		}
491
492		if (expr) \|result\| {
493		if (result.toIndex()) \|index\| {
494		if (t.insts.items(.tag)[@intFromEnum(index)] == .block) {
495		return result;
496		}
497		}
498		}
499
500		const ret_block = if (block) \|b\|
501		b
502		else
503		try t.addRootInst(.{
504		.tag = .block,
505		.data = undefined,
506		});
507
508		if (expr) \|result\| {
509		_ = try t.addInstRef(.{
510		.tag = .@"break",
511		.data = EncodedData.encode(.@"break", .{
512		.src_node = decl.nodeOffset(node),
513		.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
514		.block = ret_block,
515		.operand = result,
516		}),
517		}),
518		});
519		}
520
521		const content = try t.scratchToIndexList(scratch_top);
522
523		t.getInstData(ret_block).* = EncodedData.encode(.block, .{
524		.src_node = decl.nodeOffset(node),
525		.extra_index = try t.extra.append(t.gpa, IndexList, content),
526		});
527
528		return ret_block.toRef();
529		}
530
531		fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Allocator.Error!Inst.Ref {
532		const trace = tracer.trace(@src());
533		defer trace.end();
534
535		const param_decl = t.ast.fullParamDecl(param);
536		const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
537
538		// add `sym` into the scope so that the next param's type
539		// can use this param's value, if known at compile time.
540		const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
541		if (sym.found_existing) {
542		try t.report.err(.duplicate_param, .{
543		.node = param,
544		.prev = sym.value_ptr.node,
545		});
546		return .void;
547		}
548
549		sym.value_ptr.* = .{
550		.node = param,
551		.data = .param,
552		};
553
554		return value;
555		}
556
557		const Builtin = enum {
558		js,
559		import,
560		fetch,
561		compile_error,
562
563		const map = std.ComptimeStringMap(Builtin, .{
564		.{ "js", .js },
565		.{ "import", .import },
566		.{ "fetch", .fetch },
567		.{ "compileError", .compile_error },
568		});
569
570		fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref {
571		const trace = tracer.trace(@src());
572		defer trace.end();
573
574		if (args.len != 1) {
575		// try t.report.err(.invalid_builtin_args, .{ .id = "js" });
576		return .void;
577		}
578
579		const source = try t.transformExpression(decl, args[0]) orelse return null;
580
581		return try t.addInstRef(.{
582		.tag = .js,
583		.data = EncodedData.encode(.js, .{
584		.src_node = decl.nodeOffset(args[0]),
585		.operand = source,
586		}),
587		});
588		}
589		};
590
591		/// This is the primary transformation function: converting an AST expression into instructions.
592		/// `null` represents a `return` or `break`, aka this branch of code is dead.
593		fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Ref {
594		const node_tag = t.ast.nodes.items(.tag)[node];
595
596		const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
597		defer trace.end();
598
599		switch (node_tag) {
600		.root,
601		.param_decl,
602		.var_decl,
603		.fn_decl,
604		=> unreachable, // not an expression
605
606		.id => {
607		const id = t.ast.basicNodeSlice(node);
608		if (BuiltInIds.get(id)) \|ref\| {
609		return ref;
610		}
611
612		if (mem.startsWith(u8, id, "@")) {
613		@panic("TODO: string identifiers");
614		}
615
616		const lookup = t.lookupId(id) orelse {
617		try t.report.err(.unknown_identifier, .{ .node = node });
618		return .void;
619		};
620
621		lookup.used = true;
622
623		return switch (lookup.data) {
624		.declaration => try t.addInstRef(.{
625		.tag = .load_decl,
626		.data = EncodedData.encode(.load_decl, .{
627		.src_node = decl.nodeOffset(node),
628		.string = try t.addNullTerminatedString(id),
629		}),
630		}),
631		.mutable => \|idx\| idx.toRef(),
632		.immutable => \|ref\| ref,
633		.param => unreachable, //unresolved param
634		};
635		},
636
637		inline .number_int, .number_float => \|tag\| {
638		const text = t.ast.basicNodeSlice(node);
639		const value = switch (tag) {
640		.number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
641		.number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
642		else => comptime unreachable,
643		};
644		return try t.addInstRef(.{
645		.tag = .number,
646		.data = EncodedData.encode(.number, .{
647		.src_node = decl.nodeOffset(node),
648		.extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)),
649		}),
650		});
651		},
652
653		.string => {
654		const str = t.ast.basicNodeSlice(node);
655		const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
656		return try t.addInstRef(.{
657		.tag = .string,
658		.data = EncodedData.encode(.string, .{
659		.src_node = decl.nodeOffset(node),
660		.extra_index = try t.extra.append(t.gpa, SlicedString, sliced),
661		}),
662		});
663		},
664
665		.assign => {
666		const ast_bin = t.ast.nodeData(node).bin;
667		const lhs = try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse
668		return .void;
669		const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse
670		return null;
671		const assign_inst = try t.addInstRef(.{
672		.tag = .store,
673		.data = EncodedData.encode(.store, .{
674		.src_node = decl.nodeOffset(node),
675		.extra_index = try t.extra.append(t.gpa, Inst.Store, .{
676		.memory = lhs,
677		.operand = rhs,
678		}),
679		}),
680		});
681		return assign_inst;
682		},
683
684		inline .add,
685		.sub,
686		.mul,
687		.div,
688		.rem,
689		.pow,
690		.@"and",
691		.@"or",
692		.bit_and,
693		.bit_or,
694		.bit_xor,
695		.shl,
696		.shr,
697		.eq,
698		.ne,
699		.lt,
700		.le,
701		.gt,
702		.ge,
703		.add_assign,
704		.sub_assign,
705		.mul_assign,
706		.div_assign,
707		.rem_assign,
708		.pow_assign,
709		.and_assign,
710		.or_assign,
711		.bit_and_assign,
712		.bit_or_assign,
713		.bit_xor_assign,
714		.shl_assign,
715		.shr_assign,
716		=> \|tag\| {
717		// This code path implements binary operations, as well as the
718		// compound compute + assign operation.
719		const is_assign = comptime tag.isAssignBin();
720		const teg_tag = @field(
721		Teg.Inst.Tag,
722		if (is_assign)
723		@tagName(tag)[0 .. @tagName(tag).len - "_assign".len]
724		else
725		@tagName(tag),
726		);
727		const ast_bin = t.ast.nodeData(node).bin;
728		const lhs = if (is_assign)
729		try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse
730		return .void
731		else
732		try t.transformExpression(decl, ast_bin.lhs) orelse
733		return null;
734		const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
735
736		const compute_inst = try t.addInstRef(.{
737		.tag = teg_tag,
738		.data = EncodedData.encode(teg_tag, .{
739		.src_node = decl.nodeOffset(node),
740		.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
741		.lhs = if (is_assign) lhs.toRef() else lhs,
742		.rhs = rhs,
743		}),
744		}),
745		});
746
747		if (is_assign) {
748		const assign_inst = try t.addInstRef(.{
749		.tag = .store,
750		.data = EncodedData.encode(.store, .{
751		.src_node = decl.nodeOffset(node),
752		.extra_index = try t.extra.append(t.gpa, Inst.Store, .{
753		.memory = lhs,
754		.operand = compute_inst,
755		}),
756		}),
757		});
758		return assign_inst;
759		}
760
761		return compute_inst;
762		},
763
764		inline .negate,
765		.bool_not,
766		.bit_not,
767		=> \|tag\| {
768		const ast_un = t.ast.nodeData(node).un;
769		const teg_tag = @field(Teg.Inst.Tag, @tagName(tag));
770
771		const operand = try t.transformExpression(decl, ast_un.expr) orelse
772		return null;
773
774		return try t.addInstRef(.{
775		.tag = teg_tag,
776		.data = EncodedData.encode(teg_tag, .{
777		.src_node = decl.nodeOffset(node),
778		.operand = operand,
779		}),
780		});
781		},
782
783		.label => {
784		const expr = t.ast.nodeData(node).label.expr;
785		const label = t.ast.tokenSlice(t.ast.nodeToken(node));
786
787		var target: BlockContext.BreakTarget = .{
788		.parent = decl.break_target,
789		.label = label,
790		.is_loop = false,
791		};
792		const inner_ctx = decl.withBreak(&target);
793
794		const scratch_top = t.scratch.items.len;
795
796		const inner_expr = try t.transformExpression(inner_ctx, expr);
797
798		// if there are no new instructions, we dont have to create a block context
799		if (scratch_top == t.scratch.items.len)
800		return inner_expr orelse unreachable; // null means we added break or return
801
802		if (target.block_index) \|block\| {
803		if (inner_expr) \|result\| {
804		_ = try t.addInstRef(.{
805		.tag = .@"break",
806		.data = EncodedData.encode(.@"break", .{
807		.src_node = decl.nodeOffset(node),
808		.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
809		.block = block,
810		.operand = result,
811		}),
812		}),
813		});
814		}
815
816		const content = try t.scratchToIndexList(scratch_top);
817		t.scratch.shrinkRetainingCapacity(scratch_top);
818
819		t.getInstData(block).* = EncodedData.encode(.block, .{
820		.src_node = decl.nodeOffset(node),
821		.extra_index = try t.extra.append(t.gpa, IndexList, content),
822		});
823
824		try t.addScratch(Inst.Index, block);
825
826		return block.toRef();
827		} else {
828		std.debug.print("Warn, unused label {s}", .{label});
829		}
830
831		return inner_expr;
832		},
833
834		inline .block_void,
835		.block_implicit_return,
836		=> \|tag\| {
837		const statements = t.ast.nodeStatements(node);
838
839		var last: Inst.Ref = .void;
840		for (statements, 1..) \|statement, i\| {
841		const is_last = i == statements.len;
842		last = try t.transformBlockStatement(
843		decl,
844		statement,
845		tag == .block_implicit_return and is_last,
846		) orelse {
847		return null;
848		};
849		}
850
851		return if (tag == .block_implicit_return) last else .void;
852		},
853
854		.builtin_call,
855		=> {
856		const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
857		try t.report.err(.invalid_builtin, .{ .node = node });
858		return .void;
859		};
860
861		const base = t.ast.nodeData(node).list;
862		const args = base.slice(&t.ast);
863
864		return switch (builtin) {
865		.js => Builtin.builtin_js(t, decl, args),
866		// .import => Builtin.import(t, decl, args),
867		// .fetch => Builtin.fetch(t, decl, args),
868		// .compile_error => Builtin.compile_error(t, decl, args),
869		else => @panic("TODO"),
870		};
871		},
872
873		.call => {
874		const call = t.ast.nodeData(node).call;
875		const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast);
876
877		const base = try t.transformExpression(decl, call.base) orelse return null;
878
879		const args_list: RefList = brk: {
880		const args_top = t.call_args.items.len;
881		defer t.call_args.shrinkRetainingCapacity(args_top);
882
883		for (args) \|arg\| {
884		try t.call_args.append(
885		t.gpa,
886		try t.transformExpression(decl, arg) orelse
887		return null,
888		);
889		}
890
891		const items = t.call_args.items[args_top..];
892		const extra_top = t.extra.list.items.len;
893		try t.extra.list.appendSlice(t.gpa, @ptrCast(items));
894
895		break :brk .{
896		.start = @intCast(extra_top),
897		.len = @intCast(items.len),
898		};
899		};
900
901		const call_inst = try t.addInstRef(.{
902		.tag = .call,
903		.data = EncodedData.encode(.call, .{
904		.src_node = decl.nodeOffset(node),
905		.extra_index = try t.extra.append(t.gpa, Inst.Call, .{
906		.base = base,
907		.args = args_list,
908		}),
909		}),
910		});
911
912		return call_inst;
913		},
914
915		.field => {
916		const field = t.ast.nodeData(node).field;
917
918		const base = try t.transformExpression(decl, field.base) orelse return null;
919
920		// TODO: handle identifier escapes
921		const id = t.ast.tokenSlice(field.field_token);
922		const prop_name_str = try t.addSlicedString(id);
923
924		const string_ref = try t.addInstRef(.{
925		.tag = .string,
926		.data = EncodedData.encode(.string, .{
927		.src_node = decl.nodeOffset(node),
928		.extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
929		}),
930		});
931
932		return try t.addInstRef(.{
933		.tag = .field,
934		.data = EncodedData.encode(.field, .{
935		.src_node = decl.nodeOffset(node),
936		.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
937		.lhs = base,
938		.rhs = string_ref,
939		}),
940		}),
941		});
942		},
943
944		.@"if" => {
945		const if_node = t.ast.nodeData(node).@"if";
946		const if_data = t.ast.extraData(if_node.extra, Ast.If);
947
948		const cond = try t.transformExpression(decl, if_node.cond) orelse
949		return null;
950
951		const yes = try t.transformExpressionToBlock(
952		decl,
953		if_data.yes,
954		false,
955		);
956		const no = if (if_data.no.unwrap()) \|no\|
957		try t.transformExpressionToBlock(
958		decl,
959		no,
960		false,
961		)
962		else
963		Inst.Ref.void;
964
965		if (yes == .void and no == .void) {
966		// when both branches do nothing, we can omit the branch,
967		// however we still need to type-check the condition operand
968		_ = try t.addInstRef(.{
969		.tag = .ensure_bool,
970		.data = EncodedData.encode(.ensure_bool, .{
971		.src_node = decl.nodeOffset(node),
972		.operand = cond,
973		}),
974		});
975		return .void;
976		}
977
978		return try t.addInstRef(.{
979		.tag = .@"if",
980		.data = EncodedData.encode(.@"if", .{
981		.src_node = decl.nodeOffset(node),
982		.extra_index = try t.extra.append(t.gpa, Inst.If, .{
983		.cond = cond,
984		.yes = yes,
985		.no = no,
986		}),
987		}),
988		});
989		},
990
991		.@"return" => {
992		const ret_node = t.ast.nodeData(node).@"return";
993
994		const ret_block_ptr = decl.return_target orelse {
995		try t.report.err(.invalid_return, .{ .node = node });
996		return .void;
997		};
998
999		const ret_block = if (ret_block_ptr.*) \|block\|
1000		block
1001		else append_block: {
1002		const block = try t.addRootInst(.{
1003		.tag = .block,
1004		.data = undefined,
1005		});
1006		ret_block_ptr.* = block;
1007		break :append_block block;
1008		};
1009
1010		const operand: Inst.Ref = if (ret_node.expr.unwrap()) \|expr\|
1011		try t.transformExpression(decl.withReturn(null), expr) orelse
1012		return null
1013		else
1014		.void;
1015
1016		_ = try t.addInstRef(.{
1017		.tag = .@"break",
1018		.data = EncodedData.encode(.@"break", .{
1019		.src_node = decl.nodeOffset(node),
1020		.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
1021		.block = ret_block,
1022		.operand = operand,
1023		}),
1024		}),
1025		});
1026
1027		return null; // control flow ends
1028		},
1029
1030		.@"break" => {
1031		const brk_node = t.ast.nodeData(node).@"break";
1032		const label = t.ast.tokenSlice(
1033		brk_node.label.unwrap() orelse
1034		@panic("TODO: unlabelled breaks"),
1035		);
1036
1037		const block = try decl.findAndMarkBreak(t, label) orelse {
1038		try t.report.err(.unresolved_label, .{ .node = node });
1039		return null;
1040		};
1041
1042		const operand: Inst.Ref = if (brk_node.value.unwrap()) \|expr\|
1043		try t.transformExpression(decl.withReturn(null), expr) orelse
1044		return null
1045		else
1046		.void;
1047
1048		_ = try t.addInstRef(.{
1049		.tag = .@"break",
1050		.data = EncodedData.encode(.@"break", .{
1051		.src_node = decl.nodeOffset(node),
1052		.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
1053		.block = block,
1054		.operand = operand,
1055		}),
1056		}),
1057		});
1058
1059		return null; // control flow ends
1060		},
1061
1062		else => \|tag\| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}),
1063		}
1064		}
1065
1066		fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Index {
1067		const node_tag = t.ast.nodes.items(.tag)[node];
1068
1069		const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
1070		defer trace.end();
1071
1072		switch (node_tag) {
1073		.root,
1074		.param_decl,
1075		.var_decl,
1076		.fn_decl,
1077		=> unreachable, // not an expression
1078
1079		.id => {
1080		const id = t.ast.basicNodeSlice(node);
1081		assert(!mem.eql(u8, id, "_")); // discards are not assignment targets
1082
1083		if (BuiltInIds.has(id)) {
1084		try t.report.err(.assign_builtin, .{ .node = node });
1085		return null;
1086		}
1087
1088		if (mem.startsWith(u8, id, "@")) {
1089		@panic("TODO: string identifiers");
1090		}
1091
1092		const lookup = t.lookupId(id) orelse {
1093		try t.report.err(.unknown_identifier, .{ .node = node });
1094		return null;
1095		};
1096
1097		return switch (lookup.data) {
1098		.mutable => \|inst\| {
1099		lookup.mutated = true;
1100		return inst;
1101		},
1102		else => {
1103		lookup.used = true;
1104
1105		try t.report.err(.invalid_assignment_symbol, .{
1106		.node = node,
1107		.decl = lookup.node,
1108		.kind = switch (lookup.data) {
1109		.mutable => unreachable,
1110		.declaration => .function,
1111		.param => .param,
1112		.immutable => switch (t.ast.nodeTag(lookup.node)) {
1113		.param_decl => .param,
1114		else => .constant,
1115		},
1116		},
1117		});
1118		return null;
1119		},
1120		};
1121		},
1122
1123		// TODO: allow field expressions, but these need to emit a different store node
1124
1125		else => {
1126		_ = try t.transformExpression(decl, node);
1127		try t.report.err(.invalid_assignment_target, .{
1128		.node = node,
1129		});
1130		return null;
1131		},
1132		}
1133		}
1134
1135		fn transformBlockStatement(
1136		t: *Transform,
1137		decl: BlockContext,
1138		node: NodeIndex,
1139		is_implicit_return_expression: bool,
1140		) !?Inst.Ref {
1141		const node_tag = t.ast.nodes.items(.tag)[node];
1142
1143		const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
1144		defer trace.end();
1145
1146		switch (node_tag) {
1147		.root => unreachable, // cannot appear in a block
1148		.param_decl => unreachable, // cannot appear in a block
1149
1150		.var_decl => {
1151		const var_decl = t.ast.fullVarDecl(node);
1152
1153		if (mem.eql(u8, var_decl.id, "_")) {
1154		try t.report.warn(.invalid_discard, .{ .node = node });
1155
1156		const ref = switch (t.ast.nodeTag(var_decl.init)) {
1157		.id => {
1158		const id = t.ast.basicNodeSlice(node);
1159		if (BuiltInIds.has(id)) {
1160		try t.report.warn(.discard_builtin, .{ .node = node });
1161		return .void;
1162		}
1163
1164		if (mem.startsWith(u8, id, "@")) {
1165		@panic("TODO: string identifiers");
1166		}
1167
1168		const lookup = t.lookupId(id) orelse {
1169		try t.report.err(.unknown_identifier, .{ .node = node });
1170		return .void;
1171		};
1172
1173		lookup.discarded = node;
1174
1175		return .void;
1176		},
1177		else => try t.transformExpression(decl, var_decl.init) orelse
1178		return null,
1179		};
1180
1181		return try t.addInstRef(.{
1182		.tag = .discard,
1183		.data = EncodedData.encode(.discard, .{
1184		.src_node = decl.nodeOffset(node),
1185		.operand = ref,
1186		}),
1187		});
1188		}
1189
1190		if (BuiltInIds.has(var_decl.id)) {
1191		try t.report.err(.shadow_builtin_variable, .{ .node = node });
1192		return .void;
1193		}
1194
1195		const type_ref = if (var_decl.type) \|data_type\|
1196		try t.transformExpression(decl, data_type)
1197		else
1198		null;
1199		const data_ref = try t.transformExpression(decl, var_decl.init) orelse
1200		return null;
1201		const data_coerced_ref = if (type_ref) \|ref\|
1202		try t.addInstRef(.{
1203		.tag = .as,
1204		.data = EncodedData.encode(.as, .{
1205		.src_node = decl.nodeOffset(node),
1206		.extra_index = try t.extra.append(t.gpa, Inst.As, .{
1207		.type = ref,
1208		.operand = data_ref,
1209		}),
1210		}),
1211		})
1212		else
1213		data_ref;
1214
1215		const mem_ref = if (var_decl.is_const)
1216		data_coerced_ref
1217		else
1218		try t.addInstRef(.{
1219		.tag = .memory,
1220		.data = EncodedData.encode(.memory, .{
1221		.src_node = decl.nodeOffset(node),
1222		.operand = data_coerced_ref,
1223		}),
1224		});
1225
1226		const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id);
1227
1228		// if re-declaring, apply unused warnings now
1229		if (gop.found_existing) {
1230		try gop.value_ptr.addWarnings(t);
1231		}
1232
1233		gop.value_ptr.* = .{
1234		.node = node,
1235		.data = switch (var_decl.is_const) {
1236		true => .{ .immutable = mem_ref },
1237		false => .{ .mutable = mem_ref.toIndex().? },
1238		},
1239		};
1240
1241		return .void;
1242		},
1243
1244		.fn_decl,
1245		=> \|tag\| {
1246		std.debug.panic("TODO: block statement {s}", .{@tagName(tag)});
1247		},
1248
1249		.assign,
1250		=> {
1251		const bin = t.ast.nodeData(node).bin;
1252		const lhs = bin.lhs;
1253
1254		if (t.ast.nodeTag(lhs) == .id and
1255		mem.eql(u8, t.ast.basicNodeSlice(lhs), "_"))
1256		{
1257		// discard, instead of appending an .ensure_void check,
1258		// we will add a .discard instruction instead.
1259		// this will emit a warning if given void.
1260		const result = try t.transformExpression(decl, bin.rhs) orelse
1261		return null;
1262		return try t.addInstRef(.{
1263		.tag = .discard,
1264		.data = EncodedData.encode(.discard, .{
1265		.src_node = decl.nodeOffset(node),
1266		.operand = result,
1267		}),
1268		});
1269		} else {
1270		// process this assignment as usual, do not append .ensure_void
1271		return try t.transformExpression(decl, node);
1272		}
1273		},
1274
1275		.add_assign,
1276		.sub_assign,
1277		.mul_assign,
1278		.div_assign,
1279		.rem_assign,
1280		.pow_assign,
1281		.and_assign,
1282		.or_assign,
1283		.bit_and_assign,
1284		.bit_or_assign,
1285		.bit_xor_assign,
1286		.shl_assign,
1287		.shr_assign,
1288		=> {
1289		// do not append a .ensure_void check
1290		return try t.transformExpression(decl, node);
1291		},
1292
1293		else => {
1294		const result = try t.transformExpression(decl, node) orelse
1295		return null;
1296
1297		if (!is_implicit_return_expression and result != .void) {
1298		if (t.ast.nodeTag(node) == .id) {
1299		try t.report.warn(.dangling_identifier, .{
1300		.node = node,
1301		});
1302		return result;
1303		}
1304
1305		return try t.addInstRef(.{
1306		.tag = .ensure_void,
1307		.data = EncodedData.encode(.ensure_void, .{
1308		.src_node = decl.nodeOffset(node),
1309		.operand = result,
1310		}),
1311		});
1312		}
1313
1314		return result;
1315		},
1316		}
1317		}
1318
1319		fn addRootInst(t: *Transform, inst: Inst) !Inst.Index {
1320		try t.insts.append(t.gpa, inst);
1321		return @enumFromInt(t.insts.len - 1);
1322		}
1323
1324		fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref {
1325		try t.scratch.ensureUnusedCapacity(t.gpa, 1);
1326		try t.insts.append(t.gpa, inst);
1327		const index: Inst.Index = @enumFromInt(t.insts.len - 1);
1328		t.scratch.appendAssumeCapacity(@intFromEnum(index));
1329		return index.toRef();
1330		}
1331
1332		fn pushScope(t: *Transform) !void {
1333		try t.scopes.append(t.gpa, .{});
1334		}
1335
1336		fn popScope(t: *Transform) void {
1337		t.scope().deinit(t.gpa);
1338		t.scopes.items[t.scopes.items.len - 1] = undefined;
1339		t.scopes.items.len -= 1;
1340		}
1341
1342		fn scope(t: Transform) Scope {
1343		return &t.scopes.items[t.scopes.items.len - 1];
1344		}
1345
1346		fn getInstData(t: Transform, index: Inst.Index) EncodedData {
1347		return &t.insts.items(.data)[@intFromEnum(index)];
1348		}
1349
1350		fn addScratch(t: *Transform, comptime T: type, value: T) !void {
1351		try t.scratch.append(t.gpa, @intFromEnum(value));
1352		}
1353
1354		fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
1355		const items = t.scratch.items[scratch_top..];
1356		const extra_top = t.extra.list.items.len;
1357		try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]);
1358		return .{
1359		.start = @intCast(extra_top),
1360		.len = @intCast(items.len),
1361		};
1362		}
1363
1364		fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList {
1365		const items = t.scratch.items[scratch_top..];
1366		const extra_top = t.extra.list.items.len;
1367		try t.extra.list.appendSlice(t.gpa, items);
1368		return .{
1369		.start = @intCast(extra_top),
1370		.len = @intCast(items.len),
1371		};
1372		}
1373
1374		fn lookupId(t: Transform, id: []const u8) ?Scope.Symbol {
1375		var i = t.scopes.items.len;
1376		while (i > 0) {
1377		i -= 1;
1378		if (t.scopes.items[i].symbols.getPtr(id)) \|sym\| {
1379		return sym;
1380		}
1381		}
1382		return null;
1383		}
1384
1385		fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString {
1386		if (str.len == 0) {
1387		return .empty;
1388		}
1389		// loop through strings trying to find an existing match, and dedupe memory
1390		var pos: usize = 0;
1391		while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
1392		const start_index = i - str.len;
1393		if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1394		const string: NullTerminatedString = @enumFromInt(pos);
1395		assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1396		return string;
1397		}
1398		}
1399
1400		const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len);
1401		try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1);
1402		t.string_bytes.appendSliceAssumeCapacity(str);
1403		t.string_bytes.appendAssumeCapacity(0);
1404		assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1405		return string;
1406		}
1407
1408		fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
1409		if (str.len == 0) {
1410		return .{ .start = 0, .len = 0 };
1411		}
1412
1413		// loop through strings trying to find an existing match, and dedupe memory
1414		var pos: usize = 0;
1415		while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
1416		const start_index = i - str.len;
1417		if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1418		const string: SlicedString = .{
1419		.start = @intCast(start_index),
1420		.len = @intCast(str.len),
1421		};
1422		assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1423		return string;
1424		}
1425		}
1426
1427		const string: SlicedString = .{
1428		.start = @intCast(t.string_bytes.items.len),
1429		.len = @intCast(str.len),
1430		};
1431		try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len);
1432		t.string_bytes.appendSliceAssumeCapacity(str);
1433		assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1434		return string;
1435		}
1436
1437		const Transform = @This();
1438		const std = @import("std");
1439		const mem = std.mem;
1440		const Allocator = mem.Allocator;
1441		const assert = std.debug.assert;
1442
1443		const Teg = @import("Teg.zig");
1444		const Inst = Teg.Inst;
1445		const NullTerminatedString = Teg.NullTerminatedString;
1446		const SlicedString = Teg.SlicedString;
1447		const RefList = Teg.RefList;
1448		const IndexList = Teg.IndexList;
1449		const EncodedData = Teg.EncodedData;
1450		const ExtraIndex = Teg.ExtraIndex;
1451
1452		const dod = @import("array.zig");
1453		const ExtraStore = dod.ExtraStore(.teg, .@"var");
1454		const Float = dod.Float;
1455
1456		const Tokenizer = @import("Tokenizer.zig");
1457		const Token = Tokenizer.Token;
1458		const ByteOffset = Tokenizer.ByteOffset;
1459
1460		const Ast = @import("Ast.zig");
1461		const Unused = Ast.Unused;
1462		const Node = Ast.Node;
1463		const Data = Node.Data;
1464		const SubList = Ast.SubList;
1465		const NodeIndex = Ast.NodeIndex;
1466		const TokenIndex = Ast.TokenIndex;
1467		const OptionalNodeIndex = Ast.OptionalNodeIndex;
1468		const OptionalTokenIndex = Ast.OptionalTokenIndex;
1469		const OptionalExtraIndex = Ast.OptionalExtraIndex;
1470		const optionalTokenIndex = OptionalTokenIndex.init;
1471		const optionalNodeIndex = OptionalNodeIndex.init;
1472		const optionalExtraIndex = OptionalExtraIndex.init;
1473
1474		const InternPool = @import("InternPool.zig");
1475
1476		const Report = @import("Report.zig");
1477		const Comp = @import("Comp.zig");
1478
1479		const tracer = @import("tracer");

src/compiler/array.zig deleted-390

...	...	@@ -1,390 +0,0 @@
1		const extra = @This();
2
3		pub const Mutability = enum { @"const", @"var" };
4
5		fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void {
6		const fields = std.meta.fields(@TypeOf(payload));
7		var i = index;
8		inline for (fields) \|field\| {
9		const value = @field(payload, field.name);
10		switch (field.type) {
11		u32 => items[i] = value,
12		i32 => items[i] = @bitCast(value),
13
14		[16]u8 => {
15		const u32s: [4]u32 = @bitCast(value);
16		items[i] = u32s[0];
17		items[i + 1] = u32s[1];
18		items[i + 2] = u32s[2];
19		items[i + 3] = u32s[3];
20		},
21
22		else => brk: {
23		const info = @typeInfo(field.type);
24		if (info == .Enum) {
25		items[i] = @intFromEnum(value);
26		break :brk;
27		}
28
29		const struct_data = info.Struct;
30		if (struct_data.layout == .@"packed") {
31		items[i] = @bitCast(value);
32		break :brk;
33		}
34
35		@call(.always_inline, setAsU32, .{ items, field.type, i, value });
36		},
37		}
38		i += extraSizeOf(field.type);
39		}
40		}
41
42		fn getAsU32(items: []const u32, comptime T: type, index: usize) T {
43		const fields = std.meta.fields(T);
44		var i = index;
45		var result: T = undefined;
46		inline for (fields) \|field\| {
47		@field(result, field.name) = switch (field.type) {
48		u32 => items[i],
49		i32 => @bitCast(items[i]),
50
51		[16]u8 => @bitCast(items[i..][0..4].*),
52
53		else => brk: {
54		const info = @typeInfo(field.type);
55		if (info == .Enum) {
56		break :brk @enumFromInt(items[i]);
57		}
58
59		const struct_data = info.Struct;
60		if (struct_data.layout == .@"packed") {
61		break :brk @bitCast(items[i]);
62		}
63
64		break :brk @call(
65		.always_inline,
66		getAsU32,
67		.{ items, field.type, i },
68		);
69		},
70		};
71		i += extraSizeOf(field.type);
72		}
73		return result;
74		}
75
76		pub const StoreKind = enum {
77		intern_pool,
78		teg,
79		air,
80		tfr,
81
82		pub fn resolveStoreConst(
83		comptime kind: StoreKind,
84		owner: anytype,
85		) *const kind.ResolvedStore(@TypeOf(owner)) {
86		return &owner.extra;
87		}
88
89		pub fn resolveStoreMut(
90		comptime kind: StoreKind,
91		owner: anytype,
92		) *kind.ResolvedStore(@TypeOf(owner)) {
93		return &owner.extra;
94		}
95
96		pub fn ResolvedStore(
97		comptime kind: StoreKind,
98		comptime Owner: type,
99		) type {
100		const ptr = @typeInfo(Owner).Pointer;
101		assert(ptr.size == .One);
102		const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) \|field\| {
103		if (std.mem.eql(u8, field.name, "extra")) {
104		break :brk field.type.mutability;
105		}
106		} else unreachable;
107		return ExtraStore(kind, mutability);
108		}
109		};
110
111		fn ExtraWrap(comptime kind: StoreKind) type {
112		return struct {
113		pub fn Index(comptime T: type) type {
114		return enum(u32) {
115		_,
116
117		// Reads the data from `store.extra` and returns it.
118		pub fn get(index: @This(), owner: anytype) T {
119		return kind.resolveStoreConst(owner).get(T, @intFromEnum(index));
120		}
121
122		/// Returns data with a trailing reader to read data stored immediately after
123		/// the main data structure. What is stored should be documented on `T`
124		pub fn getTrail(index: @This(), owner: anytype) struct {
125		T,
126		kind.ResolvedStore(@TypeOf(owner)).TrailReader,
127		} {
128		const advance = comptime extraSizeOf(T);
129		return .{
130		index.get(owner),
131		.{ .index = @intFromEnum(index) + advance },
132		};
133		}
134		};
135		}
136		};
137		}
138
139		pub fn EncodedData(comptime Tag: type, backing_type: type) type {
140		return enum(backing_type) {
141		_,
142
143		pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() {
144		assert(proof == tag); // incorrect memory access
145		const Data = tag.Data();
146		const info = @typeInfo(Data);
147		if (info == .Enum) {
148		return @enumFromInt(@intFromEnum(data));
149		}
150		if (info == .Struct) {
151		assert(backing_type == u64);
152		var fields: [2]u32 = @bitCast(@intFromEnum(data));
153		return getAsU32(&fields, Data, 0);
154		}
155		return @bitCast(@intFromEnum(data));
156		}
157
158		pub fn encode(comptime tag: Tag, data: tag.Data()) @This() {
159		const Data = tag.Data();
160		const info = @typeInfo(Data);
161		if (info == .Enum) {
162		return @enumFromInt(@intFromEnum(data));
163		}
164		if (info == .Struct) {
165		assert(backing_type == u64);
166		var fields: [2]u32 = undefined;
167		setAsU32(&fields, Data, 0, data);
168		return @enumFromInt(@as(backing_type, @bitCast(fields)));
169		}
170		return @enumFromInt(@as(backing_type, @bitCast(data)));
171		}
172		};
173		}
174
175		/// The number of 'u32's this type consumes in `ip.extra`.
176		pub fn extraSizeOf(comptime T: type) usize {
177		return switch (T) {
178		u32,
179		i32,
180		=> return 1,
181
182		[16]u8 => return 4,
183
184		else => {
185		const info = @typeInfo(T);
186		if (info == .Enum) {
187		comptime assert(@sizeOf(info.Enum.tag_type) <= 32);
188		return 1;
189		}
190
191		const struct_data = info.Struct;
192		if (struct_data.layout == .@"packed") {
193		comptime assert(struct_data.backing_integer == u32);
194		return 1;
195		}
196
197		const fields = std.meta.fields(T);
198		var size: usize = 0;
199		inline for (fields) \|field\| {
200		size += extraSizeOf(field.type);
201		}
202		return size;
203		},
204		};
205		}
206
207		pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type {
208		return struct {
209		list: List = .{},
210
211		pub const kind = set_kind;
212		pub const mutability = set_mutability;
213
214		const Store = @This();
215
216		const index_holder = ExtraWrap(kind);
217		pub const Index = index_holder.Index;
218
219		pub const List = switch (mutability) {
220		.@"const" => struct {
221		items: []const u32 = &.{},
222
223		pub fn deinit(slice: *@This(), gpa: Allocator) void {
224		gpa.free(slice.items);
225		}
226		},
227		.@"var" => std.ArrayListUnmanaged(u32),
228		};
229
230		pub fn deinit(store: *Store, gpa: Allocator) void {
231		store.list.deinit(gpa);
232		store.* = undefined;
233		}
234
235		pub const TrailReader = struct {
236		index: usize,
237
238		pub fn read(trail: TrailReader, ip: const Store, comptime T: type) T {
239		defer trail.index += comptime extraSizeOf(T);
240		return ip.get(T, trail.index);
241		}
242
243		pub fn readValueSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Value.Slice {
244		defer trail.index += len;
245		return .{
246		.start = @intCast(trail.index),
247		.len = len,
248		};
249		}
250
251		pub fn readTypeSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Type.Slice {
252		defer trail.index += len;
253		return .{
254		.start = @intCast(trail.index),
255		.len = len,
256		};
257		}
258		};
259
260		pub const TrailWriter = struct {
261		store: *Store,
262		leftover: usize,
263
264		pub fn init(store: *Store, len: usize) TrailWriter {
265		return .{ .store = store, .leftover = len };
266		}
267
268		pub fn end(w: *TrailWriter) void {
269		assert(w.leftover == 0);
270		}
271
272		pub fn writeU32(w: *TrailWriter, value: u32) void {
273		assert(w.leftover > 0);
274		w.ip.extra.appendAssumeCapacity(value);
275		w.leftover -= 1;
276		}
277
278		pub fn writeSlice(w: *TrailWriter, slice: anytype) void {
279		const Child = std.meta.Child(@TypeOf(slice));
280		const child = @typeInfo(Child);
281		comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32));
282		assert(w.leftover >= slice.len);
283		w.store.list.appendSliceAssumeCapacity(@ptrCast(slice));
284		w.leftover -= slice.len;
285		}
286
287		pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void {
288		_ = appendAssumeCapacity(w.store, T, data);
289		}
290		};
291
292		pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) {
293		comptime assert(mutability == .@"var");
294		try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)));
295		return store.appendAssumeCapacity(T, payload);
296		}
297
298		pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) {
299		comptime assert(mutability == .@"var");
300		const extra_index: u32 = @intCast(ip.list.items.len);
301		ip.list.items.len += extraSizeOf(@TypeOf(payload));
302		ip.set(T, extra_index, payload);
303		return @enumFromInt(extra_index);
304		}
305
306		pub fn appendTrailing(
307		store: *Store,
308		gpa: Allocator,
309		comptime T: type,
310		payload: T,
311		trailing_len: usize,
312		) Allocator.Error!struct { Index(T), TrailWriter } {
313		comptime assert(mutability == .@"var");
314		try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len);
315		const extra_index: u32 = @intCast(store.list.items.len);
316		store.list.items.len += extraSizeOf(@TypeOf(payload));
317		store.set(T, extra_index, payload);
318		return .{
319		@enumFromInt(extra_index),
320		TrailWriter.init(store, trailing_len),
321		};
322		}
323
324		fn set(store: *Store, comptime T: type, index: usize, payload: T) void {
325		comptime assert(mutability == .@"var");
326		setAsU32(store.list.items, T, index, payload);
327		}
328
329		fn get(store: *const Store, comptime T: type, index: usize) T {
330		return getAsU32(store.list.items, T, index);
331		}
332
333		pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") {
334		comptime assert(mutability == .@"var");
335		return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } };
336		}
337
338		pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") {
339		return .{ .list = .{ .items = store.list.items } };
340		}
341		};
342		}
343
344		pub const Float = struct {
345		low: u32,
346		high: u32,
347
348		pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 };
349
350		pub fn purifyNaN(f: f64) f64 {
351		if (std.math.isNan(f)) {
352		return comptime Float.pure_nan.decode();
353		}
354		return f;
355		}
356
357		pub fn assertPure(f: f64) f64 {
358		if (std.math.isNan(f)) {
359		assert(Float.pure_nan.eql(encode(f)));
360		}
361		return f;
362		}
363
364		pub fn encode(decoded: f64) Float {
365		if (std.math.isNan(decoded)) {
366		return pure_nan;
367		}
368		const as_bytes: [2]u32 = @bitCast(decoded);
369		return .{
370		.low = as_bytes[0],
371		.high = as_bytes[1],
372		};
373		}
374
375		pub fn decode(encoded: Float) f64 {
376		const as_bytes: [2]u32 = .{
377		encoded.low,
378		encoded.high,
379		};
380		return @bitCast(as_bytes);
381		}
382
383		pub fn eql(a: Float, b: Float) bool {
384		return a.low == b.low and a.high == b.high;
385		}
386		};
387
388		const std = @import("std");
389		const assert = std.debug.assert;
390		const Allocator = std.mem.Allocator;

src/compiler/main.zig deleted-146

...	...	@@ -1,146 +0,0 @@
1		const std = @import("std");
2		const builtin = @import("builtin");
3
4		const Tokenizer = @import("Tokenizer.zig");
5		const Parse = @import("Parse.zig");
6		const Transform = @import("Transform.zig");
7		const Comp = @import("Comp.zig");
8		const unicode_props = @import("unicode_props.zig");
9
10		const tracer = @import("tracer");
11
12		pub const tracer_impl = @field(tracer, @import("build_options").tracing);
13
14		pub const Report = @import("Report.zig");
15
16		pub fn main() !void {
17		try tracer.init();
18		defer tracer.deinit();
19		try tracer.init_thread(null);
20		defer tracer.deinit_thread();
21
22		try unicode_props.init();
23
24		if (builtin.mode == .Debug) {
25		std.debug.print("\r\x1b[K", .{});
26		}
27
28		const use_gpa = builtin.mode == .Debug;
29
30		var gpa: if (use_gpa) std.heap.GeneralPurposeAllocator(.{
31		.stack_trace_frames = 100,
32		}) else struct {} = .{};
33		defer if (use_gpa) {
34		_ = gpa.deinit();
35		};
36		const alloc = if (use_gpa)
37		gpa.allocator()
38		else
39		std.heap.c_allocator;
40
41		var maybe_file_path: ?[]const u8 = null;
42		defer if (maybe_file_path) \|path\| alloc.free(path);
43
44		var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
45		const cwd = try std.posix.getcwd(&cwd_buf);
46
47		var exe_name: []const u8 = undefined;
48		var dump_ir = false;
49
50		{
51		var args = try std.process.argsWithAllocator(alloc);
52		defer args.deinit();
53
54		exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
55		errdefer alloc.free(exe_name);
56
57		while (args.next()) \|arg\| {
58		if (std.mem.startsWith(u8, arg, "-")) {
59		if (builtin.mode == .Debug and std.mem.eql(u8, arg, "--report-trace")) {
60		Report.debug_show_report_trace = true;
61		continue;
62		}
63		if (std.mem.eql(u8, arg, "--dump-ir")) {
64		dump_ir = true;
65		continue;
66		}
67		std.debug.print("error: unknown flag '{s}'\n", .{arg});
68		maybe_file_path = null;
69		break;
70		}
71
72		if (maybe_file_path == null) {
73		maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
74		} else {
75		std.debug.print("error: multiple files specified\n", .{});
76		maybe_file_path = null;
77		break;
78		}
79		}
80		}
81		defer alloc.free(exe_name);
82
83		const file_path = maybe_file_path orelse {
84		std.debug.print("Usage: {s} [flags] <input file>\n", .{exe_name});
85		{
86		std.debug.print(
87		// \| 60 chars wrap point\|
88		\\
89		\\Flags:
90		\\ --dump-ir Dump intermediate representations.
91		\\
92		, .{});
93		}
94		if (builtin.mode == .Debug) {
95		std.debug.print(
96		\\
97		\\Debug Mode Flags:
98		\\ --report-trace Print stack traces per error. Very noisy
99		\\
100		, .{});
101		}
102		std.process.exit(1);
103		};
104
105		var comp: Comp = .{
106		.gpa = tracer.tracingAllocator(&alloc),
107		.options = .{
108		.dump_ir = dump_ir,
109		},
110		};
111		try comp.init();
112		defer comp.deinit();
113		const main_file = try comp.getFile(file_path);
114
115		var fail_report: Report = undefined;
116		const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
117		error.TransformFail => {
118		std.debug.print("Transform failed:\n", .{});
119		try fail_report.dumpToStdErr(&comp);
120		std.process.exit(1);
121		},
122		error.ParseFail => {
123		// Already printed.
124		std.process.exit(1);
125		},
126		else => return err,
127		};
128
129		if (comp.options.dump_ir) {
130		const stderr = std.io.getStdErr().writer();
131		try teg.dumpToWriter(stderr);
132		}
133
134		comp.semaEntryPoint(main_file) catch \|err\| switch (err) {
135		error.AnalysisFail => std.process.exit(1),
136		else => return err,
137		};
138
139		const output = std.io.getStdOut().writer();
140		var bufw = std.io.bufferedWriter(output);
141		const writer = bufw.writer();
142		try comp.link(writer);
143		try bufw.flush();
144
145		std.process.cleanExit();
146		}

src/compiler/unicode_props.zig deleted-103

...	...	@@ -1,103 +0,0 @@
1		//! Wrapper around the 'zg' PropsData library.
2		const std = @import("std");
3		const PropsData = @import("zg:PropsData");
4		const T = @import("tracer");
5
6		var props: ?PropsData = undefined;
7		var props_memory: []u8 = undefined;
8
9		const raw_allocator = if (@import("builtin").target.isWasm())
10		std.heap.wasm_allocator
11		else
12		std.heap.page_allocator;
13
14		pub fn init() !void {
15		if (props != null) return;
16
17		const trace = T.trace(@src());
18		defer trace.end();
19
20		// We know exactly how much this uses, so let's preallocate these pages and use a FBA.
21		const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856;
22
23		if (allocation_size == 0) {
24		var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator);
25		props = try PropsData.init(logging_alloc.allocator());
26		std.log.info("please write the above numbers into unicode.zig to prealloc this", .{});
27		std.process.exit(1);
28		}
29
30		props_memory = try raw_allocator.alloc(u8, allocation_size);
31		errdefer std.heap.page_allocator.free(props_memory);
32
33		var fba = std.heap.FixedBufferAllocator.init(props_memory);
34		props = try PropsData.init(fba.allocator());
35		std.debug.assert(fba.end_index == allocation_size);
36		}
37
38		pub fn deinit() void {
39		std.debug.assert(props != null);
40		raw_allocator.free(props_memory);
41		props = null;
42		props_memory = undefined;
43		}
44
45		/// True if `cp` is a mathematical symbol.
46		pub inline fn isMath(cp: u21) bool {
47		return props.?.isMath(cp);
48		}
49
50		/// True if `cp` is an alphabetic character.
51		pub inline fn isAlphabetic(cp: u21) bool {
52		return props.?.isAlphabetic(cp);
53		}
54
55		/// True if `cp` is a valid identifier start character.
56		pub inline fn isIdStart(cp: u21) bool {
57		return props.?.isIdStart(cp);
58		}
59
60		/// True if `cp` is a valid identifier continuation character.
61		pub inline fn isIdContinue(cp: u21) bool {
62		return props.?.isIdContinue(cp);
63		}
64
65		/// True if `cp` is a valid extended identifier start character.
66		pub inline fn isXidStart(cp: u21) bool {
67		return props.?.isXidStart(cp);
68		}
69
70		/// True if `cp` is a valid extended identifier continuation character.
71		pub inline fn isXidContinue(cp: u21) bool {
72		return props.?.isXidContinue(cp);
73		}
74
75		/// True if `cp` is a whitespace character.
76		pub inline fn isWhitespace(cp: u21) bool {
77		return props.?.isWhitespace(cp);
78		}
79
80		/// True if `cp` is a hexadecimal digit.
81		pub inline fn isHexDigit(cp: u21) bool {
82		return props.?.isHexDigit(cp);
83		}
84
85		/// True if `cp` is a diacritic mark.
86		pub inline fn isDiacritic(cp: u21) bool {
87		return props.?.isDiacritic(cp);
88		}
89
90		/// True if `cp` is numeric.
91		pub inline fn isNumeric(cp: u21) bool {
92		return props.?.isNumeric(cp);
93		}
94
95		/// True if `cp` is a digit.
96		pub inline fn isDigit(cp: u21) bool {
97		return props.?.isDigit(cp);
98		}
99
100		/// True if `cp` is decimal.
101		pub inline fn isDecimal(cp: u21) bool {
102		return props.?.isDecimal(cp);
103		}

src/compiler/util.zig deleted-56

...	...	@@ -1,56 +0,0 @@
1		pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
2		if (@hasField(@TypeOf(writer), "unbuffered_writer")) {
3		return detectTTYFromWriter(writer.unbuffered_writer);
4		}
5		if (@TypeOf(writer) == std.fs.File.Writer) {
6		return std.io.tty.detectConfig(writer.context);
7		}
8		if (@TypeOf(writer) == std.io.AnyWriter) {
9		const fileWriteFn = comptime (std.fs.File{ .handle = undefined })
10		.writer().any().writeFn;
11		if (writer.writeFn == fileWriteFn) {
12		const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
13		return std.io.tty.detectConfig(file.*);
14		}
15		}
16		return .no_color;
17		}
18
19		pub fn Ordinal(Int: type) type {
20		comptime assert(@typeInfo(Int).Int.signedness == .unsigned);
21		return enum(Int) {
22		start = 0,
23		_,
24
25		pub fn fromZeroBased(int: Int) @This() {
26		assert(int != std.math.maxInt(Int));
27		return @enumFromInt(int);
28		}
29		pub fn fromOneBased(int: Int) @This() {
30		assert(int > 0);
31		return @enumFromInt(int - 1);
32		}
33		pub fn zeroBased(ord: @This()) Int {
34		return @intFromEnum(ord);
35		}
36		pub fn oneBased(ord: @This()) Int {
37		return @intFromEnum(ord) + 1;
38		}
39
40		pub fn add(ord: @This(), inc: Int) @This() {
41		return fromZeroBased(ord.zeroBased() + inc);
42		}
43
44		pub fn format(
45		_: @This(),
46		comptime _: []const u8,
47		_: std.fmt.FormatOptions,
48		_: anytype,
49		) !void {
50		@compileError("Do not format `Ordinal` directly. Call oneBased or zeroBased to specify.");
51		}
52		};
53		}
54
55		const std = @import("std");
56		const assert = std.debug.assert;

src/core/Air.Dump.zig created+316

...	...	@@ -0,0 +1,316 @@
	1	//! This implements a debug formatter for Teg
	2	air: Air,
	3	ip: *const InternPool,
	4
	5	pub const State = struct {
	6	arena: std.mem.Allocator,
	7	air: *const Air,
	8	ip: *const InternPool,
	9	indent: usize = 0,
	10	tty: std.io.tty.Config,
	11	visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{},
	12
	13	fn newline(state: State, writer: anytype) !void {
	14	try writer.writeByte('\n');
	15	if (state.indent > 0)
	16	try writer.writeByteNTimes(' ', state.indent * 2);
	17	}
	18
	19	fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
	20	const air = s.air;
	21	const tty = s.tty;
	22	const tag = s.air.instTag(inst);
	23
	24	try tty.setColor(w, switch (s.air.instFlags(inst).uses) {
	25	.none => .bright_black,
	26	.multi => .red,
	27	.single => .magenta,
	28	});
	29	try w.writeByte(switch (s.air.instFlags(inst).uses) {
	30	.none => 'X',
	31	.multi => 'M',
	32	.single => 'S',
	33	});
	34	try tty.setColor(w, .cyan);
	35
	36	try w.print(" #{} = ", .{
	37	@intFromEnum(inst),
	38	});
	39
	40	try tty.setColor(w, .cyan);
	41	try w.print("{s}", .{@tagName(tag)});
	42	try tty.setColor(w, .reset);
	43
	44	if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
	45	try tty.setColor(w, .red);
	46	try w.writeAll(" [error: recursive tree]");
	47	try tty.setColor(w, .reset);
	48	return;
	49	}
	50
	51	switch (tag) {
	52	.block => {
	53	const index_list: Air.IndexList = air.instData(.block, inst);
	54	const insts = index_list.slice(air);
	55	try s.dumpInstList(w, insts, '{', '}');
	56	},
	57	.js_global_ref => {
	58	const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst);
	59	try tty.setColor(w, .magenta);
	60	try w.print(" {s}", .{name.toSlice(s.ip)});
	61	try tty.setColor(w, .reset);
	62	},
	63	.arg => {
	64	const arg: Air.Arg = air.instData(.arg, inst);
	65	try w.writeAll(" ");
	66	try tty.setColor(w, .yellow);
	67	try w.print("{d}", .{arg.arg_number});
	68	try tty.setColor(w, .reset);
	69	try w.writeAll(": ");
	70	try s.dumpValue(w, arg.type.value());
	71	},
	72	.@"break" => {
	73	const brk: Air.Break = air.instData(.@"break", inst);
	74	try w.writeAll("(");
	75	try s.dumpInstRef(w, brk.block_ref);
	76	try w.writeAll(", ");
	77	try s.dumpRef(w, brk.value);
	78	try w.writeAll(")");
	79	},
	80	.store => {
	81	const store: Air.Store = air.instData(.store, inst);
	82	try w.writeAll("(");
	83	try s.dumpInstRef(w, store.memory);
	84	try w.writeAll(", ");
	85	try s.dumpRef(w, store.operand);
	86	try w.writeAll(")");
	87	},
	88	inline .add,
	89	.mul,
	90	.sub,
	91	.div,
	92	.lt,
	93	.gt,
	94	.eq,
	95	=> \|comptime_tag\| {
	96	const bin: Air.Bin = air.instData(comptime_tag, inst);
	97	try w.writeAll("(");
	98	try s.dumpRef(w, bin.lhs);
	99	try w.writeAll(", ");
	100	try s.dumpRef(w, bin.rhs);
	101	try w.writeAll(")");
	102	},
	103	inline .negate,
	104	.bool_not,
	105	=> \|comptime_tag\| {
	106	const bin: Air.Un = air.instData(comptime_tag, inst);
	107	try w.writeAll("(");
	108	try s.dumpRef(w, bin.operand);
	109	try w.writeAll(")");
	110	},
	111	.memory => {
	112	const memory: Air.Memory = air.instData(.memory, inst);
	113	try w.writeAll("(");
	114	try s.dumpRef(w, memory.init);
	115	try w.writeAll(", type: ");
	116	try s.dumpValue(w, memory.ty.value());
	117	try w.writeAll(")");
	118	},
	119	.field => {
	120	const field: Air.Field = air.instData(.field, inst);
	121	try w.writeAll("(");
	122	try s.dumpRef(w, field.base);
	123	try w.writeAll(", ");
	124	try tty.setColor(w, .magenta);
	125	try w.print("{s}", .{field.field.toSlice(s.ip)});
	126	try tty.setColor(w, .reset);
	127	try w.writeAll(")");
	128	},
	129	.call => {
	130	const call: Air.Call = air.instData(.call, inst);
	131	const call_data = call.call_data.get(air);
	132	try w.writeAll("(");
	133	try s.dumpRef(w, call.base);
	134	try w.writeAll(", ");
	135	try s.dumpRefList(w, call_data.args.slice(air), '[', ']');
	136	try w.writeAll(")");
	137	},
	138	.@"if" => {
	139	const if_inst: Air.If = air.instData(.@"if", inst);
	140	const if_data: Air.If.Data = if_inst.data.get(air);
	141	try w.writeAll("(");
	142	try s.dumpRef(w, if_inst.cond);
	143	try w.writeAll(", {");
	144	s.indent += 1;
	145	try s.newline(w);
	146	try w.writeAll("yes: ");
	147	try s.dumpRefRecurse(w, if_data.yes);
	148	try s.newline(w);
	149	try w.writeAll("no: ");
	150	try s.dumpRefRecurse(w, if_data.no);
	151	s.indent -= 1;
	152	try s.newline(w);
	153	try w.writeAll("}");
	154	},
	155	}
	156	}
	157
	158	fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void {
	159	switch (ref.decode()) {
	160	.index => \|i\| try s.dumpInstRef(w, i),
	161	.value => \|v\| try s.dumpValue(w, v),
	162	}
	163	}
	164
	165	fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void {
	166	switch (ref.decode()) {
	167	.index => \|i\| try s.dumpInst(w, i),
	168	.value => \|v\| try s.dumpValue(w, v),
	169	}
	170	}
	171
	172	fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
	173	try s.tty.setColor(w, .red);
	174	try w.print("#{d}", .{@intFromEnum(inst)});
	175	try s.tty.setColor(w, .reset);
	176	}
	177
	178	pub fn dumpValue(s: *State, w: anytype, val: Value) !void {
	179	const tty = s.tty;
	180	switch (val.toKey(s.ip)) {
	181	.simple_type => \|t\| {
	182	try tty.setColor(w, .blue);
	183	try w.print("{s}", .{@tagName(t)});
	184	try tty.setColor(w, .reset);
	185	},
	186	.simple_value => \|t\| {
	187	try tty.setColor(w, .yellow);
	188	try w.print("{s}", .{@tagName(t)});
	189	try tty.setColor(w, .reset);
	190	},
	191	.type_function => \|ty\| {
	192	try tty.setColor(w, .blue);
	193	try w.writeAll("fn(");
	194	try tty.setColor(w, .reset);
	195	for (ty.param_types, 0..) \|param, i\| {
	196	if (i != 0) {
	197	try tty.setColor(w, .blue);
	198	try w.writeAll(",");
	199	try tty.setColor(w, .reset);
	200	}
	201	try s.dumpValue(w, param.value());
	202	}
	203	try tty.setColor(w, .blue);
	204	try w.writeAll(") -> ");
	205	try tty.setColor(w, .reset);
	206	try s.dumpValue(w, ty.return_type.value());
	207	},
	208	inline .float, .int32, .comptime_float => \|data, tag\| {
	209	try tty.setColor(w, .bright_black);
	210	try w.writeAll("<");
	211	try tty.setColor(w, .blue);
	212	try w.writeAll(@tagName(tag));
	213	try tty.setColor(w, .reset);
	214	try tty.setColor(w, .bright_black);
	215	try w.writeAll(", ");
	216	try tty.setColor(w, .yellow);
	217	try w.print("{d}", .{data});
	218	try tty.setColor(w, .bright_black);
	219	try w.writeAll(">");
	220	try tty.setColor(w, .reset);
	221	},
	222	.string => \|data\| {
	223	try w.writeAll("<");
	224	try tty.setColor(w, .blue);
	225	try w.writeAll("string");
	226	try tty.setColor(w, .reset);
	227	try w.writeAll(", ");
	228	try tty.setColor(w, .green);
	229	try w.writeAll("\"");
	230	try w.writeAll(data);
	231	try w.writeAll("\"");
	232	try tty.setColor(w, .reset);
	233	try w.writeAll(">");
	234	},
	235	.func => \|func\| {
	236	try tty.setColor(w, .bright_black);
	237	try w.writeAll("<");
	238	try s.dumpValue(w, func.ty.value());
	239	try tty.setColor(w, .bright_black);
	240	try w.writeAll(", ");
	241	const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip);
	242	try tty.setColor(w, .bright_green);
	243	try w.print("decl '{s}'", .{decl_name});
	244	try tty.setColor(w, .bright_black);
	245	try w.writeAll(">");
	246	try tty.setColor(w, .reset);
	247	},
	248	}
	249	}
	250
	251	fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void {
	252	try writer.print(" {c}", .{l});
	253	if (list.len > 0) {
	254	state.indent += 1;
	255
	256	for (list) \|inst\| {
	257	try state.newline(writer);
	258	try state.dumpInst(writer, inst);
	259	}
	260
	261	state.indent -= 1;
	262	try state.newline(writer);
	263	}
	264	try writer.print("{c}", .{r});
	265	}
	266
	267	fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void {
	268	try writer.print(" {c}", .{l});
	269	if (list.len > 0) {
	270	state.indent += 1;
	271
	272	for (list) \|inst\| {
	273	try state.newline(writer);
	274	try state.dumpRef(writer, inst);
	275	}
	276
	277	state.indent -= 1;
	278	try state.newline(writer);
	279	}
	280	try writer.print("{c}", .{r});
	281	}
	282	};
	283
	284	pub fn format(
	285	dump: @This(),
	286	comptime unused_fmt: []const u8,
	287	_: std.fmt.FormatOptions,
	288	writer: anytype,
	289	) !void {
	290	comptime assert(unused_fmt.len == 0);
	291
	292	var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
	293	defer arena.deinit();
	294
	295	const air = dump.air;
	296	var state = State{
	297	.air = &air,
	298	.ip = dump.ip,
	299	.tty = detectTTYFromWriter(writer),
	300	.arena = arena.allocator(),
	301	};
	302	try state.dumpInst(writer, .main_block);
	303	try writer.writeAll("\n");
	304	assert(state.indent == 0);
	305	}
	306
	307	const std = @import("std");
	308	const assert = std.debug.assert;
	309
	310	const Air = @import("Air.zig");
	311	const Inst = Air.Inst;
	312
	313	const InternPool = @import("InternPool.zig");
	314	const Value = InternPool.Value;
	315
	316	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/core/Air.zig created+320

...	...	@@ -0,0 +1,320 @@
	1	insts: Inst.List.Slice,
	2	extra: ExtraStore = .{},
	3
	4	pub const ExtraStore = dod.ExtraStore(.air, .@"const");
	5	pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
	6	pub const ExtraIndex = ExtraStore.Index;
	7
	8	pub const Inst = struct {
	9	tag: Tag,
	10	data: EncodedData,
	11	flags: Flags,
	12
	13	pub const List = std.MultiArrayList(Inst);
	14
	15	pub const Tag = enum {
	16	block,
	17	js_global_ref,
	18	call,
	19	@"break",
	20	@"if",
	21	arg,
	22	field,
	23
	24	memory,
	25	store,
	26
	27	// bin
	28	add,
	29	sub,
	30	mul,
	31	div,
	32
	33	eq,
	34
	35	lt,
	36	gt,
	37
	38	// un
	39	negate,
	40	bool_not,
	41
	42	pub fn Data(comptime tag: Tag) type {
	43	return switch (tag) {
	44	.block => IndexList,
	45	.js_global_ref => InternPool.String.WithLen,
	46	.@"break" => Break,
	47	.@"if" => If,
	48	.call => Call,
	49	.arg => Arg,
	50	.field => Field,
	51	.memory => Memory,
	52	.store => Store,
	53	.add,
	54	.sub,
	55	.mul,
	56	.div,
	57	.lt,
	58	.gt,
	59	.eq,
	60	=> Bin,
	61	.negate, .bool_not => Un,
	62	};
	63	}
	64	};
	65
	66	/// reference to either a value or another instruction in this air
	67	pub const Ref = packed struct(u32) {
	68	kind: Kind,
	69	data: u31,
	70
	71	pub const Kind = enum(u1) { value, index };
	72
	73	pub const Decoded = union(enum) {
	74	value: Value,
	75	index: Index,
	76	};
	77
	78	pub fn decode(ref: Ref) Decoded {
	79	return switch (ref.kind) {
	80	.value => .{ .value = @enumFromInt(ref.data) },
	81	.index => .{ .index = @enumFromInt(ref.data) },
	82	};
	83	}
	84
	85	pub fn toOptional(ref: ?Ref) Optional {
	86	return if (ref) \|r\| @enumFromInt(@as(u32, @bitCast(r))) else .none;
	87	}
	88
	89	pub const Optional = enum(u32) {
	90	none = std.math.maxInt(u32),
	91	_,
	92
	93	pub fn unwrap(opt: Optional) ?Ref {
	94	return if (opt != .none) @bitCast(@intFromEnum(opt)) else null;
	95	}
	96	};
	97
	98	pub fn isVoid(ref: Ref) bool {
	99	return ref.kind == .value and
	100	ref.decode().value == .void;
	101	}
	102
	103	pub fn eql(ref: Ref, other: Ref) bool {
	104	return @as(u32, @bitCast(ref)) == @as(u32, @bitCast(other));
	105	}
	106	};
	107
	108	/// reference to another instruction in this air
	109	/// note that these cannot point to code in other functions, such makes no sense
	110	pub const Index = enum(u32) {
	111	main_block,
	112	_,
	113
	114	pub fn toRef(index: Index) Ref {
	115	return .{
	116	.kind = .index,
	117	.data = @intCast(@intFromEnum(index)),
	118	};
	119	}
	120
	121	pub fn toOptional(index: Index) Optional {
	122	return @enumFromInt(@intFromEnum(index));
	123	}
	124
	125	pub const Optional = enum(u32) {
	126	none = std.math.maxInt(u32),
	127	_,
	128
	129	pub fn unwrap(oi: Optional) ?Index {
	130	return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null;
	131	}
	132	};
	133	};
	134	};
	135
	136	pub const Flags = packed struct {
	137	uses: Uses = .none,
	138
	139	pub const Uses = enum(u2) {
	140	none,
	141	single,
	142	multi,
	143
	144	pub fn increment(uses: Uses) Uses {
	145	return switch (uses) {
	146	.none => .single,
	147	.single, .multi => .multi,
	148	};
	149	}
	150	};
	151	};
	152
	153	pub const IndexList = struct {
	154	start: u32,
	155	len: u32,
	156
	157	pub fn slice(self: IndexList, air: *const Air) []const Inst.Index {
	158	return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
	159	}
	160	};
	161
	162	pub const RefList = struct {
	163	start: u32,
	164	len: u32,
	165
	166	pub fn slice(self: RefList, air: *const Air) []const Inst.Ref {
	167	return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
	168	}
	169	};
	170
	171	pub const Break = struct {
	172	block_ref: Inst.Index,
	173	value: Inst.Ref,
	174	};
	175
	176	pub const Bin = struct {
	177	lhs: Inst.Ref,
	178	rhs: Inst.Ref,
	179	};
	180
	181	pub const Store = struct {
	182	memory: Inst.Index,
	183	operand: Inst.Ref,
	184	};
	185
	186	pub const Un = struct {
	187	operand: Inst.Ref,
	188	unused: Unused = .zero,
	189	};
	190
	191	pub const Field = struct {
	192	base: Inst.Ref,
	193	field: InternPool.NullTerminatedString,
	194	};
	195
	196	pub const Call = struct {
	197	base: Inst.Ref,
	198	call_data: ExtraIndex(Data),
	199
	200	pub const Data = struct {
	201	args: RefList,
	202	};
	203	};
	204
	205	pub const Arg = struct {
	206	type: Type,
	207	arg_number: u32,
	208	};
	209
	210	pub const If = struct {
	211	cond: Inst.Ref,
	212	data: ExtraIndex(Data),
	213
	214	pub const Data = struct {
	215	yes: Inst.Ref,
	216	no: Inst.Ref,
	217	union_ty: Type,
	218	};
	219	};
	220
	221	pub const Memory = struct {
	222	init: Inst.Ref,
	223	ty: Type,
	224	};
	225
	226	pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
	227	defer air.* = undefined;
	228	air.insts.deinit(gpa);
	229	air.extra.deinit(gpa);
	230	}
	231
	232	pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag {
	233	return air.insts.items(.tag)[@intFromEnum(index)];
	234	}
	235
	236	pub fn instFlags(air: *const Air, index: Inst.Index) Flags {
	237	return air.insts.items(.flags)[@intFromEnum(index)];
	238	}
	239
	240	pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData {
	241	return air.insts.items(.data)[@intFromEnum(index)];
	242	}
	243
	244	pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
	245	return air.instEncodedData(index).decode(tag, air.instTag(index));
	246	}
	247
	248	pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
	249	return air.instData(tag, index).extra_index.get(air);
	250	}
	251
	252	pub fn typeOf(air: const Air, ref: Inst.Ref, ip: InternPool) InternPool.Type {
	253	return switch (ref.decode()) {
	254	.value => \|v\| v.typeOf(ip),
	255	.index => \|i\| air.typeOfIndex(i, ip),
	256	};
	257	}
	258
	259	pub fn typeOfIndex(air: const Air, index: Inst.Index, ip: InternPool) InternPool.Type {
	260	return switch (air.instTag(index)) {
	261	.js_global_ref => .@"opaque",
	262
	263	.block => {
	264	const items = air.instData(.block, index).slice(air);
	265	return air.typeOfIndex(items[items.len - 1], ip);
	266	},
	267
	268	inline .add, .sub, .mul, .div => \|t\| air.typeOf(air.instData(t, index).lhs, ip),
	269
	270	.lt, .gt, .eq, .bool_not => .bool,
	271
	272	.arg => air.instData(.arg, index).type,
	273
	274	.negate => air.typeOf(air.instData(.negate, index).operand, ip),
	275
	276	.@"break" => .noreturn,
	277
	278	// todo:
	279	.field => .@"opaque",
	280
	281	.call => {
	282	const base_type = air.typeOf(air.instData(.call, index).base, ip);
	283	if (base_type == .@"opaque") return .@"opaque";
	284	return base_type.value().toKey(ip).type_function.return_type;
	285	},
	286
	287	.memory => air.instData(.memory, index).ty,
	288
	289	.store => air.instData(.memory, air.instData(.store, index).memory).ty,
	290
	291	.@"if" => air.instData(.@"if", index).data.get(air).union_ty,
	292	};
	293	}
	294
	295	pub fn paramCount(air: Air) u32 {
	296	for (air.insts.items(.tag)[1..], 0..) \|tag, i\| {
	297	if (tag != .arg) return @intCast(i);
	298	}
	299	return @intCast(air.insts.len);
	300	}
	301
	302	pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
	303	try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
	304	}
	305
	306	pub const Dump = @import("Air.Dump.zig");
	307
	308	const Air = @This();
	309	const std = @import("std");
	310	const mem = std.mem;
	311	const assert = std.debug.assert;
	312
	313	const InternPool = @import("InternPool.zig");
	314	const Value = InternPool.Value;
	315	const Type = InternPool.Type;
	316
	317	const Teg = @import("Teg.zig");
	318	const dod = @import("array.zig");
	319
	320	const Unused = enum(u32) { zero = 0 };

src/core/Ast.Dump.zig created+300

...	...	@@ -0,0 +1,300 @@
	1	//! A flat array is not very useful for debugging, so this structure implements
	2	//! a debug formatter for Ast.
	3	ast: Ast,
	4
	5	const State = struct {
	6	ast: *const Ast,
	7	indent: usize = 0,
	8	node_tags: []const Node.Tag,
	9	node_data: []const Node.Data,
	10	node_tokens: []const TokenIndex,
	11	tty: std.io.tty.Config,
	12
	13	fn newline(state: State, writer: anytype) !void {
	14	try writer.writeByte('\n');
	15	if (state.indent > 0)
	16	try writer.writeByteNTimes(' ', state.indent * 2);
	17	}
	18	};
	19
	20	pub fn format(
	21	dump: Dump,
	22	comptime unused_fmt: []const u8,
	23	_: std.fmt.FormatOptions,
	24	writer: anytype,
	25	) !void {
	26	comptime assert(unused_fmt.len == 0);
	27	const ast = dump.ast;
	28	var state = State{
	29	.ast = &ast,
	30	.node_tags = ast.nodes.items(.tag),
	31	.node_data = ast.nodes.items(.data),
	32	.node_tokens = ast.nodes.items(.main_token),
	33	.tty = detectTTYFromWriter(writer),
	34	};
	35	try dumpNode(writer, &state, 0);
	36	try writer.writeAll("\n");
	37	assert(state.indent == 0);
	38	}
	39
	40	fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void {
	41	const tag = state.node_tags[node];
	42	const data = state.node_data[node];
	43	const data_tag = Node.data_tags[@intFromEnum(tag)];
	44	const main_token = state.node_tokens[node];
	45	const tty = state.tty;
	46
	47	try state.tty.setColor(writer, .bright_black);
	48	try writer.print("{d}", .{node});
	49	try state.tty.setColor(writer, .reset);
	50	try writer.writeAll(" ");
	51
	52	try tty.setColor(writer, .cyan);
	53	try writer.print("{s}", .{@tagName(tag)});
	54	try tty.setColor(writer, .reset);
	55
	56	switch (tag) {
	57	.builtin_call => {
	58	try tty.setColor(writer, .red);
	59	try writer.print(" {s}", .{state.ast.tokenSlice(main_token)});
	60	try tty.setColor(writer, .reset);
	61	},
	62	else => {},
	63	}
	64
	65	switch (data_tag) {
	66	.none => {
	67	switch (tag) {
	68	.id => try tty.setColor(writer, .red),
	69	.string => try tty.setColor(writer, .green),
	70	.number_int, .number_float => try tty.setColor(writer, .yellow),
	71	else => {},
	72	}
	73	switch (tag) {
	74	.id,
	75	.string,
	76	.number_int,
	77	.number_float,
	78	=> try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}),
	79
	80	else => unreachable,
	81	}
	82	try tty.setColor(writer, .reset);
	83	},
	84	.fn_decl => {
	85	try writer.writeAll(" ");
	86	const fn_decl = state.ast.fullFnDecl(node);
	87	if (fn_decl.public) {
	88	try writer.writeAll("[pub] ");
	89	}
	90	try tty.setColor(writer, .magenta);
	91	try writer.writeAll(fn_decl.id);
	92	try tty.setColor(writer, .reset);
	93	try writer.writeAll(" {");
	94	{
	95	state.indent += 1;
	96	defer state.indent -= 1;
	97	try state.newline(writer);
	98	try writer.writeAll("params:");
	99	try dumpNodeList(writer, state, fn_decl.params, '[', ']');
	100	if (fn_decl.return_type) \|return_type\| {
	101	try state.newline(writer);
	102	try writer.writeAll("return_type: ");
	103	try dumpNode(writer, state, return_type);
	104	}
	105	try state.newline(writer);
	106	try writer.writeAll("body: ");
	107	try dumpNode(writer, state, fn_decl.body);
	108	}
	109	try state.newline(writer);
	110	try writer.writeAll("}");
	111	},
	112	.list => {
	113	const list = data.list.slice(state.ast);
	114	switch (tag) {
	115	.root,
	116	.block_void,
	117	.block_implicit_return,
	118	=> try dumpNodeList(writer, state, list, '{', '}'),
	119	.builtin_call,
	120	=> try dumpNodeList(writer, state, list, '(', ')'),
	121	else => unreachable,
	122	}
	123	},
	124	.var_decl => {
	125	try writer.writeAll(" ");
	126	const var_decl = state.ast.fullVarDecl(node);
	127	if (var_decl.public) {
	128	try writer.writeAll("[pub] ");
	129	}
	130	try writer.writeAll(switch (var_decl.is_const) {
	131	true => "constant",
	132	false => "mutable",
	133	});
	134	try tty.setColor(writer, .magenta);
	135	try writer.print(" {s}", .{var_decl.id});
	136	try tty.setColor(writer, .reset);
	137	var indent = false;
	138	if (var_decl.type) \|type_node\| {
	139	state.indent += 1;
	140	indent = true;
	141	try state.newline(writer);
	142	try writer.writeAll(": ");
	143	try dumpNode(writer, state, type_node);
	144	try writer.writeAll(", ");
	145	}
	146	try writer.writeAll(" = ");
	147	try dumpNode(writer, state, var_decl.init);
	148	if (indent) {
	149	state.indent -= 1;
	150	}
	151	},
	152	.param_decl => {
	153	try writer.writeAll(" ");
	154	const param_decl = state.ast.fullParamDecl(node);
	155	try tty.setColor(writer, .magenta);
	156	try writer.writeAll(param_decl.id);
	157	try tty.setColor(writer, .reset);
	158	try writer.writeAll(": ");
	159	try dumpNode(writer, state, param_decl.type);
	160	},
	161	.bin => {
	162	try writer.writeAll(" (");
	163	{
	164	state.indent += 1;
	165	defer state.indent -= 1;
	166	try state.newline(writer);
	167	try dumpNode(writer, state, data.bin.lhs);
	168	try state.newline(writer);
	169	try dumpNode(writer, state, data.bin.rhs);
	170	}
	171	try state.newline(writer);
	172	try writer.writeAll(")");
	173	},
	174	.un => {
	175	try writer.writeAll(" (");
	176	{
	177	state.indent += 1;
	178	defer state.indent -= 1;
	179	try state.newline(writer);
	180	try dumpNode(writer, state, data.un.expr);
	181	}
	182	try state.newline(writer);
	183	try writer.writeAll(")");
	184	},
	185	.label => {
	186	const label_text = state.ast.tokenSlice(main_token);
	187	try writer.print(" {s}:", .{label_text});
	188
	189	try writer.writeAll(" (");
	190	{
	191	state.indent += 1;
	192	defer state.indent -= 1;
	193	try state.newline(writer);
	194	try dumpNode(writer, state, data.label.expr);
	195	}
	196	try state.newline(writer);
	197	try writer.writeAll(")");
	198	},
	199	.field => {
	200	try writer.writeAll(" (");
	201	{
	202	state.indent += 1;
	203	defer state.indent -= 1;
	204	try state.newline(writer);
	205	try dumpNode(writer, state, data.field.base);
	206	}
	207	try state.newline(writer);
	208	try writer.writeAll(")");
	209	},
	210	.@"return" => {
	211	if (data.@"return".expr.unwrap()) \|expr\| {
	212	try writer.writeAll(" (");
	213	try dumpNode(writer, state, expr);
	214	try writer.writeAll(")");
	215	}
	216	},
	217	.@"break" => {
	218	const label = data.@"break".label;
	219	const value = data.@"break".value;
	220	if (label != .none or value != .none) {
	221	// try writer.writeAll("(");
	222	// try writer.writeAll(")");
	223	if (label.unwrap()) \|l\| {
	224	const label_text = state.ast.tokenSlice(l);
	225	try writer.print(" :{s}", .{label_text});
	226	}
	227	if (value.unwrap()) \|expr\| {
	228	try writer.writeAll(" (");
	229	try dumpNode(writer, state, expr);
	230	try writer.writeAll(")");
	231	}
	232	}
	233	},
	234	.call => {
	235	const base = data.call.base;
	236	const extra = state.ast.extraData(data.call.extra, Call);
	237	const arg_list = extra.args.slice(state.ast);
	238
	239	try writer.writeAll("(");
	240	try writer.writeAll("base: ");
	241	try dumpNode(writer, state, base);
	242	try writer.writeAll(", ");
	243	try writer.writeAll("arg:");
	244	try dumpNodeList(writer, state, arg_list, '[', ']');
	245	try writer.writeAll(")");
	246	},
	247	.@"if" => {
	248	const branch = data.@"if";
	249	const if_data = state.ast.extraData(branch.extra, Ast.If);
	250
	251	try writer.writeAll(" (");
	252	try dumpNode(writer, state, branch.cond);
	253	try writer.writeAll(") {");
	254	{
	255	state.indent += 1;
	256	defer state.indent -= 1;
	257	try state.newline(writer);
	258	try writer.writeAll("yes: ");
	259	try dumpNode(writer, state, if_data.yes);
	260	try state.newline(writer);
	261	if (if_data.no.unwrap()) \|no\| {
	262	try writer.writeAll("no: ");
	263	try dumpNode(writer, state, no);
	264	} else {
	265	try writer.writeAll("no: [none]");
	266	}
	267	}
	268	try state.newline(writer);
	269	try writer.writeAll("}");
	270	},
	271	}
	272	}
	273
	274	fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void {
	275	try writer.print(" {c}", .{l});
	276	{
	277	state.indent += 1;
	278	defer state.indent -= 1;
	279
	280	for (list) \|node\| {
	281	try state.newline(writer);
	282	try dumpNode(writer, state, node);
	283	}
	284	}
	285	try state.newline(writer);
	286	try writer.print("{c}", .{r});
	287	}
	288
	289	const Dump = @This();
	290
	291	const std = @import("std");
	292	const assert = std.debug.assert;
	293
	294	const Ast = @import("Ast.zig");
	295	const NodeIndex = Ast.NodeIndex;
	296	const Node = Ast.Node;
	297	const TokenIndex = Ast.TokenIndex;
	298	const Call = Ast.Call;
	299
	300	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/core/Ast.zig created+678

...	...	@@ -0,0 +1,678 @@
	1	//! The Abstract Syntax Tree preserves most details of the source code,
	2	//! as it is used for formatting. Before semantic analysis, this
	3	//! structure is normalized into `Teg`, which removes constructs like
	4	//! grouping, unneded let bindings, and other transforms.
	5	//!
	6	//! `ast.nodes` contains nodes stored as three parts:
	7	//! - tag: the type of the node
	8	//! - main_token: index to it's primary token, depends on the tag
	9	//! - data: two u32 values, depends on the tag
	10	//!
	11	//! Many nodes encode all their metadata with just `node.data`, using
	12	//! facts about the language grammar to be able to infer data without
	13	//! storing it.
	14	//!
	15	//! For example, `var_decl` has the following encoding:
	16	//! - `main_token` is either a `kw_var` or `kw_let`
	17	//! - the token after `main_token` is always the identifier
	18	//! - `data.var_decl` contains `NodeIndex`es pointing at the type
	19	//! and the initializer, which are both expressions.
	20	//! - If the keyword before `main_token` is `kw_pub` it is a public variable.
	21	//!
	22	//! To make decoding these easier, many nodes contain a "full" variant,
	23	//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`.
	24	//!
	25	//! For nodes that must store more than 64 bits of payload data, an "extra"
	26	//! struct is stored out-of-line in the `extra` field. These can be read
	27	//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use
	28	//! the `full` structures with their convenience functions.
	29	//!
	30	//! Documentation comments are not stored anywhere, but you can find them
	31	//! by iterating backwards from a declaration and seeing if a `.doc_comment`
	32	//! is present. This is provided by `ast.declDocComment(id)`.
	33	//!
	34	//! The `Ast` owns token and node data, but not the source code.
	35
	36	/// Externally owned source code.
	37	utf8_source: [:0]const u8,
	38
	39	tokens: Token.List.Slice,
	40	nodes: Node.List.Slice,
	41	extra: []const u32,
	42
	43	pub const Node = struct {
	44	tag: Tag,
	45	main_token: TokenIndex,
	46	data: Data,
	47
	48	pub const List = std.MultiArrayList(Node);
	49
	50	pub const Tag = enum {
	51	/// Read with `ast.rootNodes()`
	52	/// data is .list (containing stmt slice)
	53	/// TODO: rename to `module`
	54	root,
	55	/// main_token is the {
	56	/// data is .list (containing stmt slice)
	57	///
	58	/// This guarantees that the block either empty, or the last statement
	59	/// ends with a semicolon.
	60	block_void,
	61	/// main_token is the {
	62	/// data is .list (containing stmt slice)
	63	///
	64	/// This guarantees that the block is not empty and that the last
	65	/// statement is an expression statement without a trailing semicolon.
	66	block_implicit_return,
	67	/// Variable or constant declaration
	68	/// `var x: type = init;`
	69	/// `let y = x;`
	70	/// main_token is the keyword
	71	/// data is .var_decl
	72	var_decl,
	73	/// Function declaration
	74	/// `fn foo(x: i32) -> i32 { rhs }`
	75	/// Params and return type are stored in extra data
	76	fn_decl,
	77	/// 'name: type' declaration
	78	/// Main token is the identifier
	79	param_decl,
	80	/// An identifier. main_token is the identifier.
	81	id,
	82	/// Main token is the data. May not fit in i32 range,
	83	/// but will contain no dots or e's.
	84	number_int,
	85	/// Main token is data. Will contain '.' and/or 'e'.
	86	number_float,
	87	/// A string literal
	88	string,
	89	/// A multi-line string literal
	90	/// data is .token_list of the tokens, which is guaranteed to be contiguous
	91	multiline_string,
	92	/// Call to a function
	93	/// `foo(x, y);`
	94	/// main_token is the `(`
	95	/// data is .call
	96	call,
	97	/// Call to a builtin function
	98	/// `@raw("hello");`
	99	/// data is .list (containing arg slice)
	100	builtin_call,
	101	/// `if cond { ... } else ...`
	102	/// main_token is the `if`
	103	/// data is .@"if"
	104	@"if",
	105	/// `if let id = ... { ... } else ...`
	106	/// main_token is the `if`
	107	/// data is .@"if"
	108	// if_let_nonvoid,
	109	/// `id: ...`
	110	/// data is un
	111	label,
	112	/// `expr.field`
	113	/// main_token is the `.`
	114	/// data is .field
	115	field,
	116	/// `lhs[rhs]`
	117	/// main_token is the `[`
	118	/// data is .bin
	119	index,
	120	/// `return expr;`
	121	/// main_token is the `return`
	122	/// data is .@"return"
	123	@"return",
	124	/// `break :brk expr;`
	125	/// main_token is the `break`
	126	/// data is .@"break"
	127	@"break",
	128	/// `(expr)`
	129	/// main_token is the `(`
	130	/// data is .un
	131	group,
	132	/// `lhs + rhs`. main_token is the operator.
	133	add,
	134	/// `lhs - rhs`. main_token is the operator.
	135	sub,
	136	/// `lhs * rhs`. main_token is the operator.
	137	mul,
	138	/// `lhs / rhs`. main_token is the operator.
	139	div,
	140	/// `lhs % rhs`. main_token is the operator.
	141	rem,
	142	/// `lhs ** rhs`. main_token is the operator.
	143	pow,
	144	/// `lhs && rhs`. main_token is the operator.
	145	@"and",
	146	/// `lhs \|\| rhs`. main_token is the operator.
	147	@"or",
	148	/// `lhs & rhs`. main_token is the operator.
	149	bit_and,
	150	/// `lhs \| rhs`. main_token is the operator.
	151	bit_or,
	152	/// `lhs ^ rhs`. main_token is the operator.
	153	bit_xor,
	154	/// `lhs << rhs`. main_token is the operator.
	155	shl,
	156	/// `lhs >> rhs`. main_token is the operator.
	157	shr,
	158	/// `lhs == rhs`. main_token is the operator.
	159	eq,
	160	/// `lhs != rhs`. main_token is the operator.
	161	ne,
	162	/// `lhs < rhs`. main_token is the operator.
	163	lt,
	164	/// `lhs <= rhs`. main_token is the operator.
	165	le,
	166	/// `lhs > rhs`. main_token is the operator.
	167	gt,
	168	/// `lhs >= rhs`. main_token is the operator.
	169	ge,
	170	/// `lhs = rhs`. main_token is the operator.
	171	assign,
	172	/// `lhs += rhs`. main_token is the operator.
	173	add_assign,
	174	/// `lhs -= rhs`. main_token is the operator.
	175	sub_assign,
	176	/// `lhs *= rhs`. main_token is the operator.
	177	mul_assign,
	178	/// `lhs /= rhs`. main_token is the operator.
	179	div_assign,
	180	/// `lhs %= rhs`. main_token is the operator.
	181	rem_assign,
	182	/// `lhs **= rhs`. main_token is the operator.
	183	pow_assign,
	184	/// `lhs &&= rhs`. main_token is the operator.
	185	and_assign,
	186	/// `lhs \|\|= rhs`. main_token is the operator.
	187	or_assign,
	188	/// `lhs &= rhs`. main_token is the operator.
	189	bit_and_assign,
	190	/// `lhs \|= rhs`. main_token is the operator.
	191	bit_or_assign,
	192	/// `lhs ^= rhs`. main_token is the operator.
	193	bit_xor_assign,
	194	/// `lhs <<= rhs`. main_token is the operator.
	195	shl_assign,
	196	/// `lhs >>= rhs`. main_token is the operator.
	197	shr_assign,
	198	/// `!expr`. main_token is the operator.
	199	bool_not,
	200	/// `-expr`. main_token is the operator.
	201	negate,
	202	/// `~expr`. main_token is the operator.
	203	bit_not,
	204	/// `comptime expr`. main_token is the operator.
	205	@"comptime",
	206	/// `pure expr`. main_token is the operator.
	207	// pure,
	208	/// `assert expr`. main_token is the operator.
	209	// assert,
	210
	211	pub fn isAssignBin(op: Tag) bool {
	212	return switch (op) {
	213	.assign,
	214	.add_assign,
	215	.sub_assign,
	216	.mul_assign,
	217	.div_assign,
	218	.rem_assign,
	219	.pow_assign,
	220	.and_assign,
	221	.or_assign,
	222	.bit_and_assign,
	223	.bit_or_assign,
	224	.bit_xor_assign,
	225	.shl_assign,
	226	.shr_assign,
	227	=> true,
	228	.add,
	229	.sub,
	230	.mul,
	231	.div,
	232	.rem,
	233	.pow,
	234	.@"and",
	235	.@"or",
	236	.bit_and,
	237	.bit_or,
	238	.bit_xor,
	239	.shl,
	240	.shr,
	241	.eq,
	242	.ne,
	243	.lt,
	244	.le,
	245	.gt,
	246	.ge,
	247	=> false,
	248	else => unreachable, // not a binary operator
	249	};
	250	}
	251
	252	pub fn isBasic(op: Tag) bool {
	253	return switch (op) {
	254	.id,
	255	.string,
	256	.number_int,
	257	.number_float,
	258	.builtin_call,
	259	=> true,
	260	else => false,
	261	};
	262	}
	263	};
	264
	265	pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{
	266	.root = .list,
	267	.block_void = .list,
	268	.block_implicit_return = .list,
	269	.var_decl = .var_decl,
	270	.fn_decl = .fn_decl,
	271	.param_decl = .param_decl,
	272	.id = .none,
	273	.number_int = .none,
	274	.number_float = .none,
	275	.string = .none,
	276	.multiline_string = .list,
	277	.call = .call,
	278	.builtin_call = .list,
	279	.@"if" = .@"if",
	280	.field = .field,
	281	.index = .bin,
	282	.@"return" = .@"return",
	283	.@"break" = .@"break",
	284	.group = .un,
	285	.add = .bin,
	286	.sub = .bin,
	287	.mul = .bin,
	288	.div = .bin,
	289	.rem = .bin,
	290	.pow = .bin,
	291	.@"and" = .bin,
	292	.@"or" = .bin,
	293	.bit_and = .bin,
	294	.bit_or = .bin,
	295	.bit_xor = .bin,
	296	.shl = .bin,
	297	.shr = .bin,
	298	.eq = .bin,
	299	.ne = .bin,
	300	.lt = .bin,
	301	.le = .bin,
	302	.gt = .bin,
	303	.ge = .bin,
	304	.assign = .bin,
	305	.add_assign = .bin,
	306	.sub_assign = .bin,
	307	.mul_assign = .bin,
	308	.div_assign = .bin,
	309	.rem_assign = .bin,
	310	.pow_assign = .bin,
	311	.and_assign = .bin,
	312	.or_assign = .bin,
	313	.bit_and_assign = .bin,
	314	.bit_or_assign = .bin,
	315	.bit_xor_assign = .bin,
	316	.shl_assign = .bin,
	317	.shr_assign = .bin,
	318	.bool_not = .un,
	319	.negate = .un,
	320	.bit_not = .un,
	321	.@"comptime" = .un,
	322	.label = .label,
	323	});
	324
	325	pub const Data = union {
	326	none: struct { Unused = .zero, Unused = .zero },
	327	fn_decl: struct {
	328	// A `FnDecl`
	329	extra: ExtraIndex,
	330	body: NodeIndex,
	331	},
	332	/// either is a node list or a token list, depending on the node's tag
	333	list: SubList,
	334	var_decl: struct {
	335	type: OptionalNodeIndex,
	336	init: NodeIndex,
	337	},
	338	param_decl: struct {
	339	type: NodeIndex,
	340	unused: Unused = .zero,
	341	},
	342	bin: struct {
	343	lhs: NodeIndex,
	344	rhs: NodeIndex,
	345	},
	346	field: struct {
	347	base: NodeIndex,
	348	field_token: TokenIndex,
	349	},
	350	un: struct {
	351	expr: NodeIndex,
	352	unused: Unused = .zero,
	353	},
	354	label: struct {
	355	expr: NodeIndex,
	356	unused: Unused = .zero,
	357	},
	358	@"return": struct {
	359	expr: OptionalNodeIndex,
	360	unused: Unused = .zero,
	361	},
	362	@"break": struct {
	363	label: OptionalTokenIndex,
	364	value: OptionalNodeIndex,
	365	},
	366	call: struct {
	367	base: NodeIndex,
	368	/// A `Call`
	369	extra: ExtraIndex,
	370	},
	371	@"if": struct {
	372	cond: NodeIndex,
	373	/// A `If`
	374	extra: ExtraIndex,
	375	},
	376
	377	const FieldEnum = std.meta.FieldEnum(Data);
	378	};
	379	};
	380
	381	pub const Error = error{ OutOfMemory, ParseFail };
	382
	383	pub const Location = struct {
	384	line: Ordinal(u32),
	385	column: Ordinal(u32),
	386	// Byte offset of the start of the line
	387	line_start: ByteOffset,
	388	// Byte offset of the end of the line
	389	line_end: ByteOffset,
	390	};
	391
	392	pub const Span = struct {
	393	start: u32,
	394	end: u32,
	395	main: u32,
	396	};
	397
	398	// This can be removed when the following issue is implemented:
	399	// See: https://github.com/ziglang/zig/issues/104
	400	// See: https://github.com/ziglang/zig/issues/3806
	401	pub const OptionalNodeIndex = enum(u32) {
	402	none = 0,
	403	_,
	404
	405	pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
	406	assert(idx != 0);
	407	return @enumFromInt(idx orelse return .none);
	408	}
	409
	410	pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
	411	return switch (idx) {
	412	.none => null,
	413	_ => @intFromEnum(idx),
	414	};
	415	}
	416	};
	417
	418	pub const OptionalTokenIndex = enum(u32) {
	419	none = 0,
	420	_,
	421
	422	pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
	423	assert(idx != 0);
	424	return @enumFromInt(idx orelse return .none);
	425	}
	426
	427	pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
	428	return switch (idx) {
	429	.none => null,
	430	_ => @intFromEnum(idx),
	431	};
	432	}
	433	};
	434
	435	pub const OptionalExtraIndex = enum(u32) {
	436	none = std.math.maxInt(u32),
	437	_,
	438
	439	pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
	440	assert(idx != std.math.maxInt(u32));
	441	return @enumFromInt(idx orelse return .none);
	442	}
	443
	444	pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
	445	return switch (idx) {
	446	.none => null,
	447	_ => @intFromEnum(idx),
	448	};
	449	}
	450	};
	451
	452	pub const SubList = struct {
	453	/// Either NodeIndex or TokenIndex, depending on where this came from.
	454	start: u32,
	455	len: u32,
	456
	457	pub fn slice(self: SubList, ast: *const Ast) []const u32 {
	458	return ast.extra[self.start..][0..self.len];
	459	}
	460	};
	461
	462	pub const FnDecl = struct {
	463	params: SubList,
	464	return_type: OptionalNodeIndex,
	465	};
	466
	467	pub const Call = struct {
	468	args: SubList,
	469	};
	470
	471	pub const If = struct {
	472	/// either .block_void or .block_implicit_return
	473	yes: NodeIndex,
	474	/// if it exists, it is either
	475	/// .block_void, .block_implicit_return, or .@"if"
	476	no: OptionalNodeIndex,
	477	};
	478
	479	pub const full = struct {
	480	pub const FnDecl = struct {
	481	start_token: TokenIndex,
	482	public: bool,
	483	id: []const u8,
	484	params: []const NodeIndex,
	485	return_type: ?NodeIndex,
	486	body: NodeIndex,
	487	};
	488
	489	pub const VarDecl = struct {
	490	start_token: TokenIndex,
	491	public: bool,
	492	is_const: bool,
	493	id: []const u8,
	494	type: ?NodeIndex,
	495	init: NodeIndex,
	496	};
	497
	498	pub const ParamDecl = struct {
	499	start_token: TokenIndex,
	500	id: []const u8,
	501	type: NodeIndex,
	502	};
	503	};
	504
	505	pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void {
	506	ast.tokens.deinit(gpa);
	507	ast.nodes.deinit(gpa);
	508	gpa.free(ast.extra);
	509	ast.* = undefined;
	510	}
	511
	512	pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag {
	513	return ast.nodes.items(.tag)[node];
	514	}
	515
	516	pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data {
	517	return ast.nodes.items(.data)[node];
	518	}
	519
	520	pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex {
	521	return ast.nodes.items(.main_token)[node];
	522	}
	523
	524	pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl {
	525	const tok_tags = ast.tokens.items(.tag);
	526
	527	const main_token = ast.nodes.items(.main_token)[node];
	528	const public = tok_tags[main_token -\| 1] == .kw_pub;
	529	assert(tok_tags[main_token] == .kw_fn);
	530	assert(tok_tags[main_token + 1] == .id);
	531	const id = ast.tokenSlice(main_token + 1);
	532
	533	const data = ast.nodes.items(.data)[node].fn_decl;
	534	const extra = ast.extraData(data.extra, FnDecl);
	535
	536	return .{
	537	.start_token = if (public) main_token - 1 else main_token,
	538	.public = public,
	539	.id = id,
	540	.body = data.body,
	541	.params = extra.params.slice(&ast),
	542	.return_type = extra.return_type.unwrap(),
	543	};
	544	}
	545
	546	pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl {
	547	const tok_tags = ast.tokens.items(.tag);
	548
	549	const main_token = ast.nodes.items(.main_token)[node];
	550	const public = tok_tags[main_token -\| 1] == .kw_pub;
	551	const is_const = switch (tok_tags[main_token]) {
	552	.kw_let => true,
	553	.kw_var => false,
	554	else => \|tag\| {
	555	if (std.debug.runtime_safety) {
	556	std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)});
	557	}
	558	unreachable;
	559	},
	560	};
	561	assert(tok_tags[main_token + 1] == .id);
	562	const id = ast.tokenSlice(main_token + 1);
	563	const data = ast.nodes.items(.data)[node].var_decl;
	564	return .{
	565	.start_token = if (public) main_token - 1 else main_token,
	566	.public = public,
	567	.is_const = is_const,
	568	.id = id,
	569	.type = data.type.unwrap(),
	570	.init = data.init,
	571	};
	572	}
	573
	574	pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
	575	const tok_tags = ast.tokens.items(.tag);
	576
	577	const main_token = ast.nodes.items(.main_token)[node];
	578	assert(tok_tags[main_token] == .id);
	579	const id = ast.tokenSlice(main_token);
	580	const data = ast.nodes.items(.data)[node].param_decl;
	581	return .{
	582	.start_token = main_token,
	583	.id = id,
	584	.type = data.type,
	585	};
	586	}
	587
	588	pub fn idToken(ast: Ast, node: NodeIndex) TokenIndex {
	589	return switch (ast.nodeTag(node)) {
	590	.param_decl,
	591	.id,
	592	.label,
	593	=> ast.nodeToken(node),
	594	.fn_decl,
	595	.var_decl,
	596	=> ast.nodeToken(node) + 1,
	597	else => unreachable, // does not have an id
	598	};
	599	}
	600
	601	pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
	602	const data = ast.nodes.items(.data)[node].list;
	603	return data.slice(ast);
	604	}
	605
	606	pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
	607	const token_start = self.tokens.items(.offset)[token_index];
	608	return Tokenizer.tokenLocation(self.utf8_source, start_offset, token_start);
	609	}
	610
	611	pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 {
	612	return ast.tokens.get(token_index).slice(ast.utf8_source);
	613	}
	614
	615	pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
	616	assert(ast.nodes.items(.tag)[node].isBasic());
	617	const token = ast.nodes.items(.main_token)[node];
	618	return ast.tokenSlice(token);
	619	}
	620
	621	pub fn fnDeclIdToken(tree: Ast, node: NodeIndex) TokenIndex {
	622	assert(tree.nodeTag(node) == .fn_decl);
	623	return tree.nodeToken(node) + 1;
	624	}
	625
	626	pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
	627	const fields = std.meta.fields(T);
	628	var result: T = undefined;
	629	var i: usize = 0;
	630	inline for (fields) \|field\| {
	631	@field(result, field.name) = switch (field.type) {
	632	NodeIndex => tree.extra[index + i],
	633	SubList => SubList{
	634	.start = tree.extra[index + i],
	635	.len = tree.extra[index + i + 1],
	636	},
	637	OptionalNodeIndex => @enumFromInt(tree.extra[index + i]),
	638	OptionalTokenIndex => @enumFromInt(tree.extra[index + i]),
	639	OptionalExtraIndex => @enumFromInt(tree.extra[index + i]),
	640	else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)),
	641	};
	642	i += switch (field.type) {
	643	NodeIndex,
	644	OptionalNodeIndex,
	645	OptionalTokenIndex,
	646	OptionalExtraIndex,
	647	=> 1,
	648	SubList => 2,
	649	else => comptime unreachable,
	650	};
	651	}
	652	return result;
	653	}
	654
	655	pub fn dumpToWriter(ast: Ast, writer: anytype) !void {
	656	try writer.print("{}", .{Dump{ .ast = ast }});
	657	}
	658
	659	pub const Dump = @import("Ast.Dump.zig");
	660
	661	pub const Unused = enum(u32) { zero = 0 };
	662	pub const TokenIndex = u32;
	663	pub const ExtraIndex = u32;
	664	pub const NodeIndex = u32;
	665
	666	const Ast = @This();
	667	const std = @import("std");
	668	const assert = std.debug.assert;
	669
	670	const Tokenizer = @import("Tokenizer.zig");
	671	const Token = Tokenizer.Token;
	672	const TokenList = Tokenizer.Token.List;
	673	const ByteOffset = Tokenizer.ByteOffset;
	674
	675	const Report = @import("Report.zig");
	676
	677	const util = @import("util.zig");
	678	const Ordinal = util.Ordinal;

src/core/Cache.zig created+9

...	...	@@ -0,0 +1,9 @@
	1	const std = @import("std");
	2
	3	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
	4	pub const bin_digest_len = 16;
	5	pub const hex_digest_len = bin_digest_len * 2;
	6	pub const BinDigest = [bin_digest_len]u8;
	7	pub const HexDigest = [hex_digest_len]u8;
	8
	9	pub const HashHelper = std.Build.Cache.HashHelper;

src/core/Comp.zig created+443

...	...	@@ -0,0 +1,443 @@
	1	//! Compilation
	2
	3	/// General purpose allocator
	4	gpa: Allocator,
	5	/// Stores all Type and Value objects.
	6	intern_pool: InternPool = .{},
	7	/// All files in the compilation
	8	files: std.StringArrayHashMapUnmanaged(*File) = .{},
	9
	10	/// # of sema instructions in this compilation cycle
	11	analyzed_instructions: usize = 0,
	12
	13	options: Options = .{},
	14
	15	// TODO: consider computing this dynamicly per-link
	16	current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
	17
	18	pub fn init(comp: *Comp) !void {
	19	try comp.intern_pool.init(comp.gpa);
	20	}
	21
	22	pub fn deinit(comp: *Comp) void {
	23	comp.intern_pool.deinit(comp.gpa);
	24	for (comp.files.values()) \|file\| {
	25	file.deinit(comp.gpa);
	26	}
	27	comp.files.deinit(comp.gpa);
	28	comp.current_compile_decls.deinit(comp.gpa);
	29	}
	30
	31	pub fn getFile(comp: Comp, absolute_file_path: []const u8) !File {
	32	assert(std.fs.path.isAbsolute(absolute_file_path));
	33	const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path);
	34	if (gop.found_existing) {
	35	return gop.value_ptr.*;
	36	}
	37	const file = try comp.gpa.create(File);
	38	file.* = .{
	39	.file_path = absolute_file_path,
	40	.path_digest = null,
	41	.root_module = .none,
	42	.status = .never_loader,
	43	.source = null,
	44	.ast = null,
	45	.teg = null,
	46	};
	47	gop.value_ptr.* = file;
	48	return file;
	49	}
	50
	51	pub const Source = struct {
	52	code: [:0]const u8,
	53	stat: Stat,
	54	};
	55
	56	pub const File = struct {
	57	/// TODO: change this to a URL
	58	file_path: []const u8,
	59
	60	root_module: Decl.Index.Optional,
	61	status: enum {
	62	never_loader,
	63	/// A failure occurred during loading the file. Retrying might succeed.
	64	retryable_failure,
	65	/// Errors during parse
	66	parse_failure,
	67	/// Errors during transform
	68	transform_failure,
	69	/// Teg is valid
	70	success,
	71	},
	72
	73	source: ?Source,
	74	ast: ?Ast,
	75	teg: ?Teg,
	76
	77	path_digest: ?Cache.BinDigest,
	78
	79	const GetSourceError =
	80	std.fs.File.OpenError \|\|
	81	std.fs.File.ReadError \|\|
	82	std.fs.File.StatError \|\|
	83	Allocator.Error \|\| error{
	84	FileTooBig,
	85	UnexpectedEndOfFile,
	86	};
	87
	88	pub fn getSource(file: *File, gpa: Allocator) GetSourceError!Source {
	89	if (file.source) \|source\| {
	90	return source;
	91	}
	92
	93	// Keep track of inode, file size, mtime, hash so we can detect which files
	94	// have been modified when an incremental update is requested.
	95	var f = try std.fs.openFileAbsolute(file.file_path, .{});
	96	defer f.close();
	97
	98	const stat = try f.stat();
	99
	100	if (stat.size > std.math.maxInt(u32))
	101	return error.FileTooBig;
	102
	103	const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
	104	errdefer gpa.free(source);
	105	const amt = try f.readAll(source);
	106	if (amt != stat.size)
	107	return error.UnexpectedEndOfFile;
	108
	109	// Here we do not modify stat fields because this function is the one
	110	// used for error reporting. We need to keep the stat fields stale so that
	111	// transformFile can know to regenerate Teg.
	112
	113	const loaded_source = Source{
	114	.code = source,
	115	.stat = .{
	116	.size = stat.size,
	117	.inode = stat.inode,
	118	.mtime = stat.mtime,
	119	},
	120	};
	121	file.source = loaded_source;
	122	return loaded_source;
	123	}
	124
	125	pub const GetAstError = GetSourceError \|\| Parse.Error;
	126
	127	pub fn getAst(file: File, comp: Comp) GetAstError!Ast {
	128	if (file.ast) \|ast\| {
	129	return ast;
	130	}
	131
	132	var err_data: Parse.AstError.WithTrace = undefined;
	133	const ast = Parse.parse(
	134	comp.gpa,
	135	(try file.getSource(comp.gpa)).code,
	136	&err_data,
	137	) catch \|err\| switch (err) {
	138	error.OutOfMemory => return err,
	139	error.ParseFail => {
	140	// TODO: let this merge with other report
	141	var report = try err_data.toReport(comp, file);
	142	report.dumpToStdErr(comp) catch {};
	143	report.deinit(comp.gpa);
	144	return err;
	145	},
	146	};
	147
	148	if (comp.options.dump_ir) {
	149	ast.dumpToWriter(std.io.getStdErr().writer()) catch {
	150	// Simplify the error set.
	151	};
	152	}
	153
	154	file.ast = ast;
	155	return ast;
	156	}
	157
	158	pub const GetTegError = GetAstError \|\| Transform.Error;
	159
	160	pub fn getTeg(file: File, comp: Comp, fail_report: *Report) GetTegError!Teg {
	161	if (file.teg) \|teg\| {
	162	return teg;
	163	}
	164
	165	const bin_digest = hash: {
	166	var path_hash: Cache.HashHelper = .{};
	167	path_hash.addBytes("0.1.0");
	168	path_hash.addBytes(file.file_path);
	169	var bin: Cache.BinDigest = undefined;
	170	path_hash.hasher.final(&bin);
	171	break :hash bin;
	172	};
	173	file.path_digest = bin_digest;
	174
	175	const teg = try Transform.transform(
	176	try file.getAst(comp),
	177	comp.gpa,
	178	file,
	179	fail_report,
	180	);
	181
	182	file.teg = teg;
	183	return teg;
	184	}
	185
	186	pub fn unloadSource(file: *File, gpa: Allocator) void {
	187	if (file.source) \|source\| {
	188	gpa.free(source.code);
	189	file.source = null;
	190	}
	191	}
	192
	193	pub fn unloadAst(file: *File, gpa: Allocator) void {
	194	if (file.ast) \|*ast\| {
	195	ast.deinit(gpa);
	196	file.ast = null;
	197	}
	198	}
	199
	200	pub fn unloadTeg(file: *File, gpa: Allocator) void {
	201	if (file.teg) \|*teg\| {
	202	teg.deinit(gpa);
	203	file.teg = null;
	204	}
	205	}
	206
	207	pub fn deinit(file: *File, gpa: Allocator) void {
	208	file.unloadTeg(gpa);
	209	file.unloadAst(gpa);
	210	file.unloadSource(gpa);
	211	gpa.destroy(file);
	212	}
	213	};
	214
	215	pub const Decl = struct {
	216	name: InternPool.NullTerminatedString,
	217	val: InternPool.Value,
	218	/// The direct parent namespace of the Decl. In the case of the Decl
	219	/// corresponding to a file, this is the namespace of the struct, since
	220	/// there is no parent.
	221	src_namespace: Namespace.Index,
	222
	223	/// The AST node index of this declaration.
	224	/// Must be recomputed when the corresponding source file is modified.
	225	// src_node: Ast.NodeIndex,
	226	/// Line number corresponding to `src_node`. Stored separately so that source files
	227	/// do not need to be loaded into memory in order to compute debug line numbers.
	228	/// This value is absolute.
	229	// src_line: u32,
	230	/// Index of the ZIR `declaration` instruction from which this `Decl` was created.
	231	/// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`.
	232	teg_decl_index: InternPool.TrackedInst.Index.Optional,
	233
	234	/// Represents the "shallow" analysis status. For example, for decls that are functions,
	235	/// the function type is analyzed with this set to `in_progress`, however, the semantic
	236	/// analysis of the function body is performed with this value set to `success`. Functions
	237	/// have their own analysis status field.
	238	analysis: enum {
	239	/// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
	240	/// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
	241	unreferenced,
	242	/// Semantic analysis for this Decl is running right now.
	243	/// This state detects dependency loops.
	244	in_progress,
	245	/// The file corresponding to this Decl had a parse error or ZIR error.
	246	/// There will be a corresponding ErrorMsg in Zcu.failed_files.
	247	file_failure,
	248	/// This Decl might be OK but it depends on another one which did not
	249	/// successfully complete semantic analysis.
	250	dependency_failure,
	251	/// Semantic analysis failure.
	252	/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
	253	sema_failure,
	254	/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
	255	codegen_failure,
	256	/// Sematic analysis and constant value codegen of this Decl has
	257	/// succeeded. However, the Decl may be outdated due to an in-progress
	258	/// update. Note that for a function, this does not mean codegen of the
	259	/// function body succeded: that state is indicated by the function's
	260	/// `analysis` field.
	261	complete,
	262	},
	263
	264	codegen: ?JsTree = null,
	265
	266	is_pub: bool,
	267
	268	/// What kind of a declaration is this.
	269	kind: Kind,
	270
	271	pub fn deinit(decl: *Decl, gpa: Allocator) void {
	272	defer decl.* = undefined;
	273	if (decl.codegen) \|*cg\| cg.deinit(gpa);
	274	}
	275
	276	pub const Kind = enum {
	277	named,
	278	};
	279
	280	pub const Index = InternPool.DeclIndex;
	281	};
	282
	283	pub const Namespace = struct {
	284	parent: Index.Optional,
	285	file_scope: *File,
	286	/// Direct children of the namespace.
	287	/// Declaration order is preserved via entry order.
	288	/// These are only declarations named directly by the AST; anonymous
	289	/// declarations are not stored here.
	290	decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{},
	291
	292	pub const DeclContext = struct {
	293	comp: *Comp,
	294
	295	pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 {
	296	const decl = ctx.comp.intern_pool.declPtr(decl_index);
	297	return std.hash.uint32(@intFromEnum(decl.name));
	298	}
	299
	300	pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool {
	301	_ = b_index;
	302	const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index);
	303	const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index);
	304	return a_decl.name == b_decl.name;
	305	}
	306	};
	307
	308	pub const DeclNameAdapter = struct {
	309	comp: *Comp,
	310
	311	pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 {
	312	_ = self;
	313	return std.hash.uint32(@intFromEnum(s));
	314	}
	315
	316	pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool {
	317	_ = b_index;
	318	return a == self.comp.intern_pool.declPtr(b_decl_index).name;
	319	}
	320	};
	321
	322	pub fn lookupDirect(ns: Namespace, comp: Comp, name: InternPool.NullTerminatedString) ?Decl.Index {
	323	return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp });
	324	}
	325
	326	pub const Index = InternPool.NamespaceIndex;
	327	};
	328
	329	/// Index a module in a Teg.
	330	/// When a File's Teg is loaded, we call this on the root module.
	331	/// submodules are analyzed lazily.
	332	pub fn indexTegModule(comp: Comp, file: File, module_index: Teg.Inst.Index) !Namespace.Index {
	333	const teg = file.teg.?;
	334
	335	const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
	336	.parent = .none,
	337	.file_scope = file,
	338	});
	339	const ns_ptr = comp.intern_pool.namespacePtr(ns);
	340	errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns);
	341
	342	for (teg.moduleDeclarations(module_index)) \|decl_index\| {
	343	const name = try comp.intern_pool.getOrPutString(
	344	comp.gpa,
	345	teg.declName(decl_index),
	346	.no_embedded_nulls,
	347	);
	348	const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index);
	349	const decl = try comp.intern_pool.createDecl(comp.gpa, .{
	350	.name = name,
	351	.val = undefined,
	352	.src_namespace = ns,
	353	.teg_decl_index = tracked_inst.toOptional(),
	354	.analysis = .unreferenced,
	355	.is_pub = false, // todo:
	356	.kind = .named,
	357	});
	358	try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp });
	359	}
	360
	361	return ns;
	362	}
	363
	364	pub fn semaEntryPoint(comp: Comp, file: File) !void {
	365	var report: Report = undefined;
	366	const teg = try file.getTeg(comp, &report);
	367
	368	// TODO, move this elsewhere. we should print all errors and warnings at once
	369	if (teg.report.warnings.len > 0) {
	370	try teg.report.dumpToStdErr(comp);
	371	}
	372
	373	const ns = try comp.indexTegModule(file, .main_module_inst);
	374	const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
	375	@panic("no top level code?");
	376	const decl = comp.intern_pool.declPtr(decl_index);
	377
	378	assert(comp.current_compile_decls.items.len == 0);
	379
	380	var arena = std.heap.ArenaAllocator.init(comp.gpa);
	381	defer arena.deinit();
	382
	383	var sema: Sema = .{
	384	.comp = comp,
	385	.gpa = comp.gpa,
	386	.code = teg,
	387	.arena = arena.allocator(),
	388	.ip = &comp.intern_pool,
	389	.decl = decl,
	390	.decl_index = decl_index,
	391	.namespace_index = ns,
	392	};
	393	try sema.init();
	394	defer sema.deinit();
	395	_ = try sema.analyzeDeclaration();
	396	try sema.finalize();
	397	}
	398
	399	pub fn codeGenDecl(c: Comp, decl: Decl, air: *const Air) !void {
	400	const t = tracer.trace(@src());
	401	defer t.end();
	402
	403	var arena_state = std.heap.ArenaAllocator.init(c.gpa);
	404	defer arena_state.deinit();
	405	const arena = arena_state.allocator();
	406
	407	var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air);
	408	errdefer cg.deinit();
	409	try cg.build();
	410	decl.codegen = try cg.finalize();
	411	}
	412
	413	pub fn link(c: *Comp, writer: anytype) !void {
	414	try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
	415	}
	416
	417	const Options = struct {
	418	dump_ir: bool = false,
	419	};
	420
	421	const Comp = @This();
	422
	423	const std = @import("std");
	424	const Allocator = std.mem.Allocator;
	425	const assert = std.debug.assert;
	426	const Stat = std.Build.Cache.File.Stat;
	427	const log = std.log.scoped(.comp);
	428
	429	const InternPool = @import("InternPool.zig");
	430
	431	const Ast = @import("Ast.zig");
	432	const Air = @import("Air.zig");
	433	const Teg = @import("Teg.zig");
	434	const Cache = @import("Cache.zig");
	435	const Sema = @import("Sema.zig");
	436	const Transform = @import("Transform.zig");
	437	const JsNodeGen = @import("JsNodeGen.zig");
	438	const JsTree = @import("JsTree.zig");
	439	const JsLink = @import("JsLink.zig");
	440	const Report = @import("Report.zig");
	441	const Parse = @import("Parse.zig");
	442
	443	const tracer = @import("tracer");

src/core/InternPool.zig created+1196

...	...	@@ -0,0 +1,1196 @@
	1	//! Large parts of this file are based on the Zig compiler's InternPool.
	2	//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig
	3
	4	/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
	5	/// constructed lazily.
	6	map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
	7	items: std.MultiArrayList(Item) = .{},
	8	extra: ExtraStore = .{},
	9
	10	/// In order to store references to strings in fewer bytes, we copy all
	11	/// string bytes into here. String bytes can be null. It is up to whomever
	12	/// is referencing the data here whether they want to store both index and length,
	13	/// thus allowing null bytes, or store only index, and use null-termination. The
	14	/// `string_bytes` array is agnostic to either usage.
	15	string_bytes: std.ArrayListUnmanaged(u8) = .{},
	16
	17	/// Used for finding the index inside `string_bytes`.
	18	string_table: std.HashMapUnmanaged(
	19	u32,
	20	void,
	21	std.hash_map.StringIndexContext,
	22	std.hash_map.default_max_load_percentage,
	23	) = .{},
	24
	25	/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
	26	/// persists across incremental updates.
	27	tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
	28
	29	/// Rather than allocating Decl objects with an Allocator, we instead allocate
	30	/// them with this SegmentedList. This provides four advantages:
	31	/// * Stable memory so that one thread can access a Decl object while another
	32	/// thread allocates additional Decl objects from this list.
	33	/// * It allows us to use u32 indexes to reference Decl objects rather than
	34	/// pointers, saving memory in Type, Value, and dependency sets.
	35	/// * Using integers to reference Decl objects rather than pointers makes
	36	/// serialization trivial.
	37	/// * It provides a unique integer to be used for anonymous symbol names, avoiding
	38	/// multi-threaded contention on an atomic counter.
	39	allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{},
	40	/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack.
	41	decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{},
	42
	43	/// Same pattern as with `allocated_decls`.
	44	allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{},
	45	/// Same pattern as with `decls_free_list`.
	46	namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{},
	47
	48	/// How many items in the InternPool are statically known.
	49	/// This is specified with an integer literal and a corresponding comptime
	50	/// assert below to break an unfortunate and arguably incorrect dependency loop
	51	/// when compiling.
	52	pub const static_len = Teg.Inst.Index.static_len;
	53	comptime {
	54	// @compileLog(static_len, static_keys.len);
	55	assert(static_len == static_keys.len);
	56	}
	57
	58	pub const Item = struct {
	59	tag: Tag,
	60	data: EncodedData,
	61
	62	pub fn encode(comptime tag: Tag, data: tag.Data()) Item {
	63	return .{ .tag = tag, .data = EncodedData.encode(tag, data) };
	64	}
	65	};
	66
	67	pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var");
	68	pub const EncodedData = dod.EncodedData(Tag, u32);
	69	pub const ExtraIndex = ExtraStore.Index;
	70
	71	pub const Tag = enum(u8) {
	72	/// This special tag represents a value which was removed from this pool via
	73	/// `InternPool.remove`. The item remains allocated to preserve indices, but
	74	/// lookups will consider it not equal to any other item, and all queries
	75	/// assert not this tag. `data` is unused.
	76	removed,
	77
	78	/// A function type
	79	/// data is an extra index into a `FuncType`
	80	type_function,
	81	/// A type that can be represented with only an enum tag.
	82	/// data is SimpleType enum value.
	83	simple_type,
	84
	85	// --- before this are types ---
	86	// --- past this are values ---
	87
	88	/// A value that can be represented with only an enum tag.
	89	/// data is SimpleValue enum value.
	90	simple_value,
	91	/// A numeric literal that can be represented as a float.
	92	/// Integers greater than 2^53 known at compile time are currently not supported.
	93	/// data is an extra index into a `Float`
	94	comptime_float,
	95	/// An int32 value
	96	/// data is i32
	97	int32,
	98	/// A float value
	99	/// data is an extra index into a `Float`
	100	float,
	101	/// A function
	102	/// data is an extra index into a `FuncDecl`
	103	func,
	104	/// A string
	105	/// data is an extra index into a `String`
	106	string,
	107
	108	pub fn Data(comptime tag: Tag) type {
	109	return switch (tag) {
	110	.removed,
	111	=> unreachable, // does not have a data
	112
	113	.simple_type => SimpleType,
	114	.simple_value => SimpleValue,
	115	.int32 => i32,
	116
	117	else => ExtraIndex(tag.Payload()),
	118	};
	119	}
	120
	121	/// When `data` is an extra index, this is the type of the payload.
	122	pub fn Payload(comptime tag: Tag) type {
	123	return switch (tag) {
	124	.removed,
	125	.simple_type,
	126	.simple_value,
	127	.int32,
	128	=> unreachable, // does not have a payload
	129	.float, .comptime_float => Float,
	130	.func => FuncDecl,
	131	.type_function => TypeFunction,
	132	.string => String.WithLen,
	133	};
	134	}
	135
	136	/// Trailing: N/A
	137	pub const FuncDecl = struct {
	138	analysis: FuncAnalysis,
	139	owner_decl: DeclIndex,
	140	ty: Type,
	141	teg_body_inst: TrackedInst.Index,
	142	};
	143
	144	/// Trailing:
	145	/// 0. param_type: Index for each params_len
	146	pub const TypeFunction = struct {
	147	params_len: u32,
	148	return_type: Type,
	149	};
	150	};
	151
	152	pub const Value = enum(u32) {
	153	pub const first_type: Value = .opaque_type;
	154	pub const last_type: Value = .noreturn_type;
	155	pub const first_value: Value = .void;
	156	pub const last_value: Value = .empty_string;
	157
	158	opaque_type,
	159	bool_type,
	160	int32_type,
	161	float_type,
	162	string_type,
	163	noreturn_type,
	164	type_type,
	165	numeric_literal_type,
	166
	167	void,
	168	bool_true,
	169	bool_false,
	170	int32_zero,
	171	float_zero,
	172	nan,
	173	empty_string,
	174	standard_library,
	175
	176	none = std.math.maxInt(u32),
	177	_,
	178
	179	/// An array of `Index` existing within the `extra` array.
	180	/// This type exists to provide a struct with lifetime that is
	181	/// not invalidated when items are added to the `InternPool`.
	182	pub const Slice = struct {
	183	start: u32,
	184	len: u32,
	185
	186	pub fn get(slice: Slice, ip: *const InternPool) []Value {
	187	return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
	188	}
	189	};
	190
	191	/// Used for a map of `Index` values to the index within a list of `Index` values.
	192	const Adapter = struct {
	193	indexes: []const Value,
	194
	195	pub fn eql(ctx: @This(), a: Value, b_void: void, b_map_index: usize) bool {
	196	_ = b_void;
	197	return a == ctx.indexes[b_map_index];
	198	}
	199
	200	pub fn hash(ctx: @This(), a: Value) u32 {
	201	_ = ctx;
	202	return std.hash.uint32(@intFromEnum(a));
	203	}
	204	};
	205
	206	pub fn valueTag(value: Value, ip: *const InternPool) Tag {
	207	return ip.items.items(.tag)[@intFromEnum(value)];
	208	}
	209
	210	pub fn valueData(value: Value, ip: *const InternPool, comptime tag: Tag) tag.Data() {
	211	return ip.items.items(.data)[@intFromEnum(value)].decode(tag, value.valueTag(ip));
	212	}
	213
	214	pub fn toAir(value: Value) Air.Inst.Ref {
	215	return .{ .kind = .value, .data = @intCast(@intFromEnum(value)) };
	216	}
	217
	218	pub fn toKey(value: Value, ip: *const InternPool) Key {
	219	return ip.indexToKey(value);
	220	}
	221
	222	pub fn dump(value: Value, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) {
	223	return .{ .data = .{ .val = value, .ip = ip } };
	224	}
	225
	226	fn formatValue(
	227	ctx: struct { val: Value, ip: *const InternPool },
	228	comptime _: []const u8,
	229	_: std.fmt.FormatOptions,
	230	writer: anytype,
	231	) !void {
	232	// TODO: move the formatter out of Air.Dump
	233	var state: Air.Dump.State = undefined;
	234	state.ip = ctx.ip;
	235	state.tty = @import("util.zig").detectTTYFromWriter(writer);
	236	try Air.Dump.State.dumpValue(&state, writer, ctx.val);
	237	}
	238
	239	pub fn typeOf(value: Value, ip: *const InternPool) Type {
	240	return switch (value.valueTag(ip)) {
	241	.removed => unreachable, // UAF
	242
	243	.type_function => .type,
	244	.simple_type => .type,
	245
	246	.simple_value => switch (value.valueData(ip, .simple_value)) {
	247	.true => .bool,
	248	.false => .bool,
	249	.void => .void,
	250	},
	251
	252	.comptime_float => .numeric_literal,
	253	.int32 => .int32,
	254	.float => .float,
	255	.string => .string,
	256	.func => value.toKey(ip).func.ty,
	257	};
	258	}
	259
	260	/// Try to re-interpret this `Value` as a `Type`.
	261	/// Returns `null` if the value is not a type.
	262	pub fn toType(value: Value, ip: *const InternPool) ?Type {
	263	return switch (value.valueTag(ip)) {
	264	.removed => unreachable, // UAF
	265
	266	.type_function,
	267	.simple_type,
	268	=> @enumFromInt(@intFromEnum(value)),
	269
	270	// exception: void is a value AND a type
	271	.simple_value => if (value == .void) .void else null,
	272
	273	else => null,
	274	};
	275	}
	276
	277	pub fn toTypeUnsafe(value: Value, proof_ip: *const InternPool) Type {
	278	if (std.debug.runtime_safety) {
	279	return value.toType(proof_ip).?;
	280	}
	281	return @enumFromInt(@intFromEnum(value));
	282	}
	283
	284	pub fn asString(val: Value, ip: *const InternPool) InternPool.String.WithLen {
	285	return val.valueData(ip, .string).get(ip);
	286	}
	287
	288	pub fn toFloat(val: Value, ip: *const InternPool) ?f64 {
	289	return switch (val.valueTag(ip)) {
	290	inline .comptime_float, .float => \|tag\| val.valueData(ip, tag).get(ip).decode(),
	291	else => null,
	292	};
	293	}
	294	};
	295
	296	pub const Type = enum(u32) {
	297	@"opaque" = @intFromEnum(Value.opaque_type),
	298	bool = @intFromEnum(Value.bool_type),
	299	int32 = @intFromEnum(Value.int32_type),
	300	float = @intFromEnum(Value.float_type),
	301	string = @intFromEnum(Value.string_type),
	302	noreturn = @intFromEnum(Value.noreturn_type),
	303	type = @intFromEnum(Value.type_type),
	304	numeric_literal = @intFromEnum(Value.numeric_literal_type),
	305	void = @intFromEnum(Value.void),
	306	_,
	307
	308	// Restricted version of `InternPool.Tag`
	309	pub const Tag = enum(u32) {
	310	type_function = @intFromEnum(InternPool.Tag.type_function),
	311	simple_type = @intFromEnum(InternPool.Tag.simple_type),
	312	};
	313
	314	/// An array of `Index` existing within the `extra` array.
	315	/// This type exists to provide a struct with lifetime that is
	316	/// not invalidated when items are added to the `InternPool`.
	317	pub const Slice = struct {
	318	start: u32,
	319	len: u32,
	320
	321	pub fn get(slice: Slice, ip: *const InternPool) []Type {
	322	return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
	323	}
	324	};
	325
	326	pub fn value(ty: Type) Value {
	327	return @enumFromInt(@intFromEnum(ty));
	328	}
	329
	330	pub fn tag(ty: Type, ip: *const InternPool) Type.Tag {
	331	return @enumFromInt(@intFromEnum(ty.value().valueTag(ip)));
	332	}
	333
	334	pub fn toFunctionPrototype(ty: Type, ip: *const InternPool) ?Key.TypeFunction {
	335	if (ty.tag(ip) != .type_function) return null;
	336	return ip.indexToKey(ty.value()).type_function;
	337	}
	338
	339	pub fn displayName(ty: Type, ip: *const InternPool) std.fmt.Formatter(formatDisplayName) {
	340	return .{ .data = .{ .type = ty, .ip = ip } };
	341	}
	342
	343	pub fn dump(ty: Type, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) {
	344	return ty.value().dump(ip);
	345	}
	346
	347	fn formatDisplayName(
	348	ctx: struct { type: Type, ip: *const InternPool },
	349	comptime _: []const u8,
	350	_: std.fmt.FormatOptions,
	351	writer: anytype,
	352	) !void {
	353	const ty = ctx.type;
	354	const ip = ctx.ip;
	355
	356	switch (ty) {
	357	.@"opaque" => return try writer.writeAll("opaque"),
	358	.bool => return try writer.writeAll("bool"),
	359	.int32 => return try writer.writeAll("int"),
	360	.float => return try writer.writeAll("float"),
	361	.string => return try writer.writeAll("string"),
	362	.noreturn => return try writer.writeAll("noreturn"),
	363	.type => return try writer.writeAll("type"),
	364	.numeric_literal => return try writer.writeAll("numeric literal"),
	365	.void => return try writer.writeAll("void"),
	366	else => {},
	367	}
	368
	369	switch (ty.tag(ip)) {
	370	.simple_type => unreachable, // all simple types are already printed
	371	.type_function => {
	372	const type_function = ty.toFunctionPrototype(ip).?;
	373	try writer.writeAll("fn(");
	374	for (type_function.param_types, 0..) \|param, i\| {
	375	if (i != 0) try writer.writeAll(", ");
	376	try writer.print("{}", .{param.displayName(ip)});
	377	}
	378	try writer.writeAll(") -> ");
	379	try writer.print("{}", .{type_function.return_type.displayName(ip)});
	380	},
	381	}
	382	}
	383	};
	384
	385	pub const static_keys = [_]Key{
	386	SimpleType.@"opaque".key(),
	387	SimpleType.bool.key(),
	388	SimpleType.int32.key(),
	389	SimpleType.float.key(),
	390	SimpleType.string.key(),
	391	SimpleType.noreturn.key(),
	392	SimpleType.type.key(),
	393	SimpleType.numeric_literal.key(),
	394
	395	SimpleValue.void.key(),
	396	SimpleValue.true.key(),
	397	SimpleValue.false.key(),
	398	.{ .int32 = 0 },
	399	.{ .float = 0.0 },
	400	.{ .float = Float.pure_nan.decode() },
	401	.{ .string = "" },
	402	};
	403
	404	/// State that is mutable during semantic analysis. This data is not used for
	405	/// equality or hashing.
	406	pub const FuncAnalysis = packed struct(u32) {
	407	state: State,
	408
	409	_: u24 = 0,
	410
	411	pub const State = enum(u8) {
	412	/// This function has not yet undergone analysis, because we have not
	413	/// seen a potential runtime call. It may be analyzed in future.
	414	none,
	415	/// Analysis for this function has been queued, but not yet completed.
	416	queued,
	417	/// This function intentionally only has ZIR generated because it is marked
	418	/// inline, which means no runtime version of the function will be generated.
	419	inline_only,
	420	in_progress,
	421	/// There will be a corresponding ErrorMsg in Module.failed_decls
	422	sema_failure,
	423	/// This function might be OK but it depends on another Decl which did not
	424	/// successfully complete semantic analysis.
	425	dependency_failure,
	426	/// There will be a corresponding ErrorMsg in Module.failed_decls.
	427	/// Indicates that semantic analysis succeeded, but code generation for
	428	/// this function failed.
	429	codegen_failure,
	430	/// Semantic analysis and code generation of this function succeeded.
	431	success,
	432	};
	433	};
	434
	435	/// An index into `string_bytes`.
	436	pub const String = enum(u32) {
	437	/// An empty string.
	438	empty = 0,
	439	_,
	440
	441	pub const WithLen = struct {
	442	string: String,
	443	len: u32,
	444
	445	pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 {
	446	return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)];
	447	}
	448
	449	pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString {
	450	return string.string.tryToNullTerminatedString(string.len, ip, gpa);
	451	}
	452	};
	453
	454	pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 {
	455	return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)];
	456	}
	457
	458	pub fn withLen(string: String, len: u64) WithLen {
	459	return .{ .string = string, .len = @intCast(len) };
	460	}
	461
	462	pub fn at(string: String, index: u64, ip: *const InternPool) u8 {
	463	return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)];
	464	}
	465
	466	pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString {
	467	assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null);
	468	assert(string.at(len, ip) == 0);
	469	return @enumFromInt(@intFromEnum(string));
	470	}
	471
	472	pub fn tryToNullTerminatedString(
	473	string: String,
	474	len: u64,
	475	ip: *InternPool,
	476	gpa: Allocator,
	477	) !NullTerminatedString {
	478	const slice = string.toSlice(len, ip);
	479	if (std.mem.indexOfScalar(u8, slice, 0)) \|_\| {
	480	return error.HasNulls;
	481	}
	482	if (string.at(len, ip) == 0) {
	483	return @enumFromInt(@intFromEnum(string));
	484	}
	485	return ip.getOrPutString(gpa, slice, .no_embedded_nulls);
	486	}
	487	};
	488
	489	pub const NullTerminatedString = enum(u32) {
	490	empty = 0,
	491	_,
	492
	493	pub fn toString(self: NullTerminatedString) String {
	494	return @enumFromInt(@intFromEnum(self));
	495	}
	496
	497	// pub fn toOptional(self: NullTerminatedString) Optional{
	498	// return @enumFromInt(@intFromEnum(self));
	499	// }
	500
	501	pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
	502	const slice = ip.string_bytes.items[@intFromEnum(string)..];
	503	return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
	504	}
	505
	506	pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
	507	return @intCast(string.toSlice(ip).len);
	508	}
	509
	510	pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
	511	return std.mem.eql(u8, string.toSlice(ip), slice);
	512	}
	513
	514	const Adapter = struct {
	515	strings: []const NullTerminatedString,
	516
	517	pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool {
	518	_ = b_void;
	519	return a == ctx.strings[b_map_index];
	520	}
	521
	522	pub fn hash(ctx: @This(), a: NullTerminatedString) u32 {
	523	_ = ctx;
	524	return std.hash.uint32(@intFromEnum(a));
	525	}
	526	};
	527
	528	const FormatData = struct {
	529	string: NullTerminatedString,
	530	ip: *const InternPool,
	531	};
	532	fn format(
	533	data: FormatData,
	534	comptime specifier: []const u8,
	535	_: std.fmt.FormatOptions,
	536	writer: anytype,
	537	) @TypeOf(writer).Error!void {
	538	const slice = data.string.toSlice(data.ip);
	539	if (comptime std.mem.eql(u8, specifier, "")) {
	540	try writer.writeAll(slice);
	541	} else if (comptime std.mem.eql(u8, specifier, "i")) {
	542	try writer.print("{p}", .{std.zig.fmtId(slice)});
	543	} else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'");
	544	}
	545
	546	pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) {
	547	return .{ .data = .{ .string = self, .ip = ip } };
	548	}
	549	};
	550
	551	pub const TrackedInst = extern struct {
	552	path_digest: Cache.BinDigest,
	553	inst: Teg.Inst.Index,
	554
	555	comptime {
	556	// The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
	557	assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index));
	558	}
	559
	560	pub const Index = enum(u32) {
	561	_,
	562	pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index {
	563	return ip.tracked_insts.keys()[@intFromEnum(i)].inst;
	564	}
	565	pub fn toOptional(i: TrackedInst.Index) Optional {
	566	return @enumFromInt(@intFromEnum(i));
	567	}
	568	pub const Optional = enum(u32) {
	569	none = std.math.maxInt(u32),
	570	_,
	571	pub fn unwrap(opt: Optional) ?TrackedInst.Index {
	572	return switch (opt) {
	573	.none => null,
	574	_ => @enumFromInt(@intFromEnum(opt)),
	575	};
	576	}
	577	};
	578	};
	579	};
	580
	581	pub const DeclIndex = enum(u32) {
	582	_,
	583
	584	pub fn toOptional(i: DeclIndex) Optional {
	585	return @enumFromInt(@intFromEnum(i));
	586	}
	587
	588	pub const Optional = enum(u32) {
	589	none = std.math.maxInt(u32),
	590	_,
	591
	592	pub fn init(oi: ?DeclIndex) Optional {
	593	return @enumFromInt(@intFromEnum(oi orelse return .none));
	594	}
	595
	596	pub fn unwrap(oi: Optional) ?DeclIndex {
	597	if (oi == .none) return null;
	598	return @enumFromInt(@intFromEnum(oi));
	599	}
	600	};
	601	};
	602
	603	pub const NamespaceIndex = enum(u32) {
	604	_,
	605
	606	pub fn toOptional(i: DeclIndex) Optional {
	607	return @enumFromInt(@intFromEnum(i));
	608	}
	609
	610	pub const Optional = enum(u32) {
	611	none = std.math.maxInt(u32),
	612	_,
	613
	614	pub fn init(oi: ?NamespaceIndex) Optional {
	615	return @enumFromInt(@intFromEnum(oi orelse return .none));
	616	}
	617
	618	pub fn unwrap(oi: Optional) ?NamespaceIndex {
	619	if (oi == .none) return null;
	620	return @enumFromInt(@intFromEnum(oi));
	621	}
	622	};
	623	};
	624
	625	/// A single value captured in the closure of a namespace type. This is not a plain
	626	/// `Index` because we must differentiate between the following cases:
	627	/// * runtime-known value (where we store the type)
	628	/// * comptime-known value (where we store the value)
	629	/// * decl val (so that we can analyze the value lazily)
	630	/// * decl ref (so that we can analyze the reference lazily)
	631	pub const Capture = packed struct(u32) {
	632	tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref },
	633	idx: u30,
	634
	635	pub fn wrap(val: Unwrapped) Capture {
	636	return switch (val) {
	637	.@"comptime" => \|i\| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) },
	638	.runtime => \|i\| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) },
	639	.decl_val => \|i\| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) },
	640	.decl_ref => \|i\| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) },
	641	};
	642	}
	643	pub fn unwrap(val: Capture) Unwrapped {
	644	return switch (val.tag) {
	645	.@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) },
	646	.runtime => .{ .runtime = @enumFromInt(val.idx) },
	647	.decl_val => .{ .decl_val = @enumFromInt(val.idx) },
	648	.decl_ref => .{ .decl_ref = @enumFromInt(val.idx) },
	649	};
	650	}
	651
	652	pub const Unwrapped = union(enum) {
	653	/// Index refers to the value.
	654	@"comptime": Value,
	655	/// Index refers to the type.
	656	runtime: Value,
	657	decl_val: DeclIndex,
	658	decl_ref: DeclIndex,
	659	};
	660
	661	pub const Slice = struct {
	662	start: u32,
	663	len: u32,
	664	pub fn get(slice: Slice, ip: *const InternPool) []Capture {
	665	return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
	666	}
	667	};
	668	};
	669
	670	const SimpleValue = enum {
	671	true,
	672	false,
	673	void,
	674
	675	pub fn key(self: SimpleValue) Key {
	676	return .{ .simple_value = self };
	677	}
	678	};
	679
	680	const SimpleType = enum {
	681	@"opaque",
	682	int32,
	683	float,
	684	string,
	685	bool,
	686	noreturn,
	687	type,
	688	numeric_literal,
	689
	690	pub fn key(self: SimpleType) Key {
	691	return .{ .simple_type = self };
	692	}
	693	};
	694
	695	pub const Key = union(enum) {
	696	comptime_float: f64,
	697	int32: i32,
	698	float: f64,
	699	string: []const u8,
	700	simple_value: SimpleValue,
	701	simple_type: SimpleType,
	702	type_function: TypeFunction,
	703	func: Func,
	704
	705	const Adapter = struct {
	706	intern_pool: *const InternPool,
	707
	708	pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
	709	_ = b_void;
	710	if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
	711	return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
	712	}
	713
	714	pub fn hash(ctx: @This(), a: Key) u32 {
	715	return a.hash32(ctx.intern_pool);
	716	}
	717	};
	718
	719	pub const TypeFunction = struct {
	720	param_types: []Type,
	721	return_type: Type,
	722
	723	pub fn hash(self: TypeFunction, hasher: *Hash) void {
	724	const t = tracer.trace(@src());
	725	defer t.end();
	726
	727	for (self.param_types) \|param_type\| {
	728	std.hash.autoHash(hasher, param_type);
	729	}
	730	std.hash.autoHash(hasher, self.return_type);
	731	}
	732
	733	pub fn eql(a: TypeFunction, b: TypeFunction) bool {
	734	return std.mem.eql(Type, a.param_types, b.param_types) and
	735	a.return_type == b.return_type;
	736	}
	737	};
	738
	739	pub const Func = struct {
	740	ty: Type,
	741	owner_decl: DeclIndex,
	742	teg_body_inst: TrackedInst.Index,
	743
	744	pub fn hash(self: Func, hasher: *Hash) void {
	745	std.hash.autoHash(hasher, self.ty);
	746	std.hash.autoHash(hasher, self.owner_decl);
	747	std.hash.autoHash(hasher, self.teg_body_inst);
	748	}
	749	};
	750
	751	pub fn hash32(key: Key, ip: *const InternPool) u32 {
	752	return @truncate(key.hash64(ip));
	753	}
	754
	755	pub fn hash64(key: Key, ip: *const InternPool) u64 {
	756	const t = tracer.trace(@src());
	757	defer t.end();
	758
	759	_ = ip; // autofix
	760	const asBytes = std.mem.asBytes;
	761	const KeyTag = @typeInfo(Key).Union.tag_type.?;
	762	const seed = @intFromEnum(@as(KeyTag, key));
	763
	764	return switch (key) {
	765	inline .int32,
	766	.simple_value,
	767	.simple_type,
	768	=> \|x\| Hash.hash(seed, asBytes(&x)),
	769	.string,
	770	=> \|x\| Hash.hash(seed, x),
	771	.float, .comptime_float => \|x\| Hash.hash(seed, asBytes(&Float.assertPure(x))),
	772	.type_function => \|x\| {
	773	var hasher = Hash.init(seed);
	774	x.hash(&hasher);
	775	return hasher.final();
	776	},
	777	.func => \|x\| {
	778	var hasher = Hash.init(seed);
	779	x.hash(&hasher);
	780	return hasher.final();
	781	},
	782	};
	783	}
	784
	785	pub fn eql(a: Key, b: Key, ip: *const InternPool) bool {
	786	const t = tracer.trace(@src());
	787	defer t.end();
	788
	789	_ = ip; // autofix
	790	const KeyTag = @typeInfo(Key).Union.tag_type.?;
	791	const a_tag: KeyTag = a;
	792	const b_tag: KeyTag = b;
	793	if (a_tag != b_tag) return false;
	794	return switch (a) {
	795	inline .int32,
	796	.simple_value,
	797	.simple_type,
	798	=> \|x, tag\| x == @field(b, @tagName(tag)),
	799	.string,
	800	=> \|x\| std.mem.eql(u8, x, b.string),
	801	.float => \|x\| Float.assertPure(x) == Float.assertPure(b.float),
	802	.comptime_float => \|x\| Float.assertPure(x) == Float.assertPure(b.comptime_float),
	803	.type_function => \|x\| x.eql(b.type_function),
	804	.func => \|a_info\| {
	805	const b_info = b.func;
	806	if (a_info.owner_decl != b_info.owner_decl)
	807	return false;
	808	if (a_info.ty == b_info.ty)
	809	return true;
	810
	811	// TODO: zig has something extra here for inferred error sets
	812	return false;
	813	},
	814	};
	815	}
	816	};
	817
	818	const EmbeddedNulls = enum {
	819	no_embedded_nulls,
	820	maybe_embedded_nulls,
	821
	822	const OptionalNullTerminatedString = @compileError("TODO");
	823	const OptionalString = @compileError("TODO");
	824
	825	fn StringType(comptime embedded_nulls: EmbeddedNulls) type {
	826	return switch (embedded_nulls) {
	827	.no_embedded_nulls => NullTerminatedString,
	828	.maybe_embedded_nulls => String,
	829	};
	830	}
	831
	832	fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type {
	833	return switch (embedded_nulls) {
	834	.no_embedded_nulls => OptionalNullTerminatedString,
	835	.maybe_embedded_nulls => OptionalString,
	836	};
	837	}
	838	};
	839
	840	// implementations:
	841
	842	/// To construct, instantiate `.{ .gpa = gpa }` then call this.
	843	pub fn init(ip: *InternPool, gpa: Allocator) !void {
	844	const t = tracer.trace(@src());
	845	defer t.end();
	846
	847	assert(ip.items.len == 0);
	848
	849	// Reserve string index 0 for an empty string.
	850	assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty);
	851
	852	// So that we can use `catch unreachable` below.
	853	try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
	854	try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
	855	try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len);
	856
	857	// This inserts all the statically-known values into the intern pool in the
	858	// order expected.
	859	for (static_keys) \|key\| {
	860	_ = ip.get(gpa, key) catch unreachable;
	861	}
	862
	863	if (std.debug.runtime_safety) {
	864	// Sanity check.
	865	assert(ip.indexToKey(.bool_true).simple_value == .true);
	866	assert(ip.indexToKey(.bool_false).simple_value == .false);
	867	}
	868
	869	assert(ip.items.len == static_keys.len);
	870	}
	871
	872	pub fn deinit(ip: *InternPool, gpa: Allocator) void {
	873	const t = tracer.trace(@src());
	874	defer t.end();
	875
	876	ip.map.deinit(gpa);
	877	ip.items.deinit(gpa);
	878	ip.extra.deinit(gpa);
	879	// ip.limbs.deinit(gpa);
	880	ip.string_bytes.deinit(gpa);
	881
	882	var decls = ip.allocated_decls.iterator(0);
	883	while (decls.next()) \|decl\| {
	884	decl.deinit(gpa);
	885	}
	886
	887	ip.allocated_decls.deinit(gpa);
	888	ip.decls_free_list.deinit(gpa);
	889
	890	var namespaces = ip.allocated_namespaces.iterator(0);
	891	while (namespaces.next()) \|ns\| {
	892	ns.decls.deinit(gpa);
	893	}
	894
	895	ip.namespaces_free_list.deinit(gpa);
	896	ip.allocated_namespaces.deinit(gpa);
	897
	898	// for (ip.maps.items) \|*map\| map.deinit(gpa);
	899	// ip.maps.deinit(gpa);
	900
	901	ip.string_table.deinit(gpa);
	902
	903	ip.tracked_insts.deinit(gpa);
	904
	905	// ip.src_hash_deps.deinit(gpa);
	906	// ip.decl_val_deps.deinit(gpa);
	907	// ip.func_ies_deps.deinit(gpa);
	908	// ip.namespace_deps.deinit(gpa);
	909	// ip.namespace_name_deps.deinit(gpa);
	910
	911	// ip.first_dependency.deinit(gpa);
	912
	913	// ip.dep_entries.deinit(gpa);
	914	// ip.free_dep_entries.deinit(gpa);
	915
	916	ip.* = undefined;
	917	}
	918
	919	pub fn indexToKey(ip: *const InternPool, index: Value) Key {
	920	assert(index != .none);
	921	const item: Item = ip.items.get(@intFromEnum(index));
	922	const data = item.data;
	923	return switch (item.tag) {
	924	.removed => unreachable,
	925
	926	.type_function => .{
	927	.type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)),
	928	},
	929
	930	.simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) },
	931	.simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) },
	932	.int32 => .{ .int32 = data.decode(.int32, item.tag) },
	933	.float => .{ .float = data.decode(.float, item.tag).get(ip).decode() },
	934	.comptime_float => .{ .comptime_float = data.decode(.comptime_float, item.tag).get(ip).decode() },
	935	.string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) },
	936	.func => {
	937	const func = data.decode(.func, item.tag).get(ip);
	938	return .{ .func = .{
	939	.ty = func.ty,
	940	.owner_decl = func.owner_decl,
	941	.teg_body_inst = func.teg_body_inst,
	942	} };
	943	},
	944	};
	945	}
	946
	947	pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value {
	948	const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)});
	949	defer t.end();
	950
	951	const adapter: Key.Adapter = .{ .intern_pool = ip };
	952	const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
	953	if (gop.found_existing) return @enumFromInt(gop.index);
	954	if (ip.items.len >= std.math.maxInt(u31) - 1) {
	955	return error.TooManyValues;
	956	}
	957
	958	const t2 = tracer.traceFmt(@src(), "(insert)", .{});
	959	defer t2.end();
	960
	961	try ip.items.ensureUnusedCapacity(gpa, 1);
	962	const item: Item = switch (key) {
	963	inline .int32,
	964	.simple_type,
	965	.simple_value,
	966	=> \|s, tag\| Item.encode(@field(Tag, @tagName(tag)), s),
	967	.string => \|s\| Item.encode(
	968	.string,
	969	try ip.extra.append(
	970	gpa,
	971	String.WithLen,
	972	(try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len),
	973	),
	974	),
	975	.float => \|f\| Item.encode(
	976	.float,
	977	try ip.extra.append(gpa, Float, Float.encode(f)),
	978	),
	979	.comptime_float => \|f\| Item.encode(
	980	.comptime_float,
	981	try ip.extra.append(gpa, Float, Float.encode(f)),
	982	),
	983	.func => \|f\| brk: {
	984	const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{
	985	.analysis = .{
	986	.state = .none,
	987	},
	988	.owner_decl = f.owner_decl,
	989	.ty = f.ty,
	990	.teg_body_inst = f.teg_body_inst,
	991	});
	992
	993	break :brk Item.encode(.func, extra);
	994	},
	995	.type_function => \|f\| brk: {
	996	const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
	997	.params_len = @intCast(f.param_types.len),
	998	.return_type = f.return_type,
	999	}, f.param_types.len);
	1000	defer trail.end();
	1001	trail.writeSlice(f.param_types);
	1002	break :brk Item.encode(.type_function, extra);
	1003	},
	1004	};
	1005	ip.items.appendAssumeCapacity(item);
	1006	const index: Value = @enumFromInt(ip.items.len - 1);
	1007	if (std.debug.runtime_safety) {
	1008	assert(ip.indexToKey(index).eql(key, ip));
	1009	}
	1010	return index;
	1011	}
	1012
	1013	fn extraTypeFunction(
	1014	ip: *const InternPool,
	1015	extra_index: ExtraIndex(Tag.TypeFunction),
	1016	) Key.TypeFunction {
	1017	const type_function, var trail = extra_index.getTrail(ip);
	1018	const param_types = trail.readTypeSlice(type_function.params_len);
	1019	const result: Key.TypeFunction = .{
	1020	.return_type = type_function.return_type,
	1021	.param_types = param_types.get(ip),
	1022	};
	1023	return result;
	1024	}
	1025
	1026	pub fn getOrPutString(
	1027	ip: *InternPool,
	1028	gpa: Allocator,
	1029	slice: []const u8,
	1030	comptime embedded_nulls: EmbeddedNulls,
	1031	) Allocator.Error!embedded_nulls.StringType() {
	1032	const t = tracer.trace(@src());
	1033	defer t.end();
	1034
	1035	try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1);
	1036	ip.string_bytes.appendSliceAssumeCapacity(slice);
	1037	ip.string_bytes.appendAssumeCapacity(0);
	1038	return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls);
	1039	}
	1040
	1041	fn getOrPutTrailingString(
	1042	ip: *InternPool,
	1043	gpa: Allocator,
	1044	len: usize,
	1045	comptime embedded_nulls: EmbeddedNulls,
	1046	) Allocator.Error!embedded_nulls.StringType() {
	1047	const t = tracer.trace(@src());
	1048	defer t.end();
	1049
	1050	const string_bytes = &ip.string_bytes;
	1051	const str_index: u32 = @intCast(string_bytes.items.len - len);
	1052	if (len > 0 and string_bytes.getLast() == 0) {
	1053	_ = string_bytes.pop();
	1054	} else {
	1055	try string_bytes.ensureUnusedCapacity(gpa, 1);
	1056	}
	1057	const key: []const u8 = string_bytes.items[str_index..];
	1058	const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
	1059	switch (embedded_nulls) {
	1060	.no_embedded_nulls => assert(!has_embedded_null),
	1061	.maybe_embedded_nulls => if (has_embedded_null) {
	1062	string_bytes.appendAssumeCapacity(0);
	1063	return @enumFromInt(str_index);
	1064	},
	1065	}
	1066	const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{
	1067	.bytes = string_bytes,
	1068	}, std.hash_map.StringIndexContext{
	1069	.bytes = string_bytes,
	1070	});
	1071	if (gop.found_existing) {
	1072	string_bytes.shrinkRetainingCapacity(str_index);
	1073	return @enumFromInt(gop.key_ptr.*);
	1074	} else {
	1075	gop.key_ptr.* = str_index;
	1076	string_bytes.appendAssumeCapacity(0);
	1077	return @enumFromInt(str_index);
	1078	}
	1079	}
	1080
	1081	pub fn trackInst(ip: InternPool, gpa: Allocator, file: Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index {
	1082	const t = tracer.trace(@src());
	1083	defer t.end();
	1084
	1085	const key: TrackedInst = .{
	1086	.path_digest = file.path_digest.?,
	1087	.inst = inst,
	1088	};
	1089	const gop = try ip.tracked_insts.getOrPut(gpa, key);
	1090	return @enumFromInt(gop.index);
	1091	}
	1092
	1093	pub fn declPtr(ip: InternPool, index: DeclIndex) Comp.Decl {
	1094	return ip.allocated_decls.at(@intFromEnum(index));
	1095	}
	1096
	1097	pub fn declPtrConst(ip: const InternPool, index: DeclIndex) const Comp.Decl {
	1098	return ip.allocated_decls.at(@intFromEnum(index));
	1099	}
	1100
	1101	pub fn namespacePtr(ip: InternPool, index: NamespaceIndex) Comp.Namespace {
	1102	return ip.allocated_namespaces.at(@intFromEnum(index));
	1103	}
	1104
	1105	pub fn createDecl(
	1106	ip: *InternPool,
	1107	gpa: Allocator,
	1108	initialization: Comp.Decl,
	1109	) Allocator.Error!DeclIndex {
	1110	const t = tracer.trace(@src());
	1111	defer t.end();
	1112
	1113	if (ip.decls_free_list.popOrNull()) \|index\| {
	1114	ip.allocated_decls.at(@intFromEnum(index)).* = initialization;
	1115	return index;
	1116	}
	1117	const ptr = try ip.allocated_decls.addOne(gpa);
	1118	ptr.* = initialization;
	1119	return @enumFromInt(ip.allocated_decls.len - 1);
	1120	}
	1121
	1122	pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void {
	1123	const t = tracer.trace(@src());
	1124	defer t.end();
	1125
	1126	ip.declPtr(index).* = undefined;
	1127	ip.decls_free_list.append(gpa, index) catch {
	1128	// In order to keep `destroyDecl` a non-fallible function, we ignore memory
	1129	// allocation failures here, instead leaking the Decl until garbage collection.
	1130	};
	1131	}
	1132
	1133	pub fn createNamespace(
	1134	ip: *InternPool,
	1135	gpa: Allocator,
	1136	initialization: Comp.Namespace,
	1137	) Allocator.Error!NamespaceIndex {
	1138	const t = tracer.trace(@src());
	1139	defer t.end();
	1140
	1141	if (ip.namespaces_free_list.popOrNull()) \|index\| {
	1142	ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization;
	1143	return index;
	1144	}
	1145	const ptr = try ip.allocated_namespaces.addOne(gpa);
	1146	ptr.* = initialization;
	1147	return @enumFromInt(ip.allocated_namespaces.len - 1);
	1148	}
	1149
	1150	pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void {
	1151	const t = tracer.trace(@src());
	1152	defer t.end();
	1153
	1154	ip.namespacePtr(index).* = undefined;
	1155	ip.namespaces_free_list.append(gpa, index) catch {
	1156	// In order to keep `destroyNamespace` a non-fallible function, we ignore memory
	1157	// allocation failures here, instead leaking the Namespace until garbage collection.
	1158	};
	1159	}
	1160
	1161	const InternPool = @This();
	1162
	1163	const std = @import("std");
	1164	const Allocator = std.mem.Allocator;
	1165	const Hash = std.hash.Wyhash;
	1166	const assert = std.debug.assert;
	1167
	1168	const Comp = @import("Comp.zig");
	1169	const Cache = @import("Cache.zig");
	1170
	1171	const Teg = @import("Teg.zig");
	1172	const Air = @import("Air.zig");
	1173
	1174	const dod = @import("array.zig");
	1175	const Float = dod.Float;
	1176
	1177	const tracer = @import("tracer");
	1178
	1179	test InternPool {
	1180	const testing = std.testing;
	1181	std.testing.refAllDeclsRecursive(@This());
	1182
	1183	var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
	1184	const ip = &ip_struct;
	1185	defer ip.deinit();
	1186
	1187	const str = try ip.getOrPutString("hello", .no_embedded_nulls);
	1188	try testing.expectEqualStrings("hello", str.toSlice(ip));
	1189	const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
	1190	try testing.expectEqual(str, str2);
	1191
	1192	const int1 = try ip.get(.{ .int32 = 21483 });
	1193	try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
	1194	const int2 = try ip.get(.{ .int32 = 21483 });
	1195	try testing.expectEqual(int1, int2);
	1196	}

src/core/JsLink.zig created+519

...	...	@@ -0,0 +1,519 @@
	1	/// This is the JavaScript backend's linker.
	2	/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript
	3	/// bundle out of them.
	4	comp: *Comp,
	5
	6	/// Value is an index into `link_strings`
	7	decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{},
	8	link_strings: std.ArrayHashMapUnmanaged(
	9	LinkString,
	10	GlobalOwner,
	11	LinkString.Context,
	12	true,
	13	) = .{},
	14	string_bytes: std.ArrayListUnmanaged(u8) = .{},
	15
	16	/// The first declaration is the function containing top-level code.
	17	pub fn linkJsCode(
	18	comp: *Comp,
	19	decls: []const Decl.Index,
	20	writer: anytype,
	21	) !void {
	22	var t = tracer.trace(@src());
	23	defer t.end();
	24
	25	// Arena allocator is used for the main flow
	26	var link: JsLink = .{ .comp = comp };
	27	defer link.deinit();
	28
	29	// Temp allocator is used for extremely temporary string work.
	30	// var temp_sfa = std.heap.stackFallback(4096, comp.gpa);
	31	// const temp_alloc = temp_sfa.get();
	32
	33	// 1. Resolve all global identifiers. Function names must not shadow external
	34	// names. We also need this list prepared so that local identifier names do not
	35	// shadow functions or externals.
	36	const root_decl = comp.intern_pool.declPtrConst(decls[0]);
	37	assert(root_decl.kind == .named);
	38	assert(root_decl.name == .empty);
	39
	40	for (root_decl.codegen.?.extern_symbols) \|extern_symbol\| {
	41	try link.reserveExternSymbol(extern_symbol);
	42	}
	43
	44	for (decls[1..]) \|decl_index\| {
	45	assert(decl_index != decls[0]);
	46
	47	const decl = comp.intern_pool.declPtrConst(decl_index);
	48
	49	// todo: add all externs first
	50	for (decl.codegen.?.extern_symbols) \|extern_symbol\| {
	51	try link.reserveExternSymbol(extern_symbol);
	52	}
	53
	54	const name = decl.name.toSlice(&comp.intern_pool);
	55	assert(name.len > 0);
	56
	57	const gop = try link.link_strings.getOrPutContextAdapted(
	58	comp.gpa,
	59	name,
	60	LinkString.SliceContext{ .link = &link },
	61	.{ .link = &link },
	62	);
	63	if (gop.found_existing) {
	64	@panic("TODO: handle this shadowing case.");
	65	}
	66	const name_str = try link.appendLinkString(name);
	67	gop.key_ptr.* = name_str;
	68	gop.value_ptr.* = .decl;
	69	try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
	70	}
	71
	72	try writer.print("// Language Compiler v0.1.2. Alpha quality software.\n", .{});
	73
	74	// 2. Print every function decl
	75	for (decls[1..]) \|decl_index\| {
	76	try link.printDecl(decl_index, writer);
	77	}
	78
	79	// 3. Print the top-level code
	80	if (root_decl.codegen.?.nodes.len > 1) {
	81	try link.printDeclContents(&root_decl.codegen.?, writer);
	82	} else {
	83	try writer.print("// (program has no observable effects, thus an empty file)\n", .{});
	84	}
	85	}
	86
	87	fn deinit(link: *JsLink) void {
	88	var t = tracer.trace(@src());
	89	defer t.end();
	90
	91	defer link.* = undefined;
	92	link.decls.deinit(link.comp.gpa);
	93	link.link_strings.deinit(link.comp.gpa);
	94	link.string_bytes.deinit(link.comp.gpa);
	95	}
	96
	97	fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
	98	var t = tracer.trace(@src());
	99	defer t.end();
	100
	101	const name = extern_symbol.toSlice(&link.comp.intern_pool);
	102	const gop = try link.link_strings.getOrPutContextAdapted(
	103	link.comp.gpa,
	104	name,
	105	LinkString.SliceContext{ .link = link },
	106	.{ .link = link },
	107	);
	108
	109	if (gop.found_existing) {
	110	if (gop.value_ptr.* != .external) {
	111	@panic("TODO: Rename Decl");
	112	}
	113	}
	114	gop.key_ptr.* = try link.appendLinkString(name);
	115	gop.value_ptr.* = .external;
	116	}
	117
	118	fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
	119	var t = tracer.trace(@src());
	120	defer t.end();
	121
	122	const start = link.string_bytes.items.len;
	123	try link.string_bytes.appendSlice(link.comp.gpa, string);
	124	return .{ .start = @intCast(start), .len = @intCast(string.len) };
	125	}
	126
	127	fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 {
	128	var t = tracer.trace(@src());
	129	defer t.end();
	130
	131	return (link.decls.get(decl) orelse if (std.debug.runtime_safety) {
	132	std.debug.panic("Decl {d} ({}) is not present in link decl map", .{
	133	@intFromEnum(decl),
	134	link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool),
	135	});
	136	} else unreachable).toString(link).slice(link);
	137	}
	138
	139	fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void {
	140	var t = tracer.trace(@src());
	141	defer t.end();
	142
	143	try writer.writeAll("let ");
	144	try writer.writeAll(link.declIdentifier(decl_index));
	145	try writer.writeAll(" = ");
	146
	147	const decl = link.comp.intern_pool.declPtrConst(decl_index);
	148	const tree = &decl.codegen.?;
	149
	150	try writer.writeAll("(");
	151	for (0..tree.param_count) \|id\| {
	152	try link.printLocal(@intCast(id), writer);
	153	if (id != tree.param_count - 1) try writer.writeAll(", ");
	154	}
	155	try writer.writeAll(") => {\n");
	156	try link.printDeclContents(tree, writer);
	157	try writer.writeAll("};\n");
	158	}
	159
	160	fn printDeclContents(link: JsLink, tree: const JsTree, writer: anytype) !void {
	161	var t = tracer.trace(@src());
	162	defer t.end();
	163
	164	for (tree.mainStatements()) \|stmt\| {
	165	try link.printExpr(tree, stmt, writer);
	166	try writer.writeAll(";\n");
	167	}
	168	}
	169
	170	fn printBlockContents(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
	171	var t = tracer.trace(@src());
	172	defer t.end();
	173
	174	assert(list.slice(tree).len > 0);
	175	for (list.slice(tree)) \|stmt\| {
	176	try link.printExpr(tree, stmt, writer);
	177	switch (tree.nodeTag(stmt)) {
	178	else => try writer.writeAll(";"),
	179	.stmt_block,
	180	.stmt_if,
	181	=> {},
	182	}
	183	try writer.writeAll("\n");
	184	}
	185	}
	186
	187	fn printBlock(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
	188	var t = tracer.trace(@src());
	189	defer t.end();
	190
	191	if (list.len != 1) {
	192	try writer.writeAll("{");
	193	try link.printBlockContents(tree, list, writer);
	194	try writer.writeAll("}");
	195	} else {
	196	try link.printBlockContents(tree, list, writer);
	197	}
	198	}
	199
	200	fn printExpr(link: JsLink, tree: const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
	201	var t = tracer.trace(@src());
	202	defer t.end();
	203
	204	switch (tree.nodeTag(node)) {
	205	.root => unreachable,
	206
	207	.plain_value => {
	208	const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node);
	209	switch (plain_value.value.toKey(&link.comp.intern_pool)) {
	210	// TODO: minification
	211	.int32 => \|val\| try writer.print("{d}", .{val}),
	212	// TODO: correct printing of many value types
	213	.float => \|val\| try writer.print("{d}", .{val}),
	214	.simple_value => \|val\| try writer.writeAll(switch (val) {
	215	.true => "true",
	216	.false => "false",
	217	// TODO: make codegen decide from null and undefined
	218	// instead of emitting void nodes.
	219	.void => "undefined",
	220	}),
	221	// TODO: escapes
	222	.string => \|val\| try writer.print("\"{s}\"", .{val}),
	223
	224	.comptime_float => unreachable, // was not lowered to float
	225
	226	else => unreachable, // this value is not a plain_value
	227	}
	228	},
	229
	230	.local => {
	231	const ref: JsTree.LocalRef = tree.nodeData(.local, node);
	232	try link.printLocal(ref.id, writer);
	233	},
	234
	235	.global => {
	236	const ref: JsTree.GlobalRef = tree.nodeData(.global, node);
	237	try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool));
	238	},
	239
	240	.decl => {
	241	const ref: JsTree.DeclRef = tree.nodeData(.decl, node);
	242	try writer.writeAll(link.declIdentifier(ref.index));
	243	},
	244
	245	.stmt_break => {
	246	const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node);
	247	try writer.writeAll("break");
	248	if (true) {
	249	try writer.writeAll(" ");
	250	try link.printLabelIdentifier(stmt.label, writer);
	251	}
	252	},
	253
	254	.stmt_return => {
	255	const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node);
	256	try writer.writeAll("return");
	257	if (stmt.operand.unwrap()) \|operand\| {
	258	try writer.writeAll(" ");
	259	try link.printExpr(tree, operand, writer);
	260	}
	261	},
	262
	263	.stmt_declare_local => {
	264	const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node);
	265	try writer.writeAll("let ");
	266	try link.printLocal(stmt.identifier_index, writer);
	267	if (stmt.init.unwrap()) \|init\| {
	268	try writer.writeAll(" = ");
	269	try link.printExpr(tree, init, writer);
	270	}
	271	},
	272
	273	.stmt_if => {
	274	const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node);
	275	const data = stmt.data.get(tree);
	276	try writer.writeAll("if (");
	277	try link.printExpr(tree, stmt.cond, writer);
	278	try writer.writeAll(") ");
	279	if (data.flags.yes_label) {
	280	try link.printLabelIdentifier(data.flags.block_label, writer);
	281	try writer.writeAll(":");
	282	}
	283	try link.printBlock(tree, data.yes, writer);
	284	if (data.no.len > 0) {
	285	try writer.writeAll(" else ");
	286	if (data.flags.no_label) {
	287	try link.printLabelIdentifier(data.flags.block_label, writer);
	288	try writer.writeAll(":");
	289	}
	290	try link.printBlock(tree, data.no, writer);
	291	}
	292	},
	293
	294	.stmt_block => {
	295	const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node);
	296	try link.printLabelIdentifier(stmt.label, writer);
	297	try writer.writeAll(": {\n");
	298	try link.printBlockContents(tree, stmt.content.get(tree), writer);
	299	try writer.writeAll("}");
	300	},
	301
	302	inline .assign,
	303	.add,
	304	.sub,
	305	.mul,
	306	.div,
	307	.eq,
	308	.lt,
	309	.gt,
	310	.assign_add,
	311	.assign_sub,
	312	.assign_mul,
	313	.assign_div,
	314	=> \|tag\| {
	315	const bin: JsTree.Bin = tree.nodeData(tag, node);
	316	const op: Operator = @field(Operator, @tagName(tag));
	317	try link.printExpr(tree, bin.lhs, writer);
	318	try writer.writeAll(" ");
	319	try writer.writeAll(comptime op.symbol());
	320	try writer.writeAll(" ");
	321	try link.printExpr(tree, bin.rhs, writer);
	322	},
	323
	324	inline .negate,
	325	.bool_not,
	326	.pre_dec,
	327	.pre_inc,
	328	=> \|tag\| {
	329	const un: JsTree.Un = tree.nodeData(tag, node);
	330	const op: Operator = @field(Operator, @tagName(tag));
	331	try writer.writeAll(comptime op.symbol());
	332	try link.printExpr(tree, un.operand, writer);
	333	},
	334
	335	.ternary => {
	336	const ternary: JsTree.Ternary = tree.nodeData(.ternary, node);
	337	const data = ternary.data.get(tree);
	338	try writer.writeAll("(");
	339	try link.printExpr(tree, ternary.cond, writer);
	340	try writer.writeAll(" ? ");
	341	try link.printExpr(tree, data.yes, writer);
	342	try writer.writeAll(" : ");
	343	try link.printExpr(tree, data.no, writer);
	344	try writer.writeAll(")");
	345	},
	346
	347	.call => {
	348	const call: JsTree.Call = tree.nodeData(.call, node);
	349	const args = call.args.get(tree);
	350
	351	try link.printExpr(tree, call.base, writer);
	352	try writer.writeAll("(");
	353
	354	for (args.slice(tree), 0..) \|arg, i\| {
	355	if (i != 0) try writer.writeAll(", ");
	356	try link.printExpr(tree, arg, writer);
	357	}
	358
	359	try writer.writeAll(")");
	360	},
	361
	362	.field => {
	363	const field: JsTree.Field = tree.nodeData(.field, node);
	364
	365	try link.printExpr(tree, field.base, writer);
	366	try writer.writeAll(".");
	367	try writer.writeAll(field.field.toSlice(&link.comp.intern_pool));
	368	},
	369	}
	370	}
	371
	372	fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
	373	var t = tracer.trace(@src());
	374	defer t.end();
	375
	376	_ = link; // todo: be smarter about this
	377	try writer.print("v{d}", .{local});
	378	}
	379
	380	fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
	381	var t = tracer.trace(@src());
	382	defer t.end();
	383
	384	_ = link; // todo: be smarter about this
	385	try writer.print("label{d}", .{local});
	386	}
	387
	388	const GlobalOwner = enum {
	389	external,
	390	decl,
	391	};
	392
	393	// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence
	394	const Level = enum {
	395	comma,
	396	assignment,
	397	conditional,
	398	logical_or,
	399	logical_and,
	400	bitwise_or,
	401	bitwise_xor,
	402	bitwise_and,
	403	equality,
	404	relational,
	405	bitshift,
	406	additive,
	407	multiplicative,
	408	exponentiation,
	409	prefix,
	410	postfix,
	411	new,
	412	access_and_call,
	413	member,
	414	};
	415
	416	const Operator = enum {
	417	assign,
	418	assign_add,
	419	assign_sub,
	420	assign_mul,
	421	assign_div,
	422	add,
	423	sub,
	424	mul,
	425	div,
	426	eq,
	427	lt,
	428	gt,
	429	negate,
	430	bool_not,
	431	pre_inc,
	432	pre_dec,
	433
	434	pub fn symbol(op: Operator) []const u8 {
	435	return switch (op) {
	436	.assign => "=",
	437	.assign_add => "+=",
	438	.assign_sub => "-=",
	439	.assign_mul => "*=",
	440	.assign_div => "/=",
	441	.add => "+",
	442	.sub => "-",
	443	.mul => "*",
	444	.div => "/",
	445	.eq => "===",
	446	.lt => "<",
	447	.gt => ">",
	448
	449	.pre_inc => "++",
	450	.pre_dec => "++",
	451
	452	.negate => "-",
	453	.bool_not => "!",
	454	};
	455	}
	456	};
	457
	458	const LinkString = struct {
	459	start: u32,
	460	len: u32,
	461
	462	fn slice(str: LinkString, link: *const JsLink) []const u8 {
	463	return link.string_bytes.items[str.start..][0..str.len];
	464	}
	465
	466	const Index = enum(u32) {
	467	_,
	468
	469	pub fn toString(index: Index, link: *const JsLink) LinkString {
	470	return link.link_strings.keys()[@intFromEnum(index)];
	471	}
	472	};
	473
	474	const Context = struct {
	475	link: *const JsLink,
	476
	477	pub fn hash(ctx: Context, str: LinkString) u32 {
	478	// assert(str.start + str.len < ctx.link.string_bytes.items.len);
	479
	480	return @truncate(Wyhash.hash(0, str.slice(ctx.link)));
	481	}
	482
	483	pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool {
	484	assert(a.start + a.len < ctx.link.string_bytes.items.len);
	485	assert(b.start + b.len < ctx.link.string_bytes.items.len);
	486	return a.start == b.start;
	487	}
	488	};
	489
	490	const SliceContext = struct {
	491	link: *const JsLink,
	492
	493	pub fn hash(_: SliceContext, str: []const u8) u32 {
	494	return @truncate(Wyhash.hash(0, str));
	495	}
	496
	497	pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool {
	498	// since the given string is not in string_bytes we do comparisons
	499	return std.mem.eql(u8, a, b.slice(ctx.link));
	500	}
	501	};
	502	};
	503
	504	const JsLink = @This();
	505	const std = @import("std");
	506	const assert = std.debug.assert;
	507	const Wyhash = std.hash.Wyhash;
	508	const log = std.log.scoped(.js_link);
	509
	510	const Comp = @import("Comp.zig");
	511	const Decl = Comp.Decl;
	512
	513	const JsTree = @import("JsTree.zig");
	514	const Node = JsTree.Node;
	515
	516	const InternPool = @import("InternPool.zig");
	517	const String = InternPool.String;
	518
	519	const tracer = @import("tracer");

src/core/JsNodeGen.zig created+700

...	...	@@ -0,0 +1,700 @@
	1	//! Air -> JsTree
	2	gpa: Allocator,
	3	arena: Allocator,
	4	air: *const Air,
	5	ip: *const InternPool,
	6
	7	nodes: Node.List,
	8	extra: dod.ExtraStore(.tfr, .@"var"),
	9	extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void),
	10	decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void),
	11
	12	param_count: u32,
	13	air_node_map: []MapEntry,
	14	scratch: std.ArrayListUnmanaged(u32),
	15	next_id: u32,
	16	next_label: u32,
	17
	18	const Error = error{
	19	OutOfMemory,
	20	};
	21
	22	const MapEntry = union {
	23	none: void,
	24	node: Node.Index,
	25	block: packed struct {
	26	used: bool = false,
	27	index: u31,
	28	},
	29	};
	30
	31	pub fn init(gpa: Allocator, arena: Allocator, ip: const InternPool, air: const Air) !JsNodeGen {
	32	const t = tracer.trace(@src());
	33	defer t.end();
	34
	35	const air_node_map = try arena.alloc(MapEntry, air.insts.len);
	36	if (std.debug.runtime_safety) {
	37	@memset(air_node_map, .{ .none = {} });
	38	}
	39	const param_count = air.paramCount();
	40	return .{
	41	.gpa = gpa,
	42	.arena = arena,
	43	.air = air,
	44	.ip = ip,
	45	.param_count = param_count,
	46	.nodes = .{},
	47	.extern_symbols = .{},
	48	.decl_deps = .{},
	49	.air_node_map = air_node_map,
	50	.extra = .{},
	51	.scratch = .{},
	52	.next_id = param_count,
	53	.next_label = 0,
	54	};
	55	}
	56
	57	pub fn deinit(c: *JsNodeGen) void {
	58	const t = tracer.trace(@src());
	59	defer t.end();
	60
	61	defer c.* = undefined;
	62	c.nodes.deinit(c.gpa);
	63	c.extra.deinit(c.gpa);
	64	c.extern_symbols.deinit(c.gpa);
	65	c.decl_deps.deinit(c.gpa);
	66
	67	// arena can deinit the rest
	68	}
	69
	70	pub fn finalize(c: *JsNodeGen) !JsTree {
	71	const t = tracer.trace(@src());
	72	defer t.end();
	73
	74	// todo: errdefer will probably leave memory in a fully invalid state
	75	try c.nodes.setCapacity(c.gpa, c.nodes.len);
	76
	77	var extra = try c.extra.toOwnedStore(c.gpa);
	78	errdefer extra.deinit(c.gpa);
	79
	80	// todo: i dont like this dupe
	81	const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys());
	82	errdefer c.gpa.free(extern_symbols);
	83
	84	const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys());
	85	errdefer c.gpa.free(decl_deps);
	86
	87	defer c.deinit();
	88
	89	return .{
	90	.param_count = c.param_count,
	91	.nodes = c.nodes.toOwnedSlice(),
	92	.extern_symbols = extern_symbols,
	93	.decl_deps = decl_deps,
	94	.extra = extra,
	95	};
	96	}
	97
	98	pub fn build(c: *JsNodeGen) !void {
	99	const t = tracer.trace(@src());
	100	defer t.end();
	101
	102	assert(c.air.instTag(.main_block) == .block);
	103	const statements = c.air.instData(.block, .main_block).slice(c.air);
	104
	105	const scratch_top = c.scratch.items.len;
	106	defer c.scratch.shrinkRetainingCapacity(scratch_top);
	107
	108	_ = try c.add(.root, undefined);
	109
	110	if (statements.len > 0) {
	111	for (statements[0..c.param_count], 1..) \|stmt, expected_id\| {
	112	assert(@intFromEnum(stmt) == expected_id);
	113	assert(c.air.instTag(stmt) == .arg);
	114	c.air_node_map[@intFromEnum(stmt)] = .{
	115	.node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }),
	116	};
	117	}
	118
	119	for (statements[c.param_count..]) \|stmt\| {
	120	try c.codeGenInst(stmt);
	121	}
	122	}
	123
	124	const items = c.scratch.items[scratch_top..];
	125	const top = c.extra.list.items.len;
	126	try c.extra.list.appendSlice(c.gpa, items);
	127	c.nodes.items(.data)[0] = EncodedData.encode(.root, .{
	128	.start = @intCast(top),
	129	.len = @intCast(items.len),
	130	});
	131	}
	132
	133	fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
	134	const t = tracer.traceFmt(@src(), "#{d} {s}", .{ @intFromEnum(inst), @tagName(c.air.instTag(inst)) });
	135	defer t.end();
	136
	137	switch (c.air.instTag(inst)) {
	138	.arg => unreachable, // codegen must skip over these instructions.
	139
	140	.block => try c.codeGenBlock(inst),
	141
	142	.js_global_ref => {
	143	const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst);
	144	try c.extern_symbols.put(c.gpa, data, {});
	145	c.air_node_map[@intFromEnum(inst)] = .{
	146	.node = try c.add(.global, .{ .name = data }),
	147	};
	148	},
	149
	150	.memory => {
	151	const i = @intFromEnum(inst);
	152	const data: Air.Memory = c.air.instData(.memory, inst);
	153	const value: JsTree.Node.Index.Optional = if (data.init.isVoid())
	154	.none
	155	else
	156	(try c.fetchExpression(data.init)).toOptional();
	157
	158	try c.addStmt(try c.add(.stmt_declare_local, .{
	159	.identifier_index = c.next_id,
	160	.init = value,
	161	}));
	162	const id = try c.add(.local, .{ .id = c.next_id });
	163	c.next_id += 1;
	164	c.air_node_map[i] = .{ .node = id };
	165	},
	166
	167	inline .gt,
	168	.lt,
	169	.eq,
	170	.add,
	171	.sub,
	172	.mul,
	173	.div,
	174	=> \|tag\| {
	175	const data: Air.Bin = c.air.instData(tag, inst);
	176	const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
	177	.lhs = try c.fetchExpression(data.lhs),
	178	.rhs = try c.fetchExpression(data.rhs),
	179	});
	180	try c.markExpression(inst, node);
	181	},
	182
	183	.store => {
	184	const data: Air.Store = c.air.instData(.store, inst);
	185
	186	const target = try c.fetchInstExpression(data.memory);
	187
	188	if (data.operand.kind == .index) {
	189	const index = data.operand.decode().index;
	190	switch (c.air.instTag(index)) {
	191	inline .add,
	192	.sub,
	193	.mul,
	194	.div,
	195	=> \|tag\| {
	196	const inner_data: Air.Bin = c.air.instData(tag, index);
	197	if (inner_data.lhs.eql(data.memory.toRef())) {
	198	if ((tag == .add or tag == .sub) and
	199	inner_data.rhs.kind == .value)
	200	{
	201	// if (inner_data.rhs.decode().value.toFloat(c.ip)) \|f\| {
	202	// if (f == 1) {
	203	// // convert this into ++ or --
	204	// const node = try c.add(
	205	// if (tag == .add)
	206	// .pre_inc
	207	// else
	208	// .pre_dec,
	209	// .{
	210	// .operand = target,
	211	// },
	212	// );
	213	// try c.markExpression(inst, node);
	214
	215	// return;
	216	// }
	217	// }
	218	}
	219
	220	// convert into a compound assignment
	221	const node = try c.add(
	222	@field(JsTree.Tag, "assign_" ++ @tagName(tag)),
	223	.{
	224	.lhs = target,
	225	.rhs = try c.fetchExpression(inner_data.rhs),
	226	},
	227	);
	228	try c.markExpression(inst, node);
	229
	230	return;
	231	}
	232	},
	233	else => {},
	234	}
	235	}
	236
	237	const node = try c.add(.assign, .{
	238	.lhs = target,
	239	.rhs = try c.fetchExpression(data.operand),
	240	});
	241	try c.markExpression(inst, node);
	242	},
	243
	244	inline .negate,
	245	.bool_not,
	246	=> \|tag\| {
	247	const data: Air.Un = c.air.instData(tag, inst);
	248	const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
	249	.operand = try c.fetchExpression(data.operand),
	250	});
	251	try c.markExpression(inst, node);
	252	},
	253
	254	.field => {
	255	const data: Air.Field = c.air.instData(.field, inst);
	256	const node = try c.add(.field, .{
	257	.base = try c.fetchExpression(data.base),
	258	.field = data.field,
	259	});
	260	try c.markExpression(inst, node);
	261	},
	262
	263	.call => {
	264	const data: Air.Call = c.air.instData(.call, inst);
	265	const call_data = data.call_data.get(c.air);
	266
	267	try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len);
	268	const start = c.extra.list.items.len;
	269	c.extra.list.items.len += call_data.args.len;
	270
	271	const items = c.extra.list.items[start..][0..call_data.args.len];
	272
	273	for (call_data.args.slice(c.air), items) \|arg, *dest\| {
	274	dest.* = @intFromEnum(try c.fetchExpression(arg));
	275	}
	276
	277	const node = try c.add(.call, .{
	278	.base = try c.fetchExpression(data.base),
	279	.args = try c.extra.append(c.gpa, JsTree.IndexList, .{
	280	.start = @intCast(start),
	281	.len = call_data.args.len,
	282	}),
	283	});
	284	try c.markExpression(inst, node);
	285	},
	286
	287	.@"if" => {
	288	const data: Air.If = c.air.instData(.@"if", inst);
	289	const extra = data.data.get(c.air);
	290	var cond = try c.fetchExpression(data.cond);
	291
	292	if (c.canLowerToExpression(extra.yes) and
	293	c.canLowerToExpression(extra.no))
	294	{
	295	const yes = try c.lowerToExpression(extra.yes);
	296	const no = try c.lowerToExpression(extra.yes);
	297
	298	const ternary = try c.add(.ternary, .{
	299	.cond = cond,
	300	.data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{
	301	.yes = yes,
	302	.no = no,
	303	}),
	304	});
	305
	306	try c.markExpression(inst, ternary);
	307	} else {
	308	const flags = c.air.instFlags(inst);
	309	const uses = flags.uses;
	310
	311	const target = if (uses != .none)
	312	try c.addTemporary()
	313	else
	314	null;
	315
	316	var yes = try c.codeGenBranch(extra.yes, target);
	317	var no = try c.codeGenBranch(extra.no, target);
	318	assert(!((yes == null) and (no == null))); // no "void else void" branches
	319
	320	// when `yes` is void, swap the condition so that the `else` is omitted
	321	if (yes == null) {
	322	const temp_yes = yes;
	323	yes = no;
	324	no = temp_yes;
	325	cond = try c.add(.bool_not, .{
	326	.operand = cond,
	327	});
	328	}
	329
	330	const main_expr = try c.add(.stmt_if, .{
	331	.cond = cond,
	332	.data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{
	333	.yes = yes.?.list,
	334	.no = if (no) \|branch\|
	335	branch.list
	336	else
	337	.{ .start = 0, .len = 0 },
	338	.flags = .{
	339	.block_label = @intCast(c.next_label + 1),
	340	.yes_label = yes.?.requires_label,
	341	.no_label = if (no) \|branch\| branch.requires_label else false,
	342	},
	343	}),
	344	});
	345
	346	try c.addStmt(main_expr);
	347
	348	if (target) \|node\| {
	349	try c.markExpression(inst, node);
	350	}
	351	}
	352	},
	353
	354	.@"break" => {
	355	const brk: Air.Break = c.air.instData(.@"break", inst);
	356
	357	const operand = try c.fetchExpression(brk.value);
	358
	359	// break #0 -> return
	360	if (brk.block_ref == .main_block) {
	361	try c.addStmt(try c.add(.stmt_return, .{
	362	.operand = operand.toOptional(),
	363	}));
	364	} else {
	365	const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block;
	366
	367	const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air);
	368	if (block_insts[block_insts.len - 1] != inst) {
	369	block.used = true;
	370	}
	371
	372	const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]);
	373	if (assignment.unwrap()) \|target\| {
	374	try c.addStmt(try c.add(.assign, .{
	375	.lhs = target,
	376	.rhs = operand,
	377	}));
	378	}
	379
	380	try c.addStmt(try c.add(.stmt_break, .{
	381	.label = c.scratch.items[block.index + 1],
	382	}));
	383	}
	384	},
	385	}
	386	}
	387
	388	fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void {
	389	const t = tracer.trace(@src());
	390	defer t.end();
	391
	392	const flags: Air.Flags = c.air.instFlags(inst);
	393	const i = @intFromEnum(inst);
	394	switch (flags.uses) {
	395	.none => {
	396	try c.addStmt(node);
	397	},
	398	.single => {
	399	// TODO: this is not aware of side effect fences
	400	c.air_node_map[i] = .{ .node = node };
	401	},
	402	.multi => {
	403	const decl = try c.add(.stmt_declare_local, .{
	404	.identifier_index = c.next_id,
	405	.init = node.toOptional(),
	406	});
	407	try c.addStmt(decl);
	408	const id = try c.add(.local, .{ .id = c.next_id });
	409	c.next_id += 1;
	410	c.air_node_map[i] = .{ .node = id };
	411	},
	412	}
	413	}
	414
	415	fn fetchExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
	416	const t = tracer.trace(@src());
	417	defer t.end();
	418
	419	return switch (ref.decode()) {
	420	.value => \|v\| c.valueToExpression(v),
	421	.index => \|index\| fetchInstExpression(c, index),
	422	};
	423	}
	424
	425	fn fetchInstExpression(c: *JsNodeGen, index: Air.Inst.Index) !Node.Index {
	426	const t = tracer.trace(@src());
	427	defer t.end();
	428
	429	const data = c.air_node_map[@intFromEnum(index)];
	430	if (std.debug.runtime_safety) {
	431	const flags: Air.Flags = c.air.instFlags(index);
	432	switch (flags.uses) {
	433	.none => unreachable, // incorrectly marked instruction as unused,
	434	.single => {},
	435	.multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
	436	}
	437	}
	438	return data.node;
	439	}
	440
	441	fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index {
	442	const t = tracer.trace(@src());
	443	defer t.end();
	444
	445	const intern_tag = v.valueTag(c.ip);
	446	switch (intern_tag) {
	447	.removed => unreachable, // UAF
	448
	449	.type_function,
	450	.simple_type,
	451	=> unreachable, // Type values are not allowed here
	452
	453	.int32,
	454	.float,
	455	.comptime_float,
	456	.string,
	457	.simple_value,
	458	=> return try c.add(.plain_value, .{ .value = v }),
	459
	460	// funcs always point to decls
	461	.func => {
	462	const func = v.toKey(c.ip).func;
	463	try c.decl_deps.put(c.gpa, func.owner_decl, {});
	464	return try c.add(.decl, .{ .index = func.owner_decl });
	465	},
	466	}
	467	}
	468
	469	fn addTemporary(c: *JsNodeGen) !Node.Index {
	470	const t = tracer.trace(@src());
	471	defer t.end();
	472
	473	try c.addStmt(try c.add(.stmt_declare_local, .{
	474	.identifier_index = c.next_id,
	475	.init = .none,
	476	}));
	477	const id = try c.add(.local, .{ .id = c.next_id });
	478	c.next_id += 1;
	479	return id;
	480	}
	481
	482	const Branch = struct {
	483	requires_label: bool,
	484	list: JsTree.IndexList,
	485	};
	486
	487	fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch {
	488	const t = tracer.trace(@src());
	489	defer t.end();
	490
	491	const scratch_top = c.scratch.items.len;
	492	defer c.scratch.shrinkRetainingCapacity(scratch_top);
	493
	494	c.next_label += 1;
	495	defer c.next_label -= 1;
	496	const this_label = c.next_label;
	497
	498	var requires_label = false;
	499	var items = switch (inst.decode()) {
	500	.value => \|v\| items: {
	501	if (v == .void)
	502	return null;
	503	if (temporary) \|temp\| {
	504	const node = try c.valueToExpression(v);
	505	try c.addStmt(try c.add(.assign, .{
	506	.lhs = temp,
	507	.rhs = node,
	508	}));
	509	}
	510	break :items c.scratch.items[scratch_top..];
	511	},
	512	.index => \|i\| items: {
	513	assert(c.air.instTag(i) == .block);
	514	c.air_node_map[@intFromEnum(i)] = .{
	515	.block = .{ .index = @intCast(scratch_top) },
	516	};
	517
	518	try c.scratch.appendSlice(c.arena, &.{
	519	@intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
	520	this_label,
	521	});
	522
	523	const block: Air.IndexList = c.air.instData(.block, i);
	524	for (block.slice(c.air)) \|item\| {
	525	try c.codeGenInst(item);
	526	}
	527
	528	requires_label = c.air_node_map[@intFromEnum(i)].block.used;
	529
	530	break :items c.scratch.items[scratch_top + 2 ..];
	531	},
	532	};
	533
	534	const last_node = c.scratch.items[c.scratch.items.len - 1];
	535	if (c.nodes.items(.tag)[last_node] == .stmt_break and
	536	c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label)
	537	{
	538	items.len -= 1;
	539	c.nodes.len -= 1;
	540	}
	541
	542	const top = c.extra.list.items.len;
	543	try c.extra.list.appendSlice(c.gpa, items);
	544	return .{
	545	.requires_label = requires_label,
	546	.list = .{
	547	.start = @intCast(top),
	548	.len = @intCast(items.len),
	549	},
	550	};
	551	}
	552
	553	fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void {
	554	const t = tracer.trace(@src());
	555	defer t.end();
	556
	557	const scratch_top = c.scratch.items.len;
	558	errdefer c.scratch.shrinkRetainingCapacity(scratch_top);
	559
	560	c.next_label += 1;
	561	defer c.next_label -= 1;
	562
	563	const next_label = c.next_label;
	564
	565	try c.scratch.appendSlice(c.arena, &.{
	566	// @intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
	567	@intFromEnum(Node.Index.Optional.none),
	568	next_label,
	569	});
	570
	571	c.air_node_map[@intFromEnum(inst)] = .{
	572	.block = .{ .index = @intCast(scratch_top) },
	573	};
	574	const block: Air.IndexList = c.air.instData(.block, inst);
	575
	576	for (block.slice(c.air)) \|item\| {
	577	try c.codeGenInst(item);
	578	}
	579
	580	const items = c.scratch.items[scratch_top + 2 ..];
	581	const top = c.extra.list.items.len;
	582	try c.extra.list.appendSlice(c.gpa, items);
	583	c.scratch.shrinkRetainingCapacity(scratch_top);
	584	try c.addStmt(try c.add(.stmt_block, .{
	585	.label = c.next_label,
	586	.content = try c.extra.append(c.gpa, JsTree.IndexList, .{
	587	.start = @intCast(top),
	588	.len = @intCast(items.len),
	589	}),
	590	}));
	591	}
	592
	593	// Ideally, we convert everything to an expression. This allows for much
	594	// more compact code. When that isnt possible (`return`, `while`, etc),
	595	// we detect that here and ensure to lower `if` to actual if statements,
	596	// and so on.
	597	fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool {
	598	const t = tracer.trace(@src());
	599	defer t.end();
	600
	601	return switch (inst.decode()) {
	602	.value => true,
	603	.index => \|index\| switch (c.air.instTag(index)) {
	604	.js_global_ref,
	605	.call,
	606	.arg,
	607	.field,
	608	.add,
	609	.sub,
	610	.mul,
	611	.div,
	612	.eq,
	613	.lt,
	614	.gt,
	615	.negate,
	616	.bool_not,
	617	.store,
	618	=> true,
	619
	620	// JavaScript variable declarations require statements.
	621	// Later, this could be made true if we can hoist temporaries up.
	622	.memory => false,
	623
	624	// break/return must be a js statement
	625	.@"break",
	626	=> false,
	627
	628	.block => {
	629	// TODO: blocks that are
	630	// - all statements except last are lowerable
	631	// - the last statement is lowerable OR a break of this block
	632	return false;
	633	},
	634
	635	.@"if" => {
	636	const call = c.air.instData(.@"if", index);
	637	const data = call.data.get(c.air);
	638
	639	return c.canLowerToExpression(call.cond) and
	640	c.canLowerToExpression(data.yes) and
	641	c.canLowerToExpression(data.no);
	642	},
	643	},
	644	};
	645	}
	646
	647	fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
	648	const t = tracer.trace(@src());
	649	defer t.end();
	650
	651	assert(c.canLowerToExpression(ref));
	652	return switch (ref.decode()) {
	653	.value => \|v\| try c.valueToExpression(v),
	654	.index => \|index\| {
	655	try c.codeGenInst(index);
	656	return c.fetchExpression(ref);
	657	},
	658	};
	659	}
	660
	661	fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index {
	662	const t = tracer.trace(@src());
	663	defer t.end();
	664
	665	try c.nodes.append(c.gpa, .{
	666	.tag = tag,
	667	.data = EncodedData.encode(tag, data),
	668	});
	669	return @enumFromInt(c.nodes.len - 1);
	670	}
	671
	672	fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void {
	673	const t = tracer.trace(@src());
	674	defer t.end();
	675
	676	try c.scratch.append(c.arena, @intFromEnum(stmt));
	677	}
	678
	679	const JsNodeGen = @This();
	680
	681	const std = @import("std");
	682	const assert = std.debug.assert;
	683	const Allocator = std.mem.Allocator;
	684
	685	const InternPool = @import("InternPool.zig");
	686	const Value = InternPool.Value;
	687	const NullTerminatedString = InternPool.NullTerminatedString;
	688
	689	const Comp = @import("Comp.zig");
	690	const Decl = Comp.Decl;
	691
	692	const JsTree = @import("JsTree.zig");
	693	const EncodedData = JsTree.EncodedData;
	694	const Node = JsTree.Node;
	695	const LocalIndex = JsTree.LocalIndex;
	696
	697	const Air = @import("Air.zig");
	698
	699	const dod = @import("array.zig");
	700	const tracer = @import("tracer");

src/core/JsTree.zig created+257

...	...	@@ -0,0 +1,257 @@
	1	//! JavaScript Function Representation is the last representation before javascript
	2	//! source bytes are created. One notable thing here is that all local identifiers
	3	//! are numbered, instead of being named. The first `param_count` local identifiers
	4	//! refer to parameters, and the rest refer to the respective block that defined them.
	5
	6	param_count: u32,
	7	nodes: Node.List.Slice,
	8	extern_symbols: []String.WithLen,
	9	decl_deps: []Decl.Index,
	10	extra: ExtraStore,
	11
	12	pub const LocalIndex = u32;
	13	pub const ExtraStore = dod.ExtraStore(.tfr, .@"const");
	14	pub const EncodedData = dod.EncodedData(Tag, u64);
	15	pub const ExtraIndex = ExtraStore.Index;
	16
	17	pub const Tag = enum {
	18	root,
	19	/// Only allow simple value types: int, float, string, bool
	20	plain_value,
	21	/// Reference to a local identifier (give an VarId)
	22	local,
	23	/// Reference to a global identifier (give a NullTerminatedString)
	24	global,
	25	/// Reference to a declaration (contains a Decl.Index)
	26	decl,
	27	// bin ops
	28	assign,
	29	assign_add,
	30	assign_sub,
	31	assign_mul,
	32	assign_div,
	33	add,
	34	sub,
	35	mul,
	36	div,
	37	eq,
	38	lt,
	39	gt,
	40	// exprs
	41	call,
	42	field,
	43	ternary,
	44	// prefix ops
	45	negate,
	46	bool_not,
	47	pre_inc,
	48	pre_dec,
	49	// stmts
	50	/// `let ... = ...`
	51	stmt_declare_local,
	52	/// `if (...) ... else ...`
	53	stmt_if,
	54	/// `break ... ...`
	55	stmt_break,
	56	/// `return ...` (break from block #0)
	57	stmt_return,
	58	/// `...: { ... }`
	59	stmt_block,
	60
	61	pub fn Data(comptime tag: Tag) type {
	62	return switch (tag) {
	63	.root => IndexList,
	64	.plain_value => PlainValue,
	65	.local => LocalRef,
	66	.global => GlobalRef,
	67	.decl => DeclRef,
	68	.assign,
	69	.assign_add,
	70	.assign_sub,
	71	.assign_mul,
	72	.assign_div,
	73	.add,
	74	.sub,
	75	.mul,
	76	.div,
	77	.eq,
	78	.lt,
	79	.gt,
	80	=> Bin,
	81	.ternary => Ternary,
	82	.negate,
	83	.bool_not,
	84	.pre_inc,
	85	.pre_dec,
	86	=> Un,
	87	.call => Call,
	88	.field => Field,
	89	.stmt_if => StmtIf,
	90	.stmt_break => StmtBreak,
	91	.stmt_return => StmtReturn,
	92	.stmt_declare_local => StmtDeclareLocal,
	93	.stmt_block => StmtBlock,
	94	};
	95	}
	96	};
	97
	98	pub const Node = struct {
	99	tag: Tag,
	100	data: EncodedData,
	101
	102	pub const List = std.MultiArrayList(Node);
	103
	104	pub const Index = enum(u32) {
	105	root = 0,
	106	_,
	107
	108	pub fn toOptional(i: Index) Optional {
	109	return @enumFromInt(@intFromEnum(i));
	110	}
	111
	112	pub const Optional = enum(u32) {
	113	none = std.math.maxInt(u32),
	114	_,
	115
	116	pub fn unwrap(oi: Optional) ?Index {
	117	return if (oi == .none)
	118	null
	119	else
	120	@enumFromInt(@intFromEnum(oi));
	121	}
	122	};
	123	};
	124	};
	125
	126	pub const PlainValue = struct {
	127	value: Value,
	128	unused: Unused = .zero,
	129	};
	130
	131	pub const LocalRef = struct {
	132	id: LocalIndex,
	133	unused: Unused = .zero,
	134	};
	135
	136	pub const GlobalRef = struct {
	137	name: InternPool.String.WithLen,
	138	};
	139
	140	pub const DeclRef = struct {
	141	index: Decl.Index,
	142	unused: Unused = .zero,
	143	};
	144
	145	pub const StmtDeclareLocal = struct {
	146	identifier_index: u32,
	147	init: Node.Index.Optional,
	148	};
	149
	150	pub const Bin = struct {
	151	lhs: Node.Index,
	152	rhs: Node.Index,
	153	};
	154
	155	pub const Un = struct {
	156	operand: Node.Index,
	157	unused: Unused = .zero,
	158	};
	159
	160	pub const Call = struct {
	161	base: Node.Index,
	162	args: ExtraIndex(IndexList),
	163	};
	164
	165	pub const Field = struct {
	166	base: Node.Index,
	167	field: NullTerminatedString,
	168	};
	169
	170	pub const IndexList = struct {
	171	start: u32,
	172	len: u32,
	173
	174	pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index {
	175	return @ptrCast(tree.extra.list.items[list.start..][0..list.len]);
	176	}
	177	};
	178
	179	pub const StmtIf = struct {
	180	cond: Node.Index,
	181	data: ExtraIndex(Data),
	182
	183	pub const Data = struct {
	184	yes: IndexList,
	185	no: IndexList,
	186	flags: Flags,
	187	};
	188
	189	pub const Flags = packed struct(u32) {
	190	block_label: u16,
	191	yes_label: bool,
	192	no_label: bool,
	193	_: u14 = 0,
	194	};
	195	};
	196
	197	pub const Ternary = struct {
	198	cond: Node.Index,
	199	data: ExtraIndex(Data),
	200
	201	pub const Data = struct {
	202	yes: Node.Index,
	203	no: Node.Index,
	204	};
	205	};
	206
	207	pub const StmtBreak = struct {
	208	label: u32,
	209	unused: Unused = .zero,
	210	};
	211
	212	pub const StmtReturn = struct {
	213	operand: Node.Index.Optional,
	214	unused: Unused = .zero,
	215	};
	216
	217	pub const StmtBlock = struct {
	218	label: u32,
	219	content: ExtraIndex(IndexList),
	220	};
	221
	222	pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void {
	223	tree.nodes.deinit(gpa);
	224	gpa.free(tree.decl_deps);
	225	gpa.free(tree.extern_symbols);
	226	tree.extra.deinit(gpa);
	227	}
	228
	229	pub fn nodeTag(tree: JsTree, index: Node.Index) Tag {
	230	return tree.nodes.items(.tag)[@intFromEnum(index)];
	231	}
	232
	233	pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() {
	234	return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index));
	235	}
	236
	237	pub fn mainStatements(tree: *const JsTree) []const Node.Index {
	238	return tree.nodeData(.root, .root).slice(tree);
	239	}
	240
	241	const JsTree = @This();
	242
	243	const std = @import("std");
	244	const assert = std.debug.assert;
	245
	246	const InternPool = @import("InternPool.zig");
	247	const Value = InternPool.Value;
	248
	249	const NullTerminatedString = InternPool.NullTerminatedString;
	250	const String = InternPool.String;
	251
	252	const Comp = @import("Comp.zig");
	253	const Decl = Comp.Decl;
	254
	255	const dod = @import("array.zig");
	256
	257	const Unused = @import("Ast.zig").Unused;

src/core/Parse.zig created+1146

...	...	@@ -0,0 +1,1146 @@
	1	//! Parser takes a Token list and creates a Node list. It does not own the source code or
	2	//! token data, however transferring into an `Ast` does move token list ownership.
	3	gpa: Allocator,
	4	source: [:0]const u8,
	5	token_tags: []const Token.Tag,
	6	token_starts: []const ByteOffset,
	7
	8	pos: TokenIndex = 0,
	9	nodes: Node.List = .{},
	10	extra: ArrayListUnmanaged(u32) = .{},
	11	scratch: ArrayListUnmanaged(NodeIndex) = .{},
	12
	13	ast_error: ?AstError.WithTrace = null,
	14
	15	depth: u32 = 0,
	16
	17	/// Instead of implementing a non-recursive parser, the stack size is limited by this constant.
	18	const depth_limit = 4096;
	19
	20	pub fn parse(gpa: std.mem.Allocator, source: [:0]const u8, ast_error: *AstError.WithTrace) Parse.Error!Ast {
	21	var err_byte: ByteOffset = 0;
	22	var tokens = Tokenizer.tokenize(
	23	source,
	24	gpa,
	25	&err_byte,
	26	) catch \|err\| switch (err) {
	27	error.OutOfMemory => return error.OutOfMemory,
	28	else => {
	29	ast_error.* = .{
	30	.err = .{ .tokenization = .{
	31	.err = err,
	32	.byte = err_byte,
	33	} },
	34	.trace = Report.DebugStoredTrace.from(@errorReturnTrace()),
	35	};
	36	return error.ParseFail;
	37	},
	38	};
	39	errdefer tokens.deinit(gpa);
	40
	41	var parser = Parse.init(gpa, source, tokens);
	42	errdefer parser.deinit();
	43
	44	try parser.nodes.ensureUnusedCapacity(
	45	gpa,
	46	@max(1, @divFloor(tokens.len, 4)),
	47	);
	48
	49	parser.parseRoot() catch \|err\| switch (err) {
	50	error.OutOfMemory => return error.OutOfMemory,
	51	error.ParseFail => {
	52	ast_error.* = parser.ast_error.?;
	53	return error.ParseFail;
	54	},
	55	};
	56
	57	return try parser.finalize(tokens);
	58	}
	59
	60	fn init(gpa: Allocator, source: [:0]const u8, tokens: Token.List.Slice) Parse {
	61	return Parse{
	62	.gpa = gpa,
	63	.source = source,
	64	.token_tags = tokens.items(.tag),
	65	.token_starts = tokens.items(.offset),
	66	};
	67	}
	68
	69	/// This transfers the parse result into an Ast
	70	fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
	71	const extra = try p.extra.toOwnedSlice(p.gpa);
	72	defer p.* = undefined;
	73	assert(token_slice.items(.tag).ptr == p.token_tags.ptr);
	74	assert(token_slice.items(.offset).ptr == p.token_starts.ptr);
	75	p.scratch.deinit(p.gpa);
	76
	77	const ast: Ast = .{
	78	.utf8_source = p.source,
	79	.tokens = token_slice,
	80	.nodes = p.nodes.toOwnedSlice(),
	81	.extra = extra,
	82	};
	83
	84	return ast;
	85	}
	86
	87	/// This discards all parser work
	88	fn deinit(p: *Parse) void {
	89	defer p.* = undefined;
	90	p.nodes.deinit(p.gpa);
	91	p.extra.deinit(p.gpa);
	92	p.scratch.deinit(p.gpa);
	93	}
	94
	95	pub const Error = error{
	96	OutOfMemory,
	97	ParseFail,
	98	};
	99
	100	pub const ExpectedContext = enum {
	101	param,
	102	argument,
	103	statement,
	104	expression,
	105	else_clause,
	106
	107	pub fn description(tag: ExpectedContext) []const u8 {
	108	return switch (tag) {
	109	.param => "a parameter",
	110	.argument => "an argument",
	111	.statement => "a statement",
	112	.expression => "an expression",
	113	.else_clause => "an else clause",
	114	};
	115	}
	116	};
	117
	118	/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
	119	/// When printing these, token list and Ast is not actually available, so full tokens
	120	/// are stored in the errors.
	121	pub const AstError = union(enum) {
	122	/// Failure during tokenization
	123	tokenization: struct {
	124	err: Tokenizer.TokenizeError,
	125	byte: ByteOffset,
	126	},
	127	/// Unexpected data, expected a token.
	128	unexpected_token: struct {
	129	expected: Token.Tag,
	130	got: Token,
	131	},
	132	/// Unexpected data, expected <ExpectedContext>
	133	unexpected_context: struct {
	134	expected: ExpectedContext,
	135	got: Token,
	136	},
	137	/// Went past depth maximum
	138	depth_limit: Token,
	139
	140	pub fn toReport(err: AstError, comp: Comp, file: Comp.File, trace: Report.DebugStoredTrace) !Report {
	141	return (Report.Err{ .ast = err }).toReport(comp, file, trace);
	142	}
	143
	144	pub fn build(err: AstError, ctx: *Report.MsgContext) !void {
	145	switch (err) {
	146	.tokenization => \|data\| {
	147	// TODO: consider making tokenizer emit AstError more directly,
	148	// so that it can just annotate start and ends
	149	try ctx.msg(switch (data.err) {
	150	error.UnterminatedString => .{ .codepoint_eol = data.byte },
	151	else => .{ .codepoint = data.byte },
	152	}, "{s}", .{switch (data.err) {
	153	error.OutOfMemory => unreachable,
	154	error.InvalidSyntax => "unexpected token",
	155	error.TrailingNumericSeparator => "trailing numberic separator",
	156	error.UnterminatedString => "unterminated string literal",
	157	error.InvalidEscapeSequence => "invalid escape sequence",
	158	error.InvalidEscapeCodepoint => "invalid escape codepoint",
	159	error.InvalidUtf8 => "file is not valid utf8",
	160	error.GitConflict => "git merge conflict encountered",
	161	}});
	162	},
	163	.unexpected_token => \|data\| {
	164	try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{
	165	data.expected.description(),
	166	data.got.tag.description(),
	167	});
	168	},
	169	.unexpected_context => \|data\| {
	170	try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{
	171	data.expected.description(),
	172	data.got.tag.description(),
	173	});
	174	},
	175	.depth_limit => \|data\| {
	176	try ctx.msg(.{ .token_stored = data }, "syntax tree is nested too deeply (limit: {d})", .{depth_limit});
	177	},
	178	}
	179	}
	180
	181	pub const WithTrace = struct {
	182	err: AstError,
	183	trace: Report.DebugStoredTrace,
	184
	185	pub fn toReport(err: WithTrace, comp: Comp, file: Comp.File) !Report {
	186	return err.err.toReport(comp, file, err.trace);
	187	}
	188	};
	189	};
	190
	191	pub fn parseRoot(p: *Parse) Error!void {
	192	const trace = t.trace(@src());
	193	defer trace.end();
	194
	195	assert(p.nodes.len == 0);
	196	assert(p.extra.items.len == 0);
	197	assert(p.pos == 0);
	198	assert(p.nodes.capacity > 0);
	199
	200	p.nodes.appendAssumeCapacity(.{
	201	.tag = .root,
	202	.main_token = 0,
	203	.data = undefined,
	204	});
	205
	206	const list = try p.parseModuleMembers();
	207	p.nodes.items(.data)[0] = .{
	208	.list = list,
	209	};
	210	}
	211
	212	fn parseModuleMembers(p: *Parse) Error!SubList {
	213	p.depth += 1;
	214	defer p.depth -= 1;
	215	if (p.depth > depth_limit) {
	216	return p.fail(.{ .depth_limit = p.fullToken(p.pos) });
	217	}
	218
	219	const trace = t.trace(@src());
	220	defer trace.end();
	221
	222	const scratch_top = p.scratch.items.len;
	223	defer p.scratch.shrinkRetainingCapacity(scratch_top);
	224
	225	// skip module doc comment
	226	while (p.eatToken(.module_doc_comment)) \|_\| {}
	227
	228	while (true) switch (p.currentTag()) {
	229	.kw_let,
	230	.kw_var,
	231	.kw_fn,
	232	.kw_pub,
	233	=> try p.scratch.append(p.gpa, try p.parseModuleDecl()),
	234
	235	.kw_struct,
	236	.kw_enum,
	237	.kw_trait,
	238	.kw_impl,
	239	=> \|tag\| std.debug.panic("TODO: parse {s}", .{@tagName(tag)}),
	240
	241	.eof => break,
	242
	243	else => {
	244	if (!p.isExpressionStart()) {
	245	return p.failUnexpectedToken(.statement);
	246	}
	247
	248	try p.scratch.append(p.gpa, try p.parseExpression(true));
	249	_ = try p.expect(.semi);
	250	},
	251	};
	252
	253	return p.scratchToSubList(scratch_top);
	254	}
	255
	256	fn parseModuleDecl(p: *Parse) Error!NodeIndex {
	257	const trace = t.trace(@src());
	258	defer trace.end();
	259
	260	_ = p.eatToken(.kw_pub);
	261
	262	return switch (p.currentTag()) {
	263	.kw_var, .kw_let => try p.parseVarDecl(),
	264
	265	.kw_fn => try p.parseFnDecl(),
	266
	267	else => unreachable,
	268	};
	269	}
	270
	271	fn parseFnDecl(p: *Parse) Error!NodeIndex {
	272	const trace = t.trace(@src());
	273	defer trace.end();
	274
	275	const main_token = p.assertToken(.kw_fn);
	276
	277	_ = try p.expect(.id);
	278
	279	_ = try p.expect(.start_paren);
	280
	281	const params = try p.parseParamList();
	282
	283	const return_type = if (p.eatToken(.minus_lt)) \|_\|
	284	if (!p.isExpressionStart())
	285	return p.failUnexpectedToken(.expression)
	286	else
	287	try p.parseExpression(false)
	288	else
	289	null;
	290
	291	try p.expectWithoutAdvance(.start_brace);
	292	const body = try p.parseBlock();
	293
	294	const extra = try p.addExtra(Ast.FnDecl, .{
	295	.params = params,
	296	.return_type = optionalNodeIndex(return_type),
	297	});
	298
	299	return try p.addNode(.{
	300	.tag = .fn_decl,
	301	.main_token = main_token,
	302	.data = .{ .fn_decl = .{
	303	.body = body,
	304	.extra = extra,
	305	} },
	306	});
	307	}
	308
	309	fn parseParamList(p: *Parse) Error!SubList {
	310	const trace = t.trace(@src());
	311	defer trace.end();
	312
	313	const scratch_top = p.scratch.items.len;
	314	defer p.scratch.shrinkRetainingCapacity(scratch_top);
	315
	316	while (true) {
	317	const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top});
	318	defer trace_param.end();
	319
	320	if (p.currentTag() == .end_paren) break;
	321
	322	try p.scratch.append(p.gpa, try p.parseParamDecl());
	323
	324	switch (p.currentTag()) {
	325	.comma => {
	326	_ = p.nextToken();
	327	},
	328	.end_paren => break,
	329	else => return p.failUnexpectedToken(.param),
	330	}
	331	}
	332
	333	_ = p.assertToken(.end_paren);
	334
	335	return p.scratchToSubList(scratch_top);
	336	}
	337
	338	fn parseParamDecl(p: *Parse) Error!NodeIndex {
	339	const trace = t.trace(@src());
	340	defer trace.end();
	341
	342	const main_token = p.assertToken(.id);
	343
	344	_ = try p.expect(.colon);
	345
	346	const ty = try p.parseExpression(false);
	347
	348	return try p.addNode(.{
	349	.tag = .param_decl,
	350	.main_token = main_token,
	351	.data = .{ .param_decl = .{
	352	.type = ty,
	353	} },
	354	});
	355	}
	356
	357	fn parseBlock(p: *Parse) Error!NodeIndex {
	358	const trace = t.trace(@src());
	359	defer trace.end();
	360
	361	const main_token = p.assertToken(.start_brace);
	362
	363	const tag, const list = try p.parseBlockContents();
	364	assert(p.lastTag() == .end_brace);
	365
	366	return try p.addNode(.{
	367	.tag = tag,
	368	.main_token = main_token,
	369	.data = .{
	370	.list = list,
	371	},
	372	});
	373	}
	374
	375	fn parseLabelExpression(p: *Parse) Error!NodeIndex {
	376	const trace = t.trace(@src());
	377	defer trace.end();
	378
	379	const main_token = p.assertToken(.id);
	380	_ = try p.expect(.colon);
	381	const expr = try p.parseExpression(false);
	382	return try p.addNode(.{
	383	.tag = .label,
	384	.main_token = main_token,
	385	.data = .{ .label = .{
	386	.expr = expr,
	387	} },
	388	});
	389	}
	390
	391	fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } {
	392	const trace = t.trace(@src());
	393	defer trace.end();
	394
	395	assert(p.lastTag() == .start_brace);
	396	const scratch_top = p.scratch.items.len;
	397	defer p.scratch.shrinkRetainingCapacity(scratch_top);
	398
	399	const tag: Node.Tag = while (true) switch (p.currentTag()) {
	400	.kw_let,
	401	.kw_var,
	402	=> try p.scratch.append(p.gpa, try p.parseVarDecl()),
	403
	404	.end_brace => break .block_void,
	405
	406	else => {
	407	if (!p.isExpressionStart()) {
	408	return p.failUnexpectedToken(.statement);
	409	}
	410
	411	const first_tag = p.currentTag();
	412
	413	try p.scratch.append(p.gpa, try p.parseExpression(true));
	414
	415	switch (first_tag) {
	416	// some control flow and blocks are allowed to not have a
	417	// trailing semicolons. this is a stylistic choice.
	418	.kw_if,
	419	.start_brace,
	420	=> {
	421	// ending a block here should also count as an implicit return,
	422	// even though voiding it with a semicolon is not allowed.
	423	if (p.currentTag() == .end_brace) break .block_implicit_return;
	424	},
	425
	426	// TODO: make kw_break and kw_return REQUIRE a semicolon, as
	427	// they are noreturn statements, doing implicit break on
	428	// them makes no sense
	429
	430	else => {
	431	if (p.eatToken(.semi) == null) {
	432	// this must be last statement in block if it isnt,
	433	// the p.expect(.end_brace) later will fail.
	434	break .block_implicit_return;
	435	}
	436	},
	437	}
	438	},
	439	};
	440
	441	_ = try p.expect(.end_brace);
	442
	443	return .{ tag, try p.scratchToSubList(scratch_top) };
	444	}
	445
	446	/// VarDecl <- (kw_pub)? (kw_var \| kw_let) id (colon expr)? equal expr semi
	447	fn parseVarDecl(p: *Parse) Error!NodeIndex {
	448	const trace = t.trace(@src());
	449	defer trace.end();
	450
	451	const kind = p.currentTag();
	452	assert(kind == .kw_var or kind == .kw_let);
	453
	454	const main_token = p.nextToken();
	455
	456	_ = try p.expect(.id);
	457
	458	const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) \|_\|
	459	optionalNodeIndex(try p.parseExpression(false))
	460	else
	461	.none;
	462
	463	_ = try p.expect(.equal);
	464
	465	const data_init = try p.parseExpression(true);
	466
	467	_ = try p.expect(.semi);
	468
	469	return try p.addNode(.{
	470	.tag = .var_decl,
	471	.main_token = main_token,
	472	.data = .{ .var_decl = .{
	473	.type = data_type,
	474	.init = data_init,
	475	} },
	476	});
	477	}
	478
	479	fn isExpressionStart(p: *Parse) bool {
	480	return switch (p.currentTag()) {
	481	.id,
	482	.number_int,
	483	.number_float,
	484	.string,
	485	.builtin_id,
	486	.kw_return,
	487	.kw_break,
	488	.kw_if,
	489	.start_paren,
	490	.start_brace,
	491	.bang,
	492	.tilde,
	493	.minus,
	494	=> true,
	495	else => false,
	496	};
	497	}
	498
	499	const Operator = enum(u8) {
	500	add = @intFromEnum(Token.Tag.plus),
	501	sub = @intFromEnum(Token.Tag.minus),
	502	mul = @intFromEnum(Token.Tag.star),
	503	div = @intFromEnum(Token.Tag.slash),
	504	rem = @intFromEnum(Token.Tag.percent),
	505	pow = @intFromEnum(Token.Tag.star_star),
	506	@"and" = @intFromEnum(Token.Tag.and_and),
	507	@"or" = @intFromEnum(Token.Tag.pipe_pipe),
	508	bit_and = @intFromEnum(Token.Tag.@"and"),
	509	bit_or = @intFromEnum(Token.Tag.pipe),
	510	bit_xor = @intFromEnum(Token.Tag.caret),
	511	shl = @intFromEnum(Token.Tag.lt_lt),
	512	shr = @intFromEnum(Token.Tag.gt_gt),
	513	eq = @intFromEnum(Token.Tag.equal_equal),
	514	ne = @intFromEnum(Token.Tag.bang_equal),
	515	lt = @intFromEnum(Token.Tag.lt),
	516	le = @intFromEnum(Token.Tag.lt_equal),
	517	gt = @intFromEnum(Token.Tag.gt),
	518	ge = @intFromEnum(Token.Tag.gt_equal),
	519	// Following operators are assignment operators
	520	assign = @intFromEnum(Token.Tag.equal),
	521	add_assign = @intFromEnum(Token.Tag.plus_equal),
	522	sub_assign = @intFromEnum(Token.Tag.minus_equal),
	523	mul_assign = @intFromEnum(Token.Tag.star_equal),
	524	div_assign = @intFromEnum(Token.Tag.slash_equal),
	525	rem_assign = @intFromEnum(Token.Tag.percent_equal),
	526	pow_assign = @intFromEnum(Token.Tag.star_star_equal),
	527	and_assign = @intFromEnum(Token.Tag.and_and_equal),
	528	or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal),
	529	bit_and_assign = @intFromEnum(Token.Tag.and_equal),
	530	bit_or_assign = @intFromEnum(Token.Tag.pipe_equal),
	531	bit_xor_assign = @intFromEnum(Token.Tag.caret_equal),
	532	shl_assign = @intFromEnum(Token.Tag.lt_lt_equal),
	533	shr_assign = @intFromEnum(Token.Tag.gt_gt_equal),
	534
	535	const Direction = enum { ltr, rtl };
	536
	537	/// Only the binary operator levels are actually used,
	538	/// but all are listed for reference.
	539	const Level = enum {
	540	/// e. e[] e()
	541	accessor,
	542	/// new
	543	new,
	544	/// ++e, --e, !e, ~e, +e, -e, typeof, void, delete
	545	prefix,
	546	/// **
	547	exponentiation,
	548	/// *, /, %
	549	multiplicative,
	550	/// +, -
	551	additive,
	552	/// <<, >>, >>>
	553	bitwise_shift,
	554	/// <, <=, >, >=
	555	relational,
	556	/// ==, !=
	557	equality,
	558	/// &
	559	bit_and,
	560	/// ^
	561	bit_xor,
	562	/// \|
	563	bit_or,
	564	/// &&
	565	logical_and,
	566	/// \|\|, ??
	567	logical_or,
	568	/// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
	569	assign,
	570	};
	571
	572	pub fn direction(op: Operator) Direction {
	573	return switch (op) {
	574	.pow => .rtl,
	575
	576	.assign,
	577	.add_assign,
	578	.sub_assign,
	579	.mul_assign,
	580	.div_assign,
	581	.rem_assign,
	582	.pow_assign,
	583	.and_assign,
	584	.or_assign,
	585	.bit_and_assign,
	586	.bit_or_assign,
	587	.bit_xor_assign,
	588	.shl_assign,
	589	.shr_assign,
	590	=> .rtl,
	591
	592	else => .ltr,
	593	};
	594	}
	595
	596	pub fn level(op: Operator) Level {
	597	return switch (op) {
	598	.pow => .exponentiation,
	599	.mul, .div, .rem => .multiplicative,
	600	.add, .sub => .additive,
	601	.shl, .shr => .bitwise_shift,
	602	.lt, .le, .gt, .ge => .relational,
	603	.eq, .ne => .equality,
	604	.@"and" => .logical_and,
	605	.@"or" => .logical_or,
	606	.bit_and => .bit_and,
	607	.bit_xor => .bit_xor,
	608	.bit_or => .bit_or,
	609	.assign,
	610	.add_assign,
	611	.sub_assign,
	612	.mul_assign,
	613	.div_assign,
	614	.rem_assign,
	615	.pow_assign,
	616	.and_assign,
	617	.or_assign,
	618	.bit_and_assign,
	619	.bit_or_assign,
	620	.bit_xor_assign,
	621	.shl_assign,
	622	.shr_assign,
	623	=> .assign,
	624	};
	625	}
	626
	627	const lut = brk: {
	628	const token_tags = std.enums.values(Token.Tag);
	629	const op_tags = std.enums.values(Operator);
	630
	631	var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len;
	632	for (op_tags) \|op\| {
	633	is_operator_lut[@intFromEnum(op)] = true;
	634	}
	635
	636	var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len;
	637	for (op_tags) \|op\| {
	638	ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op));
	639	}
	640
	641	break :brk struct {
	642	const is_operator = is_operator_lut;
	643	const ast_tag = ast_tag_lut;
	644	};
	645	};
	646
	647	pub fn fromToken(tag: Token.Tag) ?Operator {
	648	return if (lut.is_operator[@intFromEnum(tag)])
	649	@enumFromInt(@intFromEnum(tag))
	650	else
	651	null;
	652	}
	653
	654	pub fn astTag(op: Operator) Node.Tag {
	655	const tag = lut.ast_tag[@intFromEnum(op)];
	656	assert(tag != .root); // invalid operator or lut construction
	657	return tag;
	658	}
	659
	660	pub fn isAssign(op: Operator) bool {
	661	return switch (op) {
	662	.assign,
	663	.add_assign,
	664	.sub_assign,
	665	.mul_assign,
	666	.div_assign,
	667	.rem_assign,
	668	.pow_assign,
	669	.and_assign,
	670	.or_assign,
	671	.bit_and_assign,
	672	.bit_or_assign,
	673	.bit_xor_assign,
	674	.shl_assign,
	675	.shr_assign,
	676	=> true,
	677	else => false,
	678	};
	679	}
	680	};
	681
	682	fn parseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex {
	683	const trace = t.trace(@src());
	684	defer trace.end();
	685
	686	p.depth += 1;
	687	defer p.depth -= 1;
	688	if (p.depth > depth_limit) {
	689	return p.fail(.{ .depth_limit = p.fullToken(p.pos) });
	690	}
	691
	692	assert(p.isExpressionStart());
	693
	694	// These unary operators are greedy, so handle them first.
	695	switch (p.currentTag()) {
	696	.kw_return => return p.parseReturn(),
	697	.kw_break => return p.parseBreak(),
	698	else => {},
	699	}
	700
	701	var expr = try p.parseExpressionPart();
	702
	703	while (Operator.fromToken(p.currentTag())) \|op\| {
	704	if (!allow_assign_operators and op.isAssign()) break;
	705	expr = try p.parseBinarySuffix(expr, op, allow_assign_operators);
	706	}
	707
	708	return expr;
	709	}
	710
	711	fn parseExpressionPart(p: *Parse) Error!NodeIndex {
	712	const trace = t.trace(@src());
	713	defer trace.end();
	714
	715	var expr = switch (p.currentTag()) {
	716	// Literals
	717	.id => if (p.nextTag() == .colon)
	718	try p.parseLabelExpression()
	719	else
	720	try p.blankNode(.id),
	721	.number_int => try p.blankNode(.number_int),
	722	.number_float => try p.blankNode(.number_float),
	723	.string => try p.blankNode(.string),
	724
	725	// Complex expressions
	726	.builtin_id => try p.parseBuiltinCall(),
	727	.kw_return => try p.parseReturn(),
	728	.kw_break => try p.parseBreak(),
	729	.kw_if => try p.parseIf(),
	730	.start_paren => try p.parseGroup(),
	731	.start_brace => try p.parseBlock(),
	732
	733	// Unary operators
	734	inline .bang, .tilde, .minus => \|tag\| unary: {
	735	const main_token = p.nextToken();
	736	const expr = try p.parseExpressionPart();
	737
	738	break :unary try p.addNode(.{
	739	.tag = switch (tag) {
	740	.bang => .bool_not,
	741	.tilde => .bit_not,
	742	.minus => .negate,
	743	else => comptime unreachable,
	744	},
	745	.main_token = main_token,
	746	.data = .{ .un = .{
	747	.expr = expr,
	748	} },
	749	});
	750	},
	751
	752	else => return p.failUnexpectedToken(.expression),
	753	};
	754
	755	// parse field, calls, and index expressions
	756	while (true) {
	757	const trace2 = t.traceFmt(@src(), "(field/call)", .{});
	758	defer trace2.end();
	759
	760	switch (p.currentTag()) {
	761	.dot => {
	762	const main_token = p.nextToken();
	763	const field = try p.expect(.id);
	764
	765	expr = try p.addNode(.{
	766	.tag = .field,
	767	.main_token = main_token,
	768	.data = .{ .field = .{
	769	.base = expr,
	770	.field_token = field,
	771	} },
	772	});
	773	},
	774	.start_paren => {
	775	const main_token = p.nextToken();
	776	const args = try p.parseCallArgs();
	777
	778	const extra = try p.addExtra(Ast.Call, .{
	779	.args = args,
	780	});
	781
	782	expr = try p.addNode(.{
	783	.tag = .call,
	784	.main_token = main_token,
	785	.data = .{ .call = .{
	786	.base = expr,
	787	.extra = extra,
	788	} },
	789	});
	790	},
	791	.start_bracket => {
	792	const main_token = p.nextToken();
	793	const index = try p.parseExpression(false);
	794
	795	expr = try p.addNode(.{
	796	.tag = .index,
	797	.main_token = main_token,
	798	.data = .{ .bin = .{
	799	.lhs = expr,
	800	.rhs = index,
	801	} },
	802	});
	803	},
	804
	805	else => break,
	806	}
	807	}
	808
	809	return expr;
	810	}
	811
	812	fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator, allow_assign_operators: bool) Error!NodeIndex {
	813	const trace = t.trace(@src());
	814	defer trace.end();
	815
	816	const main_token = p.nextToken();
	817	const level = op.level();
	818	var rhs = try p.parseExpressionPart();
	819
	820	while (Operator.fromToken(p.currentTag())) \|next_op\| {
	821	if (!allow_assign_operators and next_op.isAssign()) break;
	822
	823	const new_level = next_op.level();
	824	const use = switch (op.direction()) {
	825	.ltr => @intFromEnum(new_level) < @intFromEnum(level),
	826	.rtl => @intFromEnum(new_level) <= @intFromEnum(level),
	827	};
	828	if (!use) break;
	829
	830	rhs = try p.parseBinarySuffix(rhs, next_op, allow_assign_operators);
	831	}
	832
	833	return try p.addNode(.{
	834	.tag = op.astTag(),
	835	.main_token = main_token,
	836	.data = .{ .bin = .{
	837	.lhs = lhs,
	838	.rhs = rhs,
	839	} },
	840	});
	841	}
	842
	843	fn parseReturn(p: *Parse) Error!NodeIndex {
	844	const trace = t.trace(@src());
	845	defer trace.end();
	846
	847	const main_token = p.assertToken(.kw_return);
	848
	849	const expr = if (p.isExpressionStart())
	850	try p.parseExpression(false)
	851	else
	852	null;
	853
	854	return try p.addNode(.{
	855	.tag = .@"return",
	856	.main_token = main_token,
	857	.data = .{ .@"return" = .{
	858	.expr = optionalNodeIndex(expr),
	859	} },
	860	});
	861	}
	862
	863	fn parseBreak(p: *Parse) Error!NodeIndex {
	864	const trace = t.trace(@src());
	865	defer trace.end();
	866
	867	const main_token = p.assertToken(.kw_break);
	868
	869	const label = if (p.eatToken(.colon)) \|_\|
	870	try p.expect(.id)
	871	else
	872	null;
	873
	874	const expr = if (p.isExpressionStart())
	875	try p.parseExpression(false)
	876	else
	877	null;
	878
	879	return try p.addNode(.{
	880	.tag = .@"break",
	881	.main_token = main_token,
	882	.data = .{ .@"break" = .{
	883	.label = optionalTokenIndex(label),
	884	.value = optionalNodeIndex(expr),
	885	} },
	886	});
	887	}
	888
	889	fn parseBuiltinCall(p: *Parse) Error!NodeIndex {
	890	const trace = t.trace(@src());
	891	defer trace.end();
	892
	893	const main_token = p.assertToken(.builtin_id);
	894
	895	_ = try p.expect(.start_paren);
	896
	897	const args = try p.parseCallArgs();
	898
	899	return try p.addNode(.{
	900	.tag = .builtin_call,
	901	.main_token = main_token,
	902	.data = .{ .list = args },
	903	});
	904	}
	905
	906	fn parseCallArgs(p: *Parse) Error!SubList {
	907	const trace = t.trace(@src());
	908	defer trace.end();
	909
	910	const scratch_top = p.scratch.items.len;
	911	defer p.scratch.shrinkRetainingCapacity(scratch_top);
	912
	913	while (p.isExpressionStart()) {
	914	try p.scratch.append(p.gpa, try p.parseExpression(false));
	915
	916	switch (p.currentTag()) {
	917	.end_paren => {
	918	break;
	919	},
	920	.comma => {
	921	_ = p.nextToken();
	922	if (p.currentTag() == .end_paren)
	923	break;
	924	},
	925	else => return p.failUnexpectedToken(.argument),
	926	}
	927	}
	928
	929	_ = try p.expect(.end_paren);
	930
	931	return p.scratchToSubList(scratch_top);
	932	}
	933
	934	fn parseIf(p: *Parse) Error!NodeIndex {
	935	const trace = t.trace(@src());
	936	defer trace.end();
	937
	938	const main_token = p.assertToken(.kw_if);
	939
	940	const cond = try p.parseExpression(false);
	941
	942	_ = try p.expectWithoutAdvance(.start_brace);
	943	const yes = try p.parseBlock();
	944
	945	const no = if (p.eatToken(.kw_else)) \|_\|
	946	try p.parseElseValue()
	947	else
	948	null;
	949
	950	const extra = try p.addExtra(Ast.If, .{
	951	.yes = yes,
	952	.no = optionalNodeIndex(no),
	953	});
	954
	955	return try p.addNode(.{
	956	.tag = .@"if",
	957	.main_token = main_token,
	958	.data = .{ .@"if" = .{
	959	.cond = cond,
	960	.extra = extra,
	961	} },
	962	});
	963	}
	964
	965	fn parseElseValue(p: *Parse) Error!NodeIndex {
	966	const trace = t.trace(@src());
	967	defer trace.end();
	968
	969	assert(p.lastTag() == .kw_else);
	970
	971	return switch (p.currentTag()) {
	972	.start_brace => p.parseBlock(),
	973	.kw_if => p.parseIf(),
	974	.id => p.parseLabelExpression(),
	975
	976	else => return p.failUnexpectedToken(.else_clause),
	977	};
	978	}
	979
	980	fn parseGroup(p: *Parse) Error!NodeIndex {
	981	const trace = t.trace(@src());
	982	defer trace.end();
	983
	984	const main_token = p.assertToken(.start_paren);
	985
	986	const expr = try p.parseExpression(false);
	987
	988	_ = try p.expect(.end_paren);
	989
	990	return try p.addNode(.{
	991	.tag = .group,
	992	.main_token = main_token,
	993	.data = .{ .un = .{
	994	.expr = expr,
	995	} },
	996	});
	997	}
	998
	999	// helpers
	1000
	1001	/// Append a node and return it's index.
	1002	fn addNode(p: *Parse, node: Node) !NodeIndex {
	1003	const trace = t.trace(@src());
	1004	defer trace.end();
	1005
	1006	try p.nodes.append(p.gpa, node);
	1007	// @intCast is always safe because 2^32 tokens alwayss produce less than
	1008	// 2^32 nodes, and there is a 2^32 limit on source file bytes.
	1009	return @intCast(p.nodes.len - 1);
	1010	}
	1011
	1012	fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
	1013	const trace = t.trace(@src());
	1014	defer trace.end();
	1015
	1016	const extra_top = p.extra.items.len;
	1017	const bytes = std.mem.asBytes(&extra);
	1018	try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes));
	1019	return @intCast(extra_top);
	1020	}
	1021
	1022	fn fail(p: *Parse, ast_error: AstError) Error {
	1023	p.ast_error = .{
	1024	.err = ast_error,
	1025	.trace = Report.DebugStoredTrace.capture(@returnAddress()),
	1026	};
	1027	return Error.ParseFail;
	1028	}
	1029
	1030	fn failUnexpectedToken(p: *Parse, kind: ExpectedContext) Error {
	1031	return p.fail(.{ .unexpected_context = .{
	1032	.got = p.fullToken(p.pos),
	1033	.expected = kind,
	1034	} });
	1035	}
	1036
	1037	fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
	1038	const main_token = p.assertToken(tag);
	1039	return try p.addNode(.{
	1040	.tag = @field(Node.Tag, @tagName(tag)),
	1041	.main_token = main_token,
	1042	.data = .{ .none = .{} },
	1043	});
	1044	}
	1045
	1046	fn currentTag(p: *Parse) Token.Tag {
	1047	return p.token_tags[p.pos];
	1048	}
	1049
	1050	fn lastTag(p: *Parse) Token.Tag {
	1051	return p.token_tags[p.pos - 1];
	1052	}
	1053
	1054	fn nextTag(p: *Parse) Token.Tag {
	1055	return p.token_tags[p.pos + 1];
	1056	}
	1057
	1058	fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex {
	1059	return if (p.token_tags[p.pos] == tag) p.nextToken() else null;
	1060	}
	1061
	1062	fn expect(p: *Parse, tag: Token.Tag) !TokenIndex {
	1063	if (p.currentTag() != tag) {
	1064	return p.fail(.{ .unexpected_token = .{
	1065	.got = p.fullToken(p.pos),
	1066	.expected = tag,
	1067	} });
	1068	}
	1069	return p.nextToken();
	1070	}
	1071
	1072	fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void {
	1073	if (p.currentTag() != tag) {
	1074	return p.fail(.{ .unexpected_token = .{
	1075	.got = p.fullToken(p.pos),
	1076	.expected = tag,
	1077	} });
	1078	}
	1079	}
	1080
	1081	fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex {
	1082	const token = p.nextToken();
	1083	assert(p.token_tags[token] == tag);
	1084	return token;
	1085	}
	1086
	1087	fn nextToken(p: *Parse) TokenIndex {
	1088	const result = p.pos;
	1089	p.pos += 1;
	1090	return result;
	1091	}
	1092
	1093	fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool {
	1094	return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
	1095	}
	1096
	1097	fn fullToken(p: *Parse, token: TokenIndex) Token {
	1098	return Token{
	1099	.tag = p.token_tags[token],
	1100	.offset = p.token_starts[token],
	1101	};
	1102	}
	1103
	1104	fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList {
	1105	const trace = t.trace(@src());
	1106	defer trace.end();
	1107
	1108	const items = p.scratch.items[scratch_top..];
	1109	const extra_top = p.extra.items.len;
	1110	try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]);
	1111	return .{
	1112	.start = @intCast(extra_top),
	1113	.len = @intCast(items.len),
	1114	};
	1115	}
	1116
	1117	const Parse = @This();
	1118
	1119	const std = @import("std");
	1120	const assert = std.debug.assert;
	1121	const mem = std.mem;
	1122	const Allocator = mem.Allocator;
	1123	const ArrayListUnmanaged = std.ArrayListUnmanaged;
	1124
	1125	const Tokenizer = @import("Tokenizer.zig");
	1126	const Token = Tokenizer.Token;
	1127	const ByteOffset = Tokenizer.ByteOffset;
	1128
	1129	const Ast = @import("Ast.zig");
	1130	const Node = Ast.Node;
	1131	const Data = Node.Data;
	1132	const SubList = Ast.SubList;
	1133	const NodeIndex = Ast.NodeIndex;
	1134	const TokenIndex = Ast.TokenIndex;
	1135	const ExtraIndex = Ast.ExtraIndex;
	1136	const OptionalNodeIndex = Ast.OptionalNodeIndex;
	1137	const OptionalTokenIndex = Ast.OptionalTokenIndex;
	1138	const OptionalExtraIndex = Ast.OptionalExtraIndex;
	1139	const optionalTokenIndex = OptionalTokenIndex.init;
	1140	const optionalNodeIndex = OptionalNodeIndex.init;
	1141	const optionalExtraIndex = OptionalExtraIndex.init;
	1142
	1143	const t = @import("tracer");
	1144
	1145	const Report = @import("Report.zig");
	1146	const Comp = @import("Comp.zig");

src/core/Report.zig created+864

...	...	@@ -0,0 +1,864 @@
	1	//! A Report is a list of errors and warnings for a part of a compilation.
	2	//!
	3	//! `Err/Warn` structs are unions of every possible message, the logic
	4	//! for printing them is all done here.
	5	//!
	6	//! Across the compiler, `Report.Builder`s' `.err` and `.warn` methods
	7	//! can add messages by ID.
	8	errors: []Item = &.{},
	9	warnings: []Item = &.{},
	10	string_bytes: []u8 = &.{},
	11
	12	pub var debug_show_report_trace: if (builtin.mode == .Debug) bool else void =
	13	if (builtin.mode == .Debug) false;
	14
	15	pub const Item = struct {
	16	/// What is this diagnostic
	17	data: union { @"error": Err, warn: Warn },
	18	/// What file was it in
	19	url: String,
	20
	21	debug_stack_trace: DebugStoredTrace,
	22
	23	/// Type is never stored, inferred from which list it is in.
	24	pub const Type = enum { @"error", warn };
	25	};
	26
	27	pub const Err = union(enum) {
	28	pub const FieldEnum = std.meta.FieldEnum(Err);
	29
	30	/// Constructs a `Report` with just a single error.
	31	pub fn toReport(err: Err, comp: Comp, file: File, trace: DebugStoredTrace) !Report {
	32	const string_bytes = try comp.gpa.dupe(u8, file.file_path);
	33	errdefer comp.gpa.free(string_bytes);
	34
	35	const errors = try comp.gpa.dupe(Item, &.{.{
	36	.data = .{ .@"error" = err },
	37	.url = .{ .start = 0, .len = @intCast(string_bytes.len) },
	38	.debug_stack_trace = trace,
	39	}});
	40	errdefer comp.gpa.free(errors);
	41
	42	return .{ .errors = errors, .string_bytes = string_bytes };
	43	}
	44
	45	ast: @import("Parse.zig").AstError,
	46
	47	unknown_identifier: struct {
	48	node: NodeIndex,
	49
	50	fn build(data: @This(), ctx: *MsgContext) !void {
	51	try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{
	52	ctx.ast.basicNodeSlice(data.node),
	53	});
	54	}
	55	},
	56	invalid_builtin: struct {
	57	node: NodeIndex,
	58
	59	fn build(data: @This(), ctx: *MsgContext) !void {
	60	try ctx.msgToken(
	61	ctx.ast.nodeToken(data.node),
	62	"unknown built-in function \"{s}\"",
	63	.{ctx.ast.basicNodeSlice(data.node)},
	64	);
	65	}
	66	},
	67	duplicate_function: struct {
	68	node: NodeIndex,
	69	prev: NodeIndex,
	70
	71	fn build(data: @This(), ctx: *MsgContext) !void {
	72	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
	73	ctx.ast.fullFnDecl(data.node).id,
	74	});
	75	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{});
	76	// TODO: be able to store a full list of other declarations
	77	// and add them here as "and here" and so on
	78	}
	79	},
	80	duplicate_param: struct {
	81	node: NodeIndex,
	82	prev: NodeIndex,
	83
	84	fn build(data: @This(), ctx: *MsgContext) !void {
	85	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
	86	ctx.ast.fullParamDecl(data.node).id,
	87	});
	88	try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{});
	89	// TODO: be able to store a full list of other declarations
	90	// and add them here as "and here" and so on
	91	}
	92	},
	93	invalid_return: struct {
	94	node: NodeIndex,
	95
	96	fn build(data: @This(), ctx: *MsgContext) !void {
	97	try ctx.msgNode(
	98	data.node,
	99	"return cannot be used outside of a function body",
	100	.{},
	101	);
	102	}
	103	},
	104	unresolved_label: struct {
	105	node: NodeIndex,
	106
	107	fn build(data: @This(), ctx: *MsgContext) !void {
	108	try ctx.msg(
	109	.{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 },
	110	"unknown block label \"{s}\"",
	111	.{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)},
	112	);
	113	}
	114	},
	115	shadow_builtin_variable: struct {
	116	node: NodeIndex,
	117
	118	fn build(data: @This(), ctx: *MsgContext) !void {
	119	try ctx.msgToken(
	120	ctx.ast.nodeToken(data.node) + 1,
	121	"declaration shadows reserved built-in \"{s}\"",
	122	.{ctx.ast.fullVarDecl(data.node).id},
	123	);
	124	}
	125	},
	126	comptime_float_to_int: struct {
	127	val: f64,
	128	src: LazyNode,
	129
	130	fn build(data: @This(), ctx: *MsgContext) !void {
	131	const node = try data.src.get(ctx);
	132	try ctx.msgNode(node, "numeric literal \"{d}\" cannot be coerced to \"int\"", .{data.val});
	133	}
	134	},
	135	coersion: struct {
	136	from: Type,
	137	to: Type,
	138	src: LazyNode,
	139
	140	fn build(data: @This(), ctx: *MsgContext) !void {
	141	const node = try data.src.get(ctx);
	142	try ctx.msgNode(node, "expected \"{s}\", got \"{s}\"", .{
	143	data.to.displayName(&ctx.comp.intern_pool),
	144	data.from.displayName(&ctx.comp.intern_pool),
	145	});
	146	}
	147	},
	148	call_non_function: struct {
	149	call: LazyNode,
	150	ty: Type,
	151
	152	fn build(data: @This(), ctx: *MsgContext) !void {
	153	const node = try data.call.get(ctx);
	154	try ctx.msgNode(node, "\"{s}\" is not callable", .{
	155	data.ty.displayName(&ctx.comp.intern_pool),
	156	});
	157	}
	158	},
	159	call_mismatch_args: struct {
	160	call: LazyNode,
	161	actual: u32,
	162	expected: u32,
	163
	164	fn build(data: @This(), ctx: *MsgContext) !void {
	165	const node = try data.call.get(ctx);
	166	try ctx.msgNode(node, "function expected {d} arguments, recieved {d}", .{
	167	data.expected,
	168	data.actual,
	169	});
	170	}
	171	},
	172	assign_builtin: struct {
	173	node: NodeIndex,
	174
	175	fn build(data: @This(), ctx: *MsgContext) !void {
	176	try ctx.msgNode(
	177	data.node,
	178	"cannot re-assign built-in \"{s}\"",
	179	.{ctx.ast.basicNodeSlice(data.node)},
	180	);
	181	}
	182	},
	183	invalid_assignment_symbol: struct {
	184	node: NodeIndex,
	185	decl: NodeIndex,
	186	kind: UnusedKind,
	187
	188	fn build(data: @This(), ctx: *MsgContext) !void {
	189	const id = switch (ctx.ast.nodeTag(data.decl)) {
	190	.param_decl => ctx.ast.fullParamDecl(data.decl).id,
	191	.fn_decl => ctx.ast.fullFnDecl(data.decl).id,
	192	.var_decl => ctx.ast.fullVarDecl(data.decl).id,
	193	else => unreachable, // not a valid symbol node
	194	};
	195
	196	try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{
	197	@tagName(data.kind),
	198	id,
	199	});
	200	try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{});
	201	}
	202	},
	203	invalid_assignment_target: struct {
	204	node: NodeIndex,
	205
	206	fn build(data: @This(), ctx: *MsgContext) !void {
	207	try ctx.msgNode(
	208	data.node,
	209	"invalid assignment target",
	210	.{},
	211	);
	212	}
	213	},
	214	};
	215
	216	pub const Warn = union(enum) {
	217	pub const FieldEnum = std.meta.FieldEnum(Warn);
	218
	219	unused: struct {
	220	node: NodeIndex,
	221	kind: UnusedKind,
	222
	223	fn build(data: @This(), ctx: *MsgContext) !void {
	224	const id = switch (ctx.ast.nodeTag(data.node)) {
	225	.param_decl => ctx.ast.fullParamDecl(data.node).id,
	226	.fn_decl => ctx.ast.fullFnDecl(data.node).id,
	227	.var_decl => ctx.ast.fullVarDecl(data.node).id,
	228	else => unreachable, // not a valid symbol node
	229	};
	230
	231	try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{
	232	@tagName(data.kind),
	233	id,
	234	});
	235
	236	try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id});
	237	}
	238	},
	239	not_mutated_var: struct {
	240	node: NodeIndex,
	241
	242	fn build(data: @This(), ctx: *MsgContext) !void {
	243	const id = switch (ctx.ast.nodeTag(data.node)) {
	244	.var_decl => ctx.ast.fullVarDecl(data.node).id,
	245	else => unreachable, // not a valid variable symbol node
	246	};
	247
	248	try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{
	249	id,
	250	});
	251	}
	252	},
	253	invalid_discard: struct {
	254	node: NodeIndex,
	255
	256	fn build(data: @This(), ctx: *MsgContext) !void {
	257	try ctx.msgToken(ctx.ast.nodeToken(data.node), "\"let\" is not required for a discard", .{});
	258	}
	259	},
	260	discard_builtin: struct {
	261	node: NodeIndex,
	262
	263	fn build(data: @This(), ctx: *MsgContext) !void {
	264	try ctx.msgToken(ctx.ast.nodeToken(data.node), "discarding a built-in symbol is meaningless", .{});
	265	}
	266	},
	267	unnecessary_discard: struct {
	268	node: NodeIndex,
	269
	270	fn build(data: @This(), ctx: *MsgContext) !void {
	271	try ctx.msgToken(ctx.ast.nodeToken(data.node), "useless discard", .{});
	272	}
	273	},
	274	dangling_identifier: struct {
	275	node: NodeIndex,
	276
	277	fn build(data: @This(), ctx: *MsgContext) !void {
	278	try ctx.msgToken(ctx.ast.nodeToken(data.node), "dangling identifier does nothing", .{});
	279	}
	280	},
	281	numeric_int_var_ambiguity: struct {
	282	node: LazyNode,
	283
	284	fn build(data: @This(), ctx: *MsgContext) !void {
	285	const node = try data.node.get(ctx);
	286	try ctx.msgNode(node, "constant integer in a \"var\" has an unknown type, assuming \"float\"", .{});
	287	try ctx.msgAlone("add \": float\" or \": int\" after the variable name to set the type", .{});
	288	}
	289	},
	290	};
	291
	292	/// A variant of std.builtin.StackTrace that stores it's data within itself
	293	/// instead of being a pointer. This allows capturing the stack trace and
	294	/// saving it for later printing.
	295	pub const StoredTrace = struct {
	296	data: [32]usize,
	297	index: usize,
	298
	299	pub const empty = .{
	300	.data = .{0} ** 32,
	301	.index = 0,
	302	};
	303
	304	fn trace(stored: *StoredTrace) std.builtin.StackTrace {
	305	return .{
	306	.index = stored.index,
	307	.instruction_addresses = &stored.data,
	308	};
	309	}
	310
	311	pub fn capture(begin: ?usize) StoredTrace {
	312	var stored: StoredTrace = undefined;
	313	var frame = stored.trace();
	314	std.debug.captureStackTrace(begin orelse @returnAddress(), &frame);
	315	stored.index = frame.index;
	316	return stored;
	317	}
	318
	319	pub fn from(stack_trace: ?*std.builtin.StackTrace) StoredTrace {
	320	if (stack_trace) \|stack\| {
	321	var data: [32]usize = undefined;
	322	@memset(&data, 0);
	323	const items = @min(stack.instruction_addresses.len, 32);
	324	@memcpy(data[0..items], stack.instruction_addresses[0..items]);
	325	return .{
	326	.data = data,
	327	.index = @min(items, stack.index),
	328	};
	329	} else {
	330	return empty;
	331	}
	332	}
	333	};
	334
	335	pub const DebugStoredTrace = if (builtin.mode == .Debug)
	336	StoredTrace
	337	else
	338	struct {
	339	pub fn capture(_: ?usize) @This() {
	340	return .{};
	341	}
	342
	343	pub fn from(_: ?*std.builtin.StackTrace) @This() {
	344	return .{};
	345	}
	346	};
	347
	348	pub const UnusedKind = enum {
	349	param,
	350	constant,
	351	variable,
	352	function,
	353	};
	354
	355	pub const String = struct {
	356	start: u32,
	357	len: u32,
	358
	359	pub fn slice(str: String, report: Report) []const u8 {
	360	return report.string_bytes[str.start..][0..str.len];
	361	}
	362	};
	363
	364	const Prefix = enum {
	365	@"error",
	366	warn,
	367	note,
	368
	369	fn color(prefix: Prefix) std.io.tty.Color {
	370	return switch (prefix) {
	371	.@"error" => .red,
	372	.warn => .yellow,
	373	.note => .blue,
	374	};
	375	}
	376	};
	377
	378	pub fn deinit(report: Report, gpa: Allocator) void {
	379	gpa.free(report.errors);
	380	gpa.free(report.warnings);
	381	gpa.free(report.string_bytes);
	382	}
	383
	384	pub fn dumpToWriter(report: Report, comp: *Comp, writer: anytype, tty: std.io.tty.Config) !void {
	385	// try writer.print("{d} errors, {d} warnings\n", .{ report.errors.len, report.warnings.len });
	386	for (report.errors) \|item\| {
	387	try report.formatItem(item, .@"error", comp, writer, tty);
	388	}
	389	for (report.warnings) \|item\| {
	390	try report.formatItem(item, .warn, comp, writer, tty);
	391	}
	392	}
	393
	394	pub fn formatItem(
	395	report: Report,
	396	item: Item,
	397	item_type: Item.Type,
	398	comp: *Comp,
	399	writer: anytype,
	400	tty: std.io.tty.Config,
	401	) !void {
	402	var arena = std.heap.ArenaAllocator.init(comp.gpa);
	403	defer arena.deinit();
	404	var sfa = std.heap.stackFallback(16384, arena.allocator());
	405	const alloc = sfa.get();
	406
	407	const file = try comp.getFile(item.url.slice(report));
	408
	409	const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast);
	410	const ast_data = if (need_load_ast) try file.getAst(comp) else undefined;
	411
	412	var ctx: MsgContext = .{
	413	.arena = alloc,
	414	.comp = comp,
	415	.file = file,
	416	.ast = &ast_data,
	417	.utf8_source = (try file.getSource(comp.gpa)).code,
	418	.components = ArrayListUnmanaged(Component).initCapacity(alloc, 16) catch
	419	unreachable, // stack-fallback allocator has enough space.
	420	};
	421
	422	switch (item_type) {
	423	inline else => \|tag\| switch (@field(item.data, @tagName(tag))) {
	424	inline else => \|payload\| try payload.build(&ctx),
	425	},
	426	}
	427
	428	assert(ctx.components.items.len > 0);
	429
	430	for (ctx.components.items, 1..) \|component, i\| {
	431	const prefix: Prefix = if (i == 1)
	432	switch (item_type) {
	433	inline else => \|tag\| @field(Prefix, @tagName(tag)),
	434	}
	435	else
	436	.note;
	437
	438	const is_last = i == ctx.components.items.len;
	439
	440	if (i != 1) {
	441	try tty.setColor(writer, .dim);
	442	try writer.writeAll(" │ ");
	443	try tty.setColor(writer, .reset);
	444	}
	445
	446	try tty.setColor(writer, prefix.color());
	447	try tty.setColor(writer, .bold);
	448	try writer.writeAll(@tagName(prefix));
	449	try tty.setColor(writer, .reset);
	450	try tty.setColor(writer, .dim);
	451	try writer.writeAll(": ");
	452	try tty.setColor(writer, .reset);
	453	try writer.writeAll(component.message_utf8);
	454	try writer.writeAll("\n");
	455
	456	const utf8_source = ctx.utf8_source;
	457
	458	const at = component.location orelse {
	459	if (is_last) {
	460	try tty.setColor(writer, .dim);
	461	try writer.print(" {s}", .{"┴"});
	462	try writer.writeAll("\n");
	463	try tty.setColor(writer, .reset);
	464	}
	465	continue;
	466	};
	467
	468	const line_before = utf8_source[at.loc.line_start..at.byte];
	469	const trimmed_before = std.mem.trimLeft(u8, line_before, " \t\r");
	470	const line_after = utf8_source[at.byte + at.slice.len .. at.loc.line_end + 1];
	471	const width = std.fmt.count("{d}", .{at.loc.line.oneBased()});
	472
	473	try tty.setColor(writer, .dim);
	474	try writer.writeByteNTimes(' ', width);
	475	try writer.print("{s}{s}:{d}:{d}\n", .{
	476	if (i == 1) "┬ " else "├ ",
	477	file.file_path,
	478	at.loc.line.oneBased(),
	479	at.loc.column.oneBased(),
	480	});
	481	try writer.print("{d}", .{at.loc.line.oneBased()});
	482	try writer.print("{s} ", .{"│"});
	483	try tty.setColor(writer, .reset);
	484	try writer.writeAll(trimmed_before);
	485	try tty.setColor(writer, prefix.color());
	486	try tty.setColor(writer, .bold);
	487	try writer.writeAll(at.slice);
	488	try tty.setColor(writer, .reset);
	489	try writer.writeAll(line_after);
	490	try tty.setColor(writer, .dim);
	491	try writer.writeAll(if (is_last) " ╰" else " ├");
	492	try writer.writeBytesNTimes(
	493	"─",
	494	width +
	495	(at.byte - at.loc.line_start) -
	496	(line_before.len - trimmed_before.len),
	497	);
	498	try tty.setColor(writer, .reset);
	499	try tty.setColor(writer, prefix.color());
	500	try tty.setColor(writer, .bold);
	501	try writer.writeBytesNTimes("━", at.slice.len);
	502	try tty.setColor(writer, .reset);
	503	try writer.writeAll("\n");
	504	}
	505
	506	if (builtin.mode == .Debug and debug_show_report_trace) {
	507	var trace = item.debug_stack_trace;
	508	std.debug.dumpStackTrace(trace.trace());
	509	}
	510	}
	511
	512	pub fn dumpToStdErr(report: Report, comp: *Comp) !void {
	513	const stderr = std.io.getStdErr();
	514	const tty = std.io.tty.detectConfig(stderr);
	515
	516	if (false or
	517	// Windows API uses it's own API calls, so a buffered writer
	518	// cannot be used :/
	519	(builtin.os.tag == .windows and tty == .windows_api) or
	520	// Debug stack trace printer writes directly to standard err,
	521	// so we need to be unbuffered for that.
	522	(builtin.mode == .Debug and debug_show_report_trace))
	523	{
	524	try dumpToWriter(report, comp, stderr.writer(), tty);
	525	} else {
	526	var buf = std.io.bufferedWriter(stderr.writer());
	527	const writer = buf.writer();
	528	try dumpToWriter(report, comp, writer, tty);
	529	try buf.flush();
	530	}
	531	}
	532
	533	pub const Builder = struct {
	534	gpa: Allocator,
	535	current_file: *Comp.File,
	536	errors: ArrayListUnmanaged(Item) = .{},
	537	warnings: ArrayListUnmanaged(Item) = .{},
	538	string_bytes: ArrayListUnmanaged(u8) = .{},
	539	current_file_str: ?String = null,
	540
	541	pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void {
	542	try b.errors.append(b.gpa, .{
	543	.data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) },
	544	.url = try b.currentFileString(),
	545	.debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{},
	546	});
	547	}
	548
	549	pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void {
	550	try b.warnings.append(b.gpa, .{
	551	.data = .{ .warn = @unionInit(Warn, @tagName(kind), data) },
	552	.url = try b.currentFileString(),
	553	.debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{},
	554	});
	555	}
	556
	557	pub fn addString(b: *Builder, str: []const u8) !String {
	558	const top = b.string_bytes.items.len;
	559	try b.string_bytes.appendSlice(b.gpa, str);
	560	return .{ .start = @intCast(top), .len = @intCast(str.len) };
	561	}
	562
	563	pub fn currentFileString(b: *Builder) !String {
	564	return b.current_file_str orelse {
	565	b.current_file_str = try b.addString(b.current_file.file_path);
	566	return b.current_file_str.?;
	567	};
	568	}
	569
	570	pub fn deinit(b: *Builder) void {
	571	b.errors.deinit(b.gpa);
	572	b.warnings.deinit(b.gpa);
	573	b.string_bytes.deinit(b.gpa);
	574	}
	575
	576	pub fn finalize(b: *Builder) !Report {
	577	defer b.* = undefined;
	578	errdefer b.deinit();
	579
	580	const errors = try b.errors.toOwnedSlice(b.gpa);
	581	errdefer b.gpa.free(errors);
	582
	583	const warnings = try b.warnings.toOwnedSlice(b.gpa);
	584	errdefer b.gpa.free(warnings);
	585
	586	const string_bytes = try b.string_bytes.toOwnedSlice(b.gpa);
	587	errdefer b.gpa.free(string_bytes);
	588
	589	return .{
	590	.errors = errors,
	591	.warnings = warnings,
	592	.string_bytes = string_bytes,
	593	};
	594	}
	595	};
	596
	597	pub const SliceLoc = struct {
	598	loc: Location,
	599	byte: ByteOffset,
	600	slice: []const u8,
	601	};
	602
	603	const Component = struct {
	604	message_utf8: []const u8,
	605	location: ?SliceLoc,
	606	};
	607
	608	pub const Highlight = union(enum) {
	609	// No source view
	610	alone,
	611	// Highlight one token
	612	token: TokenIndex,
	613	// Highlight one token without referring to the Ast
	614	token_stored: Tokenizer.Token,
	615	// Highlight two tokens
	616	two_tokens: TokenIndex,
	617	// Highlight a Node
	618	node: NodeIndex,
	619	// Highlight a codepoint
	620	codepoint: ByteOffset,
	621	// Highlight from the start of a codepoint to '\n'
	622	codepoint_eol: ByteOffset,
	623
	624	pub fn loc(highlight: Highlight, ast: *const Ast) ?SliceLoc {
	625	return switch (highlight) {
	626	.alone => null,
	627	inline .node,
	628	.token,
	629	.two_tokens,
	630	=> \|payload, tag\| loc: {
	631	const token_index: TokenIndex = switch (tag) {
	632	.node => ast.nodeToken(payload),
	633	.token, .two_tokens => payload,
	634	else => comptime unreachable,
	635	};
	636
	637	const location = ast.tokenLocation(0, token_index);
	638	const token = ast.tokens.get(token_index);
	639
	640	break :loc .{
	641	.loc = location,
	642	.slice = token.slice(ast.utf8_source),
	643	.byte = token.offset,
	644	};
	645	},
	646
	647	.token_stored => \|token\| loc: {
	648	const location = Tokenizer.tokenLocation(ast.utf8_source, 0, token.offset);
	649
	650	break :loc .{
	651	.loc = location,
	652	.slice = token.slice(ast.utf8_source),
	653	.byte = token.offset,
	654	};
	655	},
	656	inline .codepoint,
	657	.codepoint_eol,
	658	=> \|b, tag\| loc: {
	659	const len = switch (tag) {
	660	.codepoint => std.unicode.utf8ByteSequenceLength(ast.utf8_source[b]) catch 1,
	661	.codepoint_eol => if (std.mem.indexOfScalar(u8, ast.utf8_source[b..], '\n')) \|n\|
	662	n
	663	else
	664	(ast.utf8_source.len - b),
	665	else => comptime unreachable,
	666	};
	667	break :loc .{
	668	.byte = b,
	669	.slice = ast.utf8_source[b..][0..len],
	670	.loc = .{
	671	// TODO:
	672	.column = .start,
	673	.line = .start,
	674	.line_start = 0,
	675	.line_end = @intCast(ast.utf8_source.len),
	676	},
	677	};
	678	},
	679	};
	680	}
	681	};
	682
	683	/// This serves as a small API to make creating messages easier.
	684	pub const MsgContext = struct {
	685	file: *File,
	686	comp: *Comp,
	687	ast: *const Ast,
	688	utf8_source: [:0]const u8,
	689
	690	/// This arena is is a stack fallback with 16kb, then
	691	/// falling back to an arena allocator. This should
	692	/// make the cost of arbitrarily creating `Component`
	693	/// nodes extremely lightweight.
	694	arena: Allocator,
	695
	696	/// First component is "error" or "warn"
	697	/// Trailing components are "note"
	698	components: ArrayListUnmanaged(Component),
	699
	700	pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void {
	701	const text = try std.fmt.allocPrint(m.arena, fmt, args);
	702	try m.components.append(m.arena, .{
	703	.message_utf8 = text,
	704	.location = highlight.loc(m.ast),
	705	});
	706	}
	707
	708	pub fn msgNode(m: *MsgContext, node: NodeIndex, comptime fmt: []const u8, args: anytype) !void {
	709	return m.msg(.{ .node = node }, fmt, args);
	710	}
	711
	712	pub fn msgToken(m: *MsgContext, token: TokenIndex, comptime fmt: []const u8, args: anytype) !void {
	713	return m.msg(.{ .token = token }, fmt, args);
	714	}
	715
	716	pub fn msgAlone(m: *MsgContext, comptime fmt: []const u8, args: anytype) !void {
	717	return m.msg(.alone, fmt, args);
	718	}
	719	};
	720
	721	pub const LazyNode = struct {
	722	decl: Comp.Decl.Index,
	723	inst: Teg.Inst.Index,
	724	operand: Operand,
	725
	726	pub fn get(lazy: LazyNode, ctx: *MsgContext) !NodeIndex {
	727	const decl = ctx.comp.intern_pool.declPtrConst(lazy.decl);
	728	const ns = ctx.comp.intern_pool.namespacePtr(decl.src_namespace);
	729
	730	// TODO: This might change if we have errors like
	731	// - function arguments not matching at <callsite>
	732	// - function declared <definition>
	733	// And they are both in different files.
	734	// To resolve this, `Component` needs to store which file it is in.
	735	assert(ns.file_scope == ctx.file);
	736
	737	if (ns.parent.unwrap() != null) {
	738	@panic("TODO: LazyNode.get when the decl is in a nested namespace");
	739	}
	740
	741	const teg = &(ctx.file.teg orelse
	742	// this will be required once incremental compilation hits
	743	@panic("TODO: teg was unloaded"));
	744
	745	const decl_inst = decl.teg_decl_index.unwrap() orelse
	746	unreachable; // file was not analyzed, yet teg loaded?
	747
	748	const teg_decl = teg.instData(.declaration, decl_inst.resolve(&ctx.comp.intern_pool));
	749
	750	const node_offset = teg.instNodeOffset(lazy.inst);
	751
	752	const base_node: NodeIndex = @intCast(teg_decl.src_node + node_offset);
	753	if (lazy.operand == .main_node)
	754	return base_node;
	755
	756	const data = ctx.ast.nodeData(base_node);
	757
	758	const node_tag = ctx.ast.nodeTag(base_node);
	759
	760	switch (teg.instTag(lazy.inst)) {
	761	.as => switch (node_tag) {
	762	.var_decl => {
	763	switch (lazy.operand) {
	764	.rhs => return data.var_decl.init,
	765	else => {},
	766	}
	767	},
	768	else => {},
	769	},
	770	.call => switch (node_tag) {
	771	.call => switch (lazy.operand) {
	772	.operand => return data.call.base,
	773	else => \|op\| return ctx.ast.extraData(data.call.extra, Ast.Call)
	774	.args.slice(ctx.ast)[op.getArgId()],
	775	},
	776	else => {},
	777	},
	778	.@"break" => switch (node_tag) {
	779	.@"return" => return data.@"return".expr.unwrap().?,
	780	.@"break" => return data.@"break".value.unwrap().?,
	781	.block_implicit_return => {
	782	const statements = ctx.ast.nodeStatements(base_node);
	783	return statements[statements.len - 1];
	784	},
	785	else => return base_node,
	786	},
	787	.memory => switch (node_tag) {
	788	.var_decl => switch (lazy.operand) {
	789	.operand => return data.var_decl.init,
	790	else => {},
	791	},
	792	else => {},
	793	},
	794	.store => {
	795	assert(node_tag.isAssignBin());
	796	switch (lazy.operand) {
	797	.lhs => return data.bin.lhs,
	798	.rhs => return data.bin.rhs,
	799	else => unreachable,
	800	}
	801	},
	802	else => {},
	803	}
	804
	805	if (!std.debug.runtime_safety) unreachable;
	806
	807	std.debug.panic("TODO: operand lookup: .{s} + .{s} + {}", .{
	808	@tagName(teg.instTag(lazy.inst)),
	809	@tagName(ctx.ast.nodeTag(base_node)),
	810	lazy.operand,
	811	});
	812	}
	813	};
	814
	815	pub const Operand = enum(u32) {
	816	/// Point to the main node of an instruction
	817	main_node = 0,
	818	/// Base of a call, field, or operand to a unary instruction
	819	operand = 1,
	820	/// Return type of a function declaration or lambda node
	821	return_type = 2,
	822	/// Left hand side of a binary instruction
	823	lhs = 4,
	824	/// Right hand side of a binary instruction
	825	rhs = 5,
	826
	827	/// One of:
	828	/// - An argument of a .call node
	829	/// - A parameter of a .fn_decl node
	830	pub fn argument(id: u32) Operand {
	831	return @enumFromInt(id + 5);
	832	}
	833
	834	fn getArgId(op: Operand) u32 {
	835	return @intFromEnum(op) - 5;
	836	}
	837	};
	838
	839	const Report = @This();
	840	const std = @import("std");
	841	const ArrayListUnmanaged = std.ArrayListUnmanaged;
	842	const Allocator = std.mem.Allocator;
	843	const assert = std.debug.assert;
	844
	845	const builtin = @import("builtin");
	846
	847	const Tokenizer = @import("Tokenizer.zig");
	848	const ByteOffset = Tokenizer.ByteOffset;
	849
	850	const Ast = @import("Ast.zig");
	851	const TokenIndex = Ast.TokenIndex;
	852	const NodeIndex = Ast.NodeIndex;
	853	const Location = Ast.Location;
	854
	855	const Teg = @import("Teg.zig");
	856
	857	const Comp = @import("Comp.zig");
	858	const File = Comp.File;
	859
	860	const InternPool = @import("InternPool.zig");
	861	const Value = InternPool.Value;
	862	const Type = InternPool.Type;
	863
	864	const util = @import("util.zig");

src/core/Sema.zig created+999

...	...	@@ -0,0 +1,999 @@
	1	/// absolutely stealing zig on this one haha
	2	/// This struct is created per declaration and chills
	3
	4	// inputs:
	5	comp: *Comp,
	6	gpa: Allocator,
	7	arena: Allocator,
	8	code: Teg,
	9	ip: *InternPool,
	10	decl: *Decl,
	11	decl_index: Decl.Index,
	12	namespace_index: Namespace.Index,
	13
	14	// temp data:
	15	inst_map: InstMap = undefined,
	16	/// scratch memory. can be used in any place you might temporarily allocate an arraylist
	17	/// this is most notably used to store block contents before the block is finalized
	18	scratch: std.ArrayListUnmanaged(u32) = .{},
	19	params: []Type = undefined,
	20
	21	// outputs
	22	report: Report.Builder = undefined,
	23	air_instructions: std.MultiArrayList(Air.Inst) = .{},
	24	air_extra: dod.ExtraStore(.air, .@"var") = .{},
	25
	26	temp_result_map: std.AutoArrayHashMapUnmanaged(Teg.Inst.Index, Type) = .{},
	27
	28	pub const Error = error{
	29	/// Analysis has failed, see error list
	30	AnalysisFail,
	31	// This analysis is waiting on other dependencies to finish
	32	// Interrupt,
	33
	34	// TODO: add handling of this
	35	TooManyValues,
	36	HasNulls,
	37
	38	OutOfMemory,
	39	} \|\| Comp.File.GetTegError;
	40
	41	/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything.
	42	/// This is fine because all instructions are going to be adacent in a single declaration.
	43	const InstMap = struct {
	44	items: []Air.Inst.Ref.Optional,
	45	start: Teg.Inst.Index,
	46
	47	pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
	48	const decl = code.declData(inst_start);
	49	const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
	50	@memset(insts, .none);
	51	return .{
	52	.items = insts,
	53	.start = inst_start,
	54	};
	55	}
	56
	57	pub fn deinit(map: *InstMap, allocator: Allocator) void {
	58	allocator.free(map.items);
	59	map.* = undefined;
	60	}
	61
	62	pub fn get(map: *const InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref {
	63	const source_i = @intFromEnum(teg_code);
	64	const start_i = @intFromEnum(map.start);
	65	assert(source_i >= start_i and source_i <= (start_i + map.items.len));
	66	return map.items[source_i - start_i].unwrap();
	67	}
	68
	69	pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void {
	70	if (assertWithMessage(map.get(from) == null)) {
	71	std.debug.panic("%{d} is already mapped (to #{d})", .{
	72	@intFromEnum(from),
	73	@intFromEnum(map.get(from).?.decode().index),
	74	});
	75	}
	76	const source_i = @intFromEnum(from);
	77	const start_i = @intFromEnum(map.start);
	78	map.items[source_i - start_i] = to.toOptional();
	79	}
	80	};
	81
	82	pub fn init(s: *Sema) !void {
	83	const t = tracer.trace(@src());
	84	defer t.end();
	85
	86	s.report = .{
	87	.gpa = s.gpa,
	88	.current_file = s.ip.namespacePtr(s.namespace_index).file_scope,
	89	};
	90
	91	try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
	92
	93	s.inst_map = try InstMap.init(
	94	s.arena,
	95	&s.code,
	96	s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip),
	97	);
	98	}
	99
	100	pub fn deinit(s: *Sema) void {
	101	const t = tracer.trace(@src());
	102	defer t.end();
	103
	104	const report = s.report.finalize() catch @panic("OOM");
	105	report.dumpToStdErr(s.comp) catch @panic("Error");
	106	report.deinit(s.gpa);
	107
	108	defer s.* = undefined;
	109	s.inst_map.deinit(s.arena);
	110	s.scratch.deinit(s.arena);
	111	s.air_extra.deinit(s.gpa);
	112	s.air_instructions.deinit(s.gpa);
	113	s.temp_result_map.deinit(s.arena);
	114	}
	115
	116	pub fn finalize(s: *Sema) !void {
	117	const air: Air = s.tempAir();
	118
	119	if (s.comp.options.dump_ir) {
	120	std.debug.print("=" ** 20, .{});
	121	std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)});
	122	air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
	123	}
	124
	125	try s.comp.codeGenDecl(s.decl, &air);
	126	}
	127
	128	// --- analyze functions ---
	129
	130	/// This is the entrypoint to semantic analysis: It all starts at a declaration.
	131	pub fn analyzeDeclaration(s: *Sema) Error!Value {
	132	const t = tracer.trace(@src());
	133	defer t.end();
	134
	135	// TODO: turn this entire function into a nonrecursive state machine, so
	136	// that analysis can pause and continue at any point. for example, when a
	137	// dependency on another declaration is required, but that declaration has
	138	// not been resolved yet. or, we are waiting on a file read to happen
	139
	140	const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start);
	141	assert(decl.kind == .function);
	142
	143	s.params = try s.gpa.alloc(Type, fn_decl.params.len);
	144	defer s.gpa.free(s.params);
	145
	146	for (fn_decl.params.slice(&s.code), 0..) \|param, i\| {
	147	s.params[i] = try s.resolveType(
	148	s.inst_map.start.op(Report.Operand.argument(@intCast(i))),
	149	param,
	150	);
	151	}
	152
	153	const return_type = try s.resolveType(
	154	s.inst_map.start.op(.return_type),
	155	fn_decl.return_type,
	156	);
	157
	158	const root_block = fn_decl.body;
	159	switch (root_block.decode()) {
	160	.value => \|value\| {
	161	const coerced = try s.coerceValue(
	162	s.inst_map.start.op(.operand),
	163	value,
	164	return_type,
	165	);
	166	const block = try s.addBlock();
	167	_ = try s.addInstRef(.@"break", .{
	168	.block_ref = block.block_index,
	169	.value = coerced.toAir(),
	170	});
	171	},
	172	.index => \|index\| {
	173	try s.temp_result_map.putNoClobber(s.arena, index, return_type);
	174	_ = try s.analyzeBlock(index);
	175	},
	176	}
	177
	178	const func_prototype = (try s.ip.get(s.gpa, .{
	179	.type_function = .{
	180	.param_types = s.params,
	181	.return_type = return_type,
	182	},
	183	})).toTypeUnsafe(s.ip);
	184	const value = try s.ip.get(s.gpa, .{
	185	.func = .{
	186	.ty = func_prototype,
	187	.owner_decl = s.decl_index,
	188	.teg_body_inst = s.decl.teg_decl_index.unwrap().?,
	189	},
	190	});
	191	s.decl.val = value;
	192	return value;
	193	}
	194
	195	pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref {
	196	const t = tracer.trace(@src());
	197	defer t.end();
	198
	199	const scratch_top = s.scratch.items.len;
	200	defer s.scratch.shrinkRetainingCapacity(scratch_top);
	201
	202	const block = try s.addBlock();
	203	try s.inst_map.add(in, block.block_index.toRef());
	204
	205	var last: Air.Inst.Ref = undefined;
	206	const instructions = s.code.blockInstructions(in);
	207	assert(instructions.len != 0);
	208	for (instructions) \|inst\| {
	209	last = try s.analyzeCode(inst);
	210	}
	211
	212	if (scratch_top + 1 == s.scratch.items.len) {
	213	const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1];
	214	if (single_instruction == .@"break") {
	215	const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1]
	216	.decode(.@"break", single_instruction);
	217	if (data.block_ref == block.block_index) {
	218	// This is a block that resolved to just a single value ref with
	219	// no side effects. This means we can just lower this to the ref
	220	// itself, and give codegen an easier time.
	221
	222	// Delete the block and the single instruction
	223	// Break does not use any extra data, so this call removes both items
	224	s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index));
	225
	226	return data.value;
	227	}
	228	}
	229	}
	230
	231	return (try block.finish(s, scratch_top)).toRef();
	232	}
	233
	234	fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
	235	const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
	236	defer t.end();
	237
	238	const out = try s.analyzeCodeInner(in);
	239
	240	// Block refs are added automatically
	241	if (s.code.instTag(in) == .block) {
	242	assert(s.inst_map.get(in) != null);
	243	} else {
	244	try s.inst_map.add(in, out);
	245	}
	246
	247	return out;
	248	}
	249
	250	fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref {
	251	switch (s.code.instTag(in)) {
	252	.string => {
	253	const string: Teg.SlicedString = s.code.instPayload(.string, in);
	254	return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) });
	255	},
	256	.number => {
	257	const number: dod.Float = s.code.instPayload(.number, in);
	258	return s.makeValueAir(.{ .comptime_float = number.decode() });
	259	},
	260	.js => {
	261	const operand_ref = s.code.instData(.js, in).operand;
	262	const operand = try s.resolveRefComptimeType(in.op(.operand), operand_ref, .string);
	263	return s.addInstRef(.js_global_ref, operand.asString(s.ip));
	264	},
	265	.field => {
	266	const field: Teg.Inst.Bin = s.code.instPayload(.field, in);
	267	const base = try s.resolveRefRuntime(field.lhs);
	268
	269	const rhs = try s.resolveRefComptimeType(in.op(.rhs), field.rhs, .string);
	270	const string = rhs.asString(s.ip);
	271
	272	const base_type = s.typeOfRef(base);
	273	switch (base_type) {
	274	.@"opaque" => {
	275	return s.addInstRef(.field, .{
	276	.base = base,
	277	.field = try string.tryToNullTerminatedString(s.ip, s.gpa),
	278	});
	279	},
	280	else => \|tag\| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}),
	281	}
	282	},
	283	.load_decl => {
	284	const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in);
	285	const decl_name = try s.ip.getOrPutString(
	286	s.gpa,
	287	request.string.slice(s.code.string_bytes),
	288	.no_embedded_nulls,
	289	);
	290	const decl_index = s.lookupDecl(decl_name) orelse {
	291	std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)});
	292	return error.AnalysisFail;
	293	};
	294	const resolved = try s.resolveDecl(decl_index);
	295	return resolved.toAir();
	296	},
	297	.param => {
	298	const param: Teg.Inst.DataNode = s.code.instData(.param, in);
	299	return s.addInstRef(.arg, .{
	300	.type = s.params[param.data],
	301	.arg_number = param.data,
	302	});
	303	},
	304	inline .add,
	305	.sub,
	306	.mul,
	307	.div,
	308	.lt,
	309	.gt,
	310	.eq,
	311	=> \|tag\| {
	312	const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
	313
	314	const lhs = try s.resolveRefRuntime(bin.lhs);
	315	const rhs = try s.resolveRefRuntime(bin.rhs);
	316
	317	if (lhs.kind == .value and rhs.kind == .value) {
	318	return (try s.comptimeBin(
	319	@field(BinaryOp, @tagName(tag)),
	320	lhs.decode().value,
	321	rhs.decode().value,
	322	));
	323	}
	324
	325	const left_type = s.typeOfRef(lhs);
	326	_ = left_type; // autofix
	327	const right_type = s.typeOfRef(rhs);
	328	_ = right_type; // autofix
	329
	330	// TODO: validate type for operation
	331
	332	return s.addInstRef(tag.airTag(), .{
	333	.lhs = lhs,
	334	.rhs = rhs,
	335	});
	336	},
	337	inline .negate, .bool_not => \|tag\| {
	338	const un: Teg.Inst.RefNode = s.code.instData(tag, in);
	339
	340	const operand = try s.resolveRefRuntime(un.operand);
	341
	342	const operand_type = s.typeOfRef(operand);
	343	_ = operand_type; // autofix
	344
	345	// TODO: validate type for operation
	346
	347	return s.addInstRef(tag.airTag(), .{
	348	.operand = operand,
	349	});
	350	},
	351	.@"break" => {
	352	const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in);
	353	const operand = try s.resolveRefRuntime(brk.operand);
	354	const block_ref = s.inst_map.get(brk.block).?.decode().index; // block ref is resolved by teg
	355
	356	const coerced = if (s.temp_result_map.get(brk.block)) \|ty\|
	357	try s.coerceRef(in.op(.operand), operand, ty)
	358	else
	359	operand;
	360
	361	return s.addInstRef(.@"break", .{
	362	.block_ref = block_ref,
	363	.value = coerced,
	364	});
	365	},
	366	.call => {
	367	const call: Teg.Inst.Call = s.code.instPayload(.call, in);
	368	const base = try s.resolveRefRuntime(call.base);
	369	const base_type = s.typeOfRef(base);
	370
	371	const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len);
	372	defer s.arena.free(scratch);
	373	if (base_type != .@"opaque") {
	374	const proto = base_type.toFunctionPrototype(s.ip) orelse {
	375	return s.fail(.call_non_function, .{
	376	.call = s.nodeLoc(in, .operand),
	377	.ty = base_type,
	378	});
	379	};
	380
	381	if (proto.param_types.len != call.args.len) {
	382	// TODO: also attempt to coerce some of the params to reveal more errors.
	383	return s.fail(.call_mismatch_args, .{
	384	.call = s.nodeLoc(in, .operand),
	385	.actual = @intCast(call.args.len),
	386	.expected = @intCast(proto.param_types.len),
	387	});
	388	}
	389
	390	// validate function args
	391	for (proto.param_types, call.args.slice(&s.code), 0..) \|param, arg, i\| {
	392	const ref = try s.resolveRefRuntime(arg);
	393	const coerced = try s.coerceRef(
	394	in.op(Report.Operand.argument(@intCast(i))),
	395	ref,
	396	param,
	397	);
	398	scratch[i] = coerced;
	399	}
	400	} else {
	401	// just copy the args in without checking
	402	for (call.args.slice(&s.code), 0..) \|arg, i\| {
	403	const ref = try s.resolveRefRuntime(arg);
	404	scratch[i] = ref;
	405	}
	406	}
	407
	408	const start = s.air_extra.list.items.len;
	409	try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch));
	410
	411	return s.addInstRef(.call, .{
	412	.base = base,
	413	.call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{
	414	.args = .{
	415	.start = @intCast(start),
	416	.len = @intCast(scratch.len),
	417	},
	418	}),
	419	});
	420	},
	421	.ensure_void => {
	422	const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false);
	423	const t = s.typeOfRef(operand);
	424	_ = t;
	425	// if (!t.isEnsureVoid(s.ip)) {
	426	// std.debug.print("oh no, we are discarding {} on accident\n", .{
	427	// t.fmt(s.ip),
	428	// });
	429	// }
	430	return InternPool.Value.void.toAir();
	431	},
	432	.discard => {
	433	const operand = try s.resolveRefRuntimeExtra(s.code.instData(.discard, in).operand, false);
	434	const t = s.typeOfRef(operand);
	435	if (t == .void) {
	436	std.debug.print("oh no, we have a useless _ = void;\n", .{});
	437	}
	438	return InternPool.Value.void.toAir();
	439	},
	440	.@"if" => {
	441	const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code);
	442
	443	const cond = try s.resolveRefRuntime(data.cond);
	444
	445	// try s.validateType(s.typeOfRef(cond), .bool_type);
	446
	447	// If known at comptime, only analyze the taken path
	448	if (cond.kind == .value) {
	449	@panic("TODO: comptime IF");
	450	}
	451
	452	// Emit a runtime branch
	453	const yes_block = try s.analyzeRefToBlock(data.yes);
	454	const no_block = try s.analyzeRefToBlock(data.no);
	455
	456	return try s.addInstRef(.@"if", .{
	457	.cond = cond,
	458	.data = try s.air_extra.append(s.gpa, Air.If.Data, .{
	459	.yes = yes_block,
	460	.no = no_block,
	461	.union_ty = s.unionType(s.typeOfRef(yes_block), s.typeOfRef(no_block)),
	462	}),
	463	});
	464	},
	465	.block => {
	466	const block = try s.analyzeBlock(in);
	467	switch (block.decode()) {
	468	.index => \|t\| if (s.tempAir().instTag(t) == .block) {
	469	try s.scratch.append(s.gpa, @intFromEnum(t));
	470	},
	471	else => {},
	472	}
	473	return block;
	474	},
	475	.memory => {
	476	const data = s.code.instData(.memory, in);
	477
	478	var initial_value = try s.resolveRefRuntime(data.operand);
	479
	480	switch (initial_value.decode()) {
	481	.value => \|v\| switch (v.toKey(s.ip)) {
	482	.comptime_float => \|val\| {
	483	try s.report.warn(.numeric_int_var_ambiguity, .{
	484	.node = s.nodeLoc(in, .operand),
	485	});
	486	initial_value = (try s.makeValue(.{ .float = val })).toAir();
	487	},
	488	else => {},
	489	},
	490	else => {},
	491	}
	492
	493	return try s.addInstRef(.memory, .{
	494	.init = initial_value,
	495	.ty = s.typeOfRef(initial_value),
	496	});
	497	},
	498	.store => {
	499	const data: Teg.Inst.Store = s.code.instPayload(.store, in);
	500	assert(s.code.instTag(data.memory) == .memory); // corrupt store instruction
	501
	502	const location = s.mapTeg(data.memory.toRef()).decode().index;
	503	s.markInst(location);
	504	const operand = try s.resolveRefRuntime(data.operand);
	505	const operand_coerced = try s.coerceRef(
	506	in.op(.rhs),
	507	operand,
	508	s.typeOfRef(location.toRef()),
	509	);
	510	return try s.addInstRef(.store, .{
	511	.memory = location,
	512	.operand = operand_coerced,
	513	});
	514	},
	515	.as => {
	516	const as: Teg.Inst.As = s.code.instPayload(.as, in);
	517	const coerce_to = try s.resolveRefComptimeType(in.op(.lhs), as.type, .type);
	518	const operand = try s.resolveRefRuntime(as.operand);
	519	return try s.coerceRef(in.op(.rhs), operand, coerce_to.toType(s.ip).?);
	520	},
	521	else => \|tag\| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}),
	522	}
	523	}
	524
	525	fn markInst(s: *Sema, inst: Air.Inst.Index) void {
	526	const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
	527	flags.uses = flags.uses.increment();
	528	}
	529
	530	fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) Error!Air.Inst.Ref {
	531	const t = tracer.trace(@src());
	532	defer t.end();
	533
	534	const scratch_top = s.scratch.items.len;
	535	defer s.scratch.shrinkRetainingCapacity(scratch_top);
	536
	537	const visited = try s.resolveRefRuntime(in);
	538
	539	if (scratch_top == s.scratch.items.len and visited.kind == .value) {
	540	return visited;
	541	}
	542
	543	if (visited.kind == .index and
	544	s.air_instructions.items(.tag)[@intFromEnum(visited.decode().index)] == .block)
	545	{
	546	return visited;
	547	}
	548
	549	const block = try s.addBlock();
	550	_ = try s.addInstRef(.@"break", .{
	551	.block_ref = block.block_index,
	552	.value = visited,
	553	});
	554	return (try block.finish(s, scratch_top)).toRef();
	555	}
	556
	557	fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) Error!Air.Inst.Ref {
	558	const index = ref.toIndex() orelse {
	559	return ref.toValue().?.toAir();
	560	};
	561
	562	if (s.inst_map.get(index)) \|air\| {
	563	if (mark) {
	564	switch (air.decode()) {
	565	.index => \|i\| s.markInst(i),
	566	else => {},
	567	}
	568	}
	569	return air;
	570	}
	571
	572	return s.analyzeCode(index);
	573	}
	574
	575	fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) Error!Air.Inst.Ref {
	576	return resolveRefRuntimeExtra(s, ref, true);
	577	}
	578
	579	fn resolveRefComptimeAny(s: *Sema, ref: Teg.Inst.Ref) !Value {
	580	const index = switch (ref.decode()) {
	581	.value => \|value\| return value,
	582	.index => \|index\| index,
	583	};
	584
	585	if (s.inst_map.get(index)) \|air\| {
	586	switch (air.decode()) {
	587	.value => \|value\| return value,
	588	else => {},
	589	}
	590	}
	591
	592	switch (s.code.instTag(index)) {
	593	.string => {
	594	const string: Teg.SlicedString = s.code.instPayload(.string, index);
	595	return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
	596	},
	597
	598	else => \|tag\| std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
	599	}
	600	}
	601
	602	fn resolveRefComptimeType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref, ty: Type) !Value {
	603	const any = try s.resolveRefComptimeAny(ref);
	604	return try s.coerceValue(src, any, ty);
	605	}
	606
	607	fn resolveType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref) !Type {
	608	return (try s.resolveRefComptimeType(src, ref, .type)).toTypeUnsafe(s.ip);
	609	}
	610
	611	/// Coerce `value` runtime ref to `coerce_type`, and emit an error at `teg_ref`'s location
	612	fn coerceRef(s: *Sema, src: InstOp, value: Air.Inst.Ref, to: Type) !Air.Inst.Ref {
	613	const from = s.typeOfRef(value);
	614	if (from == to) {
	615	return value;
	616	}
	617	const index = switch (value.decode()) {
	618	.value => \|v\| return (try s.coerceValue(src, v, to)).toAir(),
	619	.index => \|index\| index,
	620	};
	621	switch (to) {
	622	.@"opaque" => {
	623	//
	624	},
	625	.bool => {
	626	//
	627	},
	628	.int32 => switch (from) {
	629	.int32 => unreachable, // same type handled
	630	.numeric_literal => unreachable, // must should comptime known
	631	else => {},
	632	},
	633	.float => switch (from) {
	634	.float => unreachable, // same type handled
	635	.int32 => return index.toRef(),
	636	else => {},
	637	},
	638	.string => {
	639	//
	640	},
	641	.noreturn => {
	642	//
	643	},
	644	.type => {
	645	//
	646	},
	647	.numeric_literal => {
	648	@panic("TODO: error message for not known at compile time");
	649	},
	650	.void => {
	651	//
	652	},
	653	else => {},
	654	}
	655	return s.coersionError(src, from, to);
	656	}
	657
	658	fn coerceValue(s: *Sema, src: InstOp, value: Value, to: Type) !Value {
	659	const from = value.typeOf(s.ip);
	660	if (from == to) {
	661	return value;
	662	}
	663	switch (to) {
	664	.@"opaque" => {
	665	//
	666	},
	667	.bool => {
	668	//
	669	},
	670	.int32 => switch (from) {
	671	.int32 => unreachable, // same type handled
	672	.numeric_literal, .float => switch (value.toKey(s.ip)) {
	673	.comptime_float, .float => \|val\| {
	674	if (!math.isFinite(val) or
	675	val < math.minInt(i32) or
	676	val > math.maxInt(i32) or
	677	(@rem(val, 1) != 0))
	678	{
	679	return s.fail(.comptime_float_to_int, .{
	680	.val = val,
	681	.src = s.lazyNode(src),
	682	});
	683	}
	684
	685	return try s.makeValue(.{
	686	.int32 = @intFromFloat(val),
	687	});
	688	},
	689	else => unreachable, // not a numeric literal
	690	},
	691	else => return s.coersionError(src, from, to),
	692	},
	693	.float => switch (from) {
	694	.float => unreachable, // same type handled
	695	.numeric_literal => if (value.toFloat(s.ip)) \|val\|
	696	return try s.makeValue(.{
	697	.float = val,
	698	}),
	699	else => {},
	700	},
	701	.string => {
	702	//
	703	},
	704	.noreturn => {
	705	//
	706	},
	707	.type => {
	708	if (value == .void) return .void;
	709	},
	710	.numeric_literal => {},
	711	.void => {
	712	if (value == .void) return .void;
	713	},
	714	else => {},
	715	}
	716	return s.coersionError(src, from, to);
	717	}
	718
	719	fn resolveDecl(s: *Sema, decl_index: Decl.Index) Error!Value {
	720	const decl = s.ip.declPtr(decl_index);
	721	switch (decl.analysis) {
	722	.unreferenced => {
	723	decl.analysis = .in_progress;
	724
	725	// TODO: make this short-circuit the current analysis, move this analyze
	726	// job into a "deferred" list, and then make this thread analyze something else
	727
	728	var arena = std.heap.ArenaAllocator.init(s.gpa);
	729	defer arena.deinit();
	730
	731	var todo: Report = undefined;
	732	var sema: Sema = .{
	733	.comp = s.comp,
	734	.gpa = s.gpa,
	735	.ip = s.ip,
	736	.arena = arena.allocator(),
	737	.code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.comp, &todo),
	738	.decl = decl,
	739	.decl_index = decl_index,
	740	.namespace_index = decl.src_namespace,
	741	};
	742
	743	try sema.init();
	744	defer sema.deinit();
	745	const value = try sema.analyzeDeclaration();
	746	try sema.finalize();
	747
	748	decl.analysis = .complete;
	749
	750	return value;
	751	},
	752	else => {
	753	return decl.val;
	754	},
	755	}
	756	}
	757
	758	fn unionType(s: *Sema, a: Type, b: Type) Type {
	759	if (a == b) {
	760	return a;
	761	}
	762
	763	if (a == .noreturn)
	764	return b;
	765
	766	if (b == .noreturn)
	767	return a;
	768
	769	std.debug.panic(
	770	"TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}",
	771	.{
	772	a.dump(s.ip),
	773	b.dump(s.ip),
	774	},
	775	);
	776	// return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } });
	777	}
	778
	779	const BinaryOp = enum {
	780	add,
	781	sub,
	782	mul,
	783	div,
	784	lt,
	785	gt,
	786	eq,
	787	};
	788
	789	fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref {
	790	switch (op) {
	791	.add => {
	792	const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
	793	const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add");
	794	return (try s.ip.get(s.gpa, .{
	795	.float = l + r,
	796	})).toAir();
	797	},
	798	inline .sub, .mul, .div => \|tag\| {
	799	const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	800	const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	801
	802	const result = switch (tag) {
	803	.sub => l - r,
	804	.mul => l * r,
	805	.div => l / r,
	806	else => comptime unreachable,
	807	};
	808
	809	return (try s.ip.get(s.gpa, .{
	810	.float = result,
	811	})).toAir();
	812	},
	813	inline .lt,
	814	.gt,
	815	=> \|tag\| {
	816	const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	817	const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck");
	818	const result = switch (tag) {
	819	.lt => l < r,
	820	.gt => l > r,
	821	else => comptime unreachable,
	822	};
	823	return @as(
	824	InternPool.Value,
	825	if (result) .bool_true else .bool_false,
	826	).toAir();
	827	},
	828	.eq,
	829	=> {
	830	const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
	831	const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq");
	832	const result = l == r;
	833	return @as(
	834	InternPool.Value,
	835	if (result) .bool_true else .bool_false,
	836	).toAir();
	837	},
	838	}
	839	}
	840
	841	/// --- helper ---
	842	const IncompleteBlock = struct {
	843	block_index: Air.Inst.Index,
	844
	845	pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index {
	846	s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top));
	847	return block.block_index;
	848	}
	849	};
	850
	851	fn addBlock(s: *Sema) !IncompleteBlock {
	852	try s.air_instructions.append(s.gpa, .{
	853	.tag = .block,
	854	.data = undefined,
	855	.flags = .{},
	856	});
	857	return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) };
	858	}
	859
	860	fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
	861	try s.air_instructions.append(s.gpa, .{
	862	.tag = tag,
	863	.data = Air.EncodedData.encode(tag, data),
	864	.flags = .{},
	865	});
	866	const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1);
	867	try s.scratch.append(s.arena, @intFromEnum(index));
	868	return index.toRef();
	869	}
	870
	871	fn setInstData(
	872	s: *Sema,
	873	comptime tag: Air.Inst.Tag,
	874	index: Air.Inst.Index,
	875	data: tag.Data(),
	876	) void {
	877	const i = @intFromEnum(index);
	878	assert(s.air_instructions.items(.tag)[i] == tag);
	879	s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data);
	880	}
	881
	882	fn makeValue(s: *Sema, key: InternPool.Key) !Value {
	883	return try s.ip.get(s.gpa, key);
	884	}
	885
	886	fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref {
	887	return (try s.makeValue(key)).toAir();
	888	}
	889
	890	fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
	891	if (inst.toValue()) \|interned\| {
	892	return interned.toAir();
	893	}
	894	const index = inst.toIndex().?;
	895	return s.inst_map.get(index) orelse {
	896	if (std.debug.runtime_safety) {
	897	std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)});
	898	}
	899	unreachable;
	900	};
	901	}
	902
	903	fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList {
	904	const items = s.scratch.items[scratch_top..];
	905	const extra_top = s.air_extra.list.items.len;
	906	try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]);
	907	return .{
	908	.start = @intCast(extra_top),
	909	.len = @intCast(items.len),
	910	};
	911	}
	912
	913	fn tempAir(s: *const Sema) Air {
	914	return .{
	915	.insts = s.air_instructions.slice(),
	916	.extra = s.air_extra.constant(),
	917	};
	918	}
	919
	920	fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Type {
	921	return s.tempAir().typeOf(ref, s.ip);
	922	}
	923
	924	fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex {
	925	var nsi = s.namespace_index;
	926	while (true) {
	927	const ns = s.ip.namespacePtr(nsi);
	928	if (ns.lookupDirect(s.comp, name)) \|decl_id\| {
	929	return decl_id;
	930	}
	931	nsi = ns.parent.unwrap() orelse
	932	unreachable; // transform ensures all identifiers get resolved
	933	}
	934	}
	935
	936	fn nodeLoc(s: *const Sema, in: Teg.Inst.Index, op: Report.Operand) Report.LazyNode {
	937	return s.lazyNode(in.op(op));
	938	}
	939
	940	fn addWarn(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void {
	941	try s.report.warn(kind, data);
	942	}
	943
	944	fn addError(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void {
	945	try s.report.err(kind, data);
	946	}
	947
	948	fn fail(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) Error {
	949	try s.report.err(kind, data);
	950	return Error.AnalysisFail;
	951	}
	952
	953	fn coersionError(s: *Sema, src: InstOp, from: Type, to: Type) Error {
	954	return s.fail(.coersion, .{
	955	.from = from,
	956	.to = to,
	957	.src = s.lazyNode(src),
	958	});
	959	}
	960
	961	fn lazyNode(s: *const Sema, src: InstOp) Report.LazyNode {
	962	return src.lazyNode(s.decl_index);
	963	}
	964
	965	const Sema = @This();
	966
	967	const std = @import("std");
	968	const mem = std.mem;
	969	const math = std.math;
	970	const Allocator = std.mem.Allocator;
	971	const assert = std.debug.assert;
	972	const log = std.log.scoped(.sema);
	973
	974	inline fn assertWithMessage(cond: bool) bool {
	975	if (std.debug.runtime_safety) {
	976	return !cond;
	977	} else if (!cond) {
	978	unreachable; // assertion failure
	979	}
	980	}
	981
	982	const Comp = @import("Comp.zig");
	983	const File = Comp.File;
	984	const Decl = Comp.Decl;
	985	const Namespace = Comp.Namespace;
	986
	987	const Teg = @import("Teg.zig");
	988	const InstOp = Teg.InstOp;
	989
	990	const Air = @import("Air.zig");
	991	const Report = @import("Report.zig");
	992	const Ast = @import("Ast.zig");
	993	const dod = @import("array.zig");
	994	const tracer = @import("tracer");
	995
	996	const InternPool = @import("InternPool.zig");
	997	const Value = InternPool.Value;
	998	const Type = InternPool.Type;
	999	const NullTerminatedString = InternPool.NullTerminatedString;

src/core/Teg.Dump.zig created+314

...	...	@@ -0,0 +1,314 @@
	1	//! This implements a debug formatter for Teg
	2	teg: Teg,
	3
	4	const State = struct {
	5	arena: std.mem.Allocator,
	6	teg: *const Teg,
	7	indent: usize = 0,
	8	tty: std.io.tty.Config,
	9	visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{},
	10
	11	fn newline(state: State, writer: anytype) !void {
	12	try writer.writeByte('\n');
	13	if (state.indent > 0)
	14	try writer.writeByteNTimes(' ', state.indent * 2);
	15	}
	16
	17	fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void {
	18	const teg = s.teg;
	19	const tty = s.tty;
	20
	21	try writer.print("%{} = ", .{@intFromEnum(inst)});
	22
	23	const tag = s.teg.instTag(inst);
	24
	25	try tty.setColor(writer, .bright_black);
	26	const node_offset = teg.instNodeOffset(inst);
	27	try writer.print("{c}{d} ", .{
	28	@as(u8, if (node_offset >= 0) '+' else '-'),
	29	@abs(node_offset),
	30	});
	31	try tty.setColor(writer, .reset);
	32
	33	try tty.setColor(writer, .cyan);
	34	try writer.print("{s}", .{@tagName(tag)});
	35	try tty.setColor(writer, .reset);
	36
	37	if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
	38	try tty.setColor(writer, .red);
	39	try writer.writeAll(" [error: recursive tree]");
	40	try tty.setColor(writer, .reset);
	41	return;
	42	}
	43
	44	switch (tag) {
	45	.module => {
	46	try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}');
	47	},
	48	.declaration => {
	49	const decl_info = teg.declData(inst);
	50	try writer.print(" ({s}) ", .{@tagName(decl_info.kind)});
	51
	52	const name = decl_info.name.slice(teg.string_bytes);
	53	if (name.len == 0) {
	54	try tty.setColor(writer, .bold);
	55	try writer.writeAll("<top level code>");
	56	} else {
	57	try tty.setColor(writer, .magenta);
	58	try writer.writeAll(name);
	59	}
	60	try tty.setColor(writer, .reset);
	61
	62	try writer.writeAll(" {");
	63	s.indent += 1;
	64	try s.newline(writer);
	65
	66	switch (decl_info.kind) {
	67	.function => {
	68	const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1";
	69	try writer.writeAll("params: ");
	70	try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']');
	71	try s.newline(writer);
	72	try writer.writeAll("return_type: ");
	73	try s.dumpRef(writer, fn_data.return_type);
	74	try s.newline(writer);
	75	try writer.writeAll("body: ");
	76	try s.dumpRef(writer, fn_data.body);
	77	},
	78	}
	79	s.indent -= 1;
	80	try s.newline(writer);
	81	try writer.writeAll("}");
	82	},
	83	.param => {
	84	const data = teg.instData(.param, inst);
	85	try writer.writeAll("(");
	86	try tty.setColor(writer, .yellow);
	87	try writer.print("{d}", .{data.data});
	88	try tty.setColor(writer, .reset);
	89	try writer.writeAll(")");
	90	},
	91	.block => {
	92	const block = teg.instData(.block, inst);
	93	const index_list = block.extra_index.get(teg);
	94	try s.dumpInstList(writer, index_list.slice(teg), '[', ']');
	95	},
	96	.number => {
	97	const number = teg.instData(.number, inst);
	98	const float = number.extra_index.get(teg).decode();
	99	try tty.setColor(writer, .yellow);
	100	try writer.print(" {d}", .{float});
	101	try tty.setColor(writer, .reset);
	102	},
	103	inline .field,
	104	.add,
	105	.sub,
	106	.mul,
	107	.div,
	108	.rem,
	109	.pow,
	110	.@"and",
	111	.@"or",
	112	.bit_and,
	113	.bit_or,
	114	.bit_xor,
	115	.shl,
	116	.shr,
	117	.eq,
	118	.ne,
	119	.lt,
	120	.le,
	121	.gt,
	122	.ge,
	123	=> \|comptime_tag\| {
	124	const data = teg.instData(comptime_tag, inst).extra_index.get(teg);
	125	try writer.writeAll("(");
	126	try s.dumpRefMarker(writer, data.lhs);
	127	try writer.writeAll(", ");
	128	try s.dumpRefMarker(writer, data.rhs);
	129	try writer.writeAll(")");
	130	},
	131	.as => {
	132	const data = teg.instData(.as, inst).extra_index.get(teg);
	133	try writer.writeAll("(");
	134	try s.dumpRefMarker(writer, data.type);
	135	try writer.writeAll(", ");
	136	try s.dumpRefMarker(writer, data.operand);
	137	try writer.writeAll(")");
	138	},
	139	.@"break" => {
	140	const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg);
	141	try writer.writeAll("(");
	142	try s.dumpRefMarker(writer, data.block.toRef());
	143	try writer.writeAll(", ");
	144	try s.dumpRefMarker(writer, data.operand);
	145	try writer.writeAll(")");
	146	},
	147	.store => {
	148	const data: Teg.Inst.Store = teg.instData(.store, inst).extra_index.get(teg);
	149	try writer.writeAll("(");
	150	try s.dumpRefMarker(writer, data.memory.toRef());
	151	try writer.writeAll(", ");
	152	try s.dumpRefMarker(writer, data.operand);
	153	try writer.writeAll(")");
	154	},
	155	.string => {
	156	const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg);
	157	try tty.setColor(writer, .green);
	158	try writer.writeAll(" \"");
	159	try writer.writeAll(data.slice(teg.string_bytes));
	160	try writer.writeAll("\"");
	161	try tty.setColor(writer, .reset);
	162	},
	163	inline .memory,
	164	.discard,
	165	.ensure_void,
	166	.ensure_bool,
	167	.negate,
	168	.bool_not,
	169	.bit_not,
	170	.js,
	171	.import,
	172	.fetch,
	173	.compile_error,
	174	=> \|comptime_tag\| {
	175	const data = teg.instData(comptime_tag, inst);
	176	try writer.writeAll("(");
	177	try s.dumpRefMarker(writer, data.operand);
	178	try writer.writeAll(")");
	179	},
	180	.call => {
	181	const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg);
	182	try writer.writeAll("(");
	183	try s.dumpRefMarker(writer, call.base);
	184	try writer.writeAll(", ");
	185	try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']');
	186	try writer.writeAll(")");
	187	},
	188	.load_decl => {
	189	const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst);
	190
	191	try tty.setColor(writer, .red);
	192	try writer.print(" {s}", .{call.string.slice(teg.string_bytes)});
	193	try tty.setColor(writer, .reset);
	194	},
	195	.@"if" => {
	196	const data: Teg.Inst.If = teg.instPayload(.@"if", inst);
	197	try writer.writeAll("(");
	198	try s.dumpRefMarker(writer, data.cond);
	199	try writer.writeAll(") {");
	200	s.indent += 1;
	201	try s.newline(writer);
	202	try writer.writeAll("yes: ");
	203	try s.dumpRef(writer, data.yes);
	204	try s.newline(writer);
	205	try writer.writeAll("no: ");
	206	try s.dumpRef(writer, data.no);
	207	s.indent -= 1;
	208	try s.newline(writer);
	209	try writer.writeAll("}");
	210	},
	211	}
	212	}
	213
	214	fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
	215	if (ref.toIndex()) \|index\| {
	216	try s.dumpInst(writer, index);
	217	} else {
	218	try s.tty.setColor(writer, .blue);
	219	try writer.print("%{s}", .{@tagName(ref)});
	220	try s.tty.setColor(writer, .reset);
	221	}
	222	}
	223
	224	fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
	225	if (ref.toIndex()) \|index\| {
	226	try s.tty.setColor(writer, .blue);
	227	try writer.print("%{d}", .{@intFromEnum(index)});
	228	try s.tty.setColor(writer, .reset);
	229	} else {
	230	try s.tty.setColor(writer, .blue);
	231	try writer.print("%{s}", .{@tagName(ref)});
	232	try s.tty.setColor(writer, .reset);
	233	}
	234	}
	235
	236	fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void {
	237	try writer.print(" {c}", .{l});
	238	if (list.len > 0) {
	239	state.indent += 1;
	240
	241	for (list) \|inst\| {
	242	try state.newline(writer);
	243	try state.dumpInst(writer, inst);
	244	}
	245
	246	state.indent -= 1;
	247	try state.newline(writer);
	248	}
	249	try writer.print("{c}", .{r});
	250	}
	251
	252	fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
	253	try writer.print(" {c}", .{l});
	254	if (list.len > 0) {
	255	state.indent += 1;
	256
	257	for (list) \|ref\| {
	258	try state.newline(writer);
	259	try state.dumpRef(writer, ref);
	260	}
	261
	262	state.indent -= 1;
	263
	264	try state.newline(writer);
	265	}
	266	try writer.print("{c}", .{r});
	267	}
	268
	269	fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
	270	try writer.print(" {c}", .{l});
	271	if (list.len > 0) {
	272	state.indent += 1;
	273
	274	for (list) \|ref\| {
	275	try state.newline(writer);
	276	try state.dumpRefMarker(writer, ref);
	277	}
	278
	279	state.indent -= 1;
	280
	281	try state.newline(writer);
	282	}
	283	try writer.print("{c}", .{r});
	284	}
	285	};
	286
	287	pub fn format(
	288	dump: @This(),
	289	comptime unused_fmt: []const u8,
	290	_: std.fmt.FormatOptions,
	291	writer: anytype,
	292	) !void {
	293	comptime assert(unused_fmt.len == 0);
	294
	295	var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
	296	defer arena.deinit();
	297
	298	const teg = dump.teg;
	299	var state = State{
	300	.teg = &teg,
	301	.tty = detectTTYFromWriter(writer),
	302	.arena = arena.allocator(),
	303	};
	304	try state.dumpInst(writer, .main_module_inst);
	305	try writer.writeAll("\n");
	306	assert(state.indent == 0);
	307	}
	308
	309	const std = @import("std");
	310	const assert = std.debug.assert;
	311
	312	const Teg = @import("Teg.zig");
	313
	314	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/core/Teg.zig created+486

...	...	@@ -0,0 +1,486 @@
	1	//! TODO: rename this to something else. Teg is not a graph
	2
	3	insts: Inst.List.Slice,
	4	string_bytes: []const u8,
	5	extra: ExtraStore = .{},
	6	report: Report,
	7
	8	pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
	9	pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
	10	pub const ExtraIndex = ExtraStore.Index;
	11	pub const Dump = @import("Teg.Dump.zig");
	12
	13	pub const Inst = struct {
	14	pub const List = std.MultiArrayList(Inst);
	15
	16	tag: Tag,
	17	data: EncodedData,
	18
	19	// todo: fully document every thing
	20	pub const Tag = enum {
	21	module,
	22	declaration,
	23
	24	block,
	25	/// Creates a memory cell in the current scope.
	26	memory,
	27	/// Performs a type coersion, such as in variable declarations.
	28	/// Emits a sema error if the type conversion fails.
	29	/// `let x: i32 = 1`
	30	as,
	31	load_decl,
	32	/// Load a parameter
	33	param,
	34	store,
	35	discard,
	36	ensure_void,
	37	ensure_bool,
	38	/// lowered from `break` or `return`
	39	/// ends a block with a resulting value
	40	@"break",
	41	number,
	42	string,
	43
	44	@"if",
	45
	46	add,
	47	sub,
	48	mul,
	49	div,
	50	rem,
	51	pow,
	52	@"and",
	53	@"or",
	54	bit_and,
	55	bit_or,
	56	bit_xor,
	57	shl,
	58	shr,
	59	eq,
	60	ne,
	61	lt,
	62	le,
	63	gt,
	64	ge,
	65
	66	negate,
	67	bool_not,
	68	bit_not,
	69	field,
	70	call,
	71
	72	/// Implements the `@js` builtin
	73	js,
	74	/// Implements the `@import` builtin
	75	import,
	76	/// Implements the `@fetch` builtin
	77	fetch,
	78	/// Implements the `@compileError` builtin
	79	compile_error,
	80
	81	pub fn Data(comptime tag: Tag) type {
	82	return switch (tag) {
	83	.module => PlNode(IndexList),
	84	.declaration => PlNode(Decl),
	85	.block => PlNode(IndexList),
	86	.as => PlNode(As),
	87	.@"break" => PlNode(Break),
	88	.number => PlNode(Float),
	89	.string => PlNode(SlicedString),
	90	.call => PlNode(Call),
	91	.@"if" => PlNode(If),
	92	.load_decl => StrNode,
	93	.param => DataNode,
	94	.store => PlNode(Store),
	95
	96	// Single instruction operand
	97	.memory,
	98	.discard,
	99	.ensure_void,
	100	.ensure_bool,
	101	.negate,
	102	.bool_not,
	103	.bit_not,
	104	.js,
	105	.import,
	106	.fetch,
	107	.compile_error,
	108	=> RefNode,
	109
	110	// Two instruction operands
	111	.field,
	112	.add,
	113	.sub,
	114	.mul,
	115	.div,
	116	.rem,
	117	.pow,
	118	.@"and",
	119	.@"or",
	120	.bit_and,
	121	.bit_or,
	122	.bit_xor,
	123	.shl,
	124	.shr,
	125	.eq,
	126	.ne,
	127	.lt,
	128	.le,
	129	.gt,
	130	.ge,
	131	=> PlNode(Bin),
	132	};
	133	}
	134
	135	pub fn airTag(comptime tag: Tag) Air.Inst.Tag {
	136	return @field(Air.Inst.Tag, @tagName(tag));
	137	}
	138	};
	139
	140	/// TODO: make enum? idk
	141	const NodeOffset = i32;
	142
	143	pub fn PlNode(comptime Payload: type) type {
	144	return struct {
	145	pub const T = Payload;
	146
	147	src_node: NodeOffset,
	148	extra_index: ExtraIndex(Payload),
	149	};
	150	}
	151
	152	pub const RefNode = struct {
	153	src_node: NodeOffset,
	154	operand: Inst.Ref,
	155	};
	156
	157	pub const StrNode = struct {
	158	src_node: NodeOffset,
	159	string: NullTerminatedString,
	160	};
	161
	162	pub const DataNode = struct {
	163	src_node: NodeOffset,
	164	data: u32,
	165	};
	166
	167	/// The position of a Teg instruction within the `Teg` instructions array.
	168	pub const Index = enum(u32) {
	169	/// Teg is structured so that the outermost "main" struct of any file
	170	/// is always at index 0.
	171	main_module_inst = 0,
	172	_,
	173
	174	pub const static_len = @intFromEnum(Ref.empty_string) + 1;
	175
	176	pub fn toRef(i: Index) Inst.Ref {
	177	return @enumFromInt(static_len + @intFromEnum(i));
	178	}
	179
	180	pub fn toOptional(i: Index) OptionalIndex {
	181	return @enumFromInt(@intFromEnum(i));
	182	}
	183
	184	pub fn op(i: Index, operand: Report.Operand) InstOp {
	185	return .{ .index = i, .op = operand };
	186	}
	187	};
	188
	189	pub const OptionalIndex = enum(u32) {
	190	/// Teg is structured so that the outermost "main" module of any file
	191	/// is always at index 0.
	192	main_module_inst = 0,
	193	ref_start_index = Index.static_len,
	194	none = std.math.maxInt(u32),
	195	_,
	196
	197	pub fn unwrap(oi: OptionalIndex) ?Index {
	198	return if (oi == .none) null else @enumFromInt(@intFromEnum(oi));
	199	}
	200	};
	201
	202	/// A reference to instruction, or to an InternPool index, or neither.
	203	///
	204	/// If the integer tag value is < InternPool.static_len, then it
	205	/// corresponds to an InternPool index. Otherwise, this refers to a ZIR
	206	/// instruction.
	207	///
	208	/// The tag type is specified so that it is safe to bitcast between `[]u32`
	209	/// and `[]Ref`.
	210	pub const Ref = enum(u32) {
	211	opaque_type,
	212	bool_type,
	213	int32_type,
	214	float_type,
	215	string_type,
	216	noreturn_type,
	217	type_type,
	218	numeric_literal_type,
	219
	220	void,
	221	bool_true,
	222	bool_false,
	223	int32_zero,
	224	float_zero,
	225	nan,
	226	empty_string,
	227	standard_library,
	228
	229	/// This Ref does not correspond to any instruction or constant
	230	/// value and may instead be used as a sentinel to indicate null.
	231	none = std.math.maxInt(u32),
	232
	233	_,
	234
	235	pub const Decoded = union(enum) {
	236	value: InternPool.Value,
	237	index: Teg.Inst.Index,
	238	};
	239
	240	pub fn decode(ref: Ref) Decoded {
	241	const ref_int = @intFromEnum(ref);
	242	return if (ref_int >= Index.static_len)
	243	.{ .index = @enumFromInt(ref_int - Index.static_len) }
	244	else
	245	.{ .value = @enumFromInt(ref_int) };
	246	}
	247
	248	pub fn toIndex(ref: Ref) ?Index {
	249	assert(ref != .none);
	250	const ref_int = @intFromEnum(ref);
	251	if (ref_int >= Index.static_len) {
	252	return @enumFromInt(ref_int - Index.static_len);
	253	} else {
	254	return null;
	255	}
	256	}
	257
	258	pub fn toIndexAllowNone(inst: Ref) ?Index {
	259	if (inst == .none) return null;
	260	return toIndex(inst);
	261	}
	262
	263	pub fn toValue(inst: Ref) ?InternPool.Value {
	264	const ref_int = @intFromEnum(inst);
	265
	266	// todo: is this > or >=
	267	if (ref_int > Index.static_len)
	268	return null;
	269
	270	return @enumFromInt(ref_int);
	271	}
	272	};
	273
	274	pub const Bin = struct {
	275	lhs: Inst.Ref,
	276	rhs: Inst.Ref,
	277	};
	278
	279	pub const Store = struct {
	280	memory: Inst.Index,
	281	operand: Inst.Ref,
	282	};
	283
	284	pub const As = struct {
	285	type: Inst.Ref,
	286	operand: Inst.Ref,
	287	};
	288
	289	pub const Break = struct {
	290	block: Inst.Index,
	291	operand: Inst.Ref,
	292	};
	293
	294	pub const Call = struct {
	295	base: Inst.Ref,
	296	args: RefList,
	297	};
	298
	299	pub const If = struct {
	300	cond: Inst.Ref,
	301	yes: Inst.Ref,
	302	no: Inst.Ref,
	303	};
	304
	305	/// This will contain trailing data depending on the .kind
	306	pub const Decl = struct {
	307	name: NullTerminatedString,
	308	hash: Cache.BinDigest,
	309	kind: Kind,
	310	instruction_count: u32,
	311
	312	pub const Kind = enum {
	313	function,
	314	// future: struct, let, var, etc
	315	};
	316	};
	317
	318	/// Trailng data for function declarations
	319	pub const DeclFn = struct {
	320	params: RefList,
	321	return_type: Ref,
	322	body: Ref,
	323	};
	324	};
	325
	326	pub const InstOp = struct {
	327	index: Inst.Index,
	328	op: Report.Operand,
	329
	330	pub fn lazyNode(inst: InstOp, decl: Comp.Decl.Index) LazyNode {
	331	return .{
	332	.inst = inst.index,
	333	.operand = inst.op,
	334	.decl = decl,
	335	};
	336	}
	337	};
	338
	339	pub const NullTerminatedString = enum(u32) {
	340	empty = 0,
	341	_,
	342
	343	pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 {
	344	if (str == .empty) return "";
	345	const sub = string_bytes[@intFromEnum(str)..];
	346	const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable;
	347
	348	return sub[0..i :0];
	349	}
	350	};
	351
	352	pub const SlicedString = struct {
	353	start: u32,
	354	len: u32,
	355
	356	pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 {
	357	return string_bytes[str.start..][0..str.len];
	358	}
	359	};
	360
	361	pub const RefList = struct {
	362	start: u32,
	363	len: u32,
	364
	365	pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref {
	366	return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
	367	}
	368	};
	369
	370	pub const IndexList = struct {
	371	start: u32,
	372	len: u32,
	373
	374	pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index {
	375	return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
	376	}
	377	};
	378
	379	pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
	380	gpa.free(teg.string_bytes);
	381	teg.extra.deinit(gpa);
	382	teg.insts.deinit(gpa);
	383	teg.report.deinit(gpa);
	384	teg.* = undefined;
	385	}
	386
	387	pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag {
	388	return teg.insts.items(.tag)[@intFromEnum(index)];
	389	}
	390
	391	pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData {
	392	return teg.insts.items(.data)[@intFromEnum(index)];
	393	}
	394
	395	pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
	396	return teg.instEncodedData(index).decode(tag, teg.instTag(index));
	397	}
	398
	399	pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
	400	return teg.instData(tag, index).extra_index.get(teg);
	401	}
	402
	403	pub fn instNodeOffset(teg: *const Teg, index: Inst.Index) i32 {
	404	switch (teg.instTag(index)) {
	405	inline else => \|tag\| {
	406	return teg.instData(tag, index).src_node;
	407	},
	408	}
	409	}
	410
	411	pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index {
	412	return teg.instData(.block, index).extra_index.get(teg).slice(teg);
	413	}
	414
	415	pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } {
	416	const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg);
	417	const decl_fn = trail.read(&teg.extra, Inst.DeclFn);
	418	return .{ decl, decl_fn };
	419	}
	420
	421	pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl {
	422	return teg.instData(.declaration, index).extra_index.get(teg);
	423	}
	424
	425	pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index {
	426	const module = teg.instData(.module, module_index);
	427	const module_data: IndexList = module.extra_index.get(teg);
	428	const slice = module_data.slice(teg);
	429	if (std.debug.runtime_safety) {
	430	for (slice) \|item\| {
	431	if (teg.instTag(item) != .declaration) {
	432	std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))});
	433	unreachable;
	434	}
	435	}
	436	}
	437	return slice;
	438	}
	439
	440	pub fn rootFunction(teg: *const Teg) Inst.Index {
	441	const decls = teg.moduleDeclarations(.main_module_inst);
	442	const root_fn = decls[decls.len - 1];
	443	assert(teg.declName(root_fn).len == 0);
	444	return root_fn;
	445	}
	446
	447	pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
	448	const decl = teg.instData(.declaration, decl_index);
	449	const decl_data: Inst.Decl = decl.extra_index.get(teg);
	450	return decl_data.name.slice(teg.string_bytes);
	451	}
	452
	453	pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
	454	try writer.print("{}", .{Dump{ .teg = teg }});
	455	}
	456
	457	const Teg = @This();
	458
	459	const std = @import("std");
	460	const mem = std.mem;
	461	const assert = std.debug.assert;
	462
	463	const Cache = @import("Cache.zig");
	464
	465	const Ast = @import("Ast.zig");
	466	const Unused = Ast.Unused;
	467	const NodeIndex = Ast.NodeIndex;
	468	const TokenIndex = Ast.TokenIndex;
	469	const OptionalNodeIndex = Ast.OptionalNodeIndex;
	470	const OptionalTokenIndex = Ast.OptionalTokenIndex;
	471	const OptionalExtraIndex = Ast.OptionalExtraIndex;
	472	const optionalTokenIndex = OptionalTokenIndex.init;
	473	const optionalNodeIndex = OptionalNodeIndex.init;
	474	const optionalExtraIndex = OptionalExtraIndex.init;
	475
	476	const dod = @import("array.zig");
	477	const Float = dod.Float;
	478
	479	const InternPool = @import("InternPool.zig");
	480
	481	const Air = @import("Air.zig");
	482
	483	const Comp = @import("Comp.zig");
	484
	485	const Report = @import("Report.zig");
	486	const LazyNode = Report.LazyNode;

src/core/Tokenizer.zig created+995

...	...	@@ -0,0 +1,995 @@
	1	//! The tokenizer is extremely simple structure that, given a view of source
	2	//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit
	3	//! offset to where that token is located.
	4	//!
	5	//! Retriving payloads like string/numeric values, identifier names, etc is
	6	//! done separately, either in Parse or Transform, when the respective data
	7	//! is actually needed. Tokenization does guarante that such data is valid.
	8	code: [:0]const u8,
	9	pos: ByteOffset = 0,
	10
	11	/// Tokenize source code into a list of tokens.
	12	/// When an error occurs, the position of the error is returned in `error_pos`.
	13	/// Caller owns returned slice.
	14	pub fn tokenize(
	15	source_code: [:0]const u8,
	16	gpa: mem.Allocator,
	17	error_pos: *ByteOffset,
	18	) TokenizeError!Token.List.Slice {
	19	const trace = T.trace(@src());
	20	defer trace.end();
	21
	22	var tokens = Token.List{};
	23	errdefer tokens.deinit(gpa);
	24
	25	// pre-allocate some space based on a very rough guess
	26	try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8));
	27
	28	var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 };
	29	errdefer error_pos.* = tokenizer.pos;
	30
	31	tokenizer.skipWhitespace();
	32	while (try tokenizer.next()) \|token\| {
	33	try tokens.append(gpa, token);
	34	tokenizer.skipWhitespace();
	35	}
	36
	37	try tokens.append(gpa, .{
	38	.tag = .eof,
	39	.offset = @intCast(source_code.len),
	40	});
	41
	42	return tokens.toOwnedSlice();
	43	}
	44
	45	pub const ByteOffset = u32;
	46
	47	pub const Token = struct {
	48	tag: Tag,
	49	offset: ByteOffset,
	50
	51	pub const List = std.MultiArrayList(Token);
	52
	53	pub const Tag = enum {
	54	id,
	55	builtin_id,
	56	number_int,
	57	number_float,
	58	string,
	59	multiline_string,
	60	doc_comment,
	61	module_doc_comment,
	62
	63	// sort-lines: start
	64	kw_assert,
	65	kw_break,
	66	kw_comptime,
	67	kw_defer,
	68	kw_else,
	69	kw_enum,
	70	kw_fn,
	71	kw_infer,
	72	kw_for,
	73	kw_while,
	74	kw_if,
	75	kw_impl,
	76	kw_json,
	77	kw_let,
	78	kw_match,
	79	kw_module,
	80	kw_pub,
	81	kw_pure,
	82	kw_return,
	83	kw_struct,
	84	kw_trait,
	85	kw_var,
	86	// sort-lines: end
	87
	88	// sort-lines: start
	89	@"and",
	90	and_and,
	91	and_and_equal,
	92	and_equal,
	93	bang,
	94	bang_equal,
	95	caret,
	96	caret_equal,
	97	colon,
	98	comma,
	99	dot,
	100	end_brace,
	101	end_bracket,
	102	end_paren,
	103	equal,
	104	equal_equal,
	105	equal_lt,
	106	gt,
	107	gt_equal,
	108	gt_gt,
	109	gt_gt_equal,
	110	gt_gt_gt,
	111	gt_gt_gt_equal,
	112	lt,
	113	lt_equal,
	114	lt_lt,
	115	lt_lt_equal,
	116	minus,
	117	minus_equal,
	118	minus_lt,
	119	percent,
	120	percent_equal,
	121	pipe,
	122	pipe_equal,
	123	pipe_pipe,
	124	pipe_pipe_equal,
	125	plus,
	126	plus_equal,
	127	semi,
	128	slash,
	129	slash_equal,
	130	star,
	131	star_equal,
	132	star_star,
	133	star_star_equal,
	134	start_brace,
	135	start_bracket,
	136	start_paren,
	137	tilde,
	138	tilde_equal,
	139	// sort-lines: end
	140
	141	/// Represents a tokenization error. Instead of reporting an error during
	142	/// tokenization, parsing will begin, and an error will be reported when
	143	/// this token is hit. That way, the error can tell you what it expected.
	144	invalid,
	145	/// End of file
	146	/// Used so that parser does not need to do out of bounds checks
	147	eof,
	148
	149	/// Most tokens have statically known lexeme, such as "struct" or "+="
	150	pub fn lexeme(tag: Tag) ?[]const u8 {
	151	return switch (tag) {
	152	.invalid => unreachable,
	153
	154	// do not have a static lexeme
	155	.eof,
	156	.id,
	157	.builtin_id,
	158	.number_int,
	159	.number_float,
	160	.string,
	161	.multiline_string,
	162	.doc_comment,
	163	.module_doc_comment,
	164	=> null,
	165
	166	.@"and" => "&",
	167	.and_and => "&&",
	168	.bang => "!",
	169	.bang_equal => "!=",
	170	.equal_lt => "=<",
	171	.caret => "^",
	172	.end_brace => "}",
	173	.end_bracket => "]",
	174	.end_paren => ")",
	175	.colon => ":",
	176	.comma => ",",
	177	.dot => ".",
	178	.equal => "=",
	179	.equal_equal => "==",
	180	.gt => ">",
	181	.gt_equal => ">=",
	182	.gt_gt => ">>",
	183	.gt_gt_gt => ">>>",
	184	.lt => "<",
	185	.lt_equal => "<=",
	186	.lt_lt => "<<",
	187	.minus => "-",
	188	.start_brace => "{",
	189	.start_bracket => "[",
	190	.start_paren => "(",
	191	.pipe => "\|",
	192	.pipe_pipe => "\|\|",
	193	.percent => "%",
	194	.plus => "+",
	195	.minus_lt => "->",
	196	.semi => ";",
	197	.slash => "/",
	198	.star => "*",
	199	.star_star => "**",
	200	.tilde => "~",
	201	.slash_equal => "/=",
	202	.percent_equal => "%=",
	203	.tilde_equal => "~=",
	204	.caret_equal => "^=",
	205	.and_equal => "&=",
	206	.pipe_equal => "\|=",
	207	.lt_lt_equal => "<<=",
	208	.gt_gt_equal => ">>=",
	209	.gt_gt_gt_equal => ">>>=",
	210	.plus_equal => "+=",
	211	.minus_equal => "-=",
	212	.star_equal => "*=",
	213	.star_star_equal => "**=",
	214	.pipe_pipe_equal => "\|\|=",
	215	.and_and_equal => "&&=",
	216
	217	inline else => \|keyword\| comptime len: {
	218	if (!mem.startsWith(u8, @tagName(keyword), "kw_")) {
	219	@compileError("Token tag is not a keyword: " ++ @tagName(keyword));
	220	}
	221	break :len @tagName(keyword)[3..];
	222	},
	223	};
	224	}
	225
	226	pub fn description(tag: Tag) []const u8 {
	227	return switch (tag) {
	228	.invalid => unreachable,
	229	.eof => "end of file",
	230	.id => "an identifier",
	231	.builtin_id => "a builtin identifier",
	232	.number_int => "a numeric literal",
	233	.number_float => "a numeric literal",
	234	.string => "a string literal",
	235	.multiline_string => "a multiline string literal",
	236	.doc_comment => "a documentation comment",
	237	.module_doc_comment => "a module documentation comment",
	238	inline else => \|t\| comptime "\"" ++ t.lexeme().? ++ "\"",
	239	};
	240	}
	241
	242	/// Most tokens have a static length, such as "struct" or "+="
	243	pub fn staticLen(tag: Tag) ?u32 {
	244	return if (tag.lexeme()) \|str\| @intCast(str.len) else null;
	245	}
	246	};
	247
	248	/// Instead of storing the length of a token, we can recompute this
	249	/// information on demand, as it is not often needed.
	250	///
	251	/// If you are computing the length of something that is knowable
	252	/// without the source code, use token.tag.staticLen() instead.
	253	pub fn len(token: Token, utf8_source: [:0]const u8) u32 {
	254	return token.tag.staticLen() orelse {
	255	var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
	256	assert(
	257	(tokenizer.next() catch
	258	unreachable // data corruption
	259	orelse unreachable // data corruption
	260	).tag == token.tag,
	261	);
	262	return tokenizer.pos - token.offset;
	263	};
	264	}
	265
	266	pub fn slice(token: Token, utf8_source: [:0]const u8) []const u8 {
	267	return token.tag.lexeme() orelse
	268	utf8_source[token.offset..][0..token.len(utf8_source)];
	269	}
	270
	271	const keywords = std.ComptimeStringMap(Tag, keywords: {
	272	const fields = std.enums.values(Tag);
	273	const KV = struct { []const u8, Tag };
	274	var kws: [fields.len]KV = undefined;
	275	var i = 0;
	276	for (fields) \|field\| {
	277	if (mem.startsWith(u8, @tagName(field), "kw_")) {
	278	kws[i] = .{ @tagName(field)[3..], field };
	279	i += 1;
	280	}
	281	}
	282	break :keywords kws[0..i];
	283	});
	284	};
	285
	286	/// When any of these error occurs, the tokenizer's position is set to
	287	/// the start of the offending token.
	288	pub const NextTokenError = error{
	289	/// Generic token syntax error
	290	InvalidSyntax,
	291	/// A trailing '_' was seen in a numeric literal.
	292	TrailingNumericSeparator,
	293	/// Newline or EOF encountered while parsing a string literal.
	294	UnterminatedString,
	295	/// A string escape sequence was invalid.
	296	InvalidEscapeSequence,
	297	/// The codepoint represented by a unicode escape sequence
	298	/// is not a valid unicode codepoint.
	299	InvalidEscapeCodepoint,
	300	/// There are unresolved git conflicts in the source code.
	301	GitConflict,
	302	/// The source code is not valid UTF-8.
	303	InvalidUtf8,
	304	};
	305
	306	pub const TokenizeError = mem.Allocator.Error \|\| NextTokenError;
	307
	308	/// Next assumes the position is already at the start of a token.
	309	/// You must call `skipWhitespace` before calling this.
	310	pub fn next(t: *Tokenizer) !?Token {
	311	while (true) {
	312	const trace = T.trace(@src());
	313	defer trace.end();
	314
	315	switch (t.byte()) {
	316	' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
	317
	318	'a'...'z', 'A'...'Z', '_', '$' => {
	319	const start = t.pos;
	320	t.pos += 1;
	321	return try t.tokenizeIdentifier(start);
	322	},
	323
	324	'@' => switch (t.nextByte()) {
	325	'"' => {
	326	const start = t.pos;
	327	t.pos += 1;
	328	try t.consumeString();
	329	return .{ .tag = .id, .offset = start };
	330	},
	331	else => {
	332	const start = t.pos;
	333	t.pos += 1;
	334	return try t.tokenizeIdentifier(start);
	335	},
	336	},
	337
	338	'0'...'9' => return try t.tokenizeNumber(),
	339
	340	'.' => switch (t.nextByte()) {
	341	'0'...'9' => return try t.tokenizeNumber(),
	342	else => return t.simpleToken(.dot),
	343	},
	344
	345	'/' => switch (t.nextByte()) {
	346	'=' => return t.simpleToken(.slash_equal),
	347	'/' => {
	348	if ((t.byteAt(2) orelse 0 == '/') and
	349	(t.byteAt(3) orelse 0 != '/'))
	350	{
	351	const start = t.pos;
	352	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len);
	353	return .{ .tag = .doc_comment, .offset = start };
	354	}
	355
	356	if (t.byteAt(2) orelse 0 == '!') {
	357	const start = t.pos;
	358	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
	359	return .{ .tag = .module_doc_comment, .offset = start };
	360	}
	361
	362	// skip the comment
	363	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
	364	t.skipWhitespace();
	365	continue;
	366	},
	367	else => return t.simpleToken(.slash),
	368	},
	369
	370	'\\' => switch (t.nextByte()) {
	371	'\\' => {
	372	const start = t.pos;
	373	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
	374	return .{ .tag = .multiline_string, .offset = start };
	375	},
	376	else => return t.maybeUTF8Error(error.InvalidSyntax),
	377	},
	378
	379	'"' => return try t.tokenizeString(),
	380
	381	'&' => switch (t.nextByte()) {
	382	'&' => return t.simpleToken(.and_and),
	383	'=' => return t.simpleToken(.and_equal),
	384	else => return t.simpleToken(.@"and"),
	385	},
	386	'\|' => switch (t.nextByte()) {
	387	'\|' => return t.simpleToken(.pipe_pipe),
	388	'=' => return t.simpleToken(.pipe_equal),
	389	else => return t.simpleToken(.pipe),
	390	},
	391	'^' => switch (t.nextByte()) {
	392	'^' => return t.simpleToken(.caret),
	393	'=' => return t.simpleToken(.caret_equal),
	394	else => return t.simpleToken(.caret),
	395	},
	396	'!' => switch (t.nextByte()) {
	397	'=' => return t.simpleToken(.equal_equal),
	398	else => return t.simpleToken(.bang_equal),
	399	},
	400	'=' => switch (t.nextByte()) {
	401	'=' => return t.simpleToken(.equal_equal),
	402	else => return t.simpleToken(.equal),
	403	},
	404	'>' => switch (t.nextByte()) {
	405	'=' => return t.simpleToken(.gt_equal),
	406	'>' => switch (t.byteAt(2) orelse 0) {
	407	'>' => return t.simpleToken(.gt_gt_gt),
	408	else => return t.simpleToken(.gt_gt),
	409	},
	410	else => return t.simpleToken(.gt),
	411	},
	412	'<' => switch (t.nextByte()) {
	413	'=' => return t.simpleToken(.lt_equal),
	414	'<' => switch (t.byteAt(2) orelse 0) {
	415	'=' => return t.simpleToken(.lt_lt_equal),
	416	else => {
	417	// Git merge conflicts are <<<<<<<
	418	const start = t.pos;
	419	t.pos += 2;
	420	var remain: u3 = 4;
	421	while (t.byte() == '<') {
	422	t.pos += 1;
	423	remain -= 1;
	424	if (remain == 0) {
	425	return error.GitConflict;
	426	}
	427	}
	428	t.pos = start;
	429	return t.simpleToken(.lt_lt);
	430	},
	431	},
	432	else => return t.simpleToken(.lt),
	433	},
	434	'+' => switch (t.nextByte()) {
	435	'=' => return t.simpleToken(.plus_equal),
	436	else => return t.simpleToken(.plus),
	437	},
	438	'-' => switch (t.nextByte()) {
	439	'=' => return t.simpleToken(.minus_equal),
	440	'>' => return t.simpleToken(.minus_lt),
	441	else => return t.simpleToken(.minus),
	442	},
	443	'*' => switch (t.nextByte()) {
	444	'*' => return t.simpleToken(.star_star),
	445	'=' => return t.simpleToken(.star_equal),
	446	else => return t.simpleToken(.star),
	447	},
	448	'%' => switch (t.nextByte()) {
	449	'=' => return t.simpleToken(.percent_equal),
	450	else => return t.simpleToken(.percent),
	451	},
	452	'~' => switch (t.nextByte()) {
	453	'=' => return t.simpleToken(.tilde_equal),
	454	else => return t.simpleToken(.tilde),
	455	},
	456
	457	';' => return t.simpleToken(.semi),
	458	':' => return t.simpleToken(.colon),
	459	'(' => return t.simpleToken(.start_paren),
	460	')' => return t.simpleToken(.end_paren),
	461	'[' => return t.simpleToken(.start_bracket),
	462	']' => return t.simpleToken(.end_bracket),
	463	'{' => return t.simpleToken(.start_brace),
	464	'}' => return t.simpleToken(.end_brace),
	465	',' => return t.simpleToken(.comma),
	466
	467	0x80...0xFF => {
	468	// need to decode utf-8
	469	const byte_len, const code_point = try t.codePoint();
	470
	471	if (unicode_props.isXidStart(code_point)) {
	472	const start = t.pos;
	473	t.pos += byte_len;
	474	return try t.tokenizeIdentifier(start);
	475	}
	476
	477	return error.InvalidSyntax;
	478	},
	479
	480	else => return error.InvalidSyntax,
	481
	482	0 => {
	483	if (t.pos == t.code.len) return null;
	484	return error.InvalidSyntax;
	485	},
	486	}
	487	}
	488	}
	489
	490	/// Source code is null terminated, so accessing this is safe
	491	fn byte(t: Tokenizer) u8 {
	492	return t.code[t.pos];
	493	}
	494
	495	/// Source code is null terminated.
	496	fn nextByte(t: Tokenizer) u8 {
	497	assert(t.byte() < 0x80);
	498	return t.code[t.pos + 1];
	499	}
	500
	501	/// Remember that using this only makes sense if you know for certain that all
	502	/// previous bytes are ASCII, otherwise you may hit UB
	503	fn byteAt(t: Tokenizer, comptime offset: u32) ?u8 {
	504	if (offset == 1) @compileError("use nextByte()");
	505
	506	// assert that the previous bytes are ASCII
	507	// current byte is allowed to be a continuation byte
	508	for (t.code[t.pos .. t.pos + offset]) \|b\| {
	509	assert(b < 0x80);
	510	}
	511	return t.code[t.pos + offset];
	512	}
	513
	514	fn byteAtRt(t: Tokenizer, offset: u32) ?u8 {
	515	// assert that the previous bytes are ASCII
	516	// current byte is allowed to be a continuation byte
	517	for (t.code[t.pos .. t.pos + offset]) \|b\| {
	518	assert(b < 0x80);
	519	}
	520	return t.code[t.pos + offset];
	521	}
	522
	523	fn codePoint(t: Tokenizer) !struct { u3, u21 } {
	524	const b = t.byte(); // must check the first byte manually
	525	assert(b >= 0x80); // must be a continuation byte
	526
	527	const byte_len = unicode.utf8ByteSequenceLength(b) catch
	528	return error.InvalidUtf8;
	529	assert(byte_len > 1);
	530
	531	const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch
	532	return error.InvalidUtf8;
	533
	534	return .{ byte_len, code_point };
	535	}
	536
	537	fn isAtIdentifierStart(t: *Tokenizer) !bool {
	538	switch (t.byte()) {
	539	'a'...'z', 'A'...'Z', '_', '$' => return true,
	540	else => return false,
	541	0x80...0xFF => {
	542	_, const code_point = try t.codePoint();
	543	return unicode_props.isXidStart(code_point);
	544	},
	545	}
	546	}
	547
	548	fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
	549	const trace = T.trace(@src());
	550	defer trace.end();
	551
	552	while (true) {
	553	switch (t.byte()) {
	554	'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
	555	t.pos += 1;
	556	},
	557	0x80...0xFF => {
	558	const byte_len, const code_point = try t.codePoint();
	559
	560	t.pos += byte_len;
	561
	562	if (!unicode_props.isXidContinue(code_point)) {
	563	break;
	564	}
	565	},
	566	else => break,
	567	}
	568	}
	569
	570	return .{
	571	.tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse
	572	if (t.code[start_byte] == '@')
	573	.builtin_id
	574	else
	575	.id,
	576	.offset = start_byte,
	577	};
	578	}
	579
	580	/// Numbers have the following rules:
	581	/// - May start with a `.`, but cannot end with `.`
	582	/// - May contain leading zeros, but transform pass will warn.
	583	/// - May contain a `_` separator, cannot have two in a row.
	584	/// - May contain `e` or `E` for exponents.
	585	/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers.
	586	/// The token is tagged as `number_int` or `number_float` depending
	587	/// on if it contained a `.` This is to help later re-parses.
	588	//
	589	// TODO: is it a zig compiler bug that `?` is required here?
	590	fn tokenizeNumber(t: *Tokenizer) !?Token {
	591	const trace = T.trace(@src());
	592	defer trace.end();
	593
	594	// Caller ensures the following invariant:
	595	switch (t.byte()) {
	596	'.' => switch (t.nextByte()) {
	597	'0'...'9' => {},
	598	else => unreachable,
	599	},
	600	'0'...'9' => {},
	601	else => unreachable,
	602	}
	603
	604	// Parse Hex, Binary, Octal numbers
	605	if (t.byte() == '0') {
	606	switch (t.nextByte()) {
	607	'x', 'X' => {
	608	t.pos += 2;
	609	try t.consumeIntegerRadix(.hex);
	610	},
	611	'b', 'B' => {
	612	t.pos += 2;
	613	try t.consumeIntegerRadix(.binary);
	614	},
	615	'o', 'O' => {
	616	t.pos += 2;
	617	try t.consumeIntegerRadix(.octal);
	618	},
	619	else => {},
	620	}
	621	}
	622
	623	const start = t.pos;
	624	var tag: Token.Tag = .number_int;
	625
	626	if (t.byte() == '.') {
	627	t.pos += 1;
	628	tag = .number_float;
	629	try t.consumeInteger();
	630	} else {
	631	try t.consumeInteger();
	632
	633	// decimal point notation
	634	switch (t.byte()) {
	635	'.' => switch (t.nextByte()) {
	636	'0'...'9' => {
	637	t.pos += 1;
	638	tag = .number_float;
	639	try t.consumeInteger();
	640	},
	641	else => {
	642	// we have not advanced the tokenizer past the `.`
	643	assert(tag == .number_int);
	644	return .{ .tag = tag, .offset = start };
	645	},
	646	},
	647	else => {},
	648	}
	649	}
	650
	651	switch (t.byte()) {
	652	'e', 'E' => {
	653	tag = .number_float;
	654	t.pos += 1;
	655
	656	switch (t.byte()) {
	657	'+', '-' => {
	658	t.pos += 1;
	659	switch (t.byte()) {
	660	'0'...'9' => {},
	661	else => return t.maybeUTF8Error(error.InvalidSyntax),
	662	}
	663	},
	664	'0'...'9' => {},
	665	else => return t.maybeUTF8Error(error.InvalidSyntax),
	666	}
	667
	668	try t.consumeInteger();
	669
	670	// decimal point notation
	671	switch (t.byte()) {
	672	'.' => switch (t.nextByte()) {
	673	'0'...'9' => {
	674	t.pos += 1;
	675	tag = .number_float;
	676	try t.consumeInteger();
	677	},
	678	else => {},
	679	},
	680	else => {},
	681	}
	682	},
	683	else => {},
	684	}
	685
	686	// after numbers must be a valid word break the above returns are right
	687	// before parsing `.` so those can be unchecked.
	688	if (try t.isAtIdentifierStart()) {
	689	return error.InvalidSyntax;
	690	}
	691
	692	return .{
	693	.tag = tag,
	694	.offset = start,
	695	};
	696	}
	697
	698	fn maybeConsumeInteger(t: *Tokenizer) !?void {
	699	const trace = T.trace(@src());
	700	defer trace.end();
	701
	702	switch (t.byte()) {
	703	'0'...'9' => {
	704	t.pos += 1;
	705	},
	706	else => return null,
	707	}
	708	t.consumeInteger();
	709	}
	710
	711	/// Requires the first byte to be a digit
	712	fn consumeInteger(t: *Tokenizer) !void {
	713	const trace = T.trace(@src());
	714	defer trace.end();
	715
	716	switch (t.byte()) {
	717	'0'...'9' => {
	718	t.pos += 1;
	719	},
	720	else => unreachable,
	721	}
	722
	723	while (true) {
	724	switch (t.byte()) {
	725	'0'...'9' => {
	726	t.pos += 1;
	727	},
	728	'_' => {
	729	t.pos += 1;
	730	switch (t.byte()) {
	731	'0'...'9' => {
	732	t.pos += 1;
	733	},
	734	else => return t.maybeUTF8Error(error.TrailingNumericSeparator),
	735	}
	736	},
	737	else => break,
	738	}
	739	}
	740	}
	741
	742	const Radix = enum { hex, binary, octal };
	743
	744	fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
	745	const trace = T.trace(@src());
	746	defer trace.end();
	747
	748	const digits = switch (radix) {
	749	.hex => "0123456789abcdefABCDEF",
	750	.binary => "01",
	751	.octal => "01234567",
	752	};
	753
	754	first: {
	755	const b = t.byte();
	756	inline for (digits) \|digit\| {
	757	if (b == digit) {
	758	break :first;
	759	}
	760	}
	761	return t.maybeUTF8Error(error.InvalidSyntax);
	762	}
	763
	764	t.pos += 1;
	765
	766	while (true) {
	767	brk: {
	768	const b = t.byte();
	769	inline for (digits) \|digit\| {
	770	if (b == digit) {
	771	break :brk;
	772	}
	773	}
	774	return;
	775	}
	776	t.pos += 1;
	777	}
	778	}
	779
	780	fn tokenizeString(t: *Tokenizer) !Token {
	781	assert(t.byte() == '"');
	782	const start = t.pos;
	783	try t.consumeString();
	784	return .{ .tag = .string, .offset = start };
	785	}
	786
	787	fn consumeString(t: *Tokenizer) !void {
	788	const trace = T.trace(@src());
	789	defer trace.end();
	790
	791	const start = t.pos;
	792	assert(t.byte() == '"');
	793
	794	t.pos += 1;
	795
	796	while (true) {
	797	switch (t.byte()) {
	798	'"' => {
	799	t.pos += 1;
	800	break;
	801	},
	802	'\\' => {
	803	switch (t.nextByte()) {
	804	'\n' => {
	805	t.pos = start;
	806	return error.UnterminatedString;
	807	},
	808	'\\' => t.pos += 2,
	809	'"' => t.pos += 2,
	810	'n' => t.pos += 2,
	811	'r' => t.pos += 2,
	812	't' => t.pos += 2,
	813	'0' => switch (t.byteAt(2) orelse {
	814	t.pos = start;
	815	return error.UnterminatedString;
	816	}) {
	817	// '\0xFF' hex escape
	818	'x' => {},
	819	// '\0' is special cased
	820	else => t.pos += 2,
	821	},
	822	'u' => {
	823	const esc_start = t.pos;
	824	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
	825	error.InvalidEscapeSequence => {
	826	if (t.byte() == '\n')
	827	return error.UnterminatedString;
	828	t.pos = esc_start;
	829	return err;
	830	},
	831	error.InvalidEscapeCodepoint => {
	832	t.pos = esc_start;
	833	return err;
	834	},
	835	// error.UnterminatedString => return err,
	836	};
	837	},
	838	else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
	839	}
	840	},
	841	'\n' => {
	842	t.pos = start;
	843	return error.UnterminatedString;
	844	},
	845	else => {
	846	t.pos += 1;
	847	},
	848	0x80...0xFF => {
	849	const byte_len, _ = try t.codePoint();
	850	t.pos += byte_len;
	851	},
	852	}
	853	}
	854	}
	855
	856	fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
	857	const trace = T.trace(@src());
	858	defer trace.end();
	859
	860	assert(t.byte() == '\\');
	861	assert(t.nextByte() == 'u');
	862	t.pos += 2;
	863
	864	if (t.byte() != '{')
	865	return error.InvalidEscapeSequence;
	866	t.pos += 1;
	867
	868	if (t.byte() == '0') {
	869	if (t.nextByte() == 'x') {
	870	t.pos += 2;
	871	}
	872	}
	873
	874	const start = t.pos;
	875	t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence;
	876
	877	const slice = t.code[start..t.pos];
	878
	879	if (t.byte() != '}')
	880	return error.InvalidEscapeSequence;
	881
	882	const code_point = std.fmt.parseInt(u21, slice, 16) catch \|err\| switch (err) {
	883	error.Overflow => return error.InvalidEscapeCodepoint,
	884	error.InvalidCharacter => unreachable,
	885	};
	886	_ = std.unicode.utf8CodepointSequenceLength(code_point) catch \|err\| switch (err) {
	887	error.CodepointTooLarge => return error.InvalidEscapeCodepoint,
	888	};
	889
	890	t.pos += 1;
	891	}
	892
	893	fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token {
	894	const start = t.pos;
	895	t.pos += comptime (tag.staticLen() orelse unreachable);
	896	return .{ .tag = tag, .offset = start };
	897	}
	898
	899	/// We are about to return an error, such as error.TrailingNumericSeparator.
	900	/// However, the character we ran into might actually be invalid UTF-8.
	901	/// This utility just checks for that to make sure we dont mask error.InvalidUtf8
	902	fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
	903	.ErrorSet = &[_]std.builtin.Type.Error{
	904	.{ .name = "InvalidUtf8" },
	905	.{ .name = @errorName(real_err) },
	906	},
	907	}) {
	908	switch (t.byte()) {
	909	0x80...0xFF => {
	910	_ = try t.codePoint();
	911	return real_err;
	912	},
	913	else => return real_err,
	914	}
	915	}
	916
	917	fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
	918	.ErrorSet = &[_]std.builtin.Type.Error{
	919	.{ .name = "InvalidUtf8" },
	920	.{ .name = @errorName(real_err) },
	921	},
	922	}) {
	923	switch (t.byteAtRt(at) orelse return real_err) {
	924	0x80...0xFF => {
	925	_ = try t.codePoint();
	926	return real_err;
	927	},
	928	else => return real_err,
	929	}
	930	}
	931
	932	pub fn skipWhitespace(t: *Tokenizer) void {
	933	const trace = T.trace(@src());
	934	defer trace.end();
	935
	936	while (true) {
	937	switch (t.byte()) {
	938	' ', '\t', '\n', '\r' => {
	939	t.pos += 1;
	940	},
	941	else => break,
	942	}
	943	}
	944	}
	945
	946	pub fn tokenLocation(
	947	utf8_source: []const u8,
	948	start_offset: ByteOffset,
	949	token_start: ByteOffset,
	950	) Location {
	951	var loc: Location = .{
	952	.line = .start,
	953	.column = .start,
	954	.line_start = start_offset,
	955	.line_end = @intCast(utf8_source.len),
	956	};
	957
	958	// Scan to by line until we go past the token start
	959	while (std.mem.indexOfScalarPos(u8, utf8_source, loc.line_start, '\n')) \|i\| {
	960	if (i >= token_start) {
	961	break; // Went past
	962	}
	963	loc.line = loc.line.add(1);
	964	loc.line_start = @intCast(i + 1);
	965	}
	966
	967	const offset = loc.line_start;
	968	for (utf8_source[offset..], 0..) \|c, i\| {
	969	if (i + offset == token_start) {
	970	loc.line_end = @intCast(i + offset);
	971	while (loc.line_end < utf8_source.len and utf8_source[loc.line_end] != '\n') {
	972	loc.line_end += 1;
	973	}
	974	return loc;
	975	}
	976	if (c == '\n') {
	977	loc.line = loc.line.add(1);
	978	loc.column = .start;
	979	loc.line_start = @intCast(i + 1);
	980	} else {
	981	loc.column = loc.column.add(1);
	982	}
	983	}
	984
	985	return loc;
	986	}
	987
	988	const Tokenizer = @This();
	989	const std = @import("std");
	990	const unicode = std.unicode;
	991	const assert = std.debug.assert;
	992	const mem = std.mem;
	993	const unicode_props = @import("unicode_props.zig");
	994	const T = @import("tracer");
	995	const Location = @import("Ast.zig").Location;

src/core/Transform.zig created+1479

...	...	@@ -0,0 +1,1479 @@
	1	//! Transform pass
	2	gpa: Allocator,
	3	ast: Ast,
	4	insts: Inst.List = .{},
	5	string_bytes: std.ArrayListUnmanaged(u8) = .{},
	6	extra: ExtraStore = .{},
	7
	8	scratch: std.ArrayListUnmanaged(u32) = .{},
	9
	10	// TODO: refactor this away
	11	// The new home for this memory should be pre-allocated on scratch,
	12	// but then remove the elements, moving scratch items on top of this
	13	// backwards a few.
	14	call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
	15	scopes: std.ArrayListUnmanaged(Scope) = .{},
	16
	17	report: Report.Builder,
	18
	19	pub const Error = error{
	20	OutOfMemory,
	21	TransformFail,
	22	};
	23
	24	pub fn transform(
	25	ast: Ast,
	26	gpa: Allocator,
	27	file: *Comp.File,
	28	fail_report: *Report,
	29	) Error!Teg {
	30	const trace = tracer.trace(@src());
	31	defer trace.end();
	32
	33	var t: Transform = .{
	34	.ast = ast,
	35	.gpa = gpa,
	36	.report = .{ .gpa = gpa, .current_file = file },
	37	};
	38	errdefer t.deinit();
	39
	40	try t.string_bytes.append(t.gpa, 0);
	41
	42	const module = t.transformModule(0) catch \|err\| switch (err) {
	43	error.OutOfMemory => return err,
	44	};
	45
	46	const report = try t.report.finalize();
	47	if (report.errors.len > 0) {
	48	fail_report.* = report;
	49	return error.TransformFail;
	50	}
	51
	52	assert(@intFromEnum(module) == 0);
	53
	54	try t.insts.setCapacity(t.gpa, t.insts.len);
	55
	56	defer t.deinit();
	57	return Teg{
	58	.insts = t.insts.toOwnedSlice(),
	59	.string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
	60	.extra = try t.extra.toOwnedStore(t.gpa),
	61	.report = report,
	62	};
	63	}
	64
	65	pub fn deinit(t: *Transform) void {
	66	t.scratch.deinit(t.gpa);
	67	t.call_args.deinit(t.gpa);
	68	assert(t.scopes.items.len == 0);
	69	t.scopes.deinit(t.gpa);
	70	t.string_bytes.deinit(t.gpa);
	71	t.insts.deinit(t.gpa);
	72	t.extra.deinit(t.gpa);
	73	t.* = undefined;
	74	}
	75
	76	const Scope = struct {
	77	symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
	78
	79	const Symbol = struct {
	80	node: NodeIndex,
	81	data: Symbol.Data,
	82	// TODO: optimize this structure
	83	used: bool = false,
	84	mutated: bool = false,
	85	discarded: ?NodeIndex = null,
	86
	87	const Data = union(enum) {
	88	declaration,
	89	param,
	90	mutable: Inst.Index,
	91	immutable: Inst.Ref,
	92	};
	93
	94	pub fn addWarnings(symbol: Symbol, t: *Transform) !void {
	95	if (!symbol.mutated and symbol.data == .mutable) {
	96	try t.report.warn(.not_mutated_var, .{
	97	.node = symbol.node,
	98	});
	99	}
	100	if (!symbol.used) {
	101	try t.report.warn(.unused, .{
	102	.node = symbol.node,
	103	.kind = switch (symbol.data) {
	104	.declaration => .function,
	105	.param => .param,
	106	.mutable => .variable,
	107	.immutable => switch (t.ast.nodeTag(symbol.node)) {
	108	.param_decl => .param,
	109	else => .constant,
	110	},
	111	},
	112	});
	113	} else if (symbol.discarded) \|discard\| {
	114	try t.report.warn(.unnecessary_discard, .{
	115	.node = discard,
	116	});
	117	}
	118	}
	119	};
	120
	121	fn addWarnings(s: Scope, t: *Transform) !void {
	122	for (s.symbols.values()) \|value\| {
	123	try value.addWarnings(t);
	124	}
	125	}
	126
	127	fn deinit(s: *Scope, gpa: Allocator) void {
	128	s.symbols.deinit(gpa);
	129	s.* = undefined;
	130	}
	131	};
	132
	133	const ScratchType = enum {
	134	ref,
	135	index,
	136	};
	137
	138	const BlockContext = struct {
	139	node_base: i32,
	140	return_target: ?*?Inst.Index = null,
	141	break_target: ?*BreakTarget = null,
	142
	143	/// This struct is always stored on the stack; the idea is to pass
	144	/// mutable pointers to it. When `break :brk` is encountered
	145	const BreakTarget = struct {
	146	label: ?[]const u8,
	147	/// This allows `continue` as well as implicit break targetting.
	148	is_loop: bool,
	149	/// If `null`, this label was unused.
	150	block_index: ?Inst.Index = null,
	151	parent: ?*BreakTarget = null,
	152
	153	fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget {
	154	return .{
	155	.parent = parent,
	156	.label = label,
	157	.is_loop = is_loop,
	158	};
	159	}
	160
	161	fn getInst(target: BreakTarget, t: Transform) !Inst.Index {
	162	return target.block_index orelse {
	163	const inst = try t.addRootInst(.{
	164	.tag = .block,
	165	.data = undefined,
	166	});
	167	target.block_index = inst;
	168	return inst;
	169	};
	170	}
	171	};
	172
	173	fn init(root_node: NodeIndex) BlockContext {
	174	return .{
	175	.node_base = @intCast(root_node),
	176	};
	177	}
	178
	179	fn nodeOffset(self: BlockContext, node: NodeIndex) i32 {
	180	return @as(i32, @intCast(node)) - self.node_base;
	181	}
	182
	183	fn declOffset(self: BlockContext, node: NodeIndex) BlockContext {
	184	return .{
	185	.node_base = self.nodeOffset(node),
	186	};
	187	}
	188
	189	fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext {
	190	return .{
	191	.node_base = self.node_base,
	192	.return_target = return_target,
	193	};
	194	}
	195
	196	fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext {
	197	return .{
	198	.node_base = self.node_base,
	199	.return_target = self.return_target,
	200	.break_target = break_target,
	201	};
	202	}
	203
	204	fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index {
	205	var break_target = self.break_target;
	206	while (break_target) \|target\| : (break_target = target.parent) {
	207	if (target.label) \|target_label\| {
	208	if (mem.eql(u8, label, target_label)) {
	209	return try target.getInst(t);
	210	}
	211	}
	212	}
	213	return null;
	214	}
	215	};
	216
	217	const Liveness = enum { alive, dead };
	218
	219	const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
	220	.{ "void", .void },
	221	.{ "true", .bool_true },
	222	.{ "false", .bool_false },
	223	.{ "int", .int32_type },
	224	.{ "float", .float_type },
	225	.{ "string", .string_type },
	226	// .{ "bigint", .bigint_type },
	227	.{ "bool", .bool_type },
	228	.{ "noreturn", .noreturn_type },
	229	.{ "type", .type_type },
	230	.{ "std", .standard_library },
	231	});
	232
	233	fn transformModule(t: *Transform, module_node: NodeIndex) Allocator.Error!Inst.Index {
	234	const trace = tracer.trace(@src());
	235	defer trace.end();
	236
	237	const module = try t.addRootInst(.{
	238	.tag = .module,
	239	.data = undefined,
	240	});
	241
	242	const decl = BlockContext.init(module_node);
	243
	244	try t.pushScope();
	245	defer t.popScope();
	246
	247	const statements = t.ast.nodeStatements(module_node);
	248	for (statements) \|statement\| {
	249	try t.previsitModuleStatement(statement);
	250	}
	251
	252	const s = t.scope();
	253
	254	const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1);
	255	defer t.gpa.free(temp);
	256
	257	{
	258	// prevent concurrent access
	259	s.symbols.lockPointers();
	260	defer s.symbols.unlockPointers();
	261	for (s.symbols.values(), 0..) \|value, i\| {
	262	temp[i] = try t.transformFunction(decl, value.node);
	263	}
	264	}
	265
	266	const top_level_fn = try t.addRootInst(.{
	267	.tag = .declaration,
	268	.data = undefined,
	269	});
	270
	271	temp[temp.len - 1] = top_level_fn;
	272
	273	const declarations: IndexList = .{
	274	.start = @intCast(t.extra.list.items.len),
	275	.len = @intCast(temp.len),
	276	};
	277	try t.extra.list.appendSlice(t.gpa, @ptrCast(temp));
	278
	279	const block = try t.addRootInst(.{
	280	.tag = .block,
	281	.data = undefined,
	282	});
	283
	284	{
	285	const scratch_top = t.scratch.items.len;
	286	defer t.scratch.shrinkRetainingCapacity(scratch_top);
	287
	288	for (statements) \|statement\| {
	289	try t.transformModuleStatement(decl, statement);
	290	}
	291
	292	t.getInstData(block).* = EncodedData.encode(.block, .{
	293	.src_node = decl.nodeOffset(module_node),
	294	.extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)),
	295	});
	296	}
	297
	298	const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{
	299	.kind = .function,
	300	.name = NullTerminatedString.empty,
	301	.hash = .{0} ** 16,
	302	.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn),
	303	}, dod.extraSizeOf(Inst.DeclFn));
	304
	305	trail.writeStruct(Inst.DeclFn, .{
	306	.params = .{ .start = 0, .len = 0 },
	307	.return_type = Inst.Ref.void,
	308	.body = block.toRef(),
	309	});
	310
	311	t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{
	312	.src_node = decl.nodeOffset(module_node),
	313	.extra_index = decl_extra,
	314	});
	315
	316	t.getInstData(module).* = EncodedData.encode(.module, .{
	317	.src_node = 0,
	318	.extra_index = try t.extra.append(t.gpa, IndexList, declarations),
	319	});
	320
	321	try t.scope().addWarnings(t);
	322
	323	return module;
	324	}
	325
	326	fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void {
	327	const trace = tracer.trace(@src());
	328	defer trace.end();
	329
	330	const tag = t.ast.nodes.items(.tag)[node];
	331	switch (tag) {
	332	.fn_decl => {
	333	const fn_decl = t.ast.fullFnDecl(node);
	334	const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
	335	if (gop.found_existing) {
	336	try t.report.err(.duplicate_function, .{
	337	.node = node,
	338	.prev = gop.value_ptr.node,
	339	});
	340	return;
	341	}
	342	gop.value_ptr.* = .{
	343	.node = node,
	344	.data = .declaration,
	345	};
	346	},
	347	else => {},
	348	}
	349	}
	350
	351	fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void {
	352	const trace = tracer.trace(@src());
	353	defer trace.end();
	354
	355	const tag = t.ast.nodes.items(.tag)[node];
	356	switch (tag) {
	357	.root,
	358	.param_decl,
	359	=> unreachable,
	360
	361	.fn_decl => {}, // functions analyzed separately
	362
	363	else => _ = try t.transformBlockStatement(decl, node, false),
	364	}
	365	}
	366
	367	fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) Allocator.Error!Inst.Index {
	368	const trace = tracer.trace(@src());
	369	defer trace.end();
	370
	371	const decl = outer_decl.declOffset(node);
	372	const fn_decl = t.ast.fullFnDecl(node);
	373
	374	const func = try t.addRootInst(.{
	375	.tag = .declaration,
	376	.data = undefined,
	377	});
	378
	379	try t.pushScope();
	380	defer t.popScope();
	381
	382	const param_list = brk: {
	383	const scratch_top = t.scratch.items.len;
	384	defer t.scratch.shrinkRetainingCapacity(scratch_top);
	385
	386	for (fn_decl.params) \|param\| {
	387	try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param));
	388	}
	389
	390	break :brk try t.scratchToRefList(scratch_top);
	391	};
	392
	393	const return_type = if (fn_decl.return_type) \|return_type_node\|
	394	try t.transformExpressionToBlock(decl, return_type_node, false)
	395	else
	396	Inst.Ref.void;
	397
	398	const body = try t.transformExpressionToBlock(decl, fn_decl.body, true);
	399
	400	const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{
	401	.kind = .function,
	402	.name = try t.addNullTerminatedString(fn_decl.id),
	403	.hash = .{0} ** 16,
	404	.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func),
	405	});
	406
	407	_ = try t.extra.append(t.gpa, Inst.DeclFn, .{
	408	.params = param_list,
	409	.return_type = return_type,
	410	.body = body,
	411	});
	412
	413	t.getInstData(func).* = EncodedData.encode(.declaration, .{
	414	.src_node = outer_decl.nodeOffset(node),
	415	.extra_index = decl_extra,
	416	});
	417
	418	try t.scope().addWarnings(t);
	419
	420	return func;
	421	}
	422
	423	const TransformFunctionCtx = struct {
	424	param_list: RefList,
	425	};
	426
	427	/// Converts expressions into blocks with a break. This is important
	428	/// for places where a block is not allowed, such as a parameter list.
	429	/// This is also used for the function body, even though it is a block.
	430	/// Simple refs turn into just the ref
	431	fn transformExpressionToBlock(
	432	t: *Transform,
	433	decl: BlockContext,
	434	node: NodeIndex,
	435	comptime is_function: bool,
	436	) Allocator.Error!Inst.Ref {
	437	const trace = tracer.trace(@src());
	438	defer trace.end();
	439
	440	const node_tag = t.ast.nodes.items(.tag)[node];
	441	switch (node_tag) {
	442	.id => {
	443	// Lower builtin ids without using a block
	444	const id = t.ast.basicNodeSlice(node);
	445	if (BuiltInIds.get(id)) \|ref\| {
	446	return ref;
	447	}
	448	},
	449
	450	else => {},
	451	}
	452
	453	var block: ?Inst.Index = null;
	454
	455	const scratch_top = t.scratch.items.len;
	456	defer t.scratch.shrinkRetainingCapacity(scratch_top);
	457
	458	if (is_function) {
	459	const s = t.scopes.getLast();
	460	for (s.symbols.values(), 0..) \|*param, i\| {
	461	assert(param.data == .param);
	462	const ref = try t.addInstRef(.{
	463	.tag = .param,
	464	.data = EncodedData.encode(.param, .{
	465	.src_node = decl.nodeOffset(node),
	466	.data = @intCast(i),
	467	}),
	468	});
	469	param.data = .{ .immutable = ref };
	470	}
	471	}
	472
	473	const expr = try t.transformExpression(
	474	if (is_function)
	475	decl.withReturn(&block)
	476	else
	477	decl,
	478	node,
	479	);
	480
	481	// if there are no new instructions, we dont have to create a block context
	482	if (scratch_top == t.scratch.items.len) {
	483	const result = expr orelse unreachable; // null means we added break or return
	484
	485	// TODO: evaluate if `if` blocks would be fine in sema with instruction refs
	486	// with this extra if check here, we filter out non-blocks.
	487	if (result.toIndex() == null) {
	488	return result;
	489	}
	490	}
	491
	492	if (expr) \|result\| {
	493	if (result.toIndex()) \|index\| {
	494	if (t.insts.items(.tag)[@intFromEnum(index)] == .block) {
	495	return result;
	496	}
	497	}
	498	}
	499
	500	const ret_block = if (block) \|b\|
	501	b
	502	else
	503	try t.addRootInst(.{
	504	.tag = .block,
	505	.data = undefined,
	506	});
	507
	508	if (expr) \|result\| {
	509	_ = try t.addInstRef(.{
	510	.tag = .@"break",
	511	.data = EncodedData.encode(.@"break", .{
	512	.src_node = decl.nodeOffset(node),
	513	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
	514	.block = ret_block,
	515	.operand = result,
	516	}),
	517	}),
	518	});
	519	}
	520
	521	const content = try t.scratchToIndexList(scratch_top);
	522
	523	t.getInstData(ret_block).* = EncodedData.encode(.block, .{
	524	.src_node = decl.nodeOffset(node),
	525	.extra_index = try t.extra.append(t.gpa, IndexList, content),
	526	});
	527
	528	return ret_block.toRef();
	529	}
	530
	531	fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Allocator.Error!Inst.Ref {
	532	const trace = tracer.trace(@src());
	533	defer trace.end();
	534
	535	const param_decl = t.ast.fullParamDecl(param);
	536	const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
	537
	538	// add `sym` into the scope so that the next param's type
	539	// can use this param's value, if known at compile time.
	540	const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
	541	if (sym.found_existing) {
	542	try t.report.err(.duplicate_param, .{
	543	.node = param,
	544	.prev = sym.value_ptr.node,
	545	});
	546	return .void;
	547	}
	548
	549	sym.value_ptr.* = .{
	550	.node = param,
	551	.data = .param,
	552	};
	553
	554	return value;
	555	}
	556
	557	const Builtin = enum {
	558	js,
	559	import,
	560	fetch,
	561	compile_error,
	562
	563	const map = std.ComptimeStringMap(Builtin, .{
	564	.{ "js", .js },
	565	.{ "import", .import },
	566	.{ "fetch", .fetch },
	567	.{ "compileError", .compile_error },
	568	});
	569
	570	fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref {
	571	const trace = tracer.trace(@src());
	572	defer trace.end();
	573
	574	if (args.len != 1) {
	575	// try t.report.err(.invalid_builtin_args, .{ .id = "js" });
	576	return .void;
	577	}
	578
	579	const source = try t.transformExpression(decl, args[0]) orelse return null;
	580
	581	return try t.addInstRef(.{
	582	.tag = .js,
	583	.data = EncodedData.encode(.js, .{
	584	.src_node = decl.nodeOffset(args[0]),
	585	.operand = source,
	586	}),
	587	});
	588	}
	589	};
	590
	591	/// This is the primary transformation function: converting an AST expression into instructions.
	592	/// `null` represents a `return` or `break`, aka this branch of code is dead.
	593	fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Ref {
	594	const node_tag = t.ast.nodes.items(.tag)[node];
	595
	596	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
	597	defer trace.end();
	598
	599	switch (node_tag) {
	600	.root,
	601	.param_decl,
	602	.var_decl,
	603	.fn_decl,
	604	=> unreachable, // not an expression
	605
	606	.id => {
	607	const id = t.ast.basicNodeSlice(node);
	608	if (BuiltInIds.get(id)) \|ref\| {
	609	return ref;
	610	}
	611
	612	if (mem.startsWith(u8, id, "@")) {
	613	@panic("TODO: string identifiers");
	614	}
	615
	616	const lookup = t.lookupId(id) orelse {
	617	try t.report.err(.unknown_identifier, .{ .node = node });
	618	return .void;
	619	};
	620
	621	lookup.used = true;
	622
	623	return switch (lookup.data) {
	624	.declaration => try t.addInstRef(.{
	625	.tag = .load_decl,
	626	.data = EncodedData.encode(.load_decl, .{
	627	.src_node = decl.nodeOffset(node),
	628	.string = try t.addNullTerminatedString(id),
	629	}),
	630	}),
	631	.mutable => \|idx\| idx.toRef(),
	632	.immutable => \|ref\| ref,
	633	.param => unreachable, //unresolved param
	634	};
	635	},
	636
	637	inline .number_int, .number_float => \|tag\| {
	638	const text = t.ast.basicNodeSlice(node);
	639	const value = switch (tag) {
	640	.number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
	641	.number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
	642	else => comptime unreachable,
	643	};
	644	return try t.addInstRef(.{
	645	.tag = .number,
	646	.data = EncodedData.encode(.number, .{
	647	.src_node = decl.nodeOffset(node),
	648	.extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)),
	649	}),
	650	});
	651	},
	652
	653	.string => {
	654	const str = t.ast.basicNodeSlice(node);
	655	const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
	656	return try t.addInstRef(.{
	657	.tag = .string,
	658	.data = EncodedData.encode(.string, .{
	659	.src_node = decl.nodeOffset(node),
	660	.extra_index = try t.extra.append(t.gpa, SlicedString, sliced),
	661	}),
	662	});
	663	},
	664
	665	.assign => {
	666	const ast_bin = t.ast.nodeData(node).bin;
	667	const lhs = try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse
	668	return .void;
	669	const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse
	670	return null;
	671	const assign_inst = try t.addInstRef(.{
	672	.tag = .store,
	673	.data = EncodedData.encode(.store, .{
	674	.src_node = decl.nodeOffset(node),
	675	.extra_index = try t.extra.append(t.gpa, Inst.Store, .{
	676	.memory = lhs,
	677	.operand = rhs,
	678	}),
	679	}),
	680	});
	681	return assign_inst;
	682	},
	683
	684	inline .add,
	685	.sub,
	686	.mul,
	687	.div,
	688	.rem,
	689	.pow,
	690	.@"and",
	691	.@"or",
	692	.bit_and,
	693	.bit_or,
	694	.bit_xor,
	695	.shl,
	696	.shr,
	697	.eq,
	698	.ne,
	699	.lt,
	700	.le,
	701	.gt,
	702	.ge,
	703	.add_assign,
	704	.sub_assign,
	705	.mul_assign,
	706	.div_assign,
	707	.rem_assign,
	708	.pow_assign,
	709	.and_assign,
	710	.or_assign,
	711	.bit_and_assign,
	712	.bit_or_assign,
	713	.bit_xor_assign,
	714	.shl_assign,
	715	.shr_assign,
	716	=> \|tag\| {
	717	// This code path implements binary operations, as well as the
	718	// compound compute + assign operation.
	719	const is_assign = comptime tag.isAssignBin();
	720	const teg_tag = @field(
	721	Teg.Inst.Tag,
	722	if (is_assign)
	723	@tagName(tag)[0 .. @tagName(tag).len - "_assign".len]
	724	else
	725	@tagName(tag),
	726	);
	727	const ast_bin = t.ast.nodeData(node).bin;
	728	const lhs = if (is_assign)
	729	try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse
	730	return .void
	731	else
	732	try t.transformExpression(decl, ast_bin.lhs) orelse
	733	return null;
	734	const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
	735
	736	const compute_inst = try t.addInstRef(.{
	737	.tag = teg_tag,
	738	.data = EncodedData.encode(teg_tag, .{
	739	.src_node = decl.nodeOffset(node),
	740	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
	741	.lhs = if (is_assign) lhs.toRef() else lhs,
	742	.rhs = rhs,
	743	}),
	744	}),
	745	});
	746
	747	if (is_assign) {
	748	const assign_inst = try t.addInstRef(.{
	749	.tag = .store,
	750	.data = EncodedData.encode(.store, .{
	751	.src_node = decl.nodeOffset(node),
	752	.extra_index = try t.extra.append(t.gpa, Inst.Store, .{
	753	.memory = lhs,
	754	.operand = compute_inst,
	755	}),
	756	}),
	757	});
	758	return assign_inst;
	759	}
	760
	761	return compute_inst;
	762	},
	763
	764	inline .negate,
	765	.bool_not,
	766	.bit_not,
	767	=> \|tag\| {
	768	const ast_un = t.ast.nodeData(node).un;
	769	const teg_tag = @field(Teg.Inst.Tag, @tagName(tag));
	770
	771	const operand = try t.transformExpression(decl, ast_un.expr) orelse
	772	return null;
	773
	774	return try t.addInstRef(.{
	775	.tag = teg_tag,
	776	.data = EncodedData.encode(teg_tag, .{
	777	.src_node = decl.nodeOffset(node),
	778	.operand = operand,
	779	}),
	780	});
	781	},
	782
	783	.label => {
	784	const expr = t.ast.nodeData(node).label.expr;
	785	const label = t.ast.tokenSlice(t.ast.nodeToken(node));
	786
	787	var target: BlockContext.BreakTarget = .{
	788	.parent = decl.break_target,
	789	.label = label,
	790	.is_loop = false,
	791	};
	792	const inner_ctx = decl.withBreak(&target);
	793
	794	const scratch_top = t.scratch.items.len;
	795
	796	const inner_expr = try t.transformExpression(inner_ctx, expr);
	797
	798	// if there are no new instructions, we dont have to create a block context
	799	if (scratch_top == t.scratch.items.len)
	800	return inner_expr orelse unreachable; // null means we added break or return
	801
	802	if (target.block_index) \|block\| {
	803	if (inner_expr) \|result\| {
	804	_ = try t.addInstRef(.{
	805	.tag = .@"break",
	806	.data = EncodedData.encode(.@"break", .{
	807	.src_node = decl.nodeOffset(node),
	808	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
	809	.block = block,
	810	.operand = result,
	811	}),
	812	}),
	813	});
	814	}
	815
	816	const content = try t.scratchToIndexList(scratch_top);
	817	t.scratch.shrinkRetainingCapacity(scratch_top);
	818
	819	t.getInstData(block).* = EncodedData.encode(.block, .{
	820	.src_node = decl.nodeOffset(node),
	821	.extra_index = try t.extra.append(t.gpa, IndexList, content),
	822	});
	823
	824	try t.addScratch(Inst.Index, block);
	825
	826	return block.toRef();
	827	} else {
	828	std.debug.print("Warn, unused label {s}", .{label});
	829	}
	830
	831	return inner_expr;
	832	},
	833
	834	inline .block_void,
	835	.block_implicit_return,
	836	=> \|tag\| {
	837	const statements = t.ast.nodeStatements(node);
	838
	839	var last: Inst.Ref = .void;
	840	for (statements, 1..) \|statement, i\| {
	841	const is_last = i == statements.len;
	842	last = try t.transformBlockStatement(
	843	decl,
	844	statement,
	845	tag == .block_implicit_return and is_last,
	846	) orelse {
	847	return null;
	848	};
	849	}
	850
	851	return if (tag == .block_implicit_return) last else .void;
	852	},
	853
	854	.builtin_call,
	855	=> {
	856	const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
	857	try t.report.err(.invalid_builtin, .{ .node = node });
	858	return .void;
	859	};
	860
	861	const base = t.ast.nodeData(node).list;
	862	const args = base.slice(&t.ast);
	863
	864	return switch (builtin) {
	865	.js => Builtin.builtin_js(t, decl, args),
	866	// .import => Builtin.import(t, decl, args),
	867	// .fetch => Builtin.fetch(t, decl, args),
	868	// .compile_error => Builtin.compile_error(t, decl, args),
	869	else => @panic("TODO"),
	870	};
	871	},
	872
	873	.call => {
	874	const call = t.ast.nodeData(node).call;
	875	const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast);
	876
	877	const base = try t.transformExpression(decl, call.base) orelse return null;
	878
	879	const args_list: RefList = brk: {
	880	const args_top = t.call_args.items.len;
	881	defer t.call_args.shrinkRetainingCapacity(args_top);
	882
	883	for (args) \|arg\| {
	884	try t.call_args.append(
	885	t.gpa,
	886	try t.transformExpression(decl, arg) orelse
	887	return null,
	888	);
	889	}
	890
	891	const items = t.call_args.items[args_top..];
	892	const extra_top = t.extra.list.items.len;
	893	try t.extra.list.appendSlice(t.gpa, @ptrCast(items));
	894
	895	break :brk .{
	896	.start = @intCast(extra_top),
	897	.len = @intCast(items.len),
	898	};
	899	};
	900
	901	const call_inst = try t.addInstRef(.{
	902	.tag = .call,
	903	.data = EncodedData.encode(.call, .{
	904	.src_node = decl.nodeOffset(node),
	905	.extra_index = try t.extra.append(t.gpa, Inst.Call, .{
	906	.base = base,
	907	.args = args_list,
	908	}),
	909	}),
	910	});
	911
	912	return call_inst;
	913	},
	914
	915	.field => {
	916	const field = t.ast.nodeData(node).field;
	917
	918	const base = try t.transformExpression(decl, field.base) orelse return null;
	919
	920	// TODO: handle identifier escapes
	921	const id = t.ast.tokenSlice(field.field_token);
	922	const prop_name_str = try t.addSlicedString(id);
	923
	924	const string_ref = try t.addInstRef(.{
	925	.tag = .string,
	926	.data = EncodedData.encode(.string, .{
	927	.src_node = decl.nodeOffset(node),
	928	.extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
	929	}),
	930	});
	931
	932	return try t.addInstRef(.{
	933	.tag = .field,
	934	.data = EncodedData.encode(.field, .{
	935	.src_node = decl.nodeOffset(node),
	936	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
	937	.lhs = base,
	938	.rhs = string_ref,
	939	}),
	940	}),
	941	});
	942	},
	943
	944	.@"if" => {
	945	const if_node = t.ast.nodeData(node).@"if";
	946	const if_data = t.ast.extraData(if_node.extra, Ast.If);
	947
	948	const cond = try t.transformExpression(decl, if_node.cond) orelse
	949	return null;
	950
	951	const yes = try t.transformExpressionToBlock(
	952	decl,
	953	if_data.yes,
	954	false,
	955	);
	956	const no = if (if_data.no.unwrap()) \|no\|
	957	try t.transformExpressionToBlock(
	958	decl,
	959	no,
	960	false,
	961	)
	962	else
	963	Inst.Ref.void;
	964
	965	if (yes == .void and no == .void) {
	966	// when both branches do nothing, we can omit the branch,
	967	// however we still need to type-check the condition operand
	968	_ = try t.addInstRef(.{
	969	.tag = .ensure_bool,
	970	.data = EncodedData.encode(.ensure_bool, .{
	971	.src_node = decl.nodeOffset(node),
	972	.operand = cond,
	973	}),
	974	});
	975	return .void;
	976	}
	977
	978	return try t.addInstRef(.{
	979	.tag = .@"if",
	980	.data = EncodedData.encode(.@"if", .{
	981	.src_node = decl.nodeOffset(node),
	982	.extra_index = try t.extra.append(t.gpa, Inst.If, .{
	983	.cond = cond,
	984	.yes = yes,
	985	.no = no,
	986	}),
	987	}),
	988	});
	989	},
	990
	991	.@"return" => {
	992	const ret_node = t.ast.nodeData(node).@"return";
	993
	994	const ret_block_ptr = decl.return_target orelse {
	995	try t.report.err(.invalid_return, .{ .node = node });
	996	return .void;
	997	};
	998
	999	const ret_block = if (ret_block_ptr.*) \|block\|
	1000	block
	1001	else append_block: {
	1002	const block = try t.addRootInst(.{
	1003	.tag = .block,
	1004	.data = undefined,
	1005	});
	1006	ret_block_ptr.* = block;
	1007	break :append_block block;
	1008	};
	1009
	1010	const operand: Inst.Ref = if (ret_node.expr.unwrap()) \|expr\|
	1011	try t.transformExpression(decl.withReturn(null), expr) orelse
	1012	return null
	1013	else
	1014	.void;
	1015
	1016	_ = try t.addInstRef(.{
	1017	.tag = .@"break",
	1018	.data = EncodedData.encode(.@"break", .{
	1019	.src_node = decl.nodeOffset(node),
	1020	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
	1021	.block = ret_block,
	1022	.operand = operand,
	1023	}),
	1024	}),
	1025	});
	1026
	1027	return null; // control flow ends
	1028	},
	1029
	1030	.@"break" => {
	1031	const brk_node = t.ast.nodeData(node).@"break";
	1032	const label = t.ast.tokenSlice(
	1033	brk_node.label.unwrap() orelse
	1034	@panic("TODO: unlabelled breaks"),
	1035	);
	1036
	1037	const block = try decl.findAndMarkBreak(t, label) orelse {
	1038	try t.report.err(.unresolved_label, .{ .node = node });
	1039	return null;
	1040	};
	1041
	1042	const operand: Inst.Ref = if (brk_node.value.unwrap()) \|expr\|
	1043	try t.transformExpression(decl.withReturn(null), expr) orelse
	1044	return null
	1045	else
	1046	.void;
	1047
	1048	_ = try t.addInstRef(.{
	1049	.tag = .@"break",
	1050	.data = EncodedData.encode(.@"break", .{
	1051	.src_node = decl.nodeOffset(node),
	1052	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
	1053	.block = block,
	1054	.operand = operand,
	1055	}),
	1056	}),
	1057	});
	1058
	1059	return null; // control flow ends
	1060	},
	1061
	1062	else => \|tag\| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}),
	1063	}
	1064	}
	1065
	1066	fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Index {
	1067	const node_tag = t.ast.nodes.items(.tag)[node];
	1068
	1069	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
	1070	defer trace.end();
	1071
	1072	switch (node_tag) {
	1073	.root,
	1074	.param_decl,
	1075	.var_decl,
	1076	.fn_decl,
	1077	=> unreachable, // not an expression
	1078
	1079	.id => {
	1080	const id = t.ast.basicNodeSlice(node);
	1081	assert(!mem.eql(u8, id, "_")); // discards are not assignment targets
	1082
	1083	if (BuiltInIds.has(id)) {
	1084	try t.report.err(.assign_builtin, .{ .node = node });
	1085	return null;
	1086	}
	1087
	1088	if (mem.startsWith(u8, id, "@")) {
	1089	@panic("TODO: string identifiers");
	1090	}
	1091
	1092	const lookup = t.lookupId(id) orelse {
	1093	try t.report.err(.unknown_identifier, .{ .node = node });
	1094	return null;
	1095	};
	1096
	1097	return switch (lookup.data) {
	1098	.mutable => \|inst\| {
	1099	lookup.mutated = true;
	1100	return inst;
	1101	},
	1102	else => {
	1103	lookup.used = true;
	1104
	1105	try t.report.err(.invalid_assignment_symbol, .{
	1106	.node = node,
	1107	.decl = lookup.node,
	1108	.kind = switch (lookup.data) {
	1109	.mutable => unreachable,
	1110	.declaration => .function,
	1111	.param => .param,
	1112	.immutable => switch (t.ast.nodeTag(lookup.node)) {
	1113	.param_decl => .param,
	1114	else => .constant,
	1115	},
	1116	},
	1117	});
	1118	return null;
	1119	},
	1120	};
	1121	},
	1122
	1123	// TODO: allow field expressions, but these need to emit a different store node
	1124
	1125	else => {
	1126	_ = try t.transformExpression(decl, node);
	1127	try t.report.err(.invalid_assignment_target, .{
	1128	.node = node,
	1129	});
	1130	return null;
	1131	},
	1132	}
	1133	}
	1134
	1135	fn transformBlockStatement(
	1136	t: *Transform,
	1137	decl: BlockContext,
	1138	node: NodeIndex,
	1139	is_implicit_return_expression: bool,
	1140	) !?Inst.Ref {
	1141	const node_tag = t.ast.nodes.items(.tag)[node];
	1142
	1143	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
	1144	defer trace.end();
	1145
	1146	switch (node_tag) {
	1147	.root => unreachable, // cannot appear in a block
	1148	.param_decl => unreachable, // cannot appear in a block
	1149
	1150	.var_decl => {
	1151	const var_decl = t.ast.fullVarDecl(node);
	1152
	1153	if (mem.eql(u8, var_decl.id, "_")) {
	1154	try t.report.warn(.invalid_discard, .{ .node = node });
	1155
	1156	const ref = switch (t.ast.nodeTag(var_decl.init)) {
	1157	.id => {
	1158	const id = t.ast.basicNodeSlice(node);
	1159	if (BuiltInIds.has(id)) {
	1160	try t.report.warn(.discard_builtin, .{ .node = node });
	1161	return .void;
	1162	}
	1163
	1164	if (mem.startsWith(u8, id, "@")) {
	1165	@panic("TODO: string identifiers");
	1166	}
	1167
	1168	const lookup = t.lookupId(id) orelse {
	1169	try t.report.err(.unknown_identifier, .{ .node = node });
	1170	return .void;
	1171	};
	1172
	1173	lookup.discarded = node;
	1174
	1175	return .void;
	1176	},
	1177	else => try t.transformExpression(decl, var_decl.init) orelse
	1178	return null,
	1179	};
	1180
	1181	return try t.addInstRef(.{
	1182	.tag = .discard,
	1183	.data = EncodedData.encode(.discard, .{
	1184	.src_node = decl.nodeOffset(node),
	1185	.operand = ref,
	1186	}),
	1187	});
	1188	}
	1189
	1190	if (BuiltInIds.has(var_decl.id)) {
	1191	try t.report.err(.shadow_builtin_variable, .{ .node = node });
	1192	return .void;
	1193	}
	1194
	1195	const type_ref = if (var_decl.type) \|data_type\|
	1196	try t.transformExpression(decl, data_type)
	1197	else
	1198	null;
	1199	const data_ref = try t.transformExpression(decl, var_decl.init) orelse
	1200	return null;
	1201	const data_coerced_ref = if (type_ref) \|ref\|
	1202	try t.addInstRef(.{
	1203	.tag = .as,
	1204	.data = EncodedData.encode(.as, .{
	1205	.src_node = decl.nodeOffset(node),
	1206	.extra_index = try t.extra.append(t.gpa, Inst.As, .{
	1207	.type = ref,
	1208	.operand = data_ref,
	1209	}),
	1210	}),
	1211	})
	1212	else
	1213	data_ref;
	1214
	1215	const mem_ref = if (var_decl.is_const)
	1216	data_coerced_ref
	1217	else
	1218	try t.addInstRef(.{
	1219	.tag = .memory,
	1220	.data = EncodedData.encode(.memory, .{
	1221	.src_node = decl.nodeOffset(node),
	1222	.operand = data_coerced_ref,
	1223	}),
	1224	});
	1225
	1226	const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id);
	1227
	1228	// if re-declaring, apply unused warnings now
	1229	if (gop.found_existing) {
	1230	try gop.value_ptr.addWarnings(t);
	1231	}
	1232
	1233	gop.value_ptr.* = .{
	1234	.node = node,
	1235	.data = switch (var_decl.is_const) {
	1236	true => .{ .immutable = mem_ref },
	1237	false => .{ .mutable = mem_ref.toIndex().? },
	1238	},
	1239	};
	1240
	1241	return .void;
	1242	},
	1243
	1244	.fn_decl,
	1245	=> \|tag\| {
	1246	std.debug.panic("TODO: block statement {s}", .{@tagName(tag)});
	1247	},
	1248
	1249	.assign,
	1250	=> {
	1251	const bin = t.ast.nodeData(node).bin;
	1252	const lhs = bin.lhs;
	1253
	1254	if (t.ast.nodeTag(lhs) == .id and
	1255	mem.eql(u8, t.ast.basicNodeSlice(lhs), "_"))
	1256	{
	1257	// discard, instead of appending an .ensure_void check,
	1258	// we will add a .discard instruction instead.
	1259	// this will emit a warning if given void.
	1260	const result = try t.transformExpression(decl, bin.rhs) orelse
	1261	return null;
	1262	return try t.addInstRef(.{
	1263	.tag = .discard,
	1264	.data = EncodedData.encode(.discard, .{
	1265	.src_node = decl.nodeOffset(node),
	1266	.operand = result,
	1267	}),
	1268	});
	1269	} else {
	1270	// process this assignment as usual, do not append .ensure_void
	1271	return try t.transformExpression(decl, node);
	1272	}
	1273	},
	1274
	1275	.add_assign,
	1276	.sub_assign,
	1277	.mul_assign,
	1278	.div_assign,
	1279	.rem_assign,
	1280	.pow_assign,
	1281	.and_assign,
	1282	.or_assign,
	1283	.bit_and_assign,
	1284	.bit_or_assign,
	1285	.bit_xor_assign,
	1286	.shl_assign,
	1287	.shr_assign,
	1288	=> {
	1289	// do not append a .ensure_void check
	1290	return try t.transformExpression(decl, node);
	1291	},
	1292
	1293	else => {
	1294	const result = try t.transformExpression(decl, node) orelse
	1295	return null;
	1296
	1297	if (!is_implicit_return_expression and result != .void) {
	1298	if (t.ast.nodeTag(node) == .id) {
	1299	try t.report.warn(.dangling_identifier, .{
	1300	.node = node,
	1301	});
	1302	return result;
	1303	}
	1304
	1305	return try t.addInstRef(.{
	1306	.tag = .ensure_void,
	1307	.data = EncodedData.encode(.ensure_void, .{
	1308	.src_node = decl.nodeOffset(node),
	1309	.operand = result,
	1310	}),
	1311	});
	1312	}
	1313
	1314	return result;
	1315	},
	1316	}
	1317	}
	1318
	1319	fn addRootInst(t: *Transform, inst: Inst) !Inst.Index {
	1320	try t.insts.append(t.gpa, inst);
	1321	return @enumFromInt(t.insts.len - 1);
	1322	}
	1323
	1324	fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref {
	1325	try t.scratch.ensureUnusedCapacity(t.gpa, 1);
	1326	try t.insts.append(t.gpa, inst);
	1327	const index: Inst.Index = @enumFromInt(t.insts.len - 1);
	1328	t.scratch.appendAssumeCapacity(@intFromEnum(index));
	1329	return index.toRef();
	1330	}
	1331
	1332	fn pushScope(t: *Transform) !void {
	1333	try t.scopes.append(t.gpa, .{});
	1334	}
	1335
	1336	fn popScope(t: *Transform) void {
	1337	t.scope().deinit(t.gpa);
	1338	t.scopes.items[t.scopes.items.len - 1] = undefined;
	1339	t.scopes.items.len -= 1;
	1340	}
	1341
	1342	fn scope(t: Transform) Scope {
	1343	return &t.scopes.items[t.scopes.items.len - 1];
	1344	}
	1345
	1346	fn getInstData(t: Transform, index: Inst.Index) EncodedData {
	1347	return &t.insts.items(.data)[@intFromEnum(index)];
	1348	}
	1349
	1350	fn addScratch(t: *Transform, comptime T: type, value: T) !void {
	1351	try t.scratch.append(t.gpa, @intFromEnum(value));
	1352	}
	1353
	1354	fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
	1355	const items = t.scratch.items[scratch_top..];
	1356	const extra_top = t.extra.list.items.len;
	1357	try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]);
	1358	return .{
	1359	.start = @intCast(extra_top),
	1360	.len = @intCast(items.len),
	1361	};
	1362	}
	1363
	1364	fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList {
	1365	const items = t.scratch.items[scratch_top..];
	1366	const extra_top = t.extra.list.items.len;
	1367	try t.extra.list.appendSlice(t.gpa, items);
	1368	return .{
	1369	.start = @intCast(extra_top),
	1370	.len = @intCast(items.len),
	1371	};
	1372	}
	1373
	1374	fn lookupId(t: Transform, id: []const u8) ?Scope.Symbol {
	1375	var i = t.scopes.items.len;
	1376	while (i > 0) {
	1377	i -= 1;
	1378	if (t.scopes.items[i].symbols.getPtr(id)) \|sym\| {
	1379	return sym;
	1380	}
	1381	}
	1382	return null;
	1383	}
	1384
	1385	fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString {
	1386	if (str.len == 0) {
	1387	return .empty;
	1388	}
	1389	// loop through strings trying to find an existing match, and dedupe memory
	1390	var pos: usize = 0;
	1391	while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
	1392	const start_index = i - str.len;
	1393	if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
	1394	const string: NullTerminatedString = @enumFromInt(pos);
	1395	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
	1396	return string;
	1397	}
	1398	}
	1399
	1400	const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len);
	1401	try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1);
	1402	t.string_bytes.appendSliceAssumeCapacity(str);
	1403	t.string_bytes.appendAssumeCapacity(0);
	1404	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
	1405	return string;
	1406	}
	1407
	1408	fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
	1409	if (str.len == 0) {
	1410	return .{ .start = 0, .len = 0 };
	1411	}
	1412
	1413	// loop through strings trying to find an existing match, and dedupe memory
	1414	var pos: usize = 0;
	1415	while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
	1416	const start_index = i - str.len;
	1417	if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
	1418	const string: SlicedString = .{
	1419	.start = @intCast(start_index),
	1420	.len = @intCast(str.len),
	1421	};
	1422	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
	1423	return string;
	1424	}
	1425	}
	1426
	1427	const string: SlicedString = .{
	1428	.start = @intCast(t.string_bytes.items.len),
	1429	.len = @intCast(str.len),
	1430	};
	1431	try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len);
	1432	t.string_bytes.appendSliceAssumeCapacity(str);
	1433	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
	1434	return string;
	1435	}
	1436
	1437	const Transform = @This();
	1438	const std = @import("std");
	1439	const mem = std.mem;
	1440	const Allocator = mem.Allocator;
	1441	const assert = std.debug.assert;
	1442
	1443	const Teg = @import("Teg.zig");
	1444	const Inst = Teg.Inst;
	1445	const NullTerminatedString = Teg.NullTerminatedString;
	1446	const SlicedString = Teg.SlicedString;
	1447	const RefList = Teg.RefList;
	1448	const IndexList = Teg.IndexList;
	1449	const EncodedData = Teg.EncodedData;
	1450	const ExtraIndex = Teg.ExtraIndex;
	1451
	1452	const dod = @import("array.zig");
	1453	const ExtraStore = dod.ExtraStore(.teg, .@"var");
	1454	const Float = dod.Float;
	1455
	1456	const Tokenizer = @import("Tokenizer.zig");
	1457	const Token = Tokenizer.Token;
	1458	const ByteOffset = Tokenizer.ByteOffset;
	1459
	1460	const Ast = @import("Ast.zig");
	1461	const Unused = Ast.Unused;
	1462	const Node = Ast.Node;
	1463	const Data = Node.Data;
	1464	const SubList = Ast.SubList;
	1465	const NodeIndex = Ast.NodeIndex;
	1466	const TokenIndex = Ast.TokenIndex;
	1467	const OptionalNodeIndex = Ast.OptionalNodeIndex;
	1468	const OptionalTokenIndex = Ast.OptionalTokenIndex;
	1469	const OptionalExtraIndex = Ast.OptionalExtraIndex;
	1470	const optionalTokenIndex = OptionalTokenIndex.init;
	1471	const optionalNodeIndex = OptionalNodeIndex.init;
	1472	const optionalExtraIndex = OptionalExtraIndex.init;
	1473
	1474	const InternPool = @import("InternPool.zig");
	1475
	1476	const Report = @import("Report.zig");
	1477	const Comp = @import("Comp.zig");
	1478
	1479	const tracer = @import("tracer");

src/core/array.zig created+390

...	...	@@ -0,0 +1,390 @@
	1	const extra = @This();
	2
	3	pub const Mutability = enum { @"const", @"var" };
	4
	5	fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void {
	6	const fields = std.meta.fields(@TypeOf(payload));
	7	var i = index;
	8	inline for (fields) \|field\| {
	9	const value = @field(payload, field.name);
	10	switch (field.type) {
	11	u32 => items[i] = value,
	12	i32 => items[i] = @bitCast(value),
	13
	14	[16]u8 => {
	15	const u32s: [4]u32 = @bitCast(value);
	16	items[i] = u32s[0];
	17	items[i + 1] = u32s[1];
	18	items[i + 2] = u32s[2];
	19	items[i + 3] = u32s[3];
	20	},
	21
	22	else => brk: {
	23	const info = @typeInfo(field.type);
	24	if (info == .Enum) {
	25	items[i] = @intFromEnum(value);
	26	break :brk;
	27	}
	28
	29	const struct_data = info.Struct;
	30	if (struct_data.layout == .@"packed") {
	31	items[i] = @bitCast(value);
	32	break :brk;
	33	}
	34
	35	@call(.always_inline, setAsU32, .{ items, field.type, i, value });
	36	},
	37	}
	38	i += extraSizeOf(field.type);
	39	}
	40	}
	41
	42	fn getAsU32(items: []const u32, comptime T: type, index: usize) T {
	43	const fields = std.meta.fields(T);
	44	var i = index;
	45	var result: T = undefined;
	46	inline for (fields) \|field\| {
	47	@field(result, field.name) = switch (field.type) {
	48	u32 => items[i],
	49	i32 => @bitCast(items[i]),
	50
	51	[16]u8 => @bitCast(items[i..][0..4].*),
	52
	53	else => brk: {
	54	const info = @typeInfo(field.type);
	55	if (info == .Enum) {
	56	break :brk @enumFromInt(items[i]);
	57	}
	58
	59	const struct_data = info.Struct;
	60	if (struct_data.layout == .@"packed") {
	61	break :brk @bitCast(items[i]);
	62	}
	63
	64	break :brk @call(
	65	.always_inline,
	66	getAsU32,
	67	.{ items, field.type, i },
	68	);
	69	},
	70	};
	71	i += extraSizeOf(field.type);
	72	}
	73	return result;
	74	}
	75
	76	pub const StoreKind = enum {
	77	intern_pool,
	78	teg,
	79	air,
	80	tfr,
	81
	82	pub fn resolveStoreConst(
	83	comptime kind: StoreKind,
	84	owner: anytype,
	85	) *const kind.ResolvedStore(@TypeOf(owner)) {
	86	return &owner.extra;
	87	}
	88
	89	pub fn resolveStoreMut(
	90	comptime kind: StoreKind,
	91	owner: anytype,
	92	) *kind.ResolvedStore(@TypeOf(owner)) {
	93	return &owner.extra;
	94	}
	95
	96	pub fn ResolvedStore(
	97	comptime kind: StoreKind,
	98	comptime Owner: type,
	99	) type {
	100	const ptr = @typeInfo(Owner).Pointer;
	101	assert(ptr.size == .One);
	102	const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) \|field\| {
	103	if (std.mem.eql(u8, field.name, "extra")) {
	104	break :brk field.type.mutability;
	105	}
	106	} else unreachable;
	107	return ExtraStore(kind, mutability);
	108	}
	109	};
	110
	111	fn ExtraWrap(comptime kind: StoreKind) type {
	112	return struct {
	113	pub fn Index(comptime T: type) type {
	114	return enum(u32) {
	115	_,
	116
	117	// Reads the data from `store.extra` and returns it.
	118	pub fn get(index: @This(), owner: anytype) T {
	119	return kind.resolveStoreConst(owner).get(T, @intFromEnum(index));
	120	}
	121
	122	/// Returns data with a trailing reader to read data stored immediately after
	123	/// the main data structure. What is stored should be documented on `T`
	124	pub fn getTrail(index: @This(), owner: anytype) struct {
	125	T,
	126	kind.ResolvedStore(@TypeOf(owner)).TrailReader,
	127	} {
	128	const advance = comptime extraSizeOf(T);
	129	return .{
	130	index.get(owner),
	131	.{ .index = @intFromEnum(index) + advance },
	132	};
	133	}
	134	};
	135	}
	136	};
	137	}
	138
	139	pub fn EncodedData(comptime Tag: type, backing_type: type) type {
	140	return enum(backing_type) {
	141	_,
	142
	143	pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() {
	144	assert(proof == tag); // incorrect memory access
	145	const Data = tag.Data();
	146	const info = @typeInfo(Data);
	147	if (info == .Enum) {
	148	return @enumFromInt(@intFromEnum(data));
	149	}
	150	if (info == .Struct) {
	151	assert(backing_type == u64);
	152	var fields: [2]u32 = @bitCast(@intFromEnum(data));
	153	return getAsU32(&fields, Data, 0);
	154	}
	155	return @bitCast(@intFromEnum(data));
	156	}
	157
	158	pub fn encode(comptime tag: Tag, data: tag.Data()) @This() {
	159	const Data = tag.Data();
	160	const info = @typeInfo(Data);
	161	if (info == .Enum) {
	162	return @enumFromInt(@intFromEnum(data));
	163	}
	164	if (info == .Struct) {
	165	assert(backing_type == u64);
	166	var fields: [2]u32 = undefined;
	167	setAsU32(&fields, Data, 0, data);
	168	return @enumFromInt(@as(backing_type, @bitCast(fields)));
	169	}
	170	return @enumFromInt(@as(backing_type, @bitCast(data)));
	171	}
	172	};
	173	}
	174
	175	/// The number of 'u32's this type consumes in `ip.extra`.
	176	pub fn extraSizeOf(comptime T: type) usize {
	177	return switch (T) {
	178	u32,
	179	i32,
	180	=> return 1,
	181
	182	[16]u8 => return 4,
	183
	184	else => {
	185	const info = @typeInfo(T);
	186	if (info == .Enum) {
	187	comptime assert(@sizeOf(info.Enum.tag_type) <= 32);
	188	return 1;
	189	}
	190
	191	const struct_data = info.Struct;
	192	if (struct_data.layout == .@"packed") {
	193	comptime assert(struct_data.backing_integer == u32);
	194	return 1;
	195	}
	196
	197	const fields = std.meta.fields(T);
	198	var size: usize = 0;
	199	inline for (fields) \|field\| {
	200	size += extraSizeOf(field.type);
	201	}
	202	return size;
	203	},
	204	};
	205	}
	206
	207	pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type {
	208	return struct {
	209	list: List = .{},
	210
	211	pub const kind = set_kind;
	212	pub const mutability = set_mutability;
	213
	214	const Store = @This();
	215
	216	const index_holder = ExtraWrap(kind);
	217	pub const Index = index_holder.Index;
	218
	219	pub const List = switch (mutability) {
	220	.@"const" => struct {
	221	items: []const u32 = &.{},
	222
	223	pub fn deinit(slice: *@This(), gpa: Allocator) void {
	224	gpa.free(slice.items);
	225	}
	226	},
	227	.@"var" => std.ArrayListUnmanaged(u32),
	228	};
	229
	230	pub fn deinit(store: *Store, gpa: Allocator) void {
	231	store.list.deinit(gpa);
	232	store.* = undefined;
	233	}
	234
	235	pub const TrailReader = struct {
	236	index: usize,
	237
	238	pub fn read(trail: TrailReader, ip: const Store, comptime T: type) T {
	239	defer trail.index += comptime extraSizeOf(T);
	240	return ip.get(T, trail.index);
	241	}
	242
	243	pub fn readValueSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Value.Slice {
	244	defer trail.index += len;
	245	return .{
	246	.start = @intCast(trail.index),
	247	.len = len,
	248	};
	249	}
	250
	251	pub fn readTypeSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Type.Slice {
	252	defer trail.index += len;
	253	return .{
	254	.start = @intCast(trail.index),
	255	.len = len,
	256	};
	257	}
	258	};
	259
	260	pub const TrailWriter = struct {
	261	store: *Store,
	262	leftover: usize,
	263
	264	pub fn init(store: *Store, len: usize) TrailWriter {
	265	return .{ .store = store, .leftover = len };
	266	}
	267
	268	pub fn end(w: *TrailWriter) void {
	269	assert(w.leftover == 0);
	270	}
	271
	272	pub fn writeU32(w: *TrailWriter, value: u32) void {
	273	assert(w.leftover > 0);
	274	w.ip.extra.appendAssumeCapacity(value);
	275	w.leftover -= 1;
	276	}
	277
	278	pub fn writeSlice(w: *TrailWriter, slice: anytype) void {
	279	const Child = std.meta.Child(@TypeOf(slice));
	280	const child = @typeInfo(Child);
	281	comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32));
	282	assert(w.leftover >= slice.len);
	283	w.store.list.appendSliceAssumeCapacity(@ptrCast(slice));
	284	w.leftover -= slice.len;
	285	}
	286
	287	pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void {
	288	_ = appendAssumeCapacity(w.store, T, data);
	289	}
	290	};
	291
	292	pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) {
	293	comptime assert(mutability == .@"var");
	294	try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)));
	295	return store.appendAssumeCapacity(T, payload);
	296	}
	297
	298	pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) {
	299	comptime assert(mutability == .@"var");
	300	const extra_index: u32 = @intCast(ip.list.items.len);
	301	ip.list.items.len += extraSizeOf(@TypeOf(payload));
	302	ip.set(T, extra_index, payload);
	303	return @enumFromInt(extra_index);
	304	}
	305
	306	pub fn appendTrailing(
	307	store: *Store,
	308	gpa: Allocator,
	309	comptime T: type,
	310	payload: T,
	311	trailing_len: usize,
	312	) Allocator.Error!struct { Index(T), TrailWriter } {
	313	comptime assert(mutability == .@"var");
	314	try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len);
	315	const extra_index: u32 = @intCast(store.list.items.len);
	316	store.list.items.len += extraSizeOf(@TypeOf(payload));
	317	store.set(T, extra_index, payload);
	318	return .{
	319	@enumFromInt(extra_index),
	320	TrailWriter.init(store, trailing_len),
	321	};
	322	}
	323
	324	fn set(store: *Store, comptime T: type, index: usize, payload: T) void {
	325	comptime assert(mutability == .@"var");
	326	setAsU32(store.list.items, T, index, payload);
	327	}
	328
	329	fn get(store: *const Store, comptime T: type, index: usize) T {
	330	return getAsU32(store.list.items, T, index);
	331	}
	332
	333	pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") {
	334	comptime assert(mutability == .@"var");
	335	return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } };
	336	}
	337
	338	pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") {
	339	return .{ .list = .{ .items = store.list.items } };
	340	}
	341	};
	342	}
	343
	344	pub const Float = struct {
	345	low: u32,
	346	high: u32,
	347
	348	pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 };
	349
	350	pub fn purifyNaN(f: f64) f64 {
	351	if (std.math.isNan(f)) {
	352	return comptime Float.pure_nan.decode();
	353	}
	354	return f;
	355	}
	356
	357	pub fn assertPure(f: f64) f64 {
	358	if (std.math.isNan(f)) {
	359	assert(Float.pure_nan.eql(encode(f)));
	360	}
	361	return f;
	362	}
	363
	364	pub fn encode(decoded: f64) Float {
	365	if (std.math.isNan(decoded)) {
	366	return pure_nan;
	367	}
	368	const as_bytes: [2]u32 = @bitCast(decoded);
	369	return .{
	370	.low = as_bytes[0],
	371	.high = as_bytes[1],
	372	};
	373	}
	374
	375	pub fn decode(encoded: Float) f64 {
	376	const as_bytes: [2]u32 = .{
	377	encoded.low,
	378	encoded.high,
	379	};
	380	return @bitCast(as_bytes);
	381	}
	382
	383	pub fn eql(a: Float, b: Float) bool {
	384	return a.low == b.low and a.high == b.high;
	385	}
	386	};
	387
	388	const std = @import("std");
	389	const assert = std.debug.assert;
	390	const Allocator = std.mem.Allocator;

src/core/main.zig created+145

...	...	@@ -0,0 +1,145 @@
	1	//! This file is used to test the compiler implementation without
	2	//! needing to build the language server and other tooling.
	3	const std = @import("std");
	4	const builtin = @import("builtin");
	5	const core = @import("core");
	6
	7	const Tokenizer = core.Tokenizer;
	8	const Parse = core.Parse;
	9	const Transform = core.Transform;
	10	const Comp = core.Comp;
	11	const unicode_props = core.unicode_props;
	12	pub const Report = core.Report;
	13
	14	const tracer = @import("tracer");
	15	pub const tracer_impl = @field(tracer, @import("build_options").tracing);
	16
	17	pub fn main() !void {
	18	try tracer.init();
	19	defer tracer.deinit();
	20	try tracer.init_thread(null);
	21	defer tracer.deinit_thread();
	22
	23	try unicode_props.init();
	24
	25	if (builtin.mode == .Debug) {
	26	// zig build system doesnt print a newline :/
	27	std.debug.print("\r\x1b[K", .{});
	28	}
	29
	30	const use_gpa = builtin.mode == .Debug;
	31	var gpa: if (use_gpa) std.heap.GeneralPurposeAllocator(.{
	32	.stack_trace_frames = 100,
	33	}) else struct {} = .{};
	34	defer _ = if (use_gpa) gpa.deinit();
	35	const alloc = if (use_gpa)
	36	gpa.allocator()
	37	else
	38	std.heap.c_allocator;
	39
	40	var maybe_file_path: ?[]const u8 = null;
	41	defer if (maybe_file_path) \|path\| alloc.free(path);
	42
	43	var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
	44	const cwd = try std.posix.getcwd(&cwd_buf);
	45
	46	var exe_name: []const u8 = undefined;
	47	var dump_ir = false;
	48
	49	{
	50	var args = try std.process.argsWithAllocator(alloc);
	51	defer args.deinit();
	52
	53	exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
	54	errdefer alloc.free(exe_name);
	55
	56	while (args.next()) \|arg\| {
	57	if (std.mem.startsWith(u8, arg, "-")) {
	58	if (builtin.mode == .Debug and std.mem.eql(u8, arg, "--report-trace")) {
	59	Report.debug_show_report_trace = true;
	60	continue;
	61	}
	62	if (std.mem.eql(u8, arg, "--dump-ir")) {
	63	dump_ir = true;
	64	continue;
	65	}
	66	std.debug.print("error: unknown flag '{s}'\n", .{arg});
	67	maybe_file_path = null;
	68	break;
	69	}
	70
	71	if (maybe_file_path == null) {
	72	maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
	73	} else {
	74	std.debug.print("error: multiple files specified\n", .{});
	75	maybe_file_path = null;
	76	break;
	77	}
	78	}
	79	}
	80	defer alloc.free(exe_name);
	81
	82	const file_path = maybe_file_path orelse {
	83	std.debug.print("Usage: {s} [flags] <input file>\n", .{exe_name});
	84	{
	85	std.debug.print(
	86	// \| 60 chars wrap point\|
	87	\\
	88	\\Flags:
	89	\\ --dump-ir Dump intermediate representations.
	90	\\
	91	, .{});
	92	}
	93	if (builtin.mode == .Debug) {
	94	std.debug.print(
	95	\\
	96	\\Debug Mode Flags:
	97	\\ --report-trace Print stack traces per error. Very noisy
	98	\\
	99	, .{});
	100	}
	101	std.process.exit(1);
	102	};
	103
	104	var comp: Comp = .{
	105	.gpa = tracer.tracingAllocator(&alloc),
	106	.options = .{
	107	.dump_ir = dump_ir,
	108	},
	109	};
	110	try comp.init();
	111	defer comp.deinit();
	112	const main_file = try comp.getFile(file_path);
	113
	114	var fail_report: Report = undefined;
	115	const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
	116	error.TransformFail => {
	117	std.debug.print("Transform failed:\n", .{});
	118	try fail_report.dumpToStdErr(&comp);
	119	std.process.exit(1);
	120	},
	121	error.ParseFail => {
	122	// Already printed.
	123	std.process.exit(1);
	124	},
	125	else => return err,
	126	};
	127
	128	if (comp.options.dump_ir) {
	129	const stderr = std.io.getStdErr().writer();
	130	try teg.dumpToWriter(stderr);
	131	}
	132
	133	comp.semaEntryPoint(main_file) catch \|err\| switch (err) {
	134	error.AnalysisFail => std.process.exit(1),
	135	else => return err,
	136	};
	137
	138	const output = std.io.getStdOut().writer();
	139	var bufw = std.io.bufferedWriter(output);
	140	const writer = bufw.writer();
	141	try comp.link(writer);
	142	try bufw.flush();
	143
	144	std.process.cleanExit();
	145	}

src/core/mod.zig created+14

...	...	@@ -0,0 +1,14 @@
	1	pub const Air = @import("Air.zig");
	2	pub const array = @import("array.zig");
	3	pub const Cache = @import("Cache.zig");
	4	pub const Comp = @import("Comp.zig");
	5	pub const InternPool = @import("InternPool.zig");
	6	pub const JsLink = @import("JsLink.zig");
	7	pub const Parse = @import("Parse.zig");
	8	pub const Report = @import("Report.zig");
	9	pub const Sema = @import("Sema.zig");
	10	pub const Teg = @import("Teg.zig");
	11	pub const Tokenizer = @import("Tokenizer.zig");
	12	pub const Transform = @import("Transform.zig");
	13	pub const unicode_props = @import("unicode_props.zig");
	14	pub const util = @import("util.zig");

src/core/unicode_props.zig created+103

...	...	@@ -0,0 +1,103 @@
	1	//! Wrapper around the 'zg' PropsData library.
	2	const std = @import("std");
	3	const PropsData = @import("zg:PropsData");
	4	const tracer = @import("tracer");
	5
	6	var props: ?PropsData = undefined;
	7	var props_memory: []u8 = undefined;
	8
	9	const raw_allocator = if (@import("builtin").target.isWasm())
	10	std.heap.wasm_allocator
	11	else
	12	std.heap.page_allocator;
	13
	14	pub fn init() !void {
	15	if (props != null) return;
	16
	17	const trace = tracer.trace(@src());
	18	defer trace.end();
	19
	20	// We know exactly how much this uses, so let's preallocate these pages and use a FBA.
	21	const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856;
	22
	23	if (allocation_size == 0) {
	24	var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator);
	25	props = try PropsData.init(logging_alloc.allocator());
	26	std.log.info("please write the above numbers into unicode.zig to prealloc this", .{});
	27	std.process.exit(1);
	28	}
	29
	30	props_memory = try raw_allocator.alloc(u8, allocation_size);
	31	errdefer std.heap.page_allocator.free(props_memory);
	32
	33	var fba = std.heap.FixedBufferAllocator.init(props_memory);
	34	props = try PropsData.init(fba.allocator());
	35	std.debug.assert(fba.end_index == allocation_size);
	36	}
	37
	38	pub fn deinit() void {
	39	std.debug.assert(props != null);
	40	raw_allocator.free(props_memory);
	41	props = null;
	42	props_memory = undefined;
	43	}
	44
	45	/// True if `cp` is a mathematical symbol.
	46	pub inline fn isMath(cp: u21) bool {
	47	return props.?.isMath(cp);
	48	}
	49
	50	/// True if `cp` is an alphabetic character.
	51	pub inline fn isAlphabetic(cp: u21) bool {
	52	return props.?.isAlphabetic(cp);
	53	}
	54
	55	/// True if `cp` is a valid identifier start character.
	56	pub inline fn isIdStart(cp: u21) bool {
	57	return props.?.isIdStart(cp);
	58	}
	59
	60	/// True if `cp` is a valid identifier continuation character.
	61	pub inline fn isIdContinue(cp: u21) bool {
	62	return props.?.isIdContinue(cp);
	63	}
	64
	65	/// True if `cp` is a valid extended identifier start character.
	66	pub inline fn isXidStart(cp: u21) bool {
	67	return props.?.isXidStart(cp);
	68	}
	69
	70	/// True if `cp` is a valid extended identifier continuation character.
	71	pub inline fn isXidContinue(cp: u21) bool {
	72	return props.?.isXidContinue(cp);
	73	}
	74
	75	/// True if `cp` is a whitespace character.
	76	pub inline fn isWhitespace(cp: u21) bool {
	77	return props.?.isWhitespace(cp);
	78	}
	79
	80	/// True if `cp` is a hexadecimal digit.
	81	pub inline fn isHexDigit(cp: u21) bool {
	82	return props.?.isHexDigit(cp);
	83	}
	84
	85	/// True if `cp` is a diacritic mark.
	86	pub inline fn isDiacritic(cp: u21) bool {
	87	return props.?.isDiacritic(cp);
	88	}
	89
	90	/// True if `cp` is numeric.
	91	pub inline fn isNumeric(cp: u21) bool {
	92	return props.?.isNumeric(cp);
	93	}
	94
	95	/// True if `cp` is a digit.
	96	pub inline fn isDigit(cp: u21) bool {
	97	return props.?.isDigit(cp);
	98	}
	99
	100	/// True if `cp` is decimal.
	101	pub inline fn isDecimal(cp: u21) bool {
	102	return props.?.isDecimal(cp);
	103	}

src/core/util.zig created+56

...	...	@@ -0,0 +1,56 @@
	1	pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
	2	if (@hasField(@TypeOf(writer), "unbuffered_writer")) {
	3	return detectTTYFromWriter(writer.unbuffered_writer);
	4	}
	5	if (@TypeOf(writer) == std.fs.File.Writer) {
	6	return std.io.tty.detectConfig(writer.context);
	7	}
	8	if (@TypeOf(writer) == std.io.AnyWriter) {
	9	const fileWriteFn = comptime (std.fs.File{ .handle = undefined })
	10	.writer().any().writeFn;
	11	if (writer.writeFn == fileWriteFn) {
	12	const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
	13	return std.io.tty.detectConfig(file.*);
	14	}
	15	}
	16	return .no_color;
	17	}
	18
	19	pub fn Ordinal(Int: type) type {
	20	comptime assert(@typeInfo(Int).Int.signedness == .unsigned);
	21	return enum(Int) {
	22	start = 0,
	23	_,
	24
	25	pub fn fromZeroBased(int: Int) @This() {
	26	assert(int != std.math.maxInt(Int));
	27	return @enumFromInt(int);
	28	}
	29	pub fn fromOneBased(int: Int) @This() {
	30	assert(int > 0);
	31	return @enumFromInt(int - 1);
	32	}
	33	pub fn zeroBased(ord: @This()) Int {
	34	return @intFromEnum(ord);
	35	}
	36	pub fn oneBased(ord: @This()) Int {
	37	return @intFromEnum(ord) + 1;
	38	}
	39
	40	pub fn add(ord: @This(), inc: Int) @This() {
	41	return fromZeroBased(ord.zeroBased() + inc);
	42	}
	43
	44	pub fn format(
	45	_: @This(),
	46	comptime _: []const u8,
	47	_: std.fmt.FormatOptions,
	48	_: anytype,
	49	) !void {
	50	@compileError("Do not format `Ordinal` directly. Call oneBased or zeroBased to specify.");
	51	}
	52	};
	53	}
	54
	55	const std = @import("std");
	56	const assert = std.debug.assert;

src/lsp/lsp.zig created+41

...	...	@@ -0,0 +1,41 @@
	1	const std = @import("std");
	2	const builtin = @import("builtin");
	3	const core = @import("core");
	4	const tracer = @import("tracer");
	5
	6	pub const tracer_impl = @field(tracer, @import("build_options").tracing);
	7
	8	pub fn main() !void {
	9	if (builtin.mode == .Debug) {
	10	// zig build system doesnt print a newline :/
	11	std.debug.print("\r\x1b[K", .{});
	12	}
	13
	14	std.debug.print("All your codebase are belong to us\n", .{});
	15
	16	const use_gpa = builtin.mode == .Debug;
	17	var gpa_state: if (use_gpa) std.heap.GeneralPurposeAllocator(.{
	18	.stack_trace_frames = 100,
	19	}) else struct {} = .{};
	20	defer _ = if (use_gpa) gpa_state.deinit();
	21	const gpa = if (use_gpa)
	22	gpa_state.allocator()
	23	else
	24	std.heap.c_allocator;
	25
	26	// if `parse` returns `error.ParseFail`, this `err` is set.
	27	//
	28	// at some point later this will change to be a `Report`
	29	// so that multiple parse errors may be reported at once.
	30	var err: core.Parse.AstError.WithTrace = undefined;
	31	var ast = try core.Parse.parse(gpa,
	32	\\let x = 1;
	33	\\
	34	\\pub fn hello() -> float {
	35	\\ 42.2
	36	\\}
	37	, &err);
	38	defer ast.deinit(gpa);
	39
	40	try ast.dumpToWriter(std.io.getStdErr().writer());
	41	}