Commit c1d54bf8b06f71d81e8614572483a69720da4ea6 | ix

author	dave caruso <me@paperdave.net> 2024-05-19 03:11:59-07:00
committer	dave caruso <me@paperdave.net> 2024-05-19 03:11:59-07:00
log	c1d54bf8b06f71d81e8614572483a69720da4ea6
tree	10775f14eb33a1b28aecbc4558bb5214ed732b94
parent	634a70376c0023f46a6aae6611ed672397eae557
signature	Commit is signed but in an unrecognized format.

better error handling

15 files changed, 960 insertions(+), 287 deletions(-)

CHANGELOG.md
demo.txt
src
- compiler

CHANGELOG.md created+8

...	...	@@ -0,0 +1,8 @@
	1	## 0.1.1-alpha
	2
	3	- introduce `Report` and show errors during Transform.
	4	- add flag --dump-ir for enabling ir dumps
	5
	6	## 0.1.0-alpha
	7
	8	initial demo of compiler pipeline in zig.

demo.txt created+1

...	...	@@ -0,0 +1 @@
	1	no "waay

src/compiler/Ast.zig+131-83

...	...	@@ -34,96 +34,54 @@
34	34	//! The `Ast` owns token and node data, but not the source code.
35	35
36	36	/// Externally owned source code.
37		utf8_source: []const u8,
	37	utf8_source: [:0]const u8,
38	38
39	39	tokens: Token.List.Slice,
40	40	nodes: Node.List.Slice,
41	41	extra: []const u32,
42	42
43		pub fn parse(gpa: std.mem.Allocator, source: []const u8) !Ast {
44		const Parse = @import("Parse.zig");
45
46		var todo: ByteOffset = 0;
47		var tokens = try Tokenizer.tokenize(source, gpa, &todo);
	43	const Parse = @import("Parse.zig");
	44	pub fn parse(
	45	gpa: std.mem.Allocator,
	46	source: [:0]const u8,
	47	ast_error: *Parse.AstError,
	48	) Parse.Error!Ast {
	49	var err_byte: ByteOffset = 0;
	50	var tokens = Tokenizer.tokenize(
	51	source,
	52	gpa,
	53	&err_byte,
	54	) catch \|err\| switch (err) {
	55	error.OutOfMemory => return error.OutOfMemory,
	56	else => {
	57	ast_error.* = .{ .tokenization = .{
	58	.err = err,
	59	.byte = err_byte,
	60	} };
	61	return error.ParseFail;
	62	},
	63	};
48	64	errdefer tokens.deinit(gpa);
49	65
50	66	var parser = Parse.init(gpa, source, tokens);
51	67	errdefer parser.deinit();
52	68
53		try parser.nodes.ensureUnusedCapacity(gpa, @divFloor(tokens.len, 4));
	69	try parser.nodes.ensureUnusedCapacity(
	70	gpa,
	71	@max(1, @divFloor(tokens.len, 4)),
	72	);
54	73
55		try parser.parseRoot();
	74	parser.parseRoot() catch \|err\| switch (err) {
	75	error.OutOfMemory => return error.OutOfMemory,
	76	error.ParseFail => {
	77	ast_error.* = parser.ast_error.?;
	78	return error.ParseFail;
	79	},
	80	};
56	81
57	82	return try parser.finalize(tokens);
58	83	}
59	84
60		pub const Location = struct {
61		line: usize,
62		column: usize,
63		line_start: usize,
64		line_end: usize,
65		};
66
67		pub const Span = struct {
68		start: u32,
69		end: u32,
70		main: u32,
71		};
72
73		// This can be removed when the following issue is implemented:
74		// See: https://github.com/ziglang/zig/issues/104
75		// See: https://github.com/ziglang/zig/issues/3806
76		pub const OptionalNodeIndex = enum(u32) {
77		none = 0,
78		_,
79
80		pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
81		assert(idx != 0);
82		return @enumFromInt(idx orelse return .none);
83		}
84
85		pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
86		return switch (idx) {
87		.none => null,
88		_ => @intFromEnum(idx),
89		};
90		}
91		};
92
93		pub const OptionalTokenIndex = enum(u32) {
94		none = 0,
95		_,
96
97		pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
98		assert(idx != 0);
99		return @enumFromInt(idx orelse return .none);
100		}
101
102		pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
103		return switch (idx) {
104		.none => null,
105		_ => @intFromEnum(idx),
106		};
107		}
108		};
109
110		pub const OptionalExtraIndex = enum(u32) {
111		none = std.math.maxInt(u32),
112		_,
113
114		pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
115		assert(idx != std.math.maxInt(u32));
116		return @enumFromInt(idx orelse return .none);
117		}
118
119		pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
120		return switch (idx) {
121		.none => null,
122		_ => @intFromEnum(idx),
123		};
124		}
125		};
126
127	85	pub const Node = struct {
128	86	tag: Tag,
129	87	main_token: TokenIndex,
...	...	@@ -462,6 +420,76 @@ pub const Node = struct {
462	420	};
463	421	};
464	422
	423	pub const Error = error{ OutOfMemory, ParseFail };
	424
	425	pub const Location = struct {
	426	// TODO: add an ordinal number type, replace with Ordinal(usize)
	427	line: usize,
	428	column: usize,
	429	line_start: usize,
	430	line_end: usize,
	431	};
	432
	433	pub const Span = struct {
	434	start: u32,
	435	end: u32,
	436	main: u32,
	437	};
	438
	439	// This can be removed when the following issue is implemented:
	440	// See: https://github.com/ziglang/zig/issues/104
	441	// See: https://github.com/ziglang/zig/issues/3806
	442	pub const OptionalNodeIndex = enum(u32) {
	443	none = 0,
	444	_,
	445
	446	pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
	447	assert(idx != 0);
	448	return @enumFromInt(idx orelse return .none);
	449	}
	450
	451	pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
	452	return switch (idx) {
	453	.none => null,
	454	_ => @intFromEnum(idx),
	455	};
	456	}
	457	};
	458
	459	pub const OptionalTokenIndex = enum(u32) {
	460	none = 0,
	461	_,
	462
	463	pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
	464	assert(idx != 0);
	465	return @enumFromInt(idx orelse return .none);
	466	}
	467
	468	pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
	469	return switch (idx) {
	470	.none => null,
	471	_ => @intFromEnum(idx),
	472	};
	473	}
	474	};
	475
	476	pub const OptionalExtraIndex = enum(u32) {
	477	none = std.math.maxInt(u32),
	478	_,
	479
	480	pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
	481	assert(idx != std.math.maxInt(u32));
	482	return @enumFromInt(idx orelse return .none);
	483	}
	484
	485	pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
	486	return switch (idx) {
	487	.none => null,
	488	_ => @intFromEnum(idx),
	489	};
	490	}
	491	};
	492
465	493	pub const SubList = struct {
466	494	/// Either NodeIndex or TokenIndex, depending on where this came from.
467	495	start: u32,
...	...	@@ -598,6 +626,19 @@ pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
598	626	};
599	627	}
600	628
	629	pub fn idToken(ast: Ast, node: NodeIndex) TokenIndex {
	630	return switch (ast.nodeTag(node)) {
	631	.param_decl,
	632	.id,
	633	.label,
	634	=> ast.nodeToken(node),
	635	.fn_decl,
	636	.var_decl,
	637	=> ast.nodeToken(node) + 1,
	638	else => unreachable, // does not have an id
	639	};
	640	}
	641
601	642	pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
602	643	const data = ast.nodes.items(.data)[node].list;
603	644	return data.slice(ast);
...	...	@@ -605,15 +646,15 @@ pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
605	646
606	647	pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
607	648	var loc = Location{
608		.line = 0,
609		.column = 0,
	649	.line = 1,
	650	.column = 1,
610	651	.line_start = start_offset,
611		.line_end = self.source.len,
	652	.line_end = self.utf8_source.len,
612	653	};
613		const token_start = self.tokens.items(.start)[token_index];
	654	const token_start = self.tokens.items(.offset)[token_index];
614	655
615	656	// Scan to by line until we go past the token start
616		while (std.mem.indexOfScalarPos(u8, self.source, loc.line_start, '\n')) \|i\| {
	657	while (std.mem.indexOfScalarPos(u8, self.utf8_source, loc.line_start, '\n')) \|i\| {
617	658	if (i >= token_start) {
618	659	break; // Went past
619	660	}
...	...	@@ -622,17 +663,17 @@ pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenInde
622	663	}
623	664
624	665	const offset = loc.line_start;
625		for (self.source[offset..], 0..) \|c, i\| {
	666	for (self.utf8_source[offset..], 0..) \|c, i\| {
626	667	if (i + offset == token_start) {
627	668	loc.line_end = i + offset;
628		while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') {
	669	while (loc.line_end < self.utf8_source.len and self.utf8_source[loc.line_end] != '\n') {
629	670	loc.line_end += 1;
630	671	}
631	672	return loc;
632	673	}
633	674	if (c == '\n') {
634	675	loc.line += 1;
635		loc.column = 0;
	676	loc.column = 1;
636	677	loc.line_start = i + 1;
637	678	} else {
638	679	loc.column += 1;
...	...	@@ -651,6 +692,11 @@ pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
651	692	return ast.tokenSlice(token);
652	693	}
653	694
	695	pub fn fnDeclIdToken(tree: Ast, node: NodeIndex) TokenIndex {
	696	assert(tree.nodeTag(node) == .fn_decl);
	697	return tree.nodeToken(node) + 1;
	698	}
	699
654	700	pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
655	701	const fields = std.meta.fields(T);
656	702	var result: T = undefined;
...	...	@@ -699,3 +745,5 @@ const Tokenizer = @import("Tokenizer.zig");
699	745	const Token = Tokenizer.Token;
700	746	const TokenList = Tokenizer.Token.List;
701	747	const ByteOffset = Tokenizer.ByteOffset;
	748
	749	const Report = @import("Report.zig");

src/compiler/Comp.zig+60-11

...	...	@@ -10,6 +10,8 @@ files: std.StringArrayHashMapUnmanaged(*File) = .{},
10	10	/// # of sema instructions in this compilation cycle
11	11	analyzed_instructions: usize = 0,
12	12
	13	options: Options = .{},
	14
13	15	// TODO: consider computing this dynamicly per-link
14	16	current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
15	17
...	...	@@ -74,7 +76,16 @@ pub const File = struct {
74	76
75	77	path_digest: ?Cache.BinDigest,
76	78
77		pub fn getSource(file: *File, gpa: Allocator) !Source {
	79	const GetSourceError =
	80	std.fs.File.OpenError \|\|
	81	std.fs.File.ReadError \|\|
	82	std.fs.File.StatError \|\|
	83	Allocator.Error \|\| error{
	84	FileTooBig,
	85	UnexpectedEndOfFile,
	86	};
	87
	88	pub fn getSource(file: *File, gpa: Allocator) GetSourceError!Source {
78	89	if (file.source) \|source\| {
79	90	return source;
80	91	}
...	...	@@ -111,17 +122,42 @@ pub const File = struct {
111	122	return loaded_source;
112	123	}
113	124
114		pub fn getAst(file: *File, gpa: Allocator) !Ast {
	125	const GetAstError = GetSourceError \|\| Parse.Error;
	126
	127	pub fn getAst(file: File, comp: Comp) GetAstError!Ast {
115	128	if (file.ast) \|ast\| {
116	129	return ast;
117	130	}
118	131
119		const ast = try Ast.parse(gpa, (try file.getSource(gpa)).code);
	132	var ast_err: Parse.AstError = undefined;
	133	const ast = Ast.parse(
	134	comp.gpa,
	135	(try file.getSource(comp.gpa)).code,
	136	&ast_err,
	137	) catch \|err\| switch (err) {
	138	error.OutOfMemory => return err,
	139	error.ParseFail => {
	140	var report = try ast_err.toReport(comp, file);
	141	// TODO: let this merge with other report
	142	report.dumpToStdErr(comp) catch {};
	143	report.deinit(comp.gpa);
	144	return err;
	145	},
	146	};
	147
	148	if (comp.options.dump_ir) {
	149	ast.dumpToWriter(std.io.getStdErr().writer()) catch {
	150	// Simplify the error set.
	151	};
	152	}
	153
120	154	file.ast = ast;
121	155	return ast;
122	156	}
123	157
124		pub fn getTeg(file: *File, gpa: Allocator) !Teg {
	158	const GetTegError = GetAstError \|\| Transform.Error;
	159
	160	pub fn getTeg(file: File, comp: Comp, fail_report: *Report) GetTegError!Teg {
125	161	if (file.teg) \|teg\| {
126	162	return teg;
127	163	}
...	...	@@ -136,7 +172,13 @@ pub const File = struct {
136	172	};
137	173	file.path_digest = bin_digest;
138	174
139		const teg = try Transform.transform(try file.getAst(gpa), gpa);
	175	const teg = try Transform.transform(
	176	try file.getAst(comp),
	177	comp.gpa,
	178	file,
	179	fail_report,
	180	);
	181
140	182	file.teg = teg;
141	183	return teg;
142	184	}
...	...	@@ -290,8 +332,6 @@ pub const Namespace = struct {
290	332	pub fn indexTegModule(comp: Comp, file: File, module_index: Teg.Inst.Index) !Namespace.Index {
291	333	const teg = file.teg.?;
292	334
293		log.info("Index {} @ {d}", .{ std.fmt.fmtSliceHexLower(file.path_digest.?[0..8]), @intFromEnum(module_index) });
294
295	335	const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
296	336	.parent = .none,
297	337	.file_scope = file,
...	...	@@ -322,7 +362,13 @@ pub fn indexTegModule(comp: Comp, file: File, module_index: Teg.Inst.Index) !N
322	362	}
323	363
324	364	pub fn semaEntryPoint(comp: Comp, file: File) !void {
325		const teg = try file.getTeg(comp.gpa);
	365	var report: Report = undefined;
	366	const teg = try file.getTeg(comp, &report);
	367
	368	// TODO, move this elsewhere. we should print all errors and warnings at once
	369	if (teg.report.warnings.len > 0) {
	370	try teg.report.dumpToStdErr(comp);
	371	}
326	372
327	373	const ns = try comp.indexTegModule(file, .main_module_inst);
328	374	const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
...	...	@@ -354,9 +400,6 @@ pub fn codeGenDecl(c: Comp, decl: Decl, air: *const Air) !void {
354	400	const t = tracer.trace(@src());
355	401	defer t.end();
356	402
357		std.debug.print("Start Codegen\n", .{});
358		defer std.debug.print("End Codegen\n", .{});
359
360	403	var arena_state = std.heap.ArenaAllocator.init(c.gpa);
361	404	defer arena_state.deinit();
362	405	const arena = arena_state.allocator();
...	...	@@ -371,6 +414,10 @@ pub fn link(c: *Comp, writer: anytype) !void {
371	414	try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
372	415	}
373	416
	417	const Options = struct {
	418	dump_ir: bool = false,
	419	};
	420
374	421	const Comp = @This();
375	422
376	423	const std = @import("std");
...	...	@@ -391,5 +438,7 @@ const Transform = @import("Transform.zig");
391	438	const JsNodeGen = @import("JsNodeGen.zig");
392	439	const JsTree = @import("JsTree.zig");
393	440	const JsLink = @import("JsLink.zig");
	441	const Report = @import("Report.zig");
	442	const Parse = @import("Parse.zig");
394	443
395	444	const tracer = @import("tracer");

src/compiler/JsLink.zig+1-4

...	...	@@ -19,7 +19,6 @@ pub fn linkJsCode(
19	19	decls: []const Decl.Index,
20	20	writer: anytype,
21	21	) !void {
22		log.info("Linking {d} decls", .{decls.len});
23	22
24	23	// Arena allocator is used for the main flow
25	24	var link: JsLink = .{ .comp = comp };
...	...	@@ -68,7 +67,7 @@ pub fn linkJsCode(
68	67	try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
69	68	}
70	69
71		try writer.print("// Compiler Demo. Do not use in production\n", .{});
	70	try writer.print("// Language Compiler v0.0.1-alpha2\n", .{});
72	71
73	72	// 2. Print every function decl
74	73	for (decls[1..]) \|decl_index\| {
...	...	@@ -106,7 +105,6 @@ fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
106	105
107	106	fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
108	107	const start = link.string_bytes.items.len;
109		log.info("Append '{s}' -> .{{ .start = {d}, .len = {d} }}", .{ string, start, string.len });
110	108	try link.string_bytes.appendSlice(link.comp.gpa, string);
111	109	return .{ .start = @intCast(start), .len = @intCast(string.len) };
112	110	}
...	...	@@ -170,7 +168,6 @@ fn printBlock(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer
170	168	}
171	169
172	170	fn printExpr(link: JsLink, tree: const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
173		log.info("print node {d}", .{@intFromEnum(node)});
174	171	switch (tree.nodeTag(node)) {
175	172	else => \|tag\| std.debug.panic("TODO: printExpr .{s}", .{@tagName(tag)}),
176	173	.root => unreachable,

src/compiler/JsNodeGen.zig-2

...	...	@@ -117,7 +117,6 @@ pub fn build(c: *JsNodeGen) !void {
117	117	}
118	118
119	119	fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
120		std.debug.print("JsNodeGen: #{d}\n", .{@intFromEnum(inst)});
121	120	switch (c.air.instTag(inst)) {
122	121	.arg => unreachable, // codegen must skip over these instructions.
123	122
...	...	@@ -400,7 +399,6 @@ fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Br
400	399	this_label,
401	400	});
402	401
403		std.debug.print("requires_label {}\n", .{requires_label});
404	402	const block: Air.IndexList = c.air.instData(.block, i);
405	403	for (block.slice(c.air)) \|item\| {
406	404	try c.codeGenInst(item);

src/compiler/Parse.zig+62-27

...	...	@@ -1,7 +1,7 @@
1	1	//! Parser takes a Token list and creates a Node list. It does not own the source code or
2	2	//! token data, however transferring into an `Ast` does move token list ownership.
3	3	gpa: Allocator,
4		source: []const u8,
	4	source: [:0]const u8,
5	5	token_tags: []const Token.Tag,
6	6	token_starts: []const ByteOffset,
7	7
...	...	@@ -12,7 +12,7 @@ scratch: ArrayListUnmanaged(NodeIndex) = .{},
12	12
13	13	ast_error: ?AstError = null,
14	14
15		pub fn init(gpa: Allocator, source: []const u8, tokens: Token.List.Slice) Parse {
	15	pub fn init(gpa: Allocator, source: [:0]const u8, tokens: Token.List.Slice) Parse {
16	16	return Parse{
17	17	.gpa = gpa,
18	18	.source = source,
...	...	@@ -36,8 +36,6 @@ pub fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
36	36	.extra = extra,
37	37	};
38	38
39		ast.dumpToWriter(std.io.getStdErr().writer()) catch {};
40
41	39	return ast;
42	40	}
43	41
...	...	@@ -50,18 +48,52 @@ pub fn deinit(p: *Parse) void {
50	48	}
51	49
52	50	pub const Error = error{
53		SyntaxError,
	51	OutOfMemory,
	52	ParseFail,
54	53	};
55	54
56	55	/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
57	56	pub const AstError = union(enum) {
	57	tokenization: struct {
	58	err: Tokenizer.TokenizeError,
	59	byte: u32,
	60	},
58	61	expected_token: Token.Tag,
59	62	expected_statement,
60	63	expected_expr,
61	64	expected_else_value,
	65
	66	pub fn toReport(parse_err: AstError, comp: Comp, file: Comp.File) !Report {
	67	return (Report.Err{ .ast = parse_err }).toReport(comp, file);
	68	}
	69
	70	pub fn build(err: AstError, ctx: *Report.MsgContext) !void {
	71	switch (err) {
	72	.tokenization => \|data\| {
	73	// TODO: consider making tokenizer emit AstError more directly,
	74	// so that it can just annotate start and ends
	75	try ctx.msg(switch (data.err) {
	76	error.UnterminatedString => .{ .codepoint_eol = data.byte },
	77	else => .{ .codepoint = data.byte },
	78	}, "{s}", .{switch (data.err) {
	79	error.OutOfMemory => unreachable,
	80	error.InvalidSyntax => "unexpected token",
	81	error.TrailingNumericSeparator => "trailing numberic separator",
	82	error.UnterminatedString => "unterminated string literal",
	83	error.InvalidEscapeSequence => "invalid escape sequence",
	84	error.InvalidEscapeCodepoint => "invalid escape codepoint",
	85	error.InvalidUtf8 => "file is not valid utf8",
	86	error.GitConflict => "git merge conflict encountered",
	87	}});
	88	},
	89
	90	else => unreachable,
	91	//
	92	}
	93	}
62	94	};
63	95
64		pub fn parseRoot(p: *Parse) !void {
	96	pub fn parseRoot(p: *Parse) Error!void {
65	97	const trace = t.trace(@src());
66	98	defer trace.end();
67	99
...	...	@@ -82,7 +114,7 @@ pub fn parseRoot(p: *Parse) !void {
82	114	};
83	115	}
84	116
85		fn parseModuleMembers(p: *Parse) !SubList {
	117	fn parseModuleMembers(p: *Parse) Error!SubList {
86	118	const trace = t.trace(@src());
87	119	defer trace.end();
88	120
...	...	@@ -103,7 +135,7 @@ fn parseModuleMembers(p: *Parse) !SubList {
103	135	.kw_enum,
104	136	.kw_trait,
105	137	.kw_impl,
106		=> return error.Todo,
	138	=> \|tag\| std.debug.panic("TODO: parse {s}", .{@tagName(tag)}),
107	139
108	140	.eof => break,
109	141
...	...	@@ -120,7 +152,7 @@ fn parseModuleMembers(p: *Parse) !SubList {
120	152	return p.scratchToSubList(scratch_top);
121	153	}
122	154
123		fn parseModuleDecl(p: *Parse) !NodeIndex {
	155	fn parseModuleDecl(p: *Parse) Error!NodeIndex {
124	156	const trace = t.trace(@src());
125	157	defer trace.end();
126	158
...	...	@@ -135,7 +167,7 @@ fn parseModuleDecl(p: *Parse) !NodeIndex {
135	167	};
136	168	}
137	169
138		fn parseFnDecl(p: *Parse) !NodeIndex {
	170	fn parseFnDecl(p: *Parse) Error!NodeIndex {
139	171	const trace = t.trace(@src());
140	172	defer trace.end();
141	173
...	...	@@ -170,7 +202,7 @@ fn parseFnDecl(p: *Parse) !NodeIndex {
170	202	});
171	203	}
172	204
173		fn parseParamList(p: *Parse) !SubList {
	205	fn parseParamList(p: *Parse) Error!SubList {
174	206	const trace = t.trace(@src());
175	207	defer trace.end();
176	208
...	...	@@ -199,7 +231,7 @@ fn parseParamList(p: *Parse) !SubList {
199	231	return p.scratchToSubList(scratch_top);
200	232	}
201	233
202		fn parseParamDecl(p: *Parse) !NodeIndex {
	234	fn parseParamDecl(p: *Parse) Error!NodeIndex {
203	235	const trace = t.trace(@src());
204	236	defer trace.end();
205	237
...	...	@@ -218,7 +250,7 @@ fn parseParamDecl(p: *Parse) !NodeIndex {
218	250	});
219	251	}
220	252
221		fn parseBlock(p: *Parse) !NodeIndex {
	253	fn parseBlock(p: *Parse) Error!NodeIndex {
222	254	const trace = t.trace(@src());
223	255	defer trace.end();
224	256
...	...	@@ -236,7 +268,7 @@ fn parseBlock(p: *Parse) !NodeIndex {
236	268	});
237	269	}
238	270
239		fn parseLabelExpression(p: *Parse) !NodeIndex {
	271	fn parseLabelExpression(p: *Parse) Error!NodeIndex {
240	272	const trace = t.trace(@src());
241	273	defer trace.end();
242	274
...	...	@@ -252,7 +284,7 @@ fn parseLabelExpression(p: *Parse) !NodeIndex {
252	284	});
253	285	}
254	286
255		fn parseBlockContents(p: *Parse) !struct { Node.Tag, SubList } {
	287	fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } {
256	288	const trace = t.trace(@src());
257	289	defer trace.end();
258	290
...	...	@@ -308,7 +340,7 @@ fn parseBlockContents(p: *Parse) !struct { Node.Tag, SubList } {
308	340	}
309	341
310	342	/// VarDecl <- (kw_pub)? (kw_var \| kw_let) id (colon expr)? equal expr semi
311		fn parseVarDecl(p: *Parse) !NodeIndex {
	343	fn parseVarDecl(p: *Parse) Error!NodeIndex {
312	344	const trace = t.trace(@src());
313	345	defer trace.end();
314	346
...	...	@@ -521,7 +553,7 @@ const Operator = enum(u8) {
521	553	}
522	554	};
523	555
524		fn parseExpression(p: *Parse) anyerror!NodeIndex {
	556	fn parseExpression(p: *Parse) Error!NodeIndex {
525	557	const trace = t.trace(@src());
526	558	defer trace.end();
527	559
...	...	@@ -543,7 +575,7 @@ fn parseExpression(p: *Parse) anyerror!NodeIndex {
543	575	return expr;
544	576	}
545	577
546		fn parseExpressionPart(p: *Parse) !NodeIndex {
	578	fn parseExpressionPart(p: *Parse) Error!NodeIndex {
547	579	const trace = t.trace(@src());
548	580	defer trace.end();
549	581
...	...	@@ -644,7 +676,7 @@ fn parseExpressionPart(p: *Parse) !NodeIndex {
644	676	return expr;
645	677	}
646	678
647		fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) !NodeIndex {
	679	fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) Error!NodeIndex {
648	680	const trace = t.trace(@src());
649	681	defer trace.end();
650	682
...	...	@@ -673,7 +705,7 @@ fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) !NodeIndex {
673	705	});
674	706	}
675	707
676		fn parseReturn(p: *Parse) !NodeIndex {
	708	fn parseReturn(p: *Parse) Error!NodeIndex {
677	709	const trace = t.trace(@src());
678	710	defer trace.end();
679	711
...	...	@@ -693,7 +725,7 @@ fn parseReturn(p: *Parse) !NodeIndex {
693	725	});
694	726	}
695	727
696		fn parseBreak(p: *Parse) !NodeIndex {
	728	fn parseBreak(p: *Parse) Error!NodeIndex {
697	729	const trace = t.trace(@src());
698	730	defer trace.end();
699	731
...	...	@@ -719,7 +751,7 @@ fn parseBreak(p: *Parse) !NodeIndex {
719	751	});
720	752	}
721	753
722		fn parseBuiltinCall(p: *Parse) !NodeIndex {
	754	fn parseBuiltinCall(p: *Parse) Error!NodeIndex {
723	755	const trace = t.trace(@src());
724	756	defer trace.end();
725	757
...	...	@@ -736,7 +768,7 @@ fn parseBuiltinCall(p: *Parse) !NodeIndex {
736	768	});
737	769	}
738	770
739		fn parseCallArgs(p: *Parse) anyerror!SubList {
	771	fn parseCallArgs(p: *Parse) Error!SubList {
740	772	const trace = t.trace(@src());
741	773	defer trace.end();
742	774
...	...	@@ -764,7 +796,7 @@ fn parseCallArgs(p: *Parse) anyerror!SubList {
764	796	return p.scratchToSubList(scratch_top);
765	797	}
766	798
767		fn parseIf(p: *Parse) !NodeIndex {
	799	fn parseIf(p: *Parse) Error!NodeIndex {
768	800	const trace = t.trace(@src());
769	801	defer trace.end();
770	802
...	...	@@ -795,7 +827,7 @@ fn parseIf(p: *Parse) !NodeIndex {
795	827	});
796	828	}
797	829
798		fn parseElseValue(p: *Parse) anyerror!NodeIndex {
	830	fn parseElseValue(p: *Parse) Error!NodeIndex {
799	831	const trace = t.trace(@src());
800	832	defer trace.end();
801	833
...	...	@@ -810,7 +842,7 @@ fn parseElseValue(p: *Parse) anyerror!NodeIndex {
810	842	};
811	843	}
812	844
813		fn parseGroup(p: *Parse) !NodeIndex {
	845	fn parseGroup(p: *Parse) Error!NodeIndex {
814	846	const trace = t.trace(@src());
815	847	defer trace.end();
816	848
...	...	@@ -855,7 +887,7 @@ fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
855	887	fn fail(p: *Parse, ast_error: AstError) Error {
856	888	p.ast_error = ast_error;
857	889	std.debug.print("got {s}", .{@tagName(p.currentTag())});
858		return Error.SyntaxError;
	890	return Error.ParseFail;
859	891	}
860	892
861	893	fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
...	...	@@ -959,3 +991,6 @@ const optionalNodeIndex = OptionalNodeIndex.init;
959	991	const optionalExtraIndex = OptionalExtraIndex.init;
960	992
961	993	const t = @import("tracer");
	994
	995	const Report = @import("Report.zig");
	996	const Comp = @import("Comp.zig");

src/compiler/Report.zig+466-15

...	...	@@ -1,38 +1,489 @@
	1	//! A Report is a list of errors and warnings for a part of a compilation.
	2	//!
	3	//! `Err/Warn` structs are unions of every possible message, the logic
	4	//! for printing them is all done here.
	5	//!
	6	//! Across the compiler, `Report.Builder`s' `.err` and `.warn` methods
	7	//! can add messages by ID.
1	8	errors: []Item = &.{},
2		warns: []Item = &.{},
	9	warnings: []Item = &.{},
3	10	string_bytes: []u8 = &.{},
4	11
5	12	pub const Item = struct {
6		offset: ByteOffset,
7		message: String,
	13	/// What is this diagnostic
	14	data: union { @"error": Err, warn: Warn },
	15	/// What file was it in
	16	url: String,
	17
	18	/// Type is never stored, inferred from which list it is in.
	19	pub const Type = enum { @"error", warn };
	20	};
	21
	22	pub const Err = union(enum) {
	23	pub const FieldEnum = std.meta.FieldEnum(Err);
	24
	25	/// Constructs a `Report` with just a single error.
	26	pub fn toReport(err: Err, comp: Comp, file: File) !Report {
	27	const string_bytes = try comp.gpa.dupe(u8, file.file_path);
	28	errdefer comp.gpa.free(string_bytes);
	29
	30	const errors = try comp.gpa.dupe(Item, &.{.{
	31	.data = .{ .@"error" = err },
	32	.url = .{ .start = 0, .len = @intCast(string_bytes.len) },
	33	}});
	34	errdefer comp.gpa.free(errors);
	35
	36	return .{ .errors = errors, .string_bytes = string_bytes };
	37	}
	38
	39	ast: @import("Parse.zig").AstError,
	40
	41	unknown_identifier: struct {
	42	node: NodeIndex,
	43
	44	fn build(data: @This(), ctx: *MsgContext) !void {
	45	try ctx.msgNode(data.node, "unknown identifier '{s}'", .{
	46	ctx.ast.basicNodeSlice(data.node),
	47	});
	48	}
	49	},
	50	invalid_builtin: struct {
	51	node: NodeIndex,
	52
	53	fn build(data: @This(), ctx: *MsgContext) !void {
	54	try ctx.msgToken(
	55	ctx.ast.nodeToken(data.node),
	56	"unknown built-in function '{s}'",
	57	.{ctx.ast.basicNodeSlice(data.node)},
	58	);
	59	}
	60	},
	61	invalid_discard: struct {
	62	node: NodeIndex,
	63
	64	fn build(data: @This(), ctx: *MsgContext) !void {
	65	try ctx.msgNode(data.node, "local cannot be named '_'", .{});
	66	}
	67	},
	68	duplicate_function: struct {
	69	node: NodeIndex,
	70	prev: NodeIndex,
	71
	72	fn build(data: @This(), ctx: *MsgContext) !void {
	73	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of '{s}'", .{
	74	ctx.ast.fullFnDecl(data.node).id,
	75	});
	76	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{});
	77	// TODO: be able to store a full list of other declarations
	78	// and add them here as "and here" and so on
	79	}
	80	},
	81	duplicate_param: struct {
	82	node: NodeIndex,
	83	prev: NodeIndex,
	84
	85	fn build(data: @This(), ctx: *MsgContext) !void {
	86	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of '{s}'", .{
	87	ctx.ast.fullParamDecl(data.node).id,
	88	});
	89	try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{});
	90	// TODO: be able to store a full list of other declarations
	91	// and add them here as "and here" and so on
	92	}
	93	},
	94	invalid_return: struct {
	95	node: NodeIndex,
	96
	97	fn build(data: @This(), ctx: *MsgContext) !void {
	98	try ctx.msgNode(
	99	data.node,
	100	"return cannot be used here",
	101	.{},
	102	);
	103	}
	104	},
	105	unresolved_label: struct {
	106	node: NodeIndex,
	107
	108	fn build(data: @This(), ctx: *MsgContext) !void {
	109	try ctx.msg(
	110	.{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 },
	111	"unknown block label '{s}'",
	112	.{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)},
	113	);
	114	}
	115	},
	116	shadow_builtin_variable: struct {
	117	node: NodeIndex,
	118
	119	fn build(data: @This(), ctx: *MsgContext) !void {
	120	try ctx.msgToken(
	121	ctx.ast.nodeToken(data.node) + 1,
	122	"declaration shadows built-in constant '{s}'",
	123	.{ctx.ast.fullVarDecl(data.node).id},
	124	);
	125	}
	126	},
	127	};
	128
	129	pub const Warn = union(enum) {
	130	pub const FieldEnum = std.meta.FieldEnum(Warn);
	131
	132	unused: struct {
	133	node: NodeIndex,
	134	kind: UnusedKind,
	135
	136	fn build(data: @This(), ctx: *MsgContext) !void {
	137	try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} '{s}'", .{
	138	@tagName(data.kind),
	139	switch (ctx.ast.nodeTag(data.node)) {
	140	.param_decl => ctx.ast.fullParamDecl(data.node).id,
	141	.fn_decl => ctx.ast.fullFnDecl(data.node).id,
	142	.var_decl => ctx.ast.fullVarDecl(data.node).id,
	143	else => unreachable, // not a valid symbol node
	144	},
	145	});
	146	}
	147	},
	148	};
	149
	150	pub const UnusedKind = enum {
	151	param,
	152	constant,
	153	variable,
	154	function,
8	155	};
9	156
10	157	pub const String = struct {
11	158	start: u32,
12	159	len: u32,
13	160
14		pub fn message(item: Item, report: Report) []const u8 {
15		return report.string_bytes[item.message_start..][0..item.message_len];
	161	pub fn slice(str: String, report: Report) []const u8 {
	162	return report.string_bytes[str.start..][0..str.len];
	163	}
	164	};
	165
	166	const Prefix = enum {
	167	@"error",
	168	warn,
	169	note,
	170
	171	fn color(prefix: Prefix) std.io.tty.Color {
	172	return switch (prefix) {
	173	.@"error" => .red,
	174	.warn => .yellow,
	175	.note => .blue,
	176	};
16	177	}
17	178	};
18	179
19		pub fn hasErrors(report: Report) bool {
20		return report.errors.len > 0;
	180	pub fn deinit(report: Report, gpa: Allocator) void {
	181	gpa.free(report.errors);
	182	gpa.free(report.warnings);
	183	gpa.free(report.string_bytes);
21	184	}
22	185
23		pub fn hasWarns(report: Report) bool {
24		return report.warns.len > 0;
	186	pub fn dumpToWriter(report: Report, comp: *Comp, writer: anytype) !void {
	187	const tty = util.detectTTYFromWriter(writer);
	188	try writer.print("{d} errors, {d} warnings\n", .{ report.errors.len, report.warnings.len });
	189	for (report.errors) \|item\| {
	190	try report.formatItem(item, .@"error", comp, writer, tty);
	191	}
	192	for (report.warnings) \|item\| {
	193	try report.formatItem(item, .warn, comp, writer, tty);
	194	}
25	195	}
26	196
27		pub const Wip = struct {
28		errors: ArrayListUnmanaged(Item),
29		warns: ArrayListUnmanaged(Item),
30		string_bytes: ArrayListUnmanaged(u8),
	197	pub fn formatItem(
	198	report: Report,
	199	item: Item,
	200	item_type: Item.Type,
	201	comp: *Comp,
	202	writer: anytype,
	203	tty: std.io.tty.Config,
	204	) !void {
	205	var arena = std.heap.ArenaAllocator.init(comp.gpa);
	206	defer arena.deinit();
	207	var sfa = std.heap.stackFallback(16384, arena.allocator());
	208	const alloc = sfa.get();
	209
	210	const file = try comp.getFile(item.url.slice(report));
	211
	212	const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast);
	213	const ast_data = if (need_load_ast) try file.getAst(comp) else undefined;
	214	const ast = if (need_load_ast) &ast_data else null;
	215
	216	var ctx: MsgContext = .{
	217	.arena = alloc,
	218	.comp = comp,
	219	.file = file,
	220	.ast = &ast_data,
	221	.components = ArrayListUnmanaged(Component).initCapacity(alloc, 16) catch
	222	unreachable, // stack-fallback allocator has enough space.
	223	};
	224
	225	switch (item_type) {
	226	inline else => \|tag\| switch (@field(item.data, @tagName(tag))) {
	227	inline else => \|payload\| try payload.build(&ctx),
	228	},
	229	}
	230
	231	assert(ctx.components.items.len > 0);
	232
	233	for (ctx.components.items, 1..) \|component, i\| {
	234	const prefix: Prefix = if (i == 1)
	235	switch (item_type) {
	236	inline else => \|tag\| @field(Prefix, @tagName(tag)),
	237	}
	238	else
	239	.note;
	240
	241	const is_last = i == ctx.components.items.len;
	242
	243	if (i != 1) {
	244	try tty.setColor(writer, .dim);
	245	try writer.writeAll(" │ ");
	246	try tty.setColor(writer, .reset);
	247	}
	248
	249	try tty.setColor(writer, prefix.color());
	250	try writer.writeAll(@tagName(prefix));
	251	try tty.setColor(writer, .reset);
	252	try tty.setColor(writer, .dim);
	253	try writer.writeAll(": ");
	254	try tty.setColor(writer, .reset);
	255	try writer.writeAll(component.message_utf8);
	256	try writer.writeAll("\n");
	257
	258	const SliceLoc = struct {
	259	loc: Location,
	260	byte: ByteOffset,
	261	slice: []const u8,
	262	};
	263
	264	const utf8_source = (try file.getSource(comp.gpa)).code;
	265
	266	const at: SliceLoc = switch (component.highlight) {
	267	inline .node,
	268	.token,
	269	.two_tokens,
	270	=> \|payload, tag\| loc: {
	271	const token_index: TokenIndex = switch (tag) {
	272	.node => ast.?.nodeToken(payload),
	273	.token, .two_tokens => payload,
	274	else => comptime unreachable,
	275	};
	276
	277	const location = ast.?.tokenLocation(0, token_index);
	278	const token = ast.?.tokens.get(token_index);
	279
	280	break :loc .{
	281	.loc = location,
	282	.slice = token.slice(ast.?.utf8_source),
	283	.byte = token.offset,
	284	};
	285	},
	286
	287	inline .codepoint,
	288	.codepoint_eol,
	289	=> \|b, tag\| loc: {
	290	const len = switch (tag) {
	291	.codepoint => std.unicode.utf8ByteSequenceLength(utf8_source[b]) catch 1,
	292	.codepoint_eol => if (std.mem.indexOfScalar(u8, utf8_source[b..], '\n')) \|n\|
	293	n
	294	else
	295	(utf8_source.len - b),
	296	else => comptime unreachable,
	297	};
	298	break :loc .{
	299	.byte = b,
	300	.slice = utf8_source[b..][0..len],
	301	.loc = .{
	302	.column = 0,
	303	.line = 0,
	304	.line_start = 0,
	305	.line_end = utf8_source.len,
	306	},
	307	};
	308	},
	309	};
	310
	311	const line_before = utf8_source[at.loc.line_start..at.byte];
	312	const trimmed_before = std.mem.trimLeft(u8, line_before, " \t\r");
	313	const line_after = utf8_source[at.byte + at.slice.len .. at.loc.line_end + 1];
	314	const width = std.fmt.count("{d}", .{at.loc.line});
	315
	316	try tty.setColor(writer, .dim);
	317	try writer.writeByteNTimes(' ', width);
	318	try writer.print("{s}{s}:{d}:{d}\n", .{
	319	if (i == 1) "┬ " else "├ ",
	320	file.file_path,
	321	at.loc.line,
	322	at.loc.column,
	323	});
	324	try writer.print("{d}", .{at.loc.line});
	325	try writer.print("{s} ", .{"│"});
	326	try tty.setColor(writer, .reset);
	327	try writer.writeAll(trimmed_before);
	328	try tty.setColor(writer, prefix.color());
	329	try tty.setColor(writer, .bold);
	330	try writer.writeAll(at.slice);
	331	try tty.setColor(writer, .reset);
	332	try writer.writeAll(line_after);
	333	try tty.setColor(writer, .dim);
	334	try writer.writeAll(if (is_last) " ╰" else " ├");
	335	try writer.writeBytesNTimes(
	336	"─",
	337	width +
	338	(at.byte - at.loc.line_start) -
	339	(line_before.len - trimmed_before.len),
	340	);
	341	try tty.setColor(writer, .reset);
	342	try tty.setColor(writer, prefix.color());
	343	try tty.setColor(writer, .bold);
	344	try writer.writeBytesNTimes("━", at.slice.len);
	345	try tty.setColor(writer, .reset);
	346	try writer.writeAll("\n");
	347	}
	348	}
	349
	350	pub fn dumpToStdErr(report: Report, comp: *Comp) !void {
	351	const stderr = std.io.getStdErr().writer();
	352
	353	// Disabled because of coloring issues.
	354	// var buf = std.io.bufferedWriter(stderr);
	355	// const writer = buf.writer();
	356	try dumpToWriter(report, comp, stderr);
	357	// try buf.flush();
	358	}
	359
	360	pub const Builder = struct {
	361	gpa: Allocator,
	362	current_file: *Comp.File,
	363	errors: ArrayListUnmanaged(Item) = .{},
	364	warnings: ArrayListUnmanaged(Item) = .{},
	365	string_bytes: ArrayListUnmanaged(u8) = .{},
	366	current_file_str: ?String = null,
	367
	368	pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void {
	369	try b.errors.append(b.gpa, .{
	370	.data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) },
	371	.url = try b.currentFileString(),
	372	});
	373	}
	374
	375	pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void {
	376	try b.warnings.append(b.gpa, .{
	377	.data = .{ .warn = @unionInit(Warn, @tagName(kind), data) },
	378	.url = try b.currentFileString(),
	379	});
	380	}
	381
	382	pub fn addString(b: *Builder, str: []const u8) !String {
	383	const top = b.string_bytes.items.len;
	384	try b.string_bytes.appendSlice(b.gpa, str);
	385	return .{ .start = @intCast(top), .len = @intCast(str.len) };
	386	}
	387
	388	pub fn currentFileString(b: *Builder) !String {
	389	return b.current_file_str orelse {
	390	b.current_file_str = try b.addString(b.current_file.file_path);
	391	return b.current_file_str.?;
	392	};
	393	}
	394
	395	pub fn deinit(b: *Builder) void {
	396	b.errors.deinit(b.gpa);
	397	b.warnings.deinit(b.gpa);
	398	}
	399
	400	pub fn finalize(b: *Builder) !Report {
	401	defer b.* = undefined;
	402	errdefer b.deinit();
	403
	404	const errors = try b.errors.toOwnedSlice(b.gpa);
	405	errdefer b.gpa.free(errors);
	406
	407	const warnings = try b.warnings.toOwnedSlice(b.gpa);
	408	errdefer b.gpa.free(warnings);
	409
	410	const string_bytes = try b.string_bytes.toOwnedSlice(b.gpa);
	411	errdefer b.gpa.free(string_bytes);
	412
	413	return .{
	414	.errors = errors,
	415	.warnings = warnings,
	416	.string_bytes = string_bytes,
	417	};
	418	}
	419	};
	420
	421	pub const Highlight = union(enum) {
	422	// Highlight one token
	423	token: TokenIndex,
	424	// Highlight two tokens
	425	two_tokens: TokenIndex,
	426	// Highlight a Node
	427	node: NodeIndex,
	428	// Highlight a codepoint
	429	codepoint: ByteOffset,
	430	// Highlight from the start of a codepoint to '\n'
	431	codepoint_eol: ByteOffset,
	432	};
	433
	434	const Component = struct {
	435	message_utf8: []const u8,
	436	highlight: Highlight,
	437	};
	438
	439	/// This serves as a small API to make creating messages easier.
	440	pub const MsgContext = struct {
	441	file: *File,
	442	comp: *Comp,
	443	ast: *const Ast,
	444
	445	/// This arena is is a stack fallback with 16kb, then
	446	/// falling back to an arena allocator. This should
	447	/// make the cost of arbitrarily creating `Component`
	448	/// nodes extremely lightweight.
	449	arena: Allocator,
	450
	451	/// First component is "error" or "warn"
	452	/// Trailing components are "note"
	453	components: ArrayListUnmanaged(Component),
	454
	455	pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void {
	456	const text = try std.fmt.allocPrint(m.arena, fmt, args);
	457	try m.components.append(m.arena, .{
	458	.message_utf8 = text,
	459	.highlight = highlight,
	460	});
	461	}
	462
	463	pub fn msgNode(m: *MsgContext, node: NodeIndex, comptime fmt: []const u8, args: anytype) !void {
	464	return m.msg(.{ .node = node }, fmt, args);
	465	}
	466
	467	pub fn msgToken(m: *MsgContext, token: TokenIndex, comptime fmt: []const u8, args: anytype) !void {
	468	return m.msg(.{ .token = token }, fmt, args);
	469	}
31	470	};
32	471
33	472	const Report = @This();
34	473	const std = @import("std");
35	474	const ArrayListUnmanaged = std.ArrayListUnmanaged;
	475	const Allocator = std.mem.Allocator;
36	476	const assert = std.debug.assert;
37		pub const Tokenizer = @import("Tokenizer.zig");
38		pub const ByteOffset = Tokenizer.ByteOffset;
	477
	478	const Tokenizer = @import("Tokenizer.zig");
	479	const ByteOffset = Tokenizer.ByteOffset;
	480
	481	const Ast = @import("Ast.zig");
	482	const TokenIndex = Ast.TokenIndex;
	483	const NodeIndex = Ast.NodeIndex;
	484	const Location = Ast.Location;
	485
	486	const Comp = @import("Comp.zig");
	487	const File = Comp.File;
	488
	489	const util = @import("util.zig");

src/compiler/Sema.zig+11-32

...	...	@@ -37,7 +37,6 @@ const InstMap = struct {
37	37
38	38	pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
39	39	const decl = code.declData(inst_start);
40		log.info("pre-allocating {d} map-indicies for InstMap", .{decl.instruction_count});
41	40	const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
42	41	@memset(insts, .none);
43	42	return .{
...	...	@@ -72,7 +71,6 @@ pub fn init(s: *Sema) !void {
72	71
73	72	try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
74	73
75		log.info("init sema on index {}", .{s.decl_index});
76	74	s.inst_map = try InstMap.init(
77	75	s.arena,
78	76	&s.code,
...	...	@@ -193,8 +191,6 @@ fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
193	191	const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
194	192	defer t.end();
195	193
196		log.info("runtime %{d} = .{s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
197
198	194	const out = try s.analyzeCodeInner(in);
199	195	if (s.code.instTag(in) != .block)
200	196	try s.inst_map.add(in, out);
...	...	@@ -226,8 +222,6 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
226	222	const string = rhs.asString(s.ip);
227	223
228	224	const base_type = s.typeOfRef(base);
229		log.info("process field with type {any} . {s}", .{ base_type, string.toSlice(s.ip) });
230
231	225	switch (base_type) {
232	226	.opaque_type => {
233	227	return s.addInstRef(.field, .{
...	...	@@ -277,18 +271,12 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
277	271	}
278	272
279	273	const left_type = s.typeOfRef(lhs);
	274	_ = left_type; // autofix
280	275	const right_type = s.typeOfRef(rhs);
	276	_ = right_type; // autofix
281	277
282	278	// TODO: validate type for operation
283	279
284		log.info("runtime {s} operation on {} and {}", .{
285		@tagName(tag),
286		left_type.toKey(s.ip),
287		right_type.toKey(s.ip),
288		});
289
290		// todo: type check
291
292	280	return s.addInstRef(tag.airTag(), .{
293	281	.lhs = lhs,
294	282	.rhs = rhs,
...	...	@@ -300,14 +288,10 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
300	288	const operand = try s.resolveRefRuntime(un.operand);
301	289
302	290	const operand_type = s.typeOfRef(operand);
	291	_ = operand_type; // autofix
303	292
304	293	// TODO: validate type for operation
305	294
306		log.info("runtime {s} operation on {}", .{
307		@tagName(tag),
308		operand_type.toKey(s.ip),
309		});
310
311	295	return s.addInstRef(tag.airTag(), .{
312	296	.operand = operand,
313	297	});
...	...	@@ -425,7 +409,6 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
425	409	fn markInst(s: *Sema, inst: Air.Inst.Index) void {
426	410	const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
427	411	flags.uses = flags.uses.increment();
428		log.info("markInst {}", .{inst});
429	412	}
430	413
431	414	fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
...	...	@@ -485,15 +468,13 @@ fn resolveRefComptime(s: *Sema, ref: Teg.Inst.Ref) !Value {
485	468	}
486	469	}
487	470
488		const tag = s.code.instTag(in);
489		log.info("comptime %{d} = .{s}", .{ @intFromEnum(in), @tagName(tag) });
490		switch (tag) {
	471	switch (s.code.instTag(in)) {
491	472	.string => {
492	473	const string: Teg.SlicedString = s.code.instPayload(.string, in);
493	474	return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
494	475	},
495	476
496		else => std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
	477	else => \|tag\| std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
497	478	}
498	479	}
499	480
...	...	@@ -509,12 +490,13 @@ fn resolveDecl(s: *Sema, decl_index: Decl.Index) !Value {
509	490	var arena = std.heap.ArenaAllocator.init(s.gpa);
510	491	defer arena.deinit();
511	492
	493	var todo: Report = undefined;
512	494	var sema: Sema = .{
513	495	.comp = s.comp,
514	496	.gpa = s.gpa,
515	497	.ip = s.ip,
516	498	.arena = arena.allocator(),
517		.code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.gpa),
	499	.code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.comp, &todo),
518	500	.decl = decl,
519	501	.decl_index = decl_index,
520	502	.namespace_index = decl.src_namespace,
...	...	@@ -529,10 +511,9 @@ fn resolveDecl(s: *Sema, decl_index: Decl.Index) !Value {
529	511
530	512	return value;
531	513	},
532		.complete => {
	514	else => {
533	515	return decl.val;
534	516	},
535		else => \|tag\| std.debug.panic("TODO: load_decl when decl is in state .{s}", .{@tagName(tag)}),
536	517	}
537	518	}
538	519
...	...	@@ -573,7 +554,6 @@ const IncompleteBlock = struct {
573	554	};
574	555
575	556	fn addBlock(s: *Sema) !IncompleteBlock {
576		log.info("start block at @{d}", .{s.air_instructions.len});
577	557	try s.air_instructions.append(s.gpa, .{
578	558	.tag = .block,
579	559	.data = undefined,
...	...	@@ -583,7 +563,6 @@ fn addBlock(s: *Sema) !IncompleteBlock {
583	563	}
584	564
585	565	fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
586		log.info("add air .{s}, {}", .{ @tagName(tag), data });
587	566	try s.air_instructions.append(s.gpa, .{
588	567	.tag = tag,
589	568	.data = Air.EncodedData.encode(tag, data),
...	...	@@ -627,11 +606,9 @@ fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
627	606	}
628	607
629	608	fn validateType(s: *Sema, checked_type: Value, wanted_type: InternPool.Index) !void {
	609	_ = s; // autofix
630	610	if (checked_type.toIntern() != wanted_type) {
631		log.info("failed type check {}, got {}", .{ Value.fromInterned(wanted_type).fmt(s.ip), checked_type.fmt(s.ip) });
632	611	return error.TypeCheckFailed;
633		} else {
634		log.info("passed type check {}", .{checked_type.fmt(s.ip)});
635	612	}
636	613	}
637	614
...	...	@@ -690,4 +667,6 @@ const InternPool = @import("InternPool.zig");
690	667	const NullTerminatedString = InternPool.NullTerminatedString;
691	668	const dod = @import("dod.zig");
692	669
	670	const Report = @import("Report.zig");
	671
693	672	const tracer = @import("tracer");

src/compiler/Teg.zig+4

...	...	@@ -3,6 +3,7 @@
3	3	insts: Inst.List.Slice,
4	4	string_bytes: []const u8,
5	5	extra: ExtraStore = .{},
	6	report: Report,
6	7
7	8	pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
8	9	pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
...	...	@@ -343,6 +344,7 @@ pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
343	344	gpa.free(teg.string_bytes);
344	345	teg.extra.deinit(gpa);
345	346	teg.insts.deinit(gpa);
	347	teg.report.deinit(gpa);
346	348	teg.* = undefined;
347	349	}
348	350
...	...	@@ -433,3 +435,5 @@ const Float = dod.Float;
433	435	const InternPool = @import("InternPool.zig");
434	436
435	437	const Air = @import("Air.zig");
	438
	439	const Report = @import("Report.zig");

src/compiler/Tokenizer.zig+96-79

...	...	@@ -5,18 +5,14 @@
5	5	//! Retriving payloads like string/numeric values, identifier names, etc is
6	6	//! done separately, either in Parse or Transform, when the respective data
7	7	//! is actually needed. Tokenization does guarante that such data is valid.
8
9		// TODO: make `code` a [:0]const u8. this doesnt disallow null bytes, but
10		// it requires the source to end with one. This can make byteAt(1) return u8
11		// instead of an optional. I am told this improves performance.
12		code: []const u8,
	8	code: [:0]const u8,
13	9	pos: ByteOffset = 0,
14	10
15	11	/// Tokenize source code into a list of tokens.
16	12	/// When an error occurs, the position of the error is returned in `error_pos`.
17	13	/// Caller owns returned slice.
18	14	pub fn tokenize(
19		source_code: []const u8,
	15	source_code: [:0]const u8,
20	16	gpa: mem.Allocator,
21	17	error_pos: *ByteOffset,
22	18	) TokenizeError!Token.List.Slice {
...	...	@@ -233,7 +229,7 @@ pub const Token = struct {
233	229	///
234	230	/// If you are computing the length of something that is knowable
235	231	/// without the source code, use token.tag.staticLen() instead.
236		pub fn len(token: Token, utf8_source: []const u8) u32 {
	232	pub fn len(token: Token, utf8_source: [:0]const u8) u32 {
237	233	return token.tag.staticLen() orelse {
238	234	var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
239	235	assert(
...	...	@@ -246,7 +242,7 @@ pub const Token = struct {
246	242	};
247	243	}
248	244
249		pub fn slice(token: Token, utf8_source: []const u8) []const u8 {
	245	pub fn slice(token: Token, utf8_source: [:0]const u8) []const u8 {
250	246	return utf8_source[token.offset..][0..token.len(utf8_source)];
251	247	}
252	248
...	...	@@ -294,7 +290,7 @@ pub fn next(t: *Tokenizer) !?Token {
294	290	const trace = T.trace(@src());
295	291	defer trace.end();
296	292
297		switch (t.byte() orelse return null) {
	293	switch (t.byte()) {
298	294	' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
299	295
300	296	'a'...'z', 'A'...'Z', '_', '$' => {
...	...	@@ -303,7 +299,7 @@ pub fn next(t: *Tokenizer) !?Token {
303	299	return try t.tokenizeIdentifier(start);
304	300	},
305	301
306		'@' => switch (t.byteAt(1) orelse 0) {
	302	'@' => switch (t.nextByte()) {
307	303	'"' => {
308	304	const start = t.pos;
309	305	t.pos += 1;
...	...	@@ -319,12 +315,12 @@ pub fn next(t: *Tokenizer) !?Token {
319	315
320	316	'0'...'9' => return try t.tokenizeNumber(),
321	317
322		'.' => switch (t.byteAt(1) orelse 0) {
	318	'.' => switch (t.nextByte()) {
323	319	'0'...'9' => return try t.tokenizeNumber(),
324	320	else => return t.simpleToken(.dot),
325	321	},
326	322
327		'/' => switch (t.byteAt(1) orelse 0) {
	323	'/' => switch (t.nextByte()) {
328	324	'=' => return t.simpleToken(.slash_equal),
329	325	'/' => {
330	326	if ((t.byteAt(2) orelse 0 == '/') and
...	...	@@ -349,7 +345,7 @@ pub fn next(t: *Tokenizer) !?Token {
349	345	else => return t.simpleToken(.slash),
350	346	},
351	347
352		'\\' => switch (t.byteAt(1) orelse 0) {
	348	'\\' => switch (t.nextByte()) {
353	349	'\\' => {
354	350	const start = t.pos;
355	351	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
...	...	@@ -360,30 +356,30 @@ pub fn next(t: *Tokenizer) !?Token {
360	356
361	357	'"' => return try t.tokenizeString(),
362	358
363		'&' => switch (t.byteAt(1) orelse 0) {
	359	'&' => switch (t.nextByte()) {
364	360	'&' => return t.simpleToken(.and_and),
365	361	'=' => return t.simpleToken(.and_equal),
366	362	else => return t.simpleToken(.@"and"),
367	363	},
368		'\|' => switch (t.byteAt(1) orelse 0) {
	364	'\|' => switch (t.nextByte()) {
369	365	'\|' => return t.simpleToken(.pipe_pipe),
370	366	'=' => return t.simpleToken(.pipe_equal),
371	367	else => return t.simpleToken(.pipe),
372	368	},
373		'^' => switch (t.byteAt(1) orelse 0) {
	369	'^' => switch (t.nextByte()) {
374	370	'^' => return t.simpleToken(.caret),
375	371	'=' => return t.simpleToken(.caret_equal),
376	372	else => return t.simpleToken(.caret),
377	373	},
378		'!' => switch (t.byteAt(1) orelse 0) {
	374	'!' => switch (t.nextByte()) {
379	375	'=' => return t.simpleToken(.equal_equal),
380	376	else => return t.simpleToken(.bang_equal),
381	377	},
382		'=' => switch (t.byteAt(1) orelse 0) {
	378	'=' => switch (t.nextByte()) {
383	379	'=' => return t.simpleToken(.equal_equal),
384	380	else => return t.simpleToken(.equal),
385	381	},
386		'>' => switch (t.byteAt(1) orelse 0) {
	382	'>' => switch (t.nextByte()) {
387	383	'=' => return t.simpleToken(.gt_equal),
388	384	'>' => switch (t.byteAt(2) orelse 0) {
389	385	'>' => return t.simpleToken(.gt_gt_gt),
...	...	@@ -391,7 +387,7 @@ pub fn next(t: *Tokenizer) !?Token {
391	387	},
392	388	else => return t.simpleToken(.gt),
393	389	},
394		'<' => switch (t.byteAt(1) orelse 0) {
	390	'<' => switch (t.nextByte()) {
395	391	'=' => return t.simpleToken(.lt_equal),
396	392	'<' => switch (t.byteAt(2) orelse 0) {
397	393	'=' => return t.simpleToken(.lt_lt_equal),
...	...	@@ -399,14 +395,12 @@ pub fn next(t: *Tokenizer) !?Token {
399	395	// Git merge conflicts are <<<<<<<
400	396	const start = t.pos;
401	397	t.pos += 2;
402		brk: {
403		var remain: u3 = 4;
404		while (t.byte() orelse break :brk == '<') {
405		t.pos += 1;
406		remain -= 1;
407		if (remain == 0) {
408		return error.GitConflict;
409		}
	398	var remain: u3 = 4;
	399	while (t.byte() == '<') {
	400	t.pos += 1;
	401	remain -= 1;
	402	if (remain == 0) {
	403	return error.GitConflict;
410	404	}
411	405	}
412	406	t.pos = start;
...	...	@@ -415,25 +409,25 @@ pub fn next(t: *Tokenizer) !?Token {
415	409	},
416	410	else => return t.simpleToken(.lt),
417	411	},
418		'+' => switch (t.byteAt(1) orelse 0) {
	412	'+' => switch (t.nextByte()) {
419	413	'=' => return t.simpleToken(.plus_equal),
420	414	else => return t.simpleToken(.plus),
421	415	},
422		'-' => switch (t.byteAt(1) orelse 0) {
	416	'-' => switch (t.nextByte()) {
423	417	'=' => return t.simpleToken(.minus_equal),
424	418	'>' => return t.simpleToken(.minus_lt),
425	419	else => return t.simpleToken(.minus),
426	420	},
427		'*' => switch (t.byteAt(1) orelse 0) {
	421	'*' => switch (t.nextByte()) {
428	422	'*' => return t.simpleToken(.star_star),
429	423	'=' => return t.simpleToken(.star_equal),
430	424	else => return t.simpleToken(.star),
431	425	},
432		'%' => switch (t.byteAt(1) orelse 0) {
	426	'%' => switch (t.nextByte()) {
433	427	'=' => return t.simpleToken(.percent_equal),
434	428	else => return t.simpleToken(.percent),
435	429	},
436		'~' => switch (t.byteAt(1) orelse 0) {
	430	'~' => switch (t.nextByte()) {
437	431	'=' => return t.simpleToken(.tilde_equal),
438	432	else => return t.simpleToken(.tilde),
439	433	},
...	...	@@ -462,19 +456,40 @@ pub fn next(t: *Tokenizer) !?Token {
462	456	},
463	457
464	458	else => return error.InvalidSyntax,
	459
	460	0 => {
	461	if (t.pos == t.code.len) return null;
	462	return error.InvalidSyntax;
	463	},
465	464	}
466	465	}
467	466	}
468	467
469		fn byte(t: Tokenizer) ?u8 {
470		if (t.pos >= t.code.len) return null;
	468	/// Source code is null terminated, so accessing this is safe
	469	fn byte(t: Tokenizer) u8 {
471	470	return t.code[t.pos];
472	471	}
473	472
	473	/// Source code is null terminated.
	474	fn nextByte(t: Tokenizer) u8 {
	475	assert(t.byte() < 0x80);
	476	return t.code[t.pos + 1];
	477	}
	478
474	479	/// Remember that using this only makes sense if you know for certain that all
475	480	/// previous bytes are ASCII, otherwise you may hit UB
476		fn byteAt(t: Tokenizer, offset: u32) ?u8 {
477		if (t.pos + offset >= t.code.len) return null;
	481	fn byteAt(t: Tokenizer, comptime offset: u32) ?u8 {
	482	if (offset == 1) @compileError("use nextByte()");
	483
	484	// assert that the previous bytes are ASCII
	485	// current byte is allowed to be a continuation byte
	486	for (t.code[t.pos .. t.pos + offset]) \|b\| {
	487	assert(b < 0x80);
	488	}
	489	return t.code[t.pos + offset];
	490	}
	491
	492	fn byteAtRt(t: Tokenizer, offset: u32) ?u8 {
478	493	// assert that the previous bytes are ASCII
479	494	// current byte is allowed to be a continuation byte
480	495	for (t.code[t.pos .. t.pos + offset]) \|b\| {
...	...	@@ -484,7 +499,7 @@ fn byteAt(t: Tokenizer, offset: u32) ?u8 {
484	499	}
485	500
486	501	fn codePoint(t: Tokenizer) !struct { u3, u21 } {
487		const b = t.byte() orelse unreachable; // must check the first byte manually
	502	const b = t.byte(); // must check the first byte manually
488	503	assert(b >= 0x80); // must be a continuation byte
489	504
490	505	const byte_len = unicode.utf8ByteSequenceLength(b) catch
...	...	@@ -498,7 +513,7 @@ fn codePoint(t: Tokenizer) !struct { u3, u21 } {
498	513	}
499	514
500	515	fn isAtIdentifierStart(t: *Tokenizer) !bool {
501		switch (t.byte() orelse return false) {
	516	switch (t.byte()) {
502	517	'a'...'z', 'A'...'Z', '_', '$' => return true,
503	518	else => return false,
504	519	0x80...0xFF => {
...	...	@@ -513,7 +528,7 @@ fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
513	528	defer trace.end();
514	529
515	530	while (true) {
516		switch (t.byte() orelse break) {
	531	switch (t.byte()) {
517	532	'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
518	533	t.pos += 1;
519	534	},
...	...	@@ -555,8 +570,8 @@ fn tokenizeNumber(t: *Tokenizer) !?Token {
555	570	defer trace.end();
556	571
557	572	// Caller ensures the following invariant:
558		switch (t.byte() orelse unreachable) {
559		'.' => switch (t.byteAt(1) orelse unreachable) {
	573	switch (t.byte()) {
	574	'.' => switch (t.nextByte()) {
560	575	'0'...'9' => {},
561	576	else => unreachable,
562	577	},
...	...	@@ -566,7 +581,7 @@ fn tokenizeNumber(t: *Tokenizer) !?Token {
566	581
567	582	// Parse Hex, Binary, Octal numbers
568	583	if (t.byte() == '0') {
569		switch (t.byteAt(1) orelse 0) {
	584	switch (t.nextByte()) {
570	585	'x', 'X' => {
571	586	t.pos += 2;
572	587	try t.consumeIntegerRadix(.hex);
...	...	@@ -594,8 +609,8 @@ fn tokenizeNumber(t: *Tokenizer) !?Token {
594	609	try t.consumeInteger();
595	610
596	611	// decimal point notation
597		switch (t.byte() orelse 0) {
598		'.' => switch (t.byteAt(1) orelse 0) {
	612	switch (t.byte()) {
	613	'.' => switch (t.nextByte()) {
599	614	'0'...'9' => {
600	615	t.pos += 1;
601	616	tag = .number_float;
...	...	@@ -611,15 +626,15 @@ fn tokenizeNumber(t: *Tokenizer) !?Token {
611	626	}
612	627	}
613	628
614		switch (t.byte() orelse 0) {
	629	switch (t.byte()) {
615	630	'e', 'E' => {
616	631	tag = .number_float;
617	632	t.pos += 1;
618	633
619		switch (t.byte() orelse 0) {
	634	switch (t.byte()) {
620	635	'+', '-' => {
621	636	t.pos += 1;
622		switch (t.byte() orelse 0) {
	637	switch (t.byte()) {
623	638	'0'...'9' => {},
624	639	else => return t.maybeUTF8Error(error.InvalidSyntax),
625	640	}
...	...	@@ -631,8 +646,8 @@ fn tokenizeNumber(t: *Tokenizer) !?Token {
631	646	try t.consumeInteger();
632	647
633	648	// decimal point notation
634		switch (t.byte() orelse 0) {
635		'.' => switch (t.byteAt(1) orelse 0) {
	649	switch (t.byte()) {
	650	'.' => switch (t.nextByte()) {
636	651	'0'...'9' => {
637	652	t.pos += 1;
638	653	tag = .number_float;
...	...	@@ -662,7 +677,7 @@ fn maybeConsumeInteger(t: *Tokenizer) !?void {
662	677	const trace = T.trace(@src());
663	678	defer trace.end();
664	679
665		switch (t.byte() orelse return null) {
	680	switch (t.byte()) {
666	681	'0'...'9' => {
667	682	t.pos += 1;
668	683	},
...	...	@@ -676,7 +691,7 @@ fn consumeInteger(t: *Tokenizer) !void {
676	691	const trace = T.trace(@src());
677	692	defer trace.end();
678	693
679		switch (t.byte() orelse unreachable) {
	694	switch (t.byte()) {
680	695	'0'...'9' => {
681	696	t.pos += 1;
682	697	},
...	...	@@ -684,13 +699,13 @@ fn consumeInteger(t: *Tokenizer) !void {
684	699	}
685	700
686	701	while (true) {
687		switch (t.byte() orelse return) {
	702	switch (t.byte()) {
688	703	'0'...'9' => {
689	704	t.pos += 1;
690	705	},
691	706	'_' => {
692	707	t.pos += 1;
693		switch (t.byte() orelse 0) {
	708	switch (t.byte()) {
694	709	'0'...'9' => {
695	710	t.pos += 1;
696	711	},
...	...	@@ -715,7 +730,7 @@ fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
715	730	};
716	731
717	732	first: {
718		const b = t.byte() orelse return;
	733	const b = t.byte();
719	734	inline for (digits) \|digit\| {
720	735	if (b == digit) {
721	736	break :first;
...	...	@@ -728,7 +743,7 @@ fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
728	743
729	744	while (true) {
730	745	brk: {
731		const b = t.byte() orelse return;
	746	const b = t.byte();
732	747	inline for (digits) \|digit\| {
733	748	if (b == digit) {
734	749	break :brk;
...	...	@@ -741,7 +756,7 @@ fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
741	756	}
742	757
743	758	fn tokenizeString(t: *Tokenizer) !Token {
744		assert(t.byte().? == '"');
	759	assert(t.byte() == '"');
745	760	const start = t.pos;
746	761	try t.consumeString();
747	762	return .{ .tag = .string, .offset = start };
...	...	@@ -751,32 +766,30 @@ fn consumeString(t: *Tokenizer) !void {
751	766	const trace = T.trace(@src());
752	767	defer trace.end();
753	768
754		assert(t.byte().? == '"');
	769	const start = t.pos;
	770	assert(t.byte() == '"');
755	771
756	772	t.pos += 1;
757	773
758	774	while (true) {
759		const b = t.byte() orelse return error.UnterminatedString;
760		switch (b) {
	775	switch (t.byte()) {
761	776	'"' => {
762	777	t.pos += 1;
763	778	break;
764	779	},
765	780	'\\' => {
766		switch (t.byteAt(1) orelse {
767		// we want error.InvalidEscapeSequence to be at the '\' char,
768		// but we want error.UnterminatedString to be at the end of the source code
769		t.pos += 1;
770		return error.UnterminatedString;
771		}) {
772		'\n' => return error.UnterminatedString,
	781	switch (t.nextByte()) {
	782	'\n' => {
	783	t.pos = start;
	784	return error.UnterminatedString;
	785	},
773	786	'\\' => t.pos += 2,
774	787	'"' => t.pos += 2,
775	788	'n' => t.pos += 2,
776	789	'r' => t.pos += 2,
777	790	't' => t.pos += 2,
778	791	'0' => switch (t.byteAt(2) orelse {
779		t.pos += 2;
	792	t.pos = start;
780	793	return error.UnterminatedString;
781	794	}) {
782	795	// '\0xFF' hex escape
...	...	@@ -788,7 +801,7 @@ fn consumeString(t: *Tokenizer) !void {
788	801	const esc_start = t.pos;
789	802	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
790	803	error.InvalidEscapeSequence => {
791		if (t.byte().? == '\n')
	804	if (t.byte() == '\n')
792	805	return error.UnterminatedString;
793	806	t.pos = esc_start;
794	807	return err;
...	...	@@ -797,12 +810,16 @@ fn consumeString(t: *Tokenizer) !void {
797	810	t.pos = esc_start;
798	811	return err;
799	812	},
800		error.UnterminatedString => return err,
	813	// error.UnterminatedString => return err,
801	814	};
802	815	},
803	816	else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
804	817	}
805	818	},
	819	'\n' => {
	820	t.pos = start;
	821	return error.UnterminatedString;
	822	},
806	823	else => {
807	824	t.pos += 1;
808	825	},
...	...	@@ -818,16 +835,16 @@ fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
818	835	const trace = T.trace(@src());
819	836	defer trace.end();
820	837
821		assert(t.byte().? == '\\');
822		assert(t.byteAt(1).? == 'u');
	838	assert(t.byte() == '\\');
	839	assert(t.nextByte() == 'u');
823	840	t.pos += 2;
824	841
825		if ((t.byte() orelse return error.UnterminatedString) != '{')
	842	if (t.byte() != '{')
826	843	return error.InvalidEscapeSequence;
827	844	t.pos += 1;
828	845
829		if ((t.byte() orelse return error.UnterminatedString) == '0') {
830		if ((t.byteAt(1) orelse return error.UnterminatedString) == 'x') {
	846	if (t.byte() == '0') {
	847	if (t.nextByte() == 'x') {
831	848	t.pos += 2;
832	849	}
833	850	}
...	...	@@ -837,7 +854,7 @@ fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
837	854
838	855	const slice = t.code[start..t.pos];
839	856
840		if ((t.byte() orelse return error.UnterminatedString) != '}')
	857	if (t.byte() != '}')
841	858	return error.InvalidEscapeSequence;
842	859
843	860	const code_point = std.fmt.parseInt(u21, slice, 16) catch \|err\| switch (err) {
...	...	@@ -866,7 +883,7 @@ fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
866	883	.{ .name = @errorName(real_err) },
867	884	},
868	885	}) {
869		switch (t.byte() orelse return real_err) {
	886	switch (t.byte()) {
870	887	0x80...0xFF => {
871	888	_ = try t.codePoint();
872	889	return real_err;
...	...	@@ -881,7 +898,7 @@ fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
881	898	.{ .name = @errorName(real_err) },
882	899	},
883	900	}) {
884		switch (t.byteAt(at) orelse return real_err) {
	901	switch (t.byteAtRt(at) orelse return real_err) {
885	902	0x80...0xFF => {
886	903	_ = try t.codePoint();
887	904	return real_err;
...	...	@@ -895,7 +912,7 @@ pub fn skipWhitespace(t: *Tokenizer) void {
895	912	defer trace.end();
896	913
897	914	while (true) {
898		switch (t.byte() orelse break) {
	915	switch (t.byte()) {
899	916	' ', '\t', '\n', '\r' => {
900	917	t.pos += 1;
901	918	},

src/compiler/Transform.zig+95-28

...	...	@@ -14,29 +14,51 @@ scratch: std.ArrayListUnmanaged(u32) = .{},
14	14	call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
15	15	scopes: std.ArrayListUnmanaged(Scope) = .{},
16	16
17		pub fn transform(ast: Ast, gpa: Allocator) !Teg {
	17	report: Report.Builder,
	18
	19	pub const Error = error{
	20	OutOfMemory,
	21	TransformFail,
	22	};
	23
	24	pub fn transform(
	25	ast: Ast,
	26	gpa: Allocator,
	27	file: *Comp.File,
	28	fail_report: *Report,
	29	) Error!Teg {
18	30	const trace = tracer.trace(@src());
19	31	defer trace.end();
20	32
21	33	var t: Transform = .{
22	34	.ast = ast,
23	35	.gpa = gpa,
	36	.report = .{ .gpa = gpa, .current_file = file },
24	37	};
25	38	errdefer t.deinit();
26	39
27	40	try t.string_bytes.append(t.gpa, 0);
28	41
29		const module = try t.transformModule(0);
	42	const module = t.transformModule(0) catch \|err\| switch (err) {
	43	error.OutOfMemory => return err,
	44	};
	45
	46	const report = try t.report.finalize();
	47	if (report.errors.len > 0) {
	48	fail_report.* = report;
	49	return error.TransformFail;
	50	}
	51
30	52	assert(@intFromEnum(module) == 0);
31	53
32	54	try t.insts.setCapacity(t.gpa, t.insts.len);
33	55
34	56	defer t.deinit();
35
36	57	return Teg{
37	58	.insts = t.insts.toOwnedSlice(),
38	59	.string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
39	60	.extra = try t.extra.toOwnedStore(t.gpa),
	61	.report = report,
40	62	};
41	63	}
42	64
...	...	@@ -45,6 +67,9 @@ pub fn deinit(t: *Transform) void {
45	67	t.call_args.deinit(t.gpa);
46	68	assert(t.scopes.items.len == 0);
47	69	t.scopes.deinit(t.gpa);
	70	t.string_bytes.deinit(t.gpa);
	71	t.insts.deinit(t.gpa);
	72	t.extra.deinit(t.gpa);
48	73	t.* = undefined;
49	74	}
50	75
...	...	@@ -65,6 +90,25 @@ const Scope = struct {
65	90	};
66	91	};
67	92
	93	fn addUnusedWarnings(s: Scope, t: *Transform) !void {
	94	for (s.symbols.values()) \|value\| {
	95	if (!value.used) {
	96	try t.report.warn(.unused, .{
	97	.node = value.node,
	98	.kind = switch (value.data) {
	99	.declaration => .function,
	100	.param => .param,
	101	.mutable => .variable,
	102	.immutable => switch (t.ast.nodeTag(value.node)) {
	103	.param_decl => .param,
	104	else => .constant,
	105	},
	106	},
	107	});
	108	}
	109	}
	110	}
	111
68	112	fn deinit(s: *Scope, gpa: Allocator) void {
69	113	s.symbols.deinit(gpa);
70	114	s.* = undefined;
...	...	@@ -171,7 +215,7 @@ const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
171	215	// todo: nan/infinity?
172	216	});
173	217
174		fn transformModule(t: *Transform, module_node: NodeIndex) !Inst.Index {
	218	fn transformModule(t: *Transform, module_node: NodeIndex) Allocator.Error!Inst.Index {
175	219	const trace = tracer.trace(@src());
176	220	defer trace.end();
177	221
...	...	@@ -259,10 +303,12 @@ fn transformModule(t: *Transform, module_node: NodeIndex) !Inst.Index {
259	303	.extra_index = try t.extra.append(t.gpa, IndexList, declarations),
260	304	});
261	305
	306	try t.scope().addUnusedWarnings(t);
	307
262	308	return module;
263	309	}
264	310
265		fn previsitModuleStatement(t: *Transform, node: NodeIndex) !void {
	311	fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void {
266	312	const trace = tracer.trace(@src());
267	313	defer trace.end();
268	314
...	...	@@ -272,7 +318,11 @@ fn previsitModuleStatement(t: *Transform, node: NodeIndex) !void {
272	318	const fn_decl = t.ast.fullFnDecl(node);
273	319	const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
274	320	if (gop.found_existing) {
275		return error.DuplicateFunction;
	321	try t.report.err(.duplicate_function, .{
	322	.node = node,
	323	.prev = gop.value_ptr.node,
	324	});
	325	return;
276	326	}
277	327	gop.value_ptr.* = .{
278	328	.node = node,
...	...	@@ -299,7 +349,7 @@ fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex)
299	349	}
300	350	}
301	351
302		fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) !Inst.Index {
	352	fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) Allocator.Error!Inst.Index {
303	353	const trace = tracer.trace(@src());
304	354	defer trace.end();
305	355
...	...	@@ -350,6 +400,8 @@ fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) !
350	400	.extra_index = decl_extra,
351	401	});
352	402
	403	try t.scope().addUnusedWarnings(t);
	404
353	405	return func;
354	406	}
355	407
...	...	@@ -366,7 +418,7 @@ fn transformExpressionToBlock(
366	418	decl: BlockContext,
367	419	node: NodeIndex,
368	420	comptime is_function: bool,
369		) anyerror!Inst.Ref {
	421	) Allocator.Error!Inst.Ref {
370	422	const trace = tracer.trace(@src());
371	423	defer trace.end();
372	424
...	...	@@ -461,7 +513,7 @@ fn transformExpressionToBlock(
461	513	return ret_block.toRef();
462	514	}
463	515
464		fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) !Inst.Ref {
	516	fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Allocator.Error!Inst.Ref {
465	517	const trace = tracer.trace(@src());
466	518	defer trace.end();
467	519
...	...	@@ -472,7 +524,11 @@ fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) !Inst
472	524	// can use this param's value, if known at compile time.
473	525	const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
474	526	if (sym.found_existing) {
475		return error.DuplicateParam;
	527	try t.report.err(.duplicate_param, .{
	528	.node = param,
	529	.prev = sym.value_ptr.node,
	530	});
	531	return .void_value;
476	532	}
477	533
478	534	sym.value_ptr.* = .{
...	...	@@ -501,7 +557,8 @@ const Builtin = enum {
501	557	defer trace.end();
502	558
503	559	if (args.len != 1) {
504		return error.InvalidBuiltin;
	560	// try t.report.err(.invalid_builtin_args, .{ .id = "js" });
	561	return .void_value;
505	562	}
506	563
507	564	const source = try t.transformExpression(decl, args[0]) orelse return null;
...	...	@@ -518,7 +575,7 @@ const Builtin = enum {
518	575
519	576	/// This is the primary transformation function: converting an AST expression into instructions.
520	577	/// `null` represents a `return` or `break`, aka this branch of code is dead.
521		fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyerror!?Inst.Ref {
	578	fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Ref {
522	579	const node_tag = t.ast.nodes.items(.tag)[node];
523	580
524	581	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
...	...	@@ -537,17 +594,13 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
537	594	return ref;
538	595	}
539	596
540		if (mem.eql(u8, id, "_")) {
541		return error.InvalidDiscard;
542		}
543
544	597	if (mem.startsWith(u8, id, "@")) {
545	598	@panic("TODO: string identifiers");
546	599	}
547	600
548	601	const lookup = t.lookupId(id) orelse {
549		std.debug.print("lookupId failed for {s}\n", .{id});
550		return error.UnknownIdentifier;
	602	try t.report.err(.unknown_identifier, .{ .node = node });
	603	return .void_value;
551	604	};
552	605
553	606	return switch (lookup.data) {
...	...	@@ -706,7 +759,7 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
706	759
707	760	.label => {
708	761	const expr = t.ast.nodeData(node).label.expr;
709		const label = t.ast.tokenSlice(t.ast.nodes.items(.main_token)[node]);
	762	const label = t.ast.tokenSlice(t.ast.nodeToken(node));
710	763
711	764	var target: BlockContext.BreakTarget = .{
712	765	.parent = decl.break_target,
...	...	@@ -778,7 +831,8 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
778	831	.builtin_call,
779	832	=> {
780	833	const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
781		return error.InvalidBuiltin;
	834	try t.report.err(.invalid_builtin, .{ .node = node });
	835	return .void_value;
782	836	};
783	837
784	838	const base = t.ast.nodeData(node).list;
...	...	@@ -847,7 +901,7 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
847	901	const string_ref = try t.addInstRef(.{
848	902	.tag = .string,
849	903	.data = EncodedData.encode(.string, .{
850		.src_node = decl.nodeOffset(node), // TODO: this is incorrect, there is no node
	904	.src_node = decl.nodeOffset(node),
851	905	.extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
852	906	}),
853	907	});
...	...	@@ -915,7 +969,8 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
915	969	const ret_node = t.ast.nodeData(node).@"return";
916	970
917	971	const ret_block_ptr = decl.return_target orelse {
918		return error.InvalidReturn;
	972	// return error.InvalidReturn;
	973	@panic("TODO");
919	974	};
920	975
921	976	const ret_block = if (ret_block_ptr.*) \|block\|
...	...	@@ -957,7 +1012,8 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyer
957	1012	);
958	1013
959	1014	const block = try decl.findAndMarkBreak(t, label) orelse {
960		return error.UnknownBreakTarget;
	1015	try t.report.err(.unresolved_label, .{ .node = node });
	1016	return null;
961	1017	};
962	1018
963	1019	const operand: Inst.Ref = if (brk_node.value.unwrap()) \|expr\|
...	...	@@ -1001,6 +1057,17 @@ fn transformBlockStatement(
1001	1057
1002	1058	.var_decl => {
1003	1059	const var_decl = t.ast.fullVarDecl(node);
	1060
	1061	if (mem.eql(u8, var_decl.id, "_")) {
	1062	try t.report.err(.invalid_discard, .{ .node = node });
	1063	return .void_value;
	1064	}
	1065
	1066	if (BuiltInIds.has(var_decl.id)) {
	1067	try t.report.err(.shadow_builtin_variable, .{ .node = node });
	1068	return .void_value;
	1069	}
	1070
1004	1071	const type_ref = if (var_decl.type) \|data_type\|
1005	1072	try t.transformExpression(decl, data_type)
1006	1073	else
...	...	@@ -1040,7 +1107,7 @@ fn transformBlockStatement(
1040	1107	},
1041	1108	});
1042	1109
1043		return mem_ref;
	1110	return .void_value;
1044	1111	},
1045	1112
1046	1113	.fn_decl,
...	...	@@ -1145,10 +1212,7 @@ fn getInstData(t: Transform, index: Inst.Index) EncodedData {
1145	1212	}
1146	1213
1147	1214	fn addScratch(t: *Transform, comptime T: type, value: T) !void {
1148		try t.scratch.append(t.gpa, switch (T) {
1149		Inst.Ref, Inst.Index => @intFromEnum(value),
1150		else => @compileError("addScratch not implemented for " ++ @typeName(T)),
1151		});
	1215	try t.scratch.append(t.gpa, @intFromEnum(value));
1152	1216	}
1153	1217
1154	1218	fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
...	...	@@ -1273,4 +1337,7 @@ const optionalExtraIndex = OptionalExtraIndex.init;
1273	1337
1274	1338	const InternPool = @import("InternPool.zig");
1275	1339
	1340	const Report = @import("Report.zig");
	1341	const Comp = @import("Comp.zig");
	1342
1276	1343	const tracer = @import("tracer");

src/compiler/Value.zig-1

...	...	@@ -32,7 +32,6 @@ pub fn asString(val: Value, ip: *InternPool) InternPool.String.WithLen {
32	32	}
33	33
34	34	pub fn @"type"(val: Value, ip: *InternPool) Value {
35		std.debug.print("typeOfValue {any}\n", .{val.toKey(ip)});
36	35	const v: InternPool.Index = switch (val.toKey(ip)) {
37	36	.simple_type, .type_function => .type_type,
38	37

src/compiler/main.zig+23-3

...	...	@@ -3,11 +3,15 @@ const Tokenizer = @import("Tokenizer.zig");
3	3	const Parse = @import("Parse.zig");
4	4	const Transform = @import("Transform.zig");
5	5	const Comp = @import("Comp.zig");
6
	6	const unicode_props = @import("unicode_props.zig");
7	7	const T = @import("tracer");
8	8	pub const tracer_impl = @field(T, "none");
9	9
	10	pub const Report = @import("Report.zig");
	11
10	12	pub fn main() !void {
	13	try unicode_props.init();
	14
11	15	std.debug.print("\r\x1b[K", .{});
12	16
13	17	try T.init();
...	...	@@ -45,7 +49,14 @@ pub fn main() !void {
45	49	defer alloc.free(exe_name);
46	50
47	51	const file_path = maybe_file_path orelse {
48		std.debug.print("Usage: {s} <input file>\n", .{exe_name});
	52	std.debug.print("Usage: {s} [flags] <input file>\n", .{exe_name});
	53	std.debug.print(
	54	// \| 60 chars wrap point\|
	55	\\
	56	\\Flags:
	57	\\ --dump-ir Dump intermediate representations.
	58	\\
	59	, .{});
49	60	std.process.exit(1);
50	61	};
51	62
...	...	@@ -54,7 +65,16 @@ pub fn main() !void {
54	65	defer comp.deinit();
55	66	const main_file = try comp.getFile(file_path);
56	67
57		const teg = try main_file.getTeg(comp.gpa);
	68	var fail_report: Report = undefined;
	69	const teg = main_file.getTeg(&comp, &fail_report) catch \|err\| switch (err) {
	70	error.TransformFail => {
	71	std.debug.print("Transform failed:\n", .{});
	72	try fail_report.dumpToStdErr(&comp);
	73	std.process.exit(1);
	74	},
	75	else => return err,
	76	};
	77
58	78	{
59	79	const stderr = std.io.getStdErr().writer();
60	80	try teg.dumpToWriter(stderr);

src/compiler/util.zig+2-2

...	...	@@ -7,9 +7,9 @@ pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
7	7	return std.io.tty.detectConfig(writer.context);
8	8	}
9	9	if (@TypeOf(writer) == std.io.AnyWriter) {
10		const writeFn = comptime (std.fs.File{ .handle = undefined })
	10	const fileWriteFn = comptime (std.fs.File{ .handle = undefined })
11	11	.writer().any().writeFn;
12		if (writer.writeFn == writeFn) {
	12	if (writer.writeFn == fileWriteFn) {
13	13	const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
14	14	return std.io.tty.detectConfig(file.*);
15	15	}