Commit 5068576f23334b555210ae97757fb76ad07d0ba1 | ix

author	Mateusz Franik <47059999+Im-Beast@users.noreply.github.com> 2024-07-12 10:59:09+02:00
committer	GitHub <noreply@github.com> 2024-07-12 01:59:09-07:00
log	5068576f23334b555210ae97757fb76ad07d0ba1
tree	e5e736b1d484a7dcf04024ad775186643afab8f9
parent	def7b01d2c3b5d21675be8938b366a263dbab586

feat: implement string (escaped) identifiers (#54)

String identifiers allow you to use any text in an identifier name,
such as keywords, or any unicode codepoints aside from null (0).
Identifiers still must be at least one character long; these
requirements and syntax are taken from the Zig compiler.

---------

Co-authored-by: dave caruso <me@paperdave.net>

4 files changed, 341 insertions(+), 53 deletions(-)

build.zig.zon
src
- core

build.zig.zon+1-1

...	...	@@ -2,7 +2,7 @@
2	2	.name = "",
3	3	.version = "0.0.0",
4	4
5		.minimum_zig_version = "0.13.0-dev.348+c564a16a0",
	5	.minimum_zig_version = "0.13.0",
6	6
7	7	.dependencies = .{
8	8	.zg = .{

src/core/JsLink.zig+3-1

...	...	@@ -446,9 +446,11 @@ pub fn canPrintWithoutEscape(c: u21) bool {
446	446
447	447	pub fn indexOfNeedsEscape(slice: []const u8) ?usize {
448	448	var it = std.unicode.Wtf8Iterator{ .bytes = slice, .i = 0 };
	449	var i: usize = 0;
449	450	while (it.nextCodepoint()) \|c\| {
450	451	if (!canPrintWithoutEscape(c))
451		return it.i;
	452	return i;
	453	i = it.i;
452	454	}
453	455	return null;
454	456	}

src/core/Report.zig+235-23

...	...	@@ -58,19 +58,32 @@ pub const Err = union(enum) {
58	58	node: NodeIndex,
59	59
60	60	fn build(data: @This(), ctx: *MsgContext) !void {
61		try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{
62		ctx.ast.basicNodeSlice(data.node),
	61	try ctx.msgNode(data.node, "unknown identifier {}", .{
	62	formatId(ctx.ast.basicNodeSlice(data.node)),
63	63	});
64	64	}
65	65	},
	66	invalid_string_identifier: struct {
	67	node: NodeIndex,
	68
	69	fn build(data: @This(), ctx: *MsgContext) !void {
	70	const slice = ctx.ast.basicNodeSlice(data.node);
	71	if (slice.len == 0) {
	72	try ctx.msgNode(data.node, "identifier cannot be empty", .{});
	73	} else {
	74	// TODO: make this point at the null escape?
	75	try ctx.msgNode(data.node, "identifier cannot include null bytes", .{});
	76	}
	77	}
	78	},
66	79	invalid_builtin: struct {
67	80	node: NodeIndex,
68	81
69	82	fn build(data: @This(), ctx: *MsgContext) !void {
70	83	try ctx.msgToken(
71	84	ctx.ast.nodeToken(data.node),
72		"unknown built-in function \"{s}\"",
73		.{ctx.ast.basicNodeSlice(data.node)},
	85	"unknown built-in function {}",
	86	.{formatId(ctx.ast.basicNodeSlice(data.node))},
74	87	);
75	88	}
76	89	},
...	...	@@ -90,8 +103,8 @@ pub const Err = union(enum) {
90	103	prev: NodeIndex,
91	104
92	105	fn build(data: @This(), ctx: *MsgContext) !void {
93		try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
94		ctx.ast.fullFnDecl(data.node).id,
	106	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of {}", .{
	107	formatId(ctx.ast.fullFnDecl(data.node).id),
95	108	});
96	109	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{});
97	110	// TODO: be able to store a full list of other declarations
...	...	@@ -103,8 +116,8 @@ pub const Err = union(enum) {
103	116	prev: NodeIndex,
104	117
105	118	fn build(data: @This(), ctx: *MsgContext) !void {
106		try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{
107		ctx.ast.fullParamDecl(data.node).id,
	119	try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of {}", .{
	120	formatId(ctx.ast.fullParamDecl(data.node).id),
108	121	});
109	122	try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{});
110	123	// TODO: be able to store a full list of other declarations
...	...	@@ -128,8 +141,8 @@ pub const Err = union(enum) {
128	141	fn build(data: @This(), ctx: *MsgContext) !void {
129	142	try ctx.msg(
130	143	.{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 },
131		"unknown block label \"{s}\"",
132		.{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)},
	144	"unknown block label {}",
	145	.{formatId(ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2))},
133	146	);
134	147	}
135	148	},
...	...	@@ -139,8 +152,8 @@ pub const Err = union(enum) {
139	152	fn build(data: @This(), ctx: *MsgContext) !void {
140	153	try ctx.msgToken(
141	154	ctx.ast.nodeToken(data.node) + 1,
142		"declaration shadows reserved built-in \"{s}\"",
143		.{ctx.ast.fullVarDecl(data.node).id},
	155	"declaration shadows reserved built-in {}",
	156	.{formatId(ctx.ast.fullVarDecl(data.node).id)},
144	157	);
145	158	}
146	159	},
...	...	@@ -218,8 +231,8 @@ pub const Err = union(enum) {
218	231	fn build(data: @This(), ctx: *MsgContext) !void {
219	232	try ctx.msgNode(
220	233	data.node,
221		"cannot re-assign built-in \"{s}\"",
222		.{ctx.ast.basicNodeSlice(data.node)},
	234	"cannot re-assign built-in {}",
	235	.{formatId(ctx.ast.basicNodeSlice(data.node))},
223	236	);
224	237	}
225	238	},
...	...	@@ -236,9 +249,9 @@ pub const Err = union(enum) {
236	249	else => unreachable, // not a valid symbol node
237	250	};
238	251
239		try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{
	252	try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} {}", .{
240	253	@tagName(data.kind),
241		id,
	254	formatId(id),
242	255	});
243	256	try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{});
244	257	}
...	...	@@ -297,12 +310,12 @@ pub const Warn = union(enum) {
297	310	else => unreachable, // not a valid symbol node
298	311	};
299	312
300		try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{
	313	try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} {}", .{
301	314	@tagName(data.kind),
302		id,
	315	formatId(id),
303	316	});
304	317
305		try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id});
	318	try ctx.msgAlone("if this was intentional, discard it with \"_ = {};\"", .{formatIdCode(id)});
306	319	}
307	320	},
308	321	not_mutated_var: struct {
...	...	@@ -314,8 +327,8 @@ pub const Warn = union(enum) {
314	327	else => unreachable, // not a valid variable symbol node
315	328	};
316	329
317		try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{
318		id,
	330	try ctx.msgToken(ctx.ast.idToken(data.node), "variable {} not mutated", .{
	331	formatId(id),
319	332	});
320	333	}
321	334	},
...	...	@@ -787,9 +800,10 @@ pub const MsgContext = struct {
787	800	components: ArrayListUnmanaged(Component),
788	801
789	802	pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void {
790		const text = try std.fmt.allocPrint(m.arena, fmt, args);
	803	var arr: std.ArrayListUnmanaged(u8) = .{};
	804	try std.fmt.format(arr.writer(m.arena), fmt, args);
791	805	try m.components.append(m.arena, .{
792		.message_utf8 = text,
	806	.message_utf8 = try arr.toOwnedSlice(m.arena),
793	807	.location = highlight.loc(m.ast),
794	808	});
795	809	}
...	...	@@ -936,6 +950,202 @@ pub const Operand = enum(u32) {
936	950	}
937	951	};
938	952
	953	/// Non-allocating formatter for taking an id token slice (raw), normalizing it
	954	/// into it's canonical form.
	955	fn formatId(id: []const u8) std.fmt.Formatter(formatIdImpl) {
	956	return .{ .data = id };
	957	}
	958
	959	fn formatIdImpl(
	960	id: []const u8,
	961	comptime format: []const u8,
	962	_: std.fmt.FormatOptions,
	963	writer: anytype,
	964	) !void {
	965	comptime assert(format.len == 0);
	966
	967	if (id[0] != '@') {
	968	// no work needed
	969	try writer.writeByte('"');
	970	try writer.writeAll(id);
	971	try writer.writeByte('"');
	972
	973	return;
	974	}
	975
	976	// determine if the entire ID can be printed without escpaing
	977	const inner = id[2 .. id.len - 1];
	978	const needs_encode = brk: {
	979	var i: usize = 0;
	980	while (i < inner.len) {
	981	switch (inner[i]) {
	982	'a'...'z',
	983	'A'...'Z',
	984	'0'...'9',
	985	'_',
	986	'$',
	987	=> {
	988	i += 1;
	989	continue;
	990	},
	991	else => break :brk true,
	992	0x80...0xFF => \|first_byte\| {
	993	const len = std.unicode.utf8ByteSequenceLength(first_byte) catch
	994	unreachable; // Invalid UTF-8
	995	const codepoint = std.unicode.utf8Decode(inner[i..][0..len]) catch
	996	unreachable; // Invalid UTF-8
	997	if (!unicode_props.isIdContinue(codepoint))
	998	break :brk true;
	999	},
	1000	}
	1001	}
	1002	break :brk false;
	1003	};
	1004
	1005	if (needs_encode) {
	1006	try writer.writeAll("@\"");
	1007	} else {
	1008	try writer.writeByte('"');
	1009	}
	1010
	1011	// TODO: these two are "reasonable upper bounds" and not scientifically proven
	1012	var decode_buf: [4096 * 2]u8 = undefined;
	1013	//var encode_buf: [4096 * 3]u8 = undefined;
	1014
	1015	var it = std.mem.window(u8, inner, 4096, 4096);
	1016	while (it.next()) \|chunk\| {
	1017	const decoded = Tokenizer.decodeStringBuf(chunk, &decode_buf);
	1018	if (needs_encode) {
	1019	// TODO: This code is kind of duplicated, but there are small
	1020	// differences in the string formatters. Ideally these are merged
	1021	// under a common function that takes an argument of the 'mode' to
	1022	// print the string in.
	1023	const JSLink = @import("JsLink.zig");
	1024	const indexOfNeedsEscape = JSLink.indexOfNeedsEscape;
	1025	const canPrintWithoutEscape = JSLink.canPrintWithoutEscape;
	1026
	1027	var text = decoded;
	1028	inner: while (text.len > 0) {
	1029	const width = std.unicode.utf8ByteSequenceLength(text[0]) catch
	1030	unreachable; // string is encoded incorrectly
	1031
	1032	const c: u21 = std.unicode.wtf8Decode(text[0..width]) catch
	1033	unreachable; // string is encoded incorrectly
	1034
	1035	if (canPrintWithoutEscape(c)) {
	1036	if (indexOfNeedsEscape(text)) \|j\| {
	1037	try writer.writeAll(text[0..j]);
	1038	text = text[j..];
	1039	continue :inner;
	1040	} else {
	1041	try writer.writeAll(text);
	1042	break :inner;
	1043	}
	1044	}
	1045
	1046	text = text[width..];
	1047
	1048	switch (c) {
	1049	'\n' => {
	1050	try writer.writeAll("\\n");
	1051	},
	1052	std.ascii.control_code.cr => {
	1053	try writer.writeAll("\\r");
	1054	},
	1055	'\\' => {
	1056	try writer.writeAll("\\\\");
	1057	},
	1058	'"' => {
	1059	try writer.writeAll("\\\"");
	1060	},
	1061	'\t' => {
	1062	try writer.writeAll("\\t");
	1063	},
	1064	else => {
	1065	const hex_chars = "0123456789ABCDEF";
	1066
	1067	if (c < 0xFF) {
	1068	try writer.writeAll(&[_]u8{
	1069	'\\',
	1070	'x',
	1071	hex_chars[(c >> 4) & 0xF],
	1072	hex_chars[c & 0xF],
	1073	});
	1074	} else if (c < 0xFFFF) {
	1075	try writer.writeAll(&[_]u8{
	1076	'\\',
	1077	'u',
	1078	'{',
	1079	'0',
	1080	'x',
	1081	hex_chars[(c >> 12) & 0xF],
	1082	hex_chars[(c >> 8) & 0xF],
	1083	hex_chars[(c >> 4) & 0xF],
	1084	hex_chars[c & 0xF],
	1085	'}',
	1086	});
	1087	} else {
	1088	const first_high_surrogate = 0xD800;
	1089	const first_low_surrogate = 0xDC00;
	1090
	1091	const k = c - 0x10000;
	1092	const lo = @as(usize, @intCast(first_high_surrogate + ((k >> 10) & 0x3FF)));
	1093	const hi = @as(usize, @intCast(first_low_surrogate + (k & 0x3FF)));
	1094
	1095	try writer.writeAll(&[_]u8{
	1096	'\\',
	1097	'u',
	1098	'{',
	1099	hex_chars[lo >> 12],
	1100	hex_chars[(lo >> 8) & 15],
	1101	hex_chars[(lo >> 4) & 15],
	1102	hex_chars[lo & 15],
	1103	'}',
	1104	'\\',
	1105	'u',
	1106	'{',
	1107	'0',
	1108	'x',
	1109	hex_chars[hi >> 12],
	1110	hex_chars[(hi >> 8) & 15],
	1111	hex_chars[(hi >> 4) & 15],
	1112	hex_chars[hi & 15],
	1113	'}',
	1114	});
	1115	}
	1116	},
	1117	}
	1118	}
	1119	} else {
	1120	try writer.writeAll(decoded);
	1121	}
	1122	}
	1123
	1124	try writer.writeByte('"');
	1125	}
	1126
	1127	fn formatIdCode(id: []const u8) std.fmt.Formatter(formatIdCodeImpl) {
	1128	return .{ .data = id };
	1129	}
	1130
	1131	fn formatIdCodeImpl(
	1132	id: []const u8,
	1133	comptime format: []const u8,
	1134	_: std.fmt.FormatOptions,
	1135	writer: anytype,
	1136	) !void {
	1137	comptime assert(format.len == 0);
	1138	if (id[0] != '@') {
	1139	// no decoding needed
	1140	try writer.writeAll(id);
	1141	return;
	1142	}
	1143
	1144	try writer.writeAll("@\"");
	1145
	1146	try writer.writeByte('"');
	1147	}
	1148
939	1149	const Report = @This();
940	1150	const std = @import("std");
941	1151	const ArrayListUnmanaged = std.ArrayListUnmanaged;
...	...	@@ -944,6 +1154,8 @@ const assert = std.debug.assert;
944	1154
945	1155	const builtin = @import("builtin");
946	1156
	1157	const unicode_props = @import("unicode_props.zig");
	1158
947	1159	const Tokenizer = @import("Tokenizer.zig");
948	1160	const ByteOffset = Tokenizer.ByteOffset;
949	1161

src/core/Transform.zig+102-28

...	...	@@ -6,6 +6,7 @@ string_bytes: std.ArrayListUnmanaged(u8) = .{},
6	6	extra: ExtraStore = .{},
7	7
8	8	scratch: std.ArrayListUnmanaged(u32) = .{},
	9	id_decode_buf: std.ArrayList(u8),
9	10
10	11	// TODO: refactor this away
11	12	// The new home for this memory should be pre-allocated on scratch,
...	...	@@ -30,12 +31,15 @@ pub fn transform(
30	31	const trace = tracer.trace(@src());
31	32	defer trace.end();
32	33
	34	var sfa = std.heap.stackFallback(4096, gpa);
	35
33	36	var t: Transform = .{
34	37	.ast = ast,
35	38	.gpa = gpa,
36	39	.report = .{ .gpa = gpa, .current_file = file },
	40	.id_decode_buf = std.ArrayList(u8).initCapacity(sfa.get(), 4096) catch unreachable,
37	41	};
38		errdefer t.deinit();
	42	defer t.deinit();
39	43
40	44	try t.string_bytes.append(t.gpa, 0);
41	45
...	...	@@ -53,7 +57,6 @@ pub fn transform(
53	57
54	58	try t.insts.setCapacity(t.gpa, t.insts.len);
55	59
56		defer t.deinit();
57	60	return Teg{
58	61	.insts = t.insts.toOwnedSlice(),
59	62	.string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
...	...	@@ -70,6 +73,7 @@ pub fn deinit(t: *Transform) void {
70	73	t.string_bytes.deinit(t.gpa);
71	74	t.insts.deinit(t.gpa);
72	75	t.extra.deinit(t.gpa);
	76	t.id_decode_buf.deinit();
73	77	t.* = undefined;
74	78	}
75	79
...	...	@@ -124,8 +128,13 @@ const Scope = struct {
124	128	}
125	129	}
126	130
127		fn deinit(s: *Scope, gpa: Allocator) void {
128		s.symbols.deinit(gpa);
	131	fn deinit(s: Scope, t: Transform) void {
	132	for (s.symbols.keys()) \|k\| {
	133	if (!isSliceInBuffer(u8, t.ast.utf8_source, k)) {
	134	t.gpa.free(k);
	135	}
	136	}
	137	s.symbols.deinit(t.gpa);
129	138	s.* = undefined;
130	139	}
131	140	};
...	...	@@ -338,8 +347,12 @@ fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void
338	347	switch (tag) {
339	348	.fn_decl => {
340	349	const fn_decl = t.ast.fullFnDecl(node);
341		const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
	350	var id, const id_allocated = try t.decodeIdentifierExtra(fn_decl.id);
	351	if (id_allocated)
	352	id = try t.gpa.dupe(u8, id);
	353	const gop = try t.scope().symbols.getOrPut(t.gpa, id);
342	354	if (gop.found_existing) {
	355	t.gpa.free(id);
343	356	try t.report.err(.duplicate_function, .{
344	357	.node = node,
345	358	.prev = gop.value_ptr.node,
...	...	@@ -448,7 +461,7 @@ fn transformExpressionToBlock(
448	461	switch (node_tag) {
449	462	.id => {
450	463	// Lower builtin ids without using a block
451		const id = t.ast.basicNodeSlice(node);
	464	const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node));
452	465	if (BuiltInIds.get(id)) \|ref\| {
453	466	return ref;
454	467	}
...	...	@@ -544,10 +557,15 @@ fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Alloc
544	557	const param_decl = t.ast.fullParamDecl(param);
545	558	const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
546	559
	560	var id, const id_allocated = try t.decodeIdentifierExtra(param_decl.id);
	561	if (id_allocated)
	562	id = try t.gpa.dupe(u8, id);
	563
547	564	// add `sym` into the scope so that the next param's type
548	565	// can use this param's value, if known at compile time.
549		const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
	566	const sym = try t.scope().symbols.getOrPut(t.gpa, id);
550	567	if (sym.found_existing) {
	568	if (id_allocated) t.gpa.free(id);
551	569	try t.report.err(.duplicate_param, .{
552	570	.node = param,
553	571	.prev = sym.value_ptr.node,
...	...	@@ -685,13 +703,14 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Alloc
685	703	=> unreachable, // not an expression
686	704
687	705	.id => {
688		const id = t.ast.basicNodeSlice(node);
689		if (BuiltInIds.get(id)) \|ref\| {
690		return ref;
	706	const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node));
	707	if (id.len == 0 or mem.containsAtLeast(u8, id, 1, &[1]u8{'\x00'})) {
	708	try t.report.err(.invalid_string_identifier, .{ .node = node });
	709	return .void;
691	710	}
692	711
693		if (mem.startsWith(u8, id, "@")) {
694		@panic("TODO: string identifiers");
	712	if (BuiltInIds.get(id)) \|ref\| {
	713	return ref;
695	714	}
696	715
697	716	const lookup = t.lookupId(id) orelse {
...	...	@@ -1178,16 +1197,17 @@ fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex)
1178	1197	=> unreachable, // not an expression
1179	1198
1180	1199	.id => {
1181		const id = t.ast.basicNodeSlice(node);
	1200	const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node));
1182	1201	assert(!mem.eql(u8, id, "_")); // discards are not assignment targets
1183	1202
1184		if (BuiltInIds.has(id)) {
1185		try t.report.err(.assign_builtin, .{ .node = node });
	1203	if (id.len == 0 or mem.indexOfScalar(u8, id, '\x00') != null) {
	1204	try t.report.err(.invalid_string_identifier, .{ .node = node });
1186	1205	return null;
1187	1206	}
1188	1207
1189		if (mem.startsWith(u8, id, "@")) {
1190		@panic("TODO: string identifiers");
	1208	if (BuiltInIds.has(id)) {
	1209	try t.report.err(.assign_builtin, .{ .node = node });
	1210	return null;
1191	1211	}
1192	1212
1193	1213	const lookup = t.lookupId(id) orelse {
...	...	@@ -1251,23 +1271,30 @@ fn transformBlockStatement(
1251	1271	.var_decl => {
1252	1272	const var_decl = t.ast.fullVarDecl(node);
1253	1273
1254		if (mem.eql(u8, var_decl.id, "_")) {
	1274	var id, const id_allocated = try t.decodeIdentifierExtra(var_decl.id);
	1275	if (id.len == 0 or mem.indexOfScalar(u8, id, '\x00') != null) {
	1276	try t.report.err(.invalid_string_identifier, .{ .node = node });
	1277	return .void;
	1278	}
	1279
	1280	if (mem.eql(u8, id, "_")) {
1255	1281	try t.report.warn(.invalid_discard, .{ .node = node });
1256	1282
1257	1283	const ref = switch (t.ast.nodeTag(var_decl.init)) {
1258	1284	.id => {
1259		const id = t.ast.basicNodeSlice(node);
1260		if (BuiltInIds.has(id)) {
1261		try t.report.warn(.discard_builtin, .{ .node = node });
	1285	const inner_id = try t.decodeIdentifier(t.ast.basicNodeSlice(var_decl.init));
	1286	if (inner_id.len == 0 or mem.indexOfScalar(u8, inner_id, '\x00') != null) {
	1287	try t.report.err(.invalid_string_identifier, .{ .node = var_decl.init });
1262	1288	return .void;
1263	1289	}
1264	1290
1265		if (mem.startsWith(u8, id, "@")) {
1266		@panic("TODO: string identifiers");
	1291	if (BuiltInIds.has(inner_id)) {
	1292	try t.report.warn(.discard_builtin, .{ .node = var_decl.init });
	1293	return .void;
1267	1294	}
1268	1295
1269		const lookup = t.lookupId(id) orelse {
1270		try t.report.err(.unknown_identifier, .{ .node = node });
	1296	const lookup = t.lookupId(inner_id) orelse {
	1297	try t.report.err(.unknown_identifier, .{ .node = var_decl.init });
1271	1298	return .void;
1272	1299	};
1273	1300
...	...	@@ -1288,7 +1315,16 @@ fn transformBlockStatement(
1288	1315	});
1289	1316	}
1290	1317
1291		if (BuiltInIds.has(var_decl.id)) {
	1318	if (id_allocated) {
	1319	// consider arena?
	1320	id = try t.gpa.dupe(u8, id);
	1321	} else {
	1322	// assert that the memory is in the source code. this is how
	1323	// we can identify later if it is owned or not.
	1324	assert(isSliceInBuffer(u8, t.ast.utf8_source, id));
	1325	}
	1326
	1327	if (BuiltInIds.has(id)) {
1292	1328	try t.report.err(.shadow_builtin_variable, .{ .node = node });
1293	1329	return .void;
1294	1330	}
...	...	@@ -1324,10 +1360,11 @@ fn transformBlockStatement(
1324	1360	}),
1325	1361	});
1326	1362
1327		const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id);
	1363	const gop = try t.scope().symbols.getOrPut(t.gpa, id);
1328	1364
1329	1365	// if re-declaring, apply unused warnings now
1330	1366	if (gop.found_existing) {
	1367	if (id_allocated) t.gpa.free(id);
1331	1368	try gop.value_ptr.addWarnings(t);
1332	1369	}
1333	1370
...	...	@@ -1435,7 +1472,7 @@ fn pushScope(t: *Transform) !void {
1435	1472	}
1436	1473
1437	1474	fn popScope(t: *Transform) void {
1438		t.scope().deinit(t.gpa);
	1475	t.scope().deinit(t);
1439	1476	t.scopes.items[t.scopes.items.len - 1] = undefined;
1440	1477	t.scopes.items.len -= 1;
1441	1478	}
...	...	@@ -1552,6 +1589,43 @@ fn addDecodedSlicedString(t: *Transform, escaped: []const u8) !SlicedString {
1552	1589	return t.addSlicedString(decoded);
1553	1590	}
1554	1591
	1592	/// Decodes an identifier, storing it into a buffer reused across the transform
	1593	/// pass Memory is essentially free for this operation, at the cost of the
	1594	/// return value only being valid until the next call.
	1595	fn decodeIdentifierExtra(t: *Transform, raw_token: []const u8) !struct { []const u8, bool } {
	1596	assert(raw_token.len > 0);
	1597
	1598	if (raw_token[0] != '@') {
	1599	return .{ raw_token, false };
	1600	}
	1601
	1602	assert(raw_token[1] == '"');
	1603	assert(raw_token[raw_token.len - 1] == '"');
	1604
	1605	const inner = raw_token[2 .. raw_token.len - 1];
	1606	const len = Tokenizer.decodeStringCount(inner);
	1607
	1608	try t.id_decode_buf.ensureTotalCapacityPrecise(len);
	1609	@memset(t.id_decode_buf.items, undefined);
	1610	t.id_decode_buf.items.len = len;
	1611
	1612	return .{
	1613	Tokenizer.decodeStringBuf(inner, t.id_decode_buf.items),
	1614	true,
	1615	};
	1616	}
	1617
	1618	fn decodeIdentifier(t: *Transform, raw_token: []const u8) ![]const u8 {
	1619	return (try t.decodeIdentifierExtra(raw_token)).@"0";
	1620	}
	1621
	1622	fn isSliceInBuffer(comptime T: type, haystack: []const T, needle: []const T) bool {
	1623	return (@intFromPtr(haystack.ptr) <=
	1624	@intFromPtr(needle.ptr) and
	1625	(@intFromPtr(needle.ptr) + needle.len * @sizeOf(T)) <=
	1626	(@intFromPtr(haystack.ptr) + haystack.len * @sizeOf(T)));
	1627	}
	1628
1555	1629	const Transform = @This();
1556	1630	const std = @import("std");
1557	1631	const mem = std.mem;