diff --git a/build.zig.zon b/build.zig.zon index 927b751a340b3ada67cce95226a172d08402fb6f..b94be8dde6b2a04eb4035309b39e497669b683e6 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -2,7 +2,7 @@ .name = "", .version = "0.0.0", - .minimum_zig_version = "0.13.0-dev.348+c564a16a0", + .minimum_zig_version = "0.13.0", .dependencies = .{ .zg = .{ diff --git a/src/core/JsLink.zig b/src/core/JsLink.zig index d60ac068a4b5996e19cb559f8099569716e90446..5a33ca3535dabf2b2a04b8450925ca189b172a3f 100644 --- a/src/core/JsLink.zig +++ b/src/core/JsLink.zig @@ -446,9 +446,11 @@ pub fn canPrintWithoutEscape(c: u21) bool { pub fn indexOfNeedsEscape(slice: []const u8) ?usize { var it = std.unicode.Wtf8Iterator{ .bytes = slice, .i = 0 }; + var i: usize = 0; while (it.nextCodepoint()) |c| { if (!canPrintWithoutEscape(c)) - return it.i; + return i; + i = it.i; } return null; } diff --git a/src/core/Report.zig b/src/core/Report.zig index 9dbd664d962779b86ef2a77ff280f9f4fc3a358a..d26557fd9ff49a5b8605362553efe55de478ea1a 100644 --- a/src/core/Report.zig +++ b/src/core/Report.zig @@ -58,19 +58,32 @@ pub const Err = union(enum) { node: NodeIndex, fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{ - ctx.ast.basicNodeSlice(data.node), + try ctx.msgNode(data.node, "unknown identifier {}", .{ + formatId(ctx.ast.basicNodeSlice(data.node)), }); } }, + invalid_string_identifier: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + const slice = ctx.ast.basicNodeSlice(data.node); + if (slice.len == 0) { + try ctx.msgNode(data.node, "identifier cannot be empty", .{}); + } else { + // TODO: make this point at the null escape? + try ctx.msgNode(data.node, "identifier cannot include null bytes", .{}); + } + } + }, invalid_builtin: struct { node: NodeIndex, fn build(data: @This(), ctx: *MsgContext) !void { try ctx.msgToken( ctx.ast.nodeToken(data.node), - "unknown built-in function \"{s}\"", - .{ctx.ast.basicNodeSlice(data.node)}, + "unknown built-in function {}", + .{formatId(ctx.ast.basicNodeSlice(data.node))}, ); } }, @@ -90,8 +103,8 @@ pub const Err = union(enum) { prev: NodeIndex, fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ - ctx.ast.fullFnDecl(data.node).id, + try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of {}", .{ + formatId(ctx.ast.fullFnDecl(data.node).id), }); try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{}); // TODO: be able to store a full list of other declarations @@ -103,8 +116,8 @@ pub const Err = union(enum) { prev: NodeIndex, fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ - ctx.ast.fullParamDecl(data.node).id, + try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of {}", .{ + formatId(ctx.ast.fullParamDecl(data.node).id), }); try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{}); // TODO: be able to store a full list of other declarations @@ -128,8 +141,8 @@ pub const Err = union(enum) { fn build(data: @This(), ctx: *MsgContext) !void { try ctx.msg( .{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 }, - "unknown block label \"{s}\"", - .{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)}, + "unknown block label {}", + .{formatId(ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2))}, ); } }, @@ -139,8 +152,8 @@ pub const Err = union(enum) { fn build(data: @This(), ctx: *MsgContext) !void { try ctx.msgToken( ctx.ast.nodeToken(data.node) + 1, - "declaration shadows reserved built-in \"{s}\"", - .{ctx.ast.fullVarDecl(data.node).id}, + "declaration shadows reserved built-in {}", + .{formatId(ctx.ast.fullVarDecl(data.node).id)}, ); } }, @@ -218,8 +231,8 @@ pub const Err = union(enum) { fn build(data: @This(), ctx: *MsgContext) !void { try ctx.msgNode( data.node, - "cannot re-assign built-in \"{s}\"", - .{ctx.ast.basicNodeSlice(data.node)}, + "cannot re-assign built-in {}", + .{formatId(ctx.ast.basicNodeSlice(data.node))}, ); } }, @@ -236,9 +249,9 @@ pub const Err = union(enum) { else => unreachable, // not a valid symbol node }; - try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{ + try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} {}", .{ @tagName(data.kind), - id, + formatId(id), }); try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{}); } @@ -297,12 +310,12 @@ pub const Warn = union(enum) { else => unreachable, // not a valid symbol node }; - try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{ + try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} {}", .{ @tagName(data.kind), - id, + formatId(id), }); - try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id}); + try ctx.msgAlone("if this was intentional, discard it with \"_ = {};\"", .{formatIdCode(id)}); } }, not_mutated_var: struct { @@ -314,8 +327,8 @@ pub const Warn = union(enum) { else => unreachable, // not a valid variable symbol node }; - try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{ - id, + try ctx.msgToken(ctx.ast.idToken(data.node), "variable {} not mutated", .{ + formatId(id), }); } }, @@ -787,9 +800,10 @@ pub const MsgContext = struct { components: ArrayListUnmanaged(Component), pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void { - const text = try std.fmt.allocPrint(m.arena, fmt, args); + var arr: std.ArrayListUnmanaged(u8) = .{}; + try std.fmt.format(arr.writer(m.arena), fmt, args); try m.components.append(m.arena, .{ - .message_utf8 = text, + .message_utf8 = try arr.toOwnedSlice(m.arena), .location = highlight.loc(m.ast), }); } @@ -936,6 +950,202 @@ pub const Operand = enum(u32) { } }; +/// Non-allocating formatter for taking an id token slice (raw), normalizing it +/// into it's canonical form. +fn formatId(id: []const u8) std.fmt.Formatter(formatIdImpl) { + return .{ .data = id }; +} + +fn formatIdImpl( + id: []const u8, + comptime format: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) !void { + comptime assert(format.len == 0); + + if (id[0] != '@') { + // no work needed + try writer.writeByte('"'); + try writer.writeAll(id); + try writer.writeByte('"'); + + return; + } + + // determine if the entire ID can be printed without escpaing + const inner = id[2 .. id.len - 1]; + const needs_encode = brk: { + var i: usize = 0; + while (i < inner.len) { + switch (inner[i]) { + 'a'...'z', + 'A'...'Z', + '0'...'9', + '_', + '$', + => { + i += 1; + continue; + }, + else => break :brk true, + 0x80...0xFF => |first_byte| { + const len = std.unicode.utf8ByteSequenceLength(first_byte) catch + unreachable; // Invalid UTF-8 + const codepoint = std.unicode.utf8Decode(inner[i..][0..len]) catch + unreachable; // Invalid UTF-8 + if (!unicode_props.isIdContinue(codepoint)) + break :brk true; + }, + } + } + break :brk false; + }; + + if (needs_encode) { + try writer.writeAll("@\""); + } else { + try writer.writeByte('"'); + } + + // TODO: these two are "reasonable upper bounds" and not scientifically proven + var decode_buf: [4096 * 2]u8 = undefined; + //var encode_buf: [4096 * 3]u8 = undefined; + + var it = std.mem.window(u8, inner, 4096, 4096); + while (it.next()) |chunk| { + const decoded = Tokenizer.decodeStringBuf(chunk, &decode_buf); + if (needs_encode) { + // TODO: This code is kind of duplicated, but there are small + // differences in the string formatters. Ideally these are merged + // under a common function that takes an argument of the 'mode' to + // print the string in. + const JSLink = @import("JsLink.zig"); + const indexOfNeedsEscape = JSLink.indexOfNeedsEscape; + const canPrintWithoutEscape = JSLink.canPrintWithoutEscape; + + var text = decoded; + inner: while (text.len > 0) { + const width = std.unicode.utf8ByteSequenceLength(text[0]) catch + unreachable; // string is encoded incorrectly + + const c: u21 = std.unicode.wtf8Decode(text[0..width]) catch + unreachable; // string is encoded incorrectly + + if (canPrintWithoutEscape(c)) { + if (indexOfNeedsEscape(text)) |j| { + try writer.writeAll(text[0..j]); + text = text[j..]; + continue :inner; + } else { + try writer.writeAll(text); + break :inner; + } + } + + text = text[width..]; + + switch (c) { + '\n' => { + try writer.writeAll("\\n"); + }, + std.ascii.control_code.cr => { + try writer.writeAll("\\r"); + }, + '\\' => { + try writer.writeAll("\\\\"); + }, + '"' => { + try writer.writeAll("\\\""); + }, + '\t' => { + try writer.writeAll("\\t"); + }, + else => { + const hex_chars = "0123456789ABCDEF"; + + if (c < 0xFF) { + try writer.writeAll(&[_]u8{ + '\\', + 'x', + hex_chars[(c >> 4) & 0xF], + hex_chars[c & 0xF], + }); + } else if (c < 0xFFFF) { + try writer.writeAll(&[_]u8{ + '\\', + 'u', + '{', + '0', + 'x', + hex_chars[(c >> 12) & 0xF], + hex_chars[(c >> 8) & 0xF], + hex_chars[(c >> 4) & 0xF], + hex_chars[c & 0xF], + '}', + }); + } else { + const first_high_surrogate = 0xD800; + const first_low_surrogate = 0xDC00; + + const k = c - 0x10000; + const lo = @as(usize, @intCast(first_high_surrogate + ((k >> 10) & 0x3FF))); + const hi = @as(usize, @intCast(first_low_surrogate + (k & 0x3FF))); + + try writer.writeAll(&[_]u8{ + '\\', + 'u', + '{', + hex_chars[lo >> 12], + hex_chars[(lo >> 8) & 15], + hex_chars[(lo >> 4) & 15], + hex_chars[lo & 15], + '}', + '\\', + 'u', + '{', + '0', + 'x', + hex_chars[hi >> 12], + hex_chars[(hi >> 8) & 15], + hex_chars[(hi >> 4) & 15], + hex_chars[hi & 15], + '}', + }); + } + }, + } + } + } else { + try writer.writeAll(decoded); + } + } + + try writer.writeByte('"'); +} + +fn formatIdCode(id: []const u8) std.fmt.Formatter(formatIdCodeImpl) { + return .{ .data = id }; +} + +fn formatIdCodeImpl( + id: []const u8, + comptime format: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) !void { + comptime assert(format.len == 0); + if (id[0] != '@') { + // no decoding needed + try writer.writeAll(id); + return; + } + + try writer.writeAll("@\""); + + try writer.writeByte('"'); +} + const Report = @This(); const std = @import("std"); const ArrayListUnmanaged = std.ArrayListUnmanaged; @@ -944,6 +1154,8 @@ const assert = std.debug.assert; const builtin = @import("builtin"); +const unicode_props = @import("unicode_props.zig"); + const Tokenizer = @import("Tokenizer.zig"); const ByteOffset = Tokenizer.ByteOffset; diff --git a/src/core/Transform.zig b/src/core/Transform.zig index 1de0a1bd0930e4d353ccc8162062919eb06047be..36bab689bb7a33830a3888379b674afee6ce5861 100644 --- a/src/core/Transform.zig +++ b/src/core/Transform.zig @@ -6,6 +6,7 @@ string_bytes: std.ArrayListUnmanaged(u8) = .{}, extra: ExtraStore = .{}, scratch: std.ArrayListUnmanaged(u32) = .{}, +id_decode_buf: std.ArrayList(u8), // TODO: refactor this away // The new home for this memory should be pre-allocated on scratch, @@ -30,12 +31,15 @@ pub fn transform( const trace = tracer.trace(@src()); defer trace.end(); + var sfa = std.heap.stackFallback(4096, gpa); + var t: Transform = .{ .ast = ast, .gpa = gpa, .report = .{ .gpa = gpa, .current_file = file }, + .id_decode_buf = std.ArrayList(u8).initCapacity(sfa.get(), 4096) catch unreachable, }; - errdefer t.deinit(); + defer t.deinit(); try t.string_bytes.append(t.gpa, 0); @@ -53,7 +57,6 @@ pub fn transform( try t.insts.setCapacity(t.gpa, t.insts.len); - defer t.deinit(); return Teg{ .insts = t.insts.toOwnedSlice(), .string_bytes = try t.string_bytes.toOwnedSlice(t.gpa), @@ -70,6 +73,7 @@ pub fn deinit(t: *Transform) void { t.string_bytes.deinit(t.gpa); t.insts.deinit(t.gpa); t.extra.deinit(t.gpa); + t.id_decode_buf.deinit(); t.* = undefined; } @@ -124,8 +128,13 @@ const Scope = struct { } } - fn deinit(s: *Scope, gpa: Allocator) void { - s.symbols.deinit(gpa); + fn deinit(s: *Scope, t: *Transform) void { + for (s.symbols.keys()) |k| { + if (!isSliceInBuffer(u8, t.ast.utf8_source, k)) { + t.gpa.free(k); + } + } + s.symbols.deinit(t.gpa); s.* = undefined; } }; @@ -338,8 +347,12 @@ fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void switch (tag) { .fn_decl => { const fn_decl = t.ast.fullFnDecl(node); - const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id); + var id, const id_allocated = try t.decodeIdentifierExtra(fn_decl.id); + if (id_allocated) + id = try t.gpa.dupe(u8, id); + const gop = try t.scope().symbols.getOrPut(t.gpa, id); if (gop.found_existing) { + t.gpa.free(id); try t.report.err(.duplicate_function, .{ .node = node, .prev = gop.value_ptr.node, @@ -448,7 +461,7 @@ fn transformExpressionToBlock( switch (node_tag) { .id => { // Lower builtin ids without using a block - const id = t.ast.basicNodeSlice(node); + const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node)); if (BuiltInIds.get(id)) |ref| { return ref; } @@ -544,10 +557,15 @@ fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Alloc const param_decl = t.ast.fullParamDecl(param); const value = try t.transformExpressionToBlock(decl, param_decl.type, false); + var id, const id_allocated = try t.decodeIdentifierExtra(param_decl.id); + if (id_allocated) + id = try t.gpa.dupe(u8, id); + // add `sym` into the scope so that the next param's type // can use this param's value, if known at compile time. - const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id); + const sym = try t.scope().symbols.getOrPut(t.gpa, id); if (sym.found_existing) { + if (id_allocated) t.gpa.free(id); try t.report.err(.duplicate_param, .{ .node = param, .prev = sym.value_ptr.node, @@ -685,13 +703,14 @@ fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Alloc => unreachable, // not an expression .id => { - const id = t.ast.basicNodeSlice(node); - if (BuiltInIds.get(id)) |ref| { - return ref; + const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node)); + if (id.len == 0 or mem.containsAtLeast(u8, id, 1, &[1]u8{'\x00'})) { + try t.report.err(.invalid_string_identifier, .{ .node = node }); + return .void; } - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); + if (BuiltInIds.get(id)) |ref| { + return ref; } const lookup = t.lookupId(id) orelse { @@ -1178,18 +1197,19 @@ fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex) => unreachable, // not an expression .id => { - const id = t.ast.basicNodeSlice(node); + const id = try t.decodeIdentifier(t.ast.basicNodeSlice(node)); assert(!mem.eql(u8, id, "_")); // discards are not assignment targets + if (id.len == 0 or mem.indexOfScalar(u8, id, '\x00') != null) { + try t.report.err(.invalid_string_identifier, .{ .node = node }); + return null; + } + if (BuiltInIds.has(id)) { try t.report.err(.assign_builtin, .{ .node = node }); return null; } - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); - } - const lookup = t.lookupId(id) orelse { try t.report.err(.unknown_identifier, .{ .node = node }); return null; @@ -1251,23 +1271,30 @@ fn transformBlockStatement( .var_decl => { const var_decl = t.ast.fullVarDecl(node); - if (mem.eql(u8, var_decl.id, "_")) { + var id, const id_allocated = try t.decodeIdentifierExtra(var_decl.id); + if (id.len == 0 or mem.indexOfScalar(u8, id, '\x00') != null) { + try t.report.err(.invalid_string_identifier, .{ .node = node }); + return .void; + } + + if (mem.eql(u8, id, "_")) { try t.report.warn(.invalid_discard, .{ .node = node }); const ref = switch (t.ast.nodeTag(var_decl.init)) { .id => { - const id = t.ast.basicNodeSlice(node); - if (BuiltInIds.has(id)) { - try t.report.warn(.discard_builtin, .{ .node = node }); + const inner_id = try t.decodeIdentifier(t.ast.basicNodeSlice(var_decl.init)); + if (inner_id.len == 0 or mem.indexOfScalar(u8, inner_id, '\x00') != null) { + try t.report.err(.invalid_string_identifier, .{ .node = var_decl.init }); return .void; } - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); + if (BuiltInIds.has(inner_id)) { + try t.report.warn(.discard_builtin, .{ .node = var_decl.init }); + return .void; } - const lookup = t.lookupId(id) orelse { - try t.report.err(.unknown_identifier, .{ .node = node }); + const lookup = t.lookupId(inner_id) orelse { + try t.report.err(.unknown_identifier, .{ .node = var_decl.init }); return .void; }; @@ -1288,7 +1315,16 @@ fn transformBlockStatement( }); } - if (BuiltInIds.has(var_decl.id)) { + if (id_allocated) { + // consider arena? + id = try t.gpa.dupe(u8, id); + } else { + // assert that the memory is in the source code. this is how + // we can identify later if it is owned or not. + assert(isSliceInBuffer(u8, t.ast.utf8_source, id)); + } + + if (BuiltInIds.has(id)) { try t.report.err(.shadow_builtin_variable, .{ .node = node }); return .void; } @@ -1324,10 +1360,11 @@ fn transformBlockStatement( }), }); - const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id); + const gop = try t.scope().symbols.getOrPut(t.gpa, id); // if re-declaring, apply unused warnings now if (gop.found_existing) { + if (id_allocated) t.gpa.free(id); try gop.value_ptr.addWarnings(t); } @@ -1435,7 +1472,7 @@ fn pushScope(t: *Transform) !void { } fn popScope(t: *Transform) void { - t.scope().deinit(t.gpa); + t.scope().deinit(t); t.scopes.items[t.scopes.items.len - 1] = undefined; t.scopes.items.len -= 1; } @@ -1552,6 +1589,43 @@ fn addDecodedSlicedString(t: *Transform, escaped: []const u8) !SlicedString { return t.addSlicedString(decoded); } +/// Decodes an identifier, storing it into a buffer reused across the transform +/// pass Memory is essentially free for this operation, at the cost of the +/// return value only being valid until the next call. +fn decodeIdentifierExtra(t: *Transform, raw_token: []const u8) !struct { []const u8, bool } { + assert(raw_token.len > 0); + + if (raw_token[0] != '@') { + return .{ raw_token, false }; + } + + assert(raw_token[1] == '"'); + assert(raw_token[raw_token.len - 1] == '"'); + + const inner = raw_token[2 .. raw_token.len - 1]; + const len = Tokenizer.decodeStringCount(inner); + + try t.id_decode_buf.ensureTotalCapacityPrecise(len); + @memset(t.id_decode_buf.items, undefined); + t.id_decode_buf.items.len = len; + + return .{ + Tokenizer.decodeStringBuf(inner, t.id_decode_buf.items), + true, + }; +} + +fn decodeIdentifier(t: *Transform, raw_token: []const u8) ![]const u8 { + return (try t.decodeIdentifierExtra(raw_token)).@"0"; +} + +fn isSliceInBuffer(comptime T: type, haystack: []const T, needle: []const T) bool { + return (@intFromPtr(haystack.ptr) <= + @intFromPtr(needle.ptr) and + (@intFromPtr(needle.ptr) + needle.len * @sizeOf(T)) <= + (@intFromPtr(haystack.ptr) + haystack.len * @sizeOf(T))); +} + const Transform = @This(); const std = @import("std"); const mem = std.mem;