diff --git a/build.zig b/build.zig index a66998bce30b70508240713d6052815013e3e2c3..33a9d274adf43ac2ae78886bc9817abd140bdbfb 100644 --- a/build.zig +++ b/build.zig @@ -19,47 +19,29 @@ pub fn build(b: *std.Build) void { }, }; - const tracing = b.option(enum { - none, - spall, - chrome, - log, - }, "trace", "Enable tracing") orelse .none; + const tracing = b.option( + TracingMode, + "trace", + "Enable tracing", + ) orelse .none; - const dep_opts = .{ .target = target, .optimize = optimize }; + const main = buildWithTarget(b, target, optimize, tracing); - const zg = b.dependency("zg", dep_opts); - const tracer = b.dependency("tracer", dep_opts); - // const ada = b.dependency("ada", dep_opts); + inline for (.{ + .{ main.compiler, "compiler" }, + .{ main.lsp, "lsp" }, + }) |entry| { + const artifact, const name = entry; - const opts = b.addOptions(); - opts.addOption([]const u8, "tracing", @tagName(tracing)); - const opts_mod = opts.createModule(); - - const exe = b.addExecutable(.{ - .name = "compiler", - .root_source_file = b.path("src/compiler/main.zig"), - .target = target, - .optimize = optimize, - .link_libc = true, - }); - b.installArtifact(exe); - - exe.root_module.addImport("build_options", opts_mod); - exe.root_module.addImport("zg:PropsData", zg.module("PropsData")); - exe.root_module.addImport("tracer", tracer.module("tracer")); - // if (!target.result.isWasm()) { - // exe.root_module.addImport("ada", ada.module("ada")); - // } - - { - const run_cmd = b.addRunArtifact(exe); - run_cmd.step.dependOn(b.getInstallStep()); - if (b.args) |args| { - run_cmd.addArgs(args); + { + const run_cmd = b.addRunArtifact(artifact); + run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| { + run_cmd.addArgs(args); + } + const run_step = b.step(name, "Run " ++ name ++ " binary"); + run_step.dependOn(&run_cmd.step); } - const run_step = b.step("compiler", "Run compiler CLI"); - run_step.dependOn(&run_cmd.step); } // const wasm_api = b.addExecutable(.{ @@ -102,30 +84,90 @@ pub fn build(b: *std.Build) void { }, ) catch @panic("OOM"); - // std.log.info("{s}\n", .{cross_target.result.zigTriple(b.allocator) catch @panic("OOM")}); + const artifacts = buildWithTarget(b, cross_target, .ReleaseSafe, .none); - const cross_exe = b.addExecutable(.{ - .name = "compiler", - .root_source_file = b.path("src/compiler/main.zig"), - .target = cross_target, - .optimize = .ReleaseSafe, - .strip = true, - .link_libc = true, - }); - cross_exe.root_module.addImport("build_options", opts_mod); - cross_exe.root_module.addImport("zg:PropsData", zg.module("PropsData")); - cross_exe.root_module.addImport("tracer", tracer.module("tracer")); - - release.dependOn(&b.addInstallFile( - cross_exe.getEmittedBin(), - b.pathJoin(&.{ - "release", - name, - std.mem.concat(b.allocator, u8, &.{ - "compiler", - cross_target.result.exeFileExt(), - }) catch @panic("OOM"), - }), - ).step); + inline for (.{ + .{ artifacts.compiler, "compiler" }, + .{ artifacts.lsp, "lsp" }, + }) |entry| { + const artifact, const bin_name = entry; + release.dependOn(&b.addInstallFile( + artifact.getEmittedBin(), + b.pathJoin(&.{ + "release", + name, + std.mem.concat(b.allocator, u8, &.{ + bin_name, + cross_target.result.exeFileExt(), + }) catch @panic("OOM"), + }), + ).step); + } } } + +const TracingMode = enum { none, spall, chrome, log }; + +const Artifacts = struct { + core: *Build.Module, + compiler: *Build.Step.Compile, + lsp: *Build.Step.Compile, +}; + +fn buildWithTarget( + b: *Build, + target: Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, + tracing: TracingMode, +) Artifacts { + const dep_opts = .{ .target = target, .optimize = optimize }; + + // Dependencies + const zg = b.dependency("zg", dep_opts); + const tracer = b.dependency("tracer", dep_opts); + + // Options + const opts = b.addOptions(); + opts.addOption([]const u8, "tracing", @tagName(tracing)); + const opts_mod = opts.createModule(); + + // Core + const core = b.addModule("compiler", .{ + .root_source_file = b.path("src/core/mod.zig"), + .optimize = optimize, + .target = target, + }); + core.addImport("build_options", opts_mod); + core.addImport("tracer", tracer.module("tracer")); + core.addImport("zg:PropsData", zg.module("PropsData")); + + // Compiler Testing Exe + const compiler_exe = b.addExecutable(.{ + .name = "compiler", + .root_source_file = b.path("src/core/main.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + compiler_exe.root_module.addImport("build_options", opts_mod); + compiler_exe.root_module.addImport("core", core); + compiler_exe.root_module.addImport("tracer", tracer.module("tracer")); + + // Language Server + const lsp_exe = b.addExecutable(.{ + .name = "lsp", + .root_source_file = b.path("src/lsp/lsp.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + lsp_exe.root_module.addImport("build_options", opts_mod); + lsp_exe.root_module.addImport("core", core); + lsp_exe.root_module.addImport("tracer", tracer.module("tracer")); + + return .{ + .core = core, + .compiler = compiler_exe, + .lsp = lsp_exe, + }; +} diff --git a/src/compiler/Air.Dump.zig b/src/compiler/Air.Dump.zig deleted file mode 100644 index 51d7af7eaf89f4021e3893f0202526b803c38115..0000000000000000000000000000000000000000 --- a/src/compiler/Air.Dump.zig +++ /dev/null @@ -1,316 +0,0 @@ -//! This implements a debug formatter for Teg -air: Air, -ip: *const InternPool, - -pub const State = struct { - arena: std.mem.Allocator, - air: *const Air, - ip: *const InternPool, - indent: usize = 0, - tty: std.io.tty.Config, - visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{}, - - fn newline(state: State, writer: anytype) !void { - try writer.writeByte('\n'); - if (state.indent > 0) - try writer.writeByteNTimes(' ', state.indent * 2); - } - - fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void { - const air = s.air; - const tty = s.tty; - const tag = s.air.instTag(inst); - - try tty.setColor(w, switch (s.air.instFlags(inst).uses) { - .none => .bright_black, - .multi => .red, - .single => .magenta, - }); - try w.writeByte(switch (s.air.instFlags(inst).uses) { - .none => 'X', - .multi => 'M', - .single => 'S', - }); - try tty.setColor(w, .cyan); - - try w.print(" #{} = ", .{ - @intFromEnum(inst), - }); - - try tty.setColor(w, .cyan); - try w.print("{s}", .{@tagName(tag)}); - try tty.setColor(w, .reset); - - if ((try s.visited.fetchPut(s.arena, inst, {})) != null) { - try tty.setColor(w, .red); - try w.writeAll(" [error: recursive tree]"); - try tty.setColor(w, .reset); - return; - } - - switch (tag) { - .block => { - const index_list: Air.IndexList = air.instData(.block, inst); - const insts = index_list.slice(air); - try s.dumpInstList(w, insts, '{', '}'); - }, - .js_global_ref => { - const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst); - try tty.setColor(w, .magenta); - try w.print(" {s}", .{name.toSlice(s.ip)}); - try tty.setColor(w, .reset); - }, - .arg => { - const arg: Air.Arg = air.instData(.arg, inst); - try w.writeAll(" "); - try tty.setColor(w, .yellow); - try w.print("{d}", .{arg.arg_number}); - try tty.setColor(w, .reset); - try w.writeAll(": "); - try s.dumpValue(w, arg.type.value()); - }, - .@"break" => { - const brk: Air.Break = air.instData(.@"break", inst); - try w.writeAll("("); - try s.dumpInstRef(w, brk.block_ref); - try w.writeAll(", "); - try s.dumpRef(w, brk.value); - try w.writeAll(")"); - }, - .store => { - const store: Air.Store = air.instData(.store, inst); - try w.writeAll("("); - try s.dumpInstRef(w, store.memory); - try w.writeAll(", "); - try s.dumpRef(w, store.operand); - try w.writeAll(")"); - }, - inline .add, - .mul, - .sub, - .div, - .lt, - .gt, - .eq, - => |comptime_tag| { - const bin: Air.Bin = air.instData(comptime_tag, inst); - try w.writeAll("("); - try s.dumpRef(w, bin.lhs); - try w.writeAll(", "); - try s.dumpRef(w, bin.rhs); - try w.writeAll(")"); - }, - inline .negate, - .bool_not, - => |comptime_tag| { - const bin: Air.Un = air.instData(comptime_tag, inst); - try w.writeAll("("); - try s.dumpRef(w, bin.operand); - try w.writeAll(")"); - }, - .memory => { - const memory: Air.Memory = air.instData(.memory, inst); - try w.writeAll("("); - try s.dumpRef(w, memory.init); - try w.writeAll(", type: "); - try s.dumpValue(w, memory.ty.value()); - try w.writeAll(")"); - }, - .field => { - const field: Air.Field = air.instData(.field, inst); - try w.writeAll("("); - try s.dumpRef(w, field.base); - try w.writeAll(", "); - try tty.setColor(w, .magenta); - try w.print("{s}", .{field.field.toSlice(s.ip)}); - try tty.setColor(w, .reset); - try w.writeAll(")"); - }, - .call => { - const call: Air.Call = air.instData(.call, inst); - const call_data = call.call_data.get(air); - try w.writeAll("("); - try s.dumpRef(w, call.base); - try w.writeAll(", "); - try s.dumpRefList(w, call_data.args.slice(air), '[', ']'); - try w.writeAll(")"); - }, - .@"if" => { - const if_inst: Air.If = air.instData(.@"if", inst); - const if_data: Air.If.Data = if_inst.data.get(air); - try w.writeAll("("); - try s.dumpRef(w, if_inst.cond); - try w.writeAll(", {"); - s.indent += 1; - try s.newline(w); - try w.writeAll("yes: "); - try s.dumpRefRecurse(w, if_data.yes); - try s.newline(w); - try w.writeAll("no: "); - try s.dumpRefRecurse(w, if_data.no); - s.indent -= 1; - try s.newline(w); - try w.writeAll("}"); - }, - } - } - - fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void { - switch (ref.decode()) { - .index => |i| try s.dumpInstRef(w, i), - .value => |v| try s.dumpValue(w, v), - } - } - - fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void { - switch (ref.decode()) { - .index => |i| try s.dumpInst(w, i), - .value => |v| try s.dumpValue(w, v), - } - } - - fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void { - try s.tty.setColor(w, .red); - try w.print("#{d}", .{@intFromEnum(inst)}); - try s.tty.setColor(w, .reset); - } - - pub fn dumpValue(s: *State, w: anytype, val: Value) !void { - const tty = s.tty; - switch (val.toKey(s.ip)) { - .simple_type => |t| { - try tty.setColor(w, .blue); - try w.print("{s}", .{@tagName(t)}); - try tty.setColor(w, .reset); - }, - .simple_value => |t| { - try tty.setColor(w, .yellow); - try w.print("{s}", .{@tagName(t)}); - try tty.setColor(w, .reset); - }, - .type_function => |ty| { - try tty.setColor(w, .blue); - try w.writeAll("fn("); - try tty.setColor(w, .reset); - for (ty.param_types, 0..) |param, i| { - if (i != 0) { - try tty.setColor(w, .blue); - try w.writeAll(","); - try tty.setColor(w, .reset); - } - try s.dumpValue(w, param.value()); - } - try tty.setColor(w, .blue); - try w.writeAll(") -> "); - try tty.setColor(w, .reset); - try s.dumpValue(w, ty.return_type.value()); - }, - inline .float, .int32, .comptime_float => |data, tag| { - try tty.setColor(w, .bright_black); - try w.writeAll("<"); - try tty.setColor(w, .blue); - try w.writeAll(@tagName(tag)); - try tty.setColor(w, .reset); - try tty.setColor(w, .bright_black); - try w.writeAll(", "); - try tty.setColor(w, .yellow); - try w.print("{d}", .{data}); - try tty.setColor(w, .bright_black); - try w.writeAll(">"); - try tty.setColor(w, .reset); - }, - .string => |data| { - try w.writeAll("<"); - try tty.setColor(w, .blue); - try w.writeAll("string"); - try tty.setColor(w, .reset); - try w.writeAll(", "); - try tty.setColor(w, .green); - try w.writeAll("\""); - try w.writeAll(data); - try w.writeAll("\""); - try tty.setColor(w, .reset); - try w.writeAll(">"); - }, - .func => |func| { - try tty.setColor(w, .bright_black); - try w.writeAll("<"); - try s.dumpValue(w, func.ty.value()); - try tty.setColor(w, .bright_black); - try w.writeAll(", "); - const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip); - try tty.setColor(w, .bright_green); - try w.print("decl '{s}'", .{decl_name}); - try tty.setColor(w, .bright_black); - try w.writeAll(">"); - try tty.setColor(w, .reset); - }, - } - } - - fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - if (list.len > 0) { - state.indent += 1; - - for (list) |inst| { - try state.newline(writer); - try state.dumpInst(writer, inst); - } - - state.indent -= 1; - try state.newline(writer); - } - try writer.print("{c}", .{r}); - } - - fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - if (list.len > 0) { - state.indent += 1; - - for (list) |inst| { - try state.newline(writer); - try state.dumpRef(writer, inst); - } - - state.indent -= 1; - try state.newline(writer); - } - try writer.print("{c}", .{r}); - } -}; - -pub fn format( - dump: @This(), - comptime unused_fmt: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, -) !void { - comptime assert(unused_fmt.len == 0); - - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const air = dump.air; - var state = State{ - .air = &air, - .ip = dump.ip, - .tty = detectTTYFromWriter(writer), - .arena = arena.allocator(), - }; - try state.dumpInst(writer, .main_block); - try writer.writeAll("\n"); - assert(state.indent == 0); -} - -const std = @import("std"); -const assert = std.debug.assert; - -const Air = @import("Air.zig"); -const Inst = Air.Inst; - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; - -const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/compiler/Air.zig b/src/compiler/Air.zig deleted file mode 100644 index 4252ffa290e9295ce7318681d943d70aad6298ed..0000000000000000000000000000000000000000 --- a/src/compiler/Air.zig +++ /dev/null @@ -1,320 +0,0 @@ -insts: Inst.List.Slice, -extra: ExtraStore = .{}, - -pub const ExtraStore = dod.ExtraStore(.air, .@"const"); -pub const EncodedData = dod.EncodedData(Inst.Tag, u64); -pub const ExtraIndex = ExtraStore.Index; - -pub const Inst = struct { - tag: Tag, - data: EncodedData, - flags: Flags, - - pub const List = std.MultiArrayList(Inst); - - pub const Tag = enum { - block, - js_global_ref, - call, - @"break", - @"if", - arg, - field, - - memory, - store, - - // bin - add, - sub, - mul, - div, - - eq, - - lt, - gt, - - // un - negate, - bool_not, - - pub fn Data(comptime tag: Tag) type { - return switch (tag) { - .block => IndexList, - .js_global_ref => InternPool.String.WithLen, - .@"break" => Break, - .@"if" => If, - .call => Call, - .arg => Arg, - .field => Field, - .memory => Memory, - .store => Store, - .add, - .sub, - .mul, - .div, - .lt, - .gt, - .eq, - => Bin, - .negate, .bool_not => Un, - }; - } - }; - - /// reference to either a value or another instruction in this air - pub const Ref = packed struct(u32) { - kind: Kind, - data: u31, - - pub const Kind = enum(u1) { value, index }; - - pub const Decoded = union(enum) { - value: Value, - index: Index, - }; - - pub fn decode(ref: Ref) Decoded { - return switch (ref.kind) { - .value => .{ .value = @enumFromInt(ref.data) }, - .index => .{ .index = @enumFromInt(ref.data) }, - }; - } - - pub fn toOptional(ref: ?Ref) Optional { - return if (ref) |r| @enumFromInt(@as(u32, @bitCast(r))) else .none; - } - - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn unwrap(opt: Optional) ?Ref { - return if (opt != .none) @bitCast(@intFromEnum(opt)) else null; - } - }; - - pub fn isVoid(ref: Ref) bool { - return ref.kind == .value and - ref.decode().value == .void; - } - - pub fn eql(ref: Ref, other: Ref) bool { - return @as(u32, @bitCast(ref)) == @as(u32, @bitCast(other)); - } - }; - - /// reference to another instruction in this air - /// note that these cannot point to code in other functions, such makes no sense - pub const Index = enum(u32) { - main_block, - _, - - pub fn toRef(index: Index) Ref { - return .{ - .kind = .index, - .data = @intCast(@intFromEnum(index)), - }; - } - - pub fn toOptional(index: Index) Optional { - return @enumFromInt(@intFromEnum(index)); - } - - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn unwrap(oi: Optional) ?Index { - return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null; - } - }; - }; -}; - -pub const Flags = packed struct { - uses: Uses = .none, - - pub const Uses = enum(u2) { - none, - single, - multi, - - pub fn increment(uses: Uses) Uses { - return switch (uses) { - .none => .single, - .single, .multi => .multi, - }; - } - }; -}; - -pub const IndexList = struct { - start: u32, - len: u32, - - pub fn slice(self: IndexList, air: *const Air) []const Inst.Index { - return @ptrCast(air.extra.list.items[self.start..][0..self.len]); - } -}; - -pub const RefList = struct { - start: u32, - len: u32, - - pub fn slice(self: RefList, air: *const Air) []const Inst.Ref { - return @ptrCast(air.extra.list.items[self.start..][0..self.len]); - } -}; - -pub const Break = struct { - block_ref: Inst.Index, - value: Inst.Ref, -}; - -pub const Bin = struct { - lhs: Inst.Ref, - rhs: Inst.Ref, -}; - -pub const Store = struct { - memory: Inst.Index, - operand: Inst.Ref, -}; - -pub const Un = struct { - operand: Inst.Ref, - unused: Unused = .zero, -}; - -pub const Field = struct { - base: Inst.Ref, - field: InternPool.NullTerminatedString, -}; - -pub const Call = struct { - base: Inst.Ref, - call_data: ExtraIndex(Data), - - pub const Data = struct { - args: RefList, - }; -}; - -pub const Arg = struct { - type: Type, - arg_number: u32, -}; - -pub const If = struct { - cond: Inst.Ref, - data: ExtraIndex(Data), - - pub const Data = struct { - yes: Inst.Ref, - no: Inst.Ref, - union_ty: Type, - }; -}; - -pub const Memory = struct { - init: Inst.Ref, - ty: Type, -}; - -pub fn deinit(air: *Air, gpa: std.mem.Allocator) void { - defer air.* = undefined; - air.insts.deinit(gpa); - air.extra.deinit(gpa); -} - -pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag { - return air.insts.items(.tag)[@intFromEnum(index)]; -} - -pub fn instFlags(air: *const Air, index: Inst.Index) Flags { - return air.insts.items(.flags)[@intFromEnum(index)]; -} - -pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData { - return air.insts.items(.data)[@intFromEnum(index)]; -} - -pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() { - return air.instEncodedData(index).decode(tag, air.instTag(index)); -} - -pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T { - return air.instData(tag, index).extra_index.get(air); -} - -pub fn typeOf(air: *const Air, ref: Inst.Ref, ip: *InternPool) InternPool.Type { - return switch (ref.decode()) { - .value => |v| v.typeOf(ip), - .index => |i| air.typeOfIndex(i, ip), - }; -} - -pub fn typeOfIndex(air: *const Air, index: Inst.Index, ip: *InternPool) InternPool.Type { - return switch (air.instTag(index)) { - .js_global_ref => .@"opaque", - - .block => { - const items = air.instData(.block, index).slice(air); - return air.typeOfIndex(items[items.len - 1], ip); - }, - - inline .add, .sub, .mul, .div => |t| air.typeOf(air.instData(t, index).lhs, ip), - - .lt, .gt, .eq, .bool_not => .bool, - - .arg => air.instData(.arg, index).type, - - .negate => air.typeOf(air.instData(.negate, index).operand, ip), - - .@"break" => .noreturn, - - // todo: - .field => .@"opaque", - - .call => { - const base_type = air.typeOf(air.instData(.call, index).base, ip); - if (base_type == .@"opaque") return .@"opaque"; - return base_type.value().toKey(ip).type_function.return_type; - }, - - .memory => air.instData(.memory, index).ty, - - .store => air.instData(.memory, air.instData(.store, index).memory).ty, - - .@"if" => air.instData(.@"if", index).data.get(air).union_ty, - }; -} - -pub fn paramCount(air: Air) u32 { - for (air.insts.items(.tag)[1..], 0..) |tag, i| { - if (tag != .arg) return @intCast(i); - } - return @intCast(air.insts.len); -} - -pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void { - try writer.print("{}", .{Dump{ .air = air, .ip = ip }}); -} - -pub const Dump = @import("Air.Dump.zig"); - -const Air = @This(); -const std = @import("std"); -const mem = std.mem; -const assert = std.debug.assert; - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; -const Type = InternPool.Type; - -const Teg = @import("Teg.zig"); -const dod = @import("array.zig"); - -const Unused = enum(u32) { zero = 0 }; diff --git a/src/compiler/Ast.Dump.zig b/src/compiler/Ast.Dump.zig deleted file mode 100644 index 33186d4e05d9b4ffaa86a9542af1b0698d9ef82c..0000000000000000000000000000000000000000 --- a/src/compiler/Ast.Dump.zig +++ /dev/null @@ -1,300 +0,0 @@ -//! A flat array is not very useful for debugging, so this structure implements -//! a debug formatter for Ast. -ast: Ast, - -const State = struct { - ast: *const Ast, - indent: usize = 0, - node_tags: []const Node.Tag, - node_data: []const Node.Data, - node_tokens: []const TokenIndex, - tty: std.io.tty.Config, - - fn newline(state: State, writer: anytype) !void { - try writer.writeByte('\n'); - if (state.indent > 0) - try writer.writeByteNTimes(' ', state.indent * 2); - } -}; - -pub fn format( - dump: Dump, - comptime unused_fmt: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, -) !void { - comptime assert(unused_fmt.len == 0); - const ast = dump.ast; - var state = State{ - .ast = &ast, - .node_tags = ast.nodes.items(.tag), - .node_data = ast.nodes.items(.data), - .node_tokens = ast.nodes.items(.main_token), - .tty = detectTTYFromWriter(writer), - }; - try dumpNode(writer, &state, 0); - try writer.writeAll("\n"); - assert(state.indent == 0); -} - -fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void { - const tag = state.node_tags[node]; - const data = state.node_data[node]; - const data_tag = Node.data_tags[@intFromEnum(tag)]; - const main_token = state.node_tokens[node]; - const tty = state.tty; - - try state.tty.setColor(writer, .bright_black); - try writer.print("{d}", .{node}); - try state.tty.setColor(writer, .reset); - try writer.writeAll(" "); - - try tty.setColor(writer, .cyan); - try writer.print("{s}", .{@tagName(tag)}); - try tty.setColor(writer, .reset); - - switch (tag) { - .builtin_call => { - try tty.setColor(writer, .red); - try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}); - try tty.setColor(writer, .reset); - }, - else => {}, - } - - switch (data_tag) { - .none => { - switch (tag) { - .id => try tty.setColor(writer, .red), - .string => try tty.setColor(writer, .green), - .number_int, .number_float => try tty.setColor(writer, .yellow), - else => {}, - } - switch (tag) { - .id, - .string, - .number_int, - .number_float, - => try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}), - - else => unreachable, - } - try tty.setColor(writer, .reset); - }, - .fn_decl => { - try writer.writeAll(" "); - const fn_decl = state.ast.fullFnDecl(node); - if (fn_decl.public) { - try writer.writeAll("[pub] "); - } - try tty.setColor(writer, .magenta); - try writer.writeAll(fn_decl.id); - try tty.setColor(writer, .reset); - try writer.writeAll(" {"); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try writer.writeAll("params:"); - try dumpNodeList(writer, state, fn_decl.params, '[', ']'); - if (fn_decl.return_type) |return_type| { - try state.newline(writer); - try writer.writeAll("return_type: "); - try dumpNode(writer, state, return_type); - } - try state.newline(writer); - try writer.writeAll("body: "); - try dumpNode(writer, state, fn_decl.body); - } - try state.newline(writer); - try writer.writeAll("}"); - }, - .list => { - const list = data.list.slice(state.ast); - switch (tag) { - .root, - .block_void, - .block_implicit_return, - => try dumpNodeList(writer, state, list, '{', '}'), - .builtin_call, - => try dumpNodeList(writer, state, list, '(', ')'), - else => unreachable, - } - }, - .var_decl => { - try writer.writeAll(" "); - const var_decl = state.ast.fullVarDecl(node); - if (var_decl.public) { - try writer.writeAll("[pub] "); - } - try writer.writeAll(switch (var_decl.is_const) { - true => "constant", - false => "mutable", - }); - try tty.setColor(writer, .magenta); - try writer.print(" {s}", .{var_decl.id}); - try tty.setColor(writer, .reset); - var indent = false; - if (var_decl.type) |type_node| { - state.indent += 1; - indent = true; - try state.newline(writer); - try writer.writeAll(": "); - try dumpNode(writer, state, type_node); - try writer.writeAll(", "); - } - try writer.writeAll(" = "); - try dumpNode(writer, state, var_decl.init); - if (indent) { - state.indent -= 1; - } - }, - .param_decl => { - try writer.writeAll(" "); - const param_decl = state.ast.fullParamDecl(node); - try tty.setColor(writer, .magenta); - try writer.writeAll(param_decl.id); - try tty.setColor(writer, .reset); - try writer.writeAll(": "); - try dumpNode(writer, state, param_decl.type); - }, - .bin => { - try writer.writeAll(" ("); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try dumpNode(writer, state, data.bin.lhs); - try state.newline(writer); - try dumpNode(writer, state, data.bin.rhs); - } - try state.newline(writer); - try writer.writeAll(")"); - }, - .un => { - try writer.writeAll(" ("); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try dumpNode(writer, state, data.un.expr); - } - try state.newline(writer); - try writer.writeAll(")"); - }, - .label => { - const label_text = state.ast.tokenSlice(main_token); - try writer.print(" {s}:", .{label_text}); - - try writer.writeAll(" ("); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try dumpNode(writer, state, data.label.expr); - } - try state.newline(writer); - try writer.writeAll(")"); - }, - .field => { - try writer.writeAll(" ("); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try dumpNode(writer, state, data.field.base); - } - try state.newline(writer); - try writer.writeAll(")"); - }, - .@"return" => { - if (data.@"return".expr.unwrap()) |expr| { - try writer.writeAll(" ("); - try dumpNode(writer, state, expr); - try writer.writeAll(")"); - } - }, - .@"break" => { - const label = data.@"break".label; - const value = data.@"break".value; - if (label != .none or value != .none) { - // try writer.writeAll("("); - // try writer.writeAll(")"); - if (label.unwrap()) |l| { - const label_text = state.ast.tokenSlice(l); - try writer.print(" :{s}", .{label_text}); - } - if (value.unwrap()) |expr| { - try writer.writeAll(" ("); - try dumpNode(writer, state, expr); - try writer.writeAll(")"); - } - } - }, - .call => { - const base = data.call.base; - const extra = state.ast.extraData(data.call.extra, Call); - const arg_list = extra.args.slice(state.ast); - - try writer.writeAll("("); - try writer.writeAll("base: "); - try dumpNode(writer, state, base); - try writer.writeAll(", "); - try writer.writeAll("arg:"); - try dumpNodeList(writer, state, arg_list, '[', ']'); - try writer.writeAll(")"); - }, - .@"if" => { - const branch = data.@"if"; - const if_data = state.ast.extraData(branch.extra, Ast.If); - - try writer.writeAll(" ("); - try dumpNode(writer, state, branch.cond); - try writer.writeAll(") {"); - { - state.indent += 1; - defer state.indent -= 1; - try state.newline(writer); - try writer.writeAll("yes: "); - try dumpNode(writer, state, if_data.yes); - try state.newline(writer); - if (if_data.no.unwrap()) |no| { - try writer.writeAll("no: "); - try dumpNode(writer, state, no); - } else { - try writer.writeAll("no: [none]"); - } - } - try state.newline(writer); - try writer.writeAll("}"); - }, - } -} - -fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - { - state.indent += 1; - defer state.indent -= 1; - - for (list) |node| { - try state.newline(writer); - try dumpNode(writer, state, node); - } - } - try state.newline(writer); - try writer.print("{c}", .{r}); -} - -const Dump = @This(); - -const std = @import("std"); -const assert = std.debug.assert; - -const Ast = @import("Ast.zig"); -const NodeIndex = Ast.NodeIndex; -const Node = Ast.Node; -const TokenIndex = Ast.TokenIndex; -const Call = Ast.Call; - -const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/compiler/Ast.zig b/src/compiler/Ast.zig deleted file mode 100644 index e11c6d8fa3ed6be8c5d0d1be924fcfe2707e062f..0000000000000000000000000000000000000000 --- a/src/compiler/Ast.zig +++ /dev/null @@ -1,678 +0,0 @@ -//! The Abstract Syntax Tree preserves most details of the source code, -//! as it is used for formatting. Before semantic analysis, this -//! structure is normalized into `Teg`, which removes constructs like -//! grouping, unneded let bindings, and other transforms. -//! -//! `ast.nodes` contains nodes stored as three parts: -//! - tag: the type of the node -//! - main_token: index to it's primary token, depends on the tag -//! - data: two u32 values, depends on the tag -//! -//! Many nodes encode all their metadata with just `node.data`, using -//! facts about the language grammar to be able to infer data without -//! storing it. -//! -//! For example, `var_decl` has the following encoding: -//! - `main_token` is either a `kw_var` or `kw_let` -//! - the token after `main_token` is always the identifier -//! - `data.var_decl` contains `NodeIndex`es pointing at the type -//! and the initializer, which are both expressions. -//! - If the keyword before `main_token` is `kw_pub` it is a public variable. -//! -//! To make decoding these easier, many nodes contain a "full" variant, -//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`. -//! -//! For nodes that must store more than 64 bits of payload data, an "extra" -//! struct is stored out-of-line in the `extra` field. These can be read -//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use -//! the `full` structures with their convenience functions. -//! -//! Documentation comments are not stored anywhere, but you can find them -//! by iterating backwards from a declaration and seeing if a `.doc_comment` -//! is present. This is provided by `ast.declDocComment(id)`. -//! -//! The `Ast` owns token and node data, but not the source code. - -/// Externally owned source code. -utf8_source: [:0]const u8, - -tokens: Token.List.Slice, -nodes: Node.List.Slice, -extra: []const u32, - -pub const Node = struct { - tag: Tag, - main_token: TokenIndex, - data: Data, - - pub const List = std.MultiArrayList(Node); - - pub const Tag = enum { - /// Read with `ast.rootNodes()` - /// data is .list (containing stmt slice) - /// TODO: rename to `module` - root, - /// main_token is the { - /// data is .list (containing stmt slice) - /// - /// This guarantees that the block either empty, or the last statement - /// ends with a semicolon. - block_void, - /// main_token is the { - /// data is .list (containing stmt slice) - /// - /// This guarantees that the block is not empty and that the last - /// statement is an expression statement without a trailing semicolon. - block_implicit_return, - /// Variable or constant declaration - /// `var x: type = init;` - /// `let y = x;` - /// main_token is the keyword - /// data is .var_decl - var_decl, - /// Function declaration - /// `fn foo(x: i32) -> i32 { rhs }` - /// Params and return type are stored in extra data - fn_decl, - /// 'name: type' declaration - /// Main token is the identifier - param_decl, - /// An identifier. main_token is the identifier. - id, - /// Main token is the data. May not fit in i32 range, - /// but will contain no dots or e's. - number_int, - /// Main token is data. Will contain '.' and/or 'e'. - number_float, - /// A string literal - string, - /// A multi-line string literal - /// data is .token_list of the tokens, which is guaranteed to be contiguous - multiline_string, - /// Call to a function - /// `foo(x, y);` - /// main_token is the `(` - /// data is .call - call, - /// Call to a builtin function - /// `@raw("hello");` - /// data is .list (containing arg slice) - builtin_call, - /// `if cond { ... } else ...` - /// main_token is the `if` - /// data is .@"if" - @"if", - /// `if let id = ... { ... } else ...` - /// main_token is the `if` - /// data is .@"if" - // if_let_nonvoid, - /// `id: ...` - /// data is un - label, - /// `expr.field` - /// main_token is the `.` - /// data is .field - field, - /// `lhs[rhs]` - /// main_token is the `[` - /// data is .bin - index, - /// `return expr;` - /// main_token is the `return` - /// data is .@"return" - @"return", - /// `break :brk expr;` - /// main_token is the `break` - /// data is .@"break" - @"break", - /// `(expr)` - /// main_token is the `(` - /// data is .un - group, - /// `lhs + rhs`. main_token is the operator. - add, - /// `lhs - rhs`. main_token is the operator. - sub, - /// `lhs * rhs`. main_token is the operator. - mul, - /// `lhs / rhs`. main_token is the operator. - div, - /// `lhs % rhs`. main_token is the operator. - rem, - /// `lhs ** rhs`. main_token is the operator. - pow, - /// `lhs && rhs`. main_token is the operator. - @"and", - /// `lhs || rhs`. main_token is the operator. - @"or", - /// `lhs & rhs`. main_token is the operator. - bit_and, - /// `lhs | rhs`. main_token is the operator. - bit_or, - /// `lhs ^ rhs`. main_token is the operator. - bit_xor, - /// `lhs << rhs`. main_token is the operator. - shl, - /// `lhs >> rhs`. main_token is the operator. - shr, - /// `lhs == rhs`. main_token is the operator. - eq, - /// `lhs != rhs`. main_token is the operator. - ne, - /// `lhs < rhs`. main_token is the operator. - lt, - /// `lhs <= rhs`. main_token is the operator. - le, - /// `lhs > rhs`. main_token is the operator. - gt, - /// `lhs >= rhs`. main_token is the operator. - ge, - /// `lhs = rhs`. main_token is the operator. - assign, - /// `lhs += rhs`. main_token is the operator. - add_assign, - /// `lhs -= rhs`. main_token is the operator. - sub_assign, - /// `lhs *= rhs`. main_token is the operator. - mul_assign, - /// `lhs /= rhs`. main_token is the operator. - div_assign, - /// `lhs %= rhs`. main_token is the operator. - rem_assign, - /// `lhs **= rhs`. main_token is the operator. - pow_assign, - /// `lhs &&= rhs`. main_token is the operator. - and_assign, - /// `lhs ||= rhs`. main_token is the operator. - or_assign, - /// `lhs &= rhs`. main_token is the operator. - bit_and_assign, - /// `lhs |= rhs`. main_token is the operator. - bit_or_assign, - /// `lhs ^= rhs`. main_token is the operator. - bit_xor_assign, - /// `lhs <<= rhs`. main_token is the operator. - shl_assign, - /// `lhs >>= rhs`. main_token is the operator. - shr_assign, - /// `!expr`. main_token is the operator. - bool_not, - /// `-expr`. main_token is the operator. - negate, - /// `~expr`. main_token is the operator. - bit_not, - /// `comptime expr`. main_token is the operator. - @"comptime", - /// `pure expr`. main_token is the operator. - // pure, - /// `assert expr`. main_token is the operator. - // assert, - - pub fn isAssignBin(op: Tag) bool { - return switch (op) { - .assign, - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => true, - .add, - .sub, - .mul, - .div, - .rem, - .pow, - .@"and", - .@"or", - .bit_and, - .bit_or, - .bit_xor, - .shl, - .shr, - .eq, - .ne, - .lt, - .le, - .gt, - .ge, - => false, - else => unreachable, // not a binary operator - }; - } - - pub fn isBasic(op: Tag) bool { - return switch (op) { - .id, - .string, - .number_int, - .number_float, - .builtin_call, - => true, - else => false, - }; - } - }; - - pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{ - .root = .list, - .block_void = .list, - .block_implicit_return = .list, - .var_decl = .var_decl, - .fn_decl = .fn_decl, - .param_decl = .param_decl, - .id = .none, - .number_int = .none, - .number_float = .none, - .string = .none, - .multiline_string = .list, - .call = .call, - .builtin_call = .list, - .@"if" = .@"if", - .field = .field, - .index = .bin, - .@"return" = .@"return", - .@"break" = .@"break", - .group = .un, - .add = .bin, - .sub = .bin, - .mul = .bin, - .div = .bin, - .rem = .bin, - .pow = .bin, - .@"and" = .bin, - .@"or" = .bin, - .bit_and = .bin, - .bit_or = .bin, - .bit_xor = .bin, - .shl = .bin, - .shr = .bin, - .eq = .bin, - .ne = .bin, - .lt = .bin, - .le = .bin, - .gt = .bin, - .ge = .bin, - .assign = .bin, - .add_assign = .bin, - .sub_assign = .bin, - .mul_assign = .bin, - .div_assign = .bin, - .rem_assign = .bin, - .pow_assign = .bin, - .and_assign = .bin, - .or_assign = .bin, - .bit_and_assign = .bin, - .bit_or_assign = .bin, - .bit_xor_assign = .bin, - .shl_assign = .bin, - .shr_assign = .bin, - .bool_not = .un, - .negate = .un, - .bit_not = .un, - .@"comptime" = .un, - .label = .label, - }); - - pub const Data = union { - none: struct { Unused = .zero, Unused = .zero }, - fn_decl: struct { - // A `FnDecl` - extra: ExtraIndex, - body: NodeIndex, - }, - /// either is a node list or a token list, depending on the node's tag - list: SubList, - var_decl: struct { - type: OptionalNodeIndex, - init: NodeIndex, - }, - param_decl: struct { - type: NodeIndex, - unused: Unused = .zero, - }, - bin: struct { - lhs: NodeIndex, - rhs: NodeIndex, - }, - field: struct { - base: NodeIndex, - field_token: TokenIndex, - }, - un: struct { - expr: NodeIndex, - unused: Unused = .zero, - }, - label: struct { - expr: NodeIndex, - unused: Unused = .zero, - }, - @"return": struct { - expr: OptionalNodeIndex, - unused: Unused = .zero, - }, - @"break": struct { - label: OptionalTokenIndex, - value: OptionalNodeIndex, - }, - call: struct { - base: NodeIndex, - /// A `Call` - extra: ExtraIndex, - }, - @"if": struct { - cond: NodeIndex, - /// A `If` - extra: ExtraIndex, - }, - - const FieldEnum = std.meta.FieldEnum(Data); - }; -}; - -pub const Error = error{ OutOfMemory, ParseFail }; - -pub const Location = struct { - line: Ordinal(u32), - column: Ordinal(u32), - // Byte offset of the start of the line - line_start: ByteOffset, - // Byte offset of the end of the line - line_end: ByteOffset, -}; - -pub const Span = struct { - start: u32, - end: u32, - main: u32, -}; - -// This can be removed when the following issue is implemented: -// See: https://github.com/ziglang/zig/issues/104 -// See: https://github.com/ziglang/zig/issues/3806 -pub const OptionalNodeIndex = enum(u32) { - none = 0, - _, - - pub fn init(idx: ?NodeIndex) OptionalNodeIndex { - assert(idx != 0); - return @enumFromInt(idx orelse return .none); - } - - pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex { - return switch (idx) { - .none => null, - _ => @intFromEnum(idx), - }; - } -}; - -pub const OptionalTokenIndex = enum(u32) { - none = 0, - _, - - pub fn init(idx: ?TokenIndex) OptionalTokenIndex { - assert(idx != 0); - return @enumFromInt(idx orelse return .none); - } - - pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex { - return switch (idx) { - .none => null, - _ => @intFromEnum(idx), - }; - } -}; - -pub const OptionalExtraIndex = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn init(idx: ?ExtraIndex) OptionalExtraIndex { - assert(idx != std.math.maxInt(u32)); - return @enumFromInt(idx orelse return .none); - } - - pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex { - return switch (idx) { - .none => null, - _ => @intFromEnum(idx), - }; - } -}; - -pub const SubList = struct { - /// Either NodeIndex or TokenIndex, depending on where this came from. - start: u32, - len: u32, - - pub fn slice(self: SubList, ast: *const Ast) []const u32 { - return ast.extra[self.start..][0..self.len]; - } -}; - -pub const FnDecl = struct { - params: SubList, - return_type: OptionalNodeIndex, -}; - -pub const Call = struct { - args: SubList, -}; - -pub const If = struct { - /// either .block_void or .block_implicit_return - yes: NodeIndex, - /// if it exists, it is either - /// .block_void, .block_implicit_return, or .@"if" - no: OptionalNodeIndex, -}; - -pub const full = struct { - pub const FnDecl = struct { - start_token: TokenIndex, - public: bool, - id: []const u8, - params: []const NodeIndex, - return_type: ?NodeIndex, - body: NodeIndex, - }; - - pub const VarDecl = struct { - start_token: TokenIndex, - public: bool, - is_const: bool, - id: []const u8, - type: ?NodeIndex, - init: NodeIndex, - }; - - pub const ParamDecl = struct { - start_token: TokenIndex, - id: []const u8, - type: NodeIndex, - }; -}; - -pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void { - ast.tokens.deinit(gpa); - ast.nodes.deinit(gpa); - gpa.free(ast.extra); - ast.* = undefined; -} - -pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag { - return ast.nodes.items(.tag)[node]; -} - -pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data { - return ast.nodes.items(.data)[node]; -} - -pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex { - return ast.nodes.items(.main_token)[node]; -} - -pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl { - const tok_tags = ast.tokens.items(.tag); - - const main_token = ast.nodes.items(.main_token)[node]; - const public = tok_tags[main_token -| 1] == .kw_pub; - assert(tok_tags[main_token] == .kw_fn); - assert(tok_tags[main_token + 1] == .id); - const id = ast.tokenSlice(main_token + 1); - - const data = ast.nodes.items(.data)[node].fn_decl; - const extra = ast.extraData(data.extra, FnDecl); - - return .{ - .start_token = if (public) main_token - 1 else main_token, - .public = public, - .id = id, - .body = data.body, - .params = extra.params.slice(&ast), - .return_type = extra.return_type.unwrap(), - }; -} - -pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl { - const tok_tags = ast.tokens.items(.tag); - - const main_token = ast.nodes.items(.main_token)[node]; - const public = tok_tags[main_token -| 1] == .kw_pub; - const is_const = switch (tok_tags[main_token]) { - .kw_let => true, - .kw_var => false, - else => |tag| { - if (std.debug.runtime_safety) { - std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)}); - } - unreachable; - }, - }; - assert(tok_tags[main_token + 1] == .id); - const id = ast.tokenSlice(main_token + 1); - const data = ast.nodes.items(.data)[node].var_decl; - return .{ - .start_token = if (public) main_token - 1 else main_token, - .public = public, - .is_const = is_const, - .id = id, - .type = data.type.unwrap(), - .init = data.init, - }; -} - -pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl { - const tok_tags = ast.tokens.items(.tag); - - const main_token = ast.nodes.items(.main_token)[node]; - assert(tok_tags[main_token] == .id); - const id = ast.tokenSlice(main_token); - const data = ast.nodes.items(.data)[node].param_decl; - return .{ - .start_token = main_token, - .id = id, - .type = data.type, - }; -} - -pub fn idToken(ast: Ast, node: NodeIndex) TokenIndex { - return switch (ast.nodeTag(node)) { - .param_decl, - .id, - .label, - => ast.nodeToken(node), - .fn_decl, - .var_decl, - => ast.nodeToken(node) + 1, - else => unreachable, // does not have an id - }; -} - -pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex { - const data = ast.nodes.items(.data)[node].list; - return data.slice(ast); -} - -pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location { - const token_start = self.tokens.items(.offset)[token_index]; - return Tokenizer.tokenLocation(self.utf8_source, start_offset, token_start); -} - -pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 { - return ast.tokens.get(token_index).slice(ast.utf8_source); -} - -pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 { - assert(ast.nodes.items(.tag)[node].isBasic()); - const token = ast.nodes.items(.main_token)[node]; - return ast.tokenSlice(token); -} - -pub fn fnDeclIdToken(tree: Ast, node: NodeIndex) TokenIndex { - assert(tree.nodeTag(node) == .fn_decl); - return tree.nodeToken(node) + 1; -} - -pub fn extraData(tree: Ast, index: usize, comptime T: type) T { - const fields = std.meta.fields(T); - var result: T = undefined; - var i: usize = 0; - inline for (fields) |field| { - @field(result, field.name) = switch (field.type) { - NodeIndex => tree.extra[index + i], - SubList => SubList{ - .start = tree.extra[index + i], - .len = tree.extra[index + i + 1], - }, - OptionalNodeIndex => @enumFromInt(tree.extra[index + i]), - OptionalTokenIndex => @enumFromInt(tree.extra[index + i]), - OptionalExtraIndex => @enumFromInt(tree.extra[index + i]), - else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)), - }; - i += switch (field.type) { - NodeIndex, - OptionalNodeIndex, - OptionalTokenIndex, - OptionalExtraIndex, - => 1, - SubList => 2, - else => comptime unreachable, - }; - } - return result; -} - -pub fn dumpToWriter(ast: Ast, writer: anytype) !void { - try writer.print("{}", .{Dump{ .ast = ast }}); -} - -pub const Dump = @import("Ast.Dump.zig"); - -pub const Unused = enum(u32) { zero = 0 }; -pub const TokenIndex = u32; -pub const ExtraIndex = u32; -pub const NodeIndex = u32; - -const Ast = @This(); -const std = @import("std"); -const assert = std.debug.assert; - -const Tokenizer = @import("Tokenizer.zig"); -const Token = Tokenizer.Token; -const TokenList = Tokenizer.Token.List; -const ByteOffset = Tokenizer.ByteOffset; - -const Report = @import("Report.zig"); - -const util = @import("util.zig"); -const Ordinal = util.Ordinal; diff --git a/src/compiler/Cache.zig b/src/compiler/Cache.zig deleted file mode 100644 index 49a4a49d428577d564b2d7495a65c21c68539943..0000000000000000000000000000000000000000 --- a/src/compiler/Cache.zig +++ /dev/null @@ -1,9 +0,0 @@ -const std = @import("std"); - -/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -pub const bin_digest_len = 16; -pub const hex_digest_len = bin_digest_len * 2; -pub const BinDigest = [bin_digest_len]u8; -pub const HexDigest = [hex_digest_len]u8; - -pub const HashHelper = std.Build.Cache.HashHelper; diff --git a/src/compiler/Comp.zig b/src/compiler/Comp.zig deleted file mode 100644 index 9c9791273104a5940a38ae8a40328ba6785112a9..0000000000000000000000000000000000000000 --- a/src/compiler/Comp.zig +++ /dev/null @@ -1,443 +0,0 @@ -//! Compilation - -/// General purpose allocator -gpa: Allocator, -/// Stores all Type and Value objects. -intern_pool: InternPool = .{}, -/// All files in the compilation -files: std.StringArrayHashMapUnmanaged(*File) = .{}, - -/// # of sema instructions in this compilation cycle -analyzed_instructions: usize = 0, - -options: Options = .{}, - -// TODO: consider computing this dynamicly per-link -current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{}, - -pub fn init(comp: *Comp) !void { - try comp.intern_pool.init(comp.gpa); -} - -pub fn deinit(comp: *Comp) void { - comp.intern_pool.deinit(comp.gpa); - for (comp.files.values()) |file| { - file.deinit(comp.gpa); - } - comp.files.deinit(comp.gpa); - comp.current_compile_decls.deinit(comp.gpa); -} - -pub fn getFile(comp: *Comp, absolute_file_path: []const u8) !*File { - assert(std.fs.path.isAbsolute(absolute_file_path)); - const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path); - if (gop.found_existing) { - return gop.value_ptr.*; - } - const file = try comp.gpa.create(File); - file.* = .{ - .file_path = absolute_file_path, - .path_digest = null, - .root_module = .none, - .status = .never_loader, - .source = null, - .ast = null, - .teg = null, - }; - gop.value_ptr.* = file; - return file; -} - -pub const Source = struct { - code: [:0]const u8, - stat: Stat, -}; - -pub const File = struct { - /// TODO: change this to a URL - file_path: []const u8, - - root_module: Decl.Index.Optional, - status: enum { - never_loader, - /// A failure occurred during loading the file. Retrying might succeed. - retryable_failure, - /// Errors during parse - parse_failure, - /// Errors during transform - transform_failure, - /// Teg is valid - success, - }, - - source: ?Source, - ast: ?Ast, - teg: ?Teg, - - path_digest: ?Cache.BinDigest, - - const GetSourceError = - std.fs.File.OpenError || - std.fs.File.ReadError || - std.fs.File.StatError || - Allocator.Error || error{ - FileTooBig, - UnexpectedEndOfFile, - }; - - pub fn getSource(file: *File, gpa: Allocator) GetSourceError!Source { - if (file.source) |source| { - return source; - } - - // Keep track of inode, file size, mtime, hash so we can detect which files - // have been modified when an incremental update is requested. - var f = try std.fs.openFileAbsolute(file.file_path, .{}); - defer f.close(); - - const stat = try f.stat(); - - if (stat.size > std.math.maxInt(u32)) - return error.FileTooBig; - - const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); - errdefer gpa.free(source); - const amt = try f.readAll(source); - if (amt != stat.size) - return error.UnexpectedEndOfFile; - - // Here we do not modify stat fields because this function is the one - // used for error reporting. We need to keep the stat fields stale so that - // transformFile can know to regenerate Teg. - - const loaded_source = Source{ - .code = source, - .stat = .{ - .size = stat.size, - .inode = stat.inode, - .mtime = stat.mtime, - }, - }; - file.source = loaded_source; - return loaded_source; - } - - pub const GetAstError = GetSourceError || Parse.Error; - - pub fn getAst(file: *File, comp: *Comp) GetAstError!Ast { - if (file.ast) |ast| { - return ast; - } - - var err_data: Parse.AstError.WithTrace = undefined; - const ast = Parse.parse( - comp.gpa, - (try file.getSource(comp.gpa)).code, - &err_data, - ) catch |err| switch (err) { - error.OutOfMemory => return err, - error.ParseFail => { - // TODO: let this merge with other report - var report = try err_data.toReport(comp, file); - report.dumpToStdErr(comp) catch {}; - report.deinit(comp.gpa); - return err; - }, - }; - - if (comp.options.dump_ir) { - ast.dumpToWriter(std.io.getStdErr().writer()) catch { - // Simplify the error set. - }; - } - - file.ast = ast; - return ast; - } - - pub const GetTegError = GetAstError || Transform.Error; - - pub fn getTeg(file: *File, comp: *Comp, fail_report: *Report) GetTegError!Teg { - if (file.teg) |teg| { - return teg; - } - - const bin_digest = hash: { - var path_hash: Cache.HashHelper = .{}; - path_hash.addBytes("0.1.0"); - path_hash.addBytes(file.file_path); - var bin: Cache.BinDigest = undefined; - path_hash.hasher.final(&bin); - break :hash bin; - }; - file.path_digest = bin_digest; - - const teg = try Transform.transform( - try file.getAst(comp), - comp.gpa, - file, - fail_report, - ); - - file.teg = teg; - return teg; - } - - pub fn unloadSource(file: *File, gpa: Allocator) void { - if (file.source) |source| { - gpa.free(source.code); - file.source = null; - } - } - - pub fn unloadAst(file: *File, gpa: Allocator) void { - if (file.ast) |*ast| { - ast.deinit(gpa); - file.ast = null; - } - } - - pub fn unloadTeg(file: *File, gpa: Allocator) void { - if (file.teg) |*teg| { - teg.deinit(gpa); - file.teg = null; - } - } - - pub fn deinit(file: *File, gpa: Allocator) void { - file.unloadTeg(gpa); - file.unloadAst(gpa); - file.unloadSource(gpa); - gpa.destroy(file); - } -}; - -pub const Decl = struct { - name: InternPool.NullTerminatedString, - val: InternPool.Value, - /// The direct parent namespace of the Decl. In the case of the Decl - /// corresponding to a file, this is the namespace of the struct, since - /// there is no parent. - src_namespace: Namespace.Index, - - /// The AST node index of this declaration. - /// Must be recomputed when the corresponding source file is modified. - // src_node: Ast.NodeIndex, - /// Line number corresponding to `src_node`. Stored separately so that source files - /// do not need to be loaded into memory in order to compute debug line numbers. - /// This value is absolute. - // src_line: u32, - /// Index of the ZIR `declaration` instruction from which this `Decl` was created. - /// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`. - teg_decl_index: InternPool.TrackedInst.Index.Optional, - - /// Represents the "shallow" analysis status. For example, for decls that are functions, - /// the function type is analyzed with this set to `in_progress`, however, the semantic - /// analysis of the function body is performed with this value set to `success`. Functions - /// have their own analysis status field. - analysis: enum { - /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore - /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced. - unreferenced, - /// Semantic analysis for this Decl is running right now. - /// This state detects dependency loops. - in_progress, - /// The file corresponding to this Decl had a parse error or ZIR error. - /// There will be a corresponding ErrorMsg in Zcu.failed_files. - file_failure, - /// This Decl might be OK but it depends on another one which did not - /// successfully complete semantic analysis. - dependency_failure, - /// Semantic analysis failure. - /// There will be a corresponding ErrorMsg in Zcu.failed_decls. - sema_failure, - /// There will be a corresponding ErrorMsg in Zcu.failed_decls. - codegen_failure, - /// Sematic analysis and constant value codegen of this Decl has - /// succeeded. However, the Decl may be outdated due to an in-progress - /// update. Note that for a function, this does not mean codegen of the - /// function body succeded: that state is indicated by the function's - /// `analysis` field. - complete, - }, - - codegen: ?JsTree = null, - - is_pub: bool, - - /// What kind of a declaration is this. - kind: Kind, - - pub fn deinit(decl: *Decl, gpa: Allocator) void { - defer decl.* = undefined; - if (decl.codegen) |*cg| cg.deinit(gpa); - } - - pub const Kind = enum { - named, - }; - - pub const Index = InternPool.DeclIndex; -}; - -pub const Namespace = struct { - parent: Index.Optional, - file_scope: *File, - /// Direct children of the namespace. - /// Declaration order is preserved via entry order. - /// These are only declarations named directly by the AST; anonymous - /// declarations are not stored here. - decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{}, - - pub const DeclContext = struct { - comp: *Comp, - - pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 { - const decl = ctx.comp.intern_pool.declPtr(decl_index); - return std.hash.uint32(@intFromEnum(decl.name)); - } - - pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool { - _ = b_index; - const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index); - const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index); - return a_decl.name == b_decl.name; - } - }; - - pub const DeclNameAdapter = struct { - comp: *Comp, - - pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 { - _ = self; - return std.hash.uint32(@intFromEnum(s)); - } - - pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool { - _ = b_index; - return a == self.comp.intern_pool.declPtr(b_decl_index).name; - } - }; - - pub fn lookupDirect(ns: *Namespace, comp: *Comp, name: InternPool.NullTerminatedString) ?Decl.Index { - return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp }); - } - - pub const Index = InternPool.NamespaceIndex; -}; - -/// Index a module in a Teg. -/// When a File's Teg is loaded, we call this on the root module. -/// submodules are analyzed lazily. -pub fn indexTegModule(comp: *Comp, file: *File, module_index: Teg.Inst.Index) !Namespace.Index { - const teg = file.teg.?; - - const ns = try comp.intern_pool.createNamespace(comp.gpa, .{ - .parent = .none, - .file_scope = file, - }); - const ns_ptr = comp.intern_pool.namespacePtr(ns); - errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns); - - for (teg.moduleDeclarations(module_index)) |decl_index| { - const name = try comp.intern_pool.getOrPutString( - comp.gpa, - teg.declName(decl_index), - .no_embedded_nulls, - ); - const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index); - const decl = try comp.intern_pool.createDecl(comp.gpa, .{ - .name = name, - .val = undefined, - .src_namespace = ns, - .teg_decl_index = tracked_inst.toOptional(), - .analysis = .unreferenced, - .is_pub = false, // todo: - .kind = .named, - }); - try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp }); - } - - return ns; -} - -pub fn semaEntryPoint(comp: *Comp, file: *File) !void { - var report: Report = undefined; - const teg = try file.getTeg(comp, &report); - - // TODO, move this elsewhere. we should print all errors and warnings at once - if (teg.report.warnings.len > 0) { - try teg.report.dumpToStdErr(comp); - } - - const ns = try comp.indexTegModule(file, .main_module_inst); - const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse - @panic("no top level code?"); - const decl = comp.intern_pool.declPtr(decl_index); - - assert(comp.current_compile_decls.items.len == 0); - - var arena = std.heap.ArenaAllocator.init(comp.gpa); - defer arena.deinit(); - - var sema: Sema = .{ - .comp = comp, - .gpa = comp.gpa, - .code = teg, - .arena = arena.allocator(), - .ip = &comp.intern_pool, - .decl = decl, - .decl_index = decl_index, - .namespace_index = ns, - }; - try sema.init(); - defer sema.deinit(); - _ = try sema.analyzeDeclaration(); - try sema.finalize(); -} - -pub fn codeGenDecl(c: *Comp, decl: *Decl, air: *const Air) !void { - const t = tracer.trace(@src()); - defer t.end(); - - var arena_state = std.heap.ArenaAllocator.init(c.gpa); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air); - errdefer cg.deinit(); - try cg.build(); - decl.codegen = try cg.finalize(); -} - -pub fn link(c: *Comp, writer: anytype) !void { - try JsLink.linkJsCode(c, c.current_compile_decls.items, writer); -} - -const Options = struct { - dump_ir: bool = false, -}; - -const Comp = @This(); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const Stat = std.Build.Cache.File.Stat; -const log = std.log.scoped(.comp); - -const InternPool = @import("InternPool.zig"); - -const Ast = @import("Ast.zig"); -const Air = @import("Air.zig"); -const Teg = @import("Teg.zig"); -const Cache = @import("Cache.zig"); -const Sema = @import("Sema.zig"); -const Transform = @import("Transform.zig"); -const JsNodeGen = @import("JsNodeGen.zig"); -const JsTree = @import("JsTree.zig"); -const JsLink = @import("JsLink.zig"); -const Report = @import("Report.zig"); -const Parse = @import("Parse.zig"); - -const tracer = @import("tracer"); diff --git a/src/compiler/InternPool.zig b/src/compiler/InternPool.zig deleted file mode 100644 index 7863bf6debc6688cc2765fa2bd99bfef73f3294d..0000000000000000000000000000000000000000 --- a/src/compiler/InternPool.zig +++ /dev/null @@ -1,1196 +0,0 @@ -//! Large parts of this file are based on the Zig compiler's InternPool. -//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig - -/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are -/// constructed lazily. -map: std.AutoArrayHashMapUnmanaged(void, void) = .{}, -items: std.MultiArrayList(Item) = .{}, -extra: ExtraStore = .{}, - -/// In order to store references to strings in fewer bytes, we copy all -/// string bytes into here. String bytes can be null. It is up to whomever -/// is referencing the data here whether they want to store both index and length, -/// thus allowing null bytes, or store only index, and use null-termination. The -/// `string_bytes` array is agnostic to either usage. -string_bytes: std.ArrayListUnmanaged(u8) = .{}, - -/// Used for finding the index inside `string_bytes`. -string_table: std.HashMapUnmanaged( - u32, - void, - std.hash_map.StringIndexContext, - std.hash_map.default_max_load_percentage, -) = .{}, - -/// An index into `tracked_insts` gives a reference to a single ZIR instruction which -/// persists across incremental updates. -tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{}, - -/// Rather than allocating Decl objects with an Allocator, we instead allocate -/// them with this SegmentedList. This provides four advantages: -/// * Stable memory so that one thread can access a Decl object while another -/// thread allocates additional Decl objects from this list. -/// * It allows us to use u32 indexes to reference Decl objects rather than -/// pointers, saving memory in Type, Value, and dependency sets. -/// * Using integers to reference Decl objects rather than pointers makes -/// serialization trivial. -/// * It provides a unique integer to be used for anonymous symbol names, avoiding -/// multi-threaded contention on an atomic counter. -allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{}, -/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack. -decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{}, - -/// Same pattern as with `allocated_decls`. -allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{}, -/// Same pattern as with `decls_free_list`. -namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{}, - -/// How many items in the InternPool are statically known. -/// This is specified with an integer literal and a corresponding comptime -/// assert below to break an unfortunate and arguably incorrect dependency loop -/// when compiling. -pub const static_len = Teg.Inst.Index.static_len; -comptime { - // @compileLog(static_len, static_keys.len); - assert(static_len == static_keys.len); -} - -pub const Item = struct { - tag: Tag, - data: EncodedData, - - pub fn encode(comptime tag: Tag, data: tag.Data()) Item { - return .{ .tag = tag, .data = EncodedData.encode(tag, data) }; - } -}; - -pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var"); -pub const EncodedData = dod.EncodedData(Tag, u32); -pub const ExtraIndex = ExtraStore.Index; - -pub const Tag = enum(u8) { - /// This special tag represents a value which was removed from this pool via - /// `InternPool.remove`. The item remains allocated to preserve indices, but - /// lookups will consider it not equal to any other item, and all queries - /// assert not this tag. `data` is unused. - removed, - - /// A function type - /// data is an extra index into a `FuncType` - type_function, - /// A type that can be represented with only an enum tag. - /// data is SimpleType enum value. - simple_type, - - // --- before this are types --- - // --- past this are values --- - - /// A value that can be represented with only an enum tag. - /// data is SimpleValue enum value. - simple_value, - /// A numeric literal that can be represented as a float. - /// Integers greater than 2^53 known at compile time are currently not supported. - /// data is an extra index into a `Float` - comptime_float, - /// An int32 value - /// data is i32 - int32, - /// A float value - /// data is an extra index into a `Float` - float, - /// A function - /// data is an extra index into a `FuncDecl` - func, - /// A string - /// data is an extra index into a `String` - string, - - pub fn Data(comptime tag: Tag) type { - return switch (tag) { - .removed, - => unreachable, // does not have a data - - .simple_type => SimpleType, - .simple_value => SimpleValue, - .int32 => i32, - - else => ExtraIndex(tag.Payload()), - }; - } - - /// When `data` is an extra index, this is the type of the payload. - pub fn Payload(comptime tag: Tag) type { - return switch (tag) { - .removed, - .simple_type, - .simple_value, - .int32, - => unreachable, // does not have a payload - .float, .comptime_float => Float, - .func => FuncDecl, - .type_function => TypeFunction, - .string => String.WithLen, - }; - } - - /// Trailing: N/A - pub const FuncDecl = struct { - analysis: FuncAnalysis, - owner_decl: DeclIndex, - ty: Type, - teg_body_inst: TrackedInst.Index, - }; - - /// Trailing: - /// 0. param_type: Index for each params_len - pub const TypeFunction = struct { - params_len: u32, - return_type: Type, - }; -}; - -pub const Value = enum(u32) { - pub const first_type: Value = .opaque_type; - pub const last_type: Value = .noreturn_type; - pub const first_value: Value = .void; - pub const last_value: Value = .empty_string; - - opaque_type, - bool_type, - int32_type, - float_type, - string_type, - noreturn_type, - type_type, - numeric_literal_type, - - void, - bool_true, - bool_false, - int32_zero, - float_zero, - nan, - empty_string, - standard_library, - - none = std.math.maxInt(u32), - _, - - /// An array of `Index` existing within the `extra` array. - /// This type exists to provide a struct with lifetime that is - /// not invalidated when items are added to the `InternPool`. - pub const Slice = struct { - start: u32, - len: u32, - - pub fn get(slice: Slice, ip: *const InternPool) []Value { - return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]); - } - }; - - /// Used for a map of `Index` values to the index within a list of `Index` values. - const Adapter = struct { - indexes: []const Value, - - pub fn eql(ctx: @This(), a: Value, b_void: void, b_map_index: usize) bool { - _ = b_void; - return a == ctx.indexes[b_map_index]; - } - - pub fn hash(ctx: @This(), a: Value) u32 { - _ = ctx; - return std.hash.uint32(@intFromEnum(a)); - } - }; - - pub fn valueTag(value: Value, ip: *const InternPool) Tag { - return ip.items.items(.tag)[@intFromEnum(value)]; - } - - pub fn valueData(value: Value, ip: *const InternPool, comptime tag: Tag) tag.Data() { - return ip.items.items(.data)[@intFromEnum(value)].decode(tag, value.valueTag(ip)); - } - - pub fn toAir(value: Value) Air.Inst.Ref { - return .{ .kind = .value, .data = @intCast(@intFromEnum(value)) }; - } - - pub fn toKey(value: Value, ip: *const InternPool) Key { - return ip.indexToKey(value); - } - - pub fn dump(value: Value, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) { - return .{ .data = .{ .val = value, .ip = ip } }; - } - - fn formatValue( - ctx: struct { val: Value, ip: *const InternPool }, - comptime _: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, - ) !void { - // TODO: move the formatter out of Air.Dump - var state: Air.Dump.State = undefined; - state.ip = ctx.ip; - state.tty = @import("util.zig").detectTTYFromWriter(writer); - try Air.Dump.State.dumpValue(&state, writer, ctx.val); - } - - pub fn typeOf(value: Value, ip: *const InternPool) Type { - return switch (value.valueTag(ip)) { - .removed => unreachable, // UAF - - .type_function => .type, - .simple_type => .type, - - .simple_value => switch (value.valueData(ip, .simple_value)) { - .true => .bool, - .false => .bool, - .void => .void, - }, - - .comptime_float => .numeric_literal, - .int32 => .int32, - .float => .float, - .string => .string, - .func => value.toKey(ip).func.ty, - }; - } - - /// Try to re-interpret this `Value` as a `Type`. - /// Returns `null` if the value is not a type. - pub fn toType(value: Value, ip: *const InternPool) ?Type { - return switch (value.valueTag(ip)) { - .removed => unreachable, // UAF - - .type_function, - .simple_type, - => @enumFromInt(@intFromEnum(value)), - - // exception: void is a value AND a type - .simple_value => if (value == .void) .void else null, - - else => null, - }; - } - - pub fn toTypeUnsafe(value: Value, proof_ip: *const InternPool) Type { - if (std.debug.runtime_safety) { - return value.toType(proof_ip).?; - } - return @enumFromInt(@intFromEnum(value)); - } - - pub fn asString(val: Value, ip: *const InternPool) InternPool.String.WithLen { - return val.valueData(ip, .string).get(ip); - } - - pub fn toFloat(val: Value, ip: *const InternPool) ?f64 { - return switch (val.valueTag(ip)) { - inline .comptime_float, .float => |tag| val.valueData(ip, tag).get(ip).decode(), - else => null, - }; - } -}; - -pub const Type = enum(u32) { - @"opaque" = @intFromEnum(Value.opaque_type), - bool = @intFromEnum(Value.bool_type), - int32 = @intFromEnum(Value.int32_type), - float = @intFromEnum(Value.float_type), - string = @intFromEnum(Value.string_type), - noreturn = @intFromEnum(Value.noreturn_type), - type = @intFromEnum(Value.type_type), - numeric_literal = @intFromEnum(Value.numeric_literal_type), - void = @intFromEnum(Value.void), - _, - - // Restricted version of `InternPool.Tag` - pub const Tag = enum(u32) { - type_function = @intFromEnum(InternPool.Tag.type_function), - simple_type = @intFromEnum(InternPool.Tag.simple_type), - }; - - /// An array of `Index` existing within the `extra` array. - /// This type exists to provide a struct with lifetime that is - /// not invalidated when items are added to the `InternPool`. - pub const Slice = struct { - start: u32, - len: u32, - - pub fn get(slice: Slice, ip: *const InternPool) []Type { - return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]); - } - }; - - pub fn value(ty: Type) Value { - return @enumFromInt(@intFromEnum(ty)); - } - - pub fn tag(ty: Type, ip: *const InternPool) Type.Tag { - return @enumFromInt(@intFromEnum(ty.value().valueTag(ip))); - } - - pub fn toFunctionPrototype(ty: Type, ip: *const InternPool) ?Key.TypeFunction { - if (ty.tag(ip) != .type_function) return null; - return ip.indexToKey(ty.value()).type_function; - } - - pub fn displayName(ty: Type, ip: *const InternPool) std.fmt.Formatter(formatDisplayName) { - return .{ .data = .{ .type = ty, .ip = ip } }; - } - - pub fn dump(ty: Type, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) { - return ty.value().dump(ip); - } - - fn formatDisplayName( - ctx: struct { type: Type, ip: *const InternPool }, - comptime _: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, - ) !void { - const ty = ctx.type; - const ip = ctx.ip; - - switch (ty) { - .@"opaque" => return try writer.writeAll("opaque"), - .bool => return try writer.writeAll("bool"), - .int32 => return try writer.writeAll("int"), - .float => return try writer.writeAll("float"), - .string => return try writer.writeAll("string"), - .noreturn => return try writer.writeAll("noreturn"), - .type => return try writer.writeAll("type"), - .numeric_literal => return try writer.writeAll("numeric literal"), - .void => return try writer.writeAll("void"), - else => {}, - } - - switch (ty.tag(ip)) { - .simple_type => unreachable, // all simple types are already printed - .type_function => { - const type_function = ty.toFunctionPrototype(ip).?; - try writer.writeAll("fn("); - for (type_function.param_types, 0..) |param, i| { - if (i != 0) try writer.writeAll(", "); - try writer.print("{}", .{param.displayName(ip)}); - } - try writer.writeAll(") -> "); - try writer.print("{}", .{type_function.return_type.displayName(ip)}); - }, - } - } -}; - -pub const static_keys = [_]Key{ - SimpleType.@"opaque".key(), - SimpleType.bool.key(), - SimpleType.int32.key(), - SimpleType.float.key(), - SimpleType.string.key(), - SimpleType.noreturn.key(), - SimpleType.type.key(), - SimpleType.numeric_literal.key(), - - SimpleValue.void.key(), - SimpleValue.true.key(), - SimpleValue.false.key(), - .{ .int32 = 0 }, - .{ .float = 0.0 }, - .{ .float = Float.pure_nan.decode() }, - .{ .string = "" }, -}; - -/// State that is mutable during semantic analysis. This data is not used for -/// equality or hashing. -pub const FuncAnalysis = packed struct(u32) { - state: State, - - _: u24 = 0, - - pub const State = enum(u8) { - /// This function has not yet undergone analysis, because we have not - /// seen a potential runtime call. It may be analyzed in future. - none, - /// Analysis for this function has been queued, but not yet completed. - queued, - /// This function intentionally only has ZIR generated because it is marked - /// inline, which means no runtime version of the function will be generated. - inline_only, - in_progress, - /// There will be a corresponding ErrorMsg in Module.failed_decls - sema_failure, - /// This function might be OK but it depends on another Decl which did not - /// successfully complete semantic analysis. - dependency_failure, - /// There will be a corresponding ErrorMsg in Module.failed_decls. - /// Indicates that semantic analysis succeeded, but code generation for - /// this function failed. - codegen_failure, - /// Semantic analysis and code generation of this function succeeded. - success, - }; -}; - -/// An index into `string_bytes`. -pub const String = enum(u32) { - /// An empty string. - empty = 0, - _, - - pub const WithLen = struct { - string: String, - len: u32, - - pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 { - return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)]; - } - - pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString { - return string.string.tryToNullTerminatedString(string.len, ip, gpa); - } - }; - - pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 { - return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)]; - } - - pub fn withLen(string: String, len: u64) WithLen { - return .{ .string = string, .len = @intCast(len) }; - } - - pub fn at(string: String, index: u64, ip: *const InternPool) u8 { - return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)]; - } - - pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString { - assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null); - assert(string.at(len, ip) == 0); - return @enumFromInt(@intFromEnum(string)); - } - - pub fn tryToNullTerminatedString( - string: String, - len: u64, - ip: *InternPool, - gpa: Allocator, - ) !NullTerminatedString { - const slice = string.toSlice(len, ip); - if (std.mem.indexOfScalar(u8, slice, 0)) |_| { - return error.HasNulls; - } - if (string.at(len, ip) == 0) { - return @enumFromInt(@intFromEnum(string)); - } - return ip.getOrPutString(gpa, slice, .no_embedded_nulls); - } -}; - -pub const NullTerminatedString = enum(u32) { - empty = 0, - _, - - pub fn toString(self: NullTerminatedString) String { - return @enumFromInt(@intFromEnum(self)); - } - - // pub fn toOptional(self: NullTerminatedString) Optional{ - // return @enumFromInt(@intFromEnum(self)); - // } - - pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { - const slice = ip.string_bytes.items[@intFromEnum(string)..]; - return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0]; - } - - pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { - return @intCast(string.toSlice(ip).len); - } - - pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { - return std.mem.eql(u8, string.toSlice(ip), slice); - } - - const Adapter = struct { - strings: []const NullTerminatedString, - - pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool { - _ = b_void; - return a == ctx.strings[b_map_index]; - } - - pub fn hash(ctx: @This(), a: NullTerminatedString) u32 { - _ = ctx; - return std.hash.uint32(@intFromEnum(a)); - } - }; - - const FormatData = struct { - string: NullTerminatedString, - ip: *const InternPool, - }; - fn format( - data: FormatData, - comptime specifier: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, - ) @TypeOf(writer).Error!void { - const slice = data.string.toSlice(data.ip); - if (comptime std.mem.eql(u8, specifier, "")) { - try writer.writeAll(slice); - } else if (comptime std.mem.eql(u8, specifier, "i")) { - try writer.print("{p}", .{std.zig.fmtId(slice)}); - } else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'"); - } - - pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) { - return .{ .data = .{ .string = self, .ip = ip } }; - } -}; - -pub const TrackedInst = extern struct { - path_digest: Cache.BinDigest, - inst: Teg.Inst.Index, - - comptime { - // The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`. - assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index)); - } - - pub const Index = enum(u32) { - _, - pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index { - return ip.tracked_insts.keys()[@intFromEnum(i)].inst; - } - pub fn toOptional(i: TrackedInst.Index) Optional { - return @enumFromInt(@intFromEnum(i)); - } - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - pub fn unwrap(opt: Optional) ?TrackedInst.Index { - return switch (opt) { - .none => null, - _ => @enumFromInt(@intFromEnum(opt)), - }; - } - }; - }; -}; - -pub const DeclIndex = enum(u32) { - _, - - pub fn toOptional(i: DeclIndex) Optional { - return @enumFromInt(@intFromEnum(i)); - } - - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn init(oi: ?DeclIndex) Optional { - return @enumFromInt(@intFromEnum(oi orelse return .none)); - } - - pub fn unwrap(oi: Optional) ?DeclIndex { - if (oi == .none) return null; - return @enumFromInt(@intFromEnum(oi)); - } - }; -}; - -pub const NamespaceIndex = enum(u32) { - _, - - pub fn toOptional(i: DeclIndex) Optional { - return @enumFromInt(@intFromEnum(i)); - } - - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn init(oi: ?NamespaceIndex) Optional { - return @enumFromInt(@intFromEnum(oi orelse return .none)); - } - - pub fn unwrap(oi: Optional) ?NamespaceIndex { - if (oi == .none) return null; - return @enumFromInt(@intFromEnum(oi)); - } - }; -}; - -/// A single value captured in the closure of a namespace type. This is not a plain -/// `Index` because we must differentiate between the following cases: -/// * runtime-known value (where we store the type) -/// * comptime-known value (where we store the value) -/// * decl val (so that we can analyze the value lazily) -/// * decl ref (so that we can analyze the reference lazily) -pub const Capture = packed struct(u32) { - tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref }, - idx: u30, - - pub fn wrap(val: Unwrapped) Capture { - return switch (val) { - .@"comptime" => |i| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) }, - .runtime => |i| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) }, - .decl_val => |i| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) }, - .decl_ref => |i| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) }, - }; - } - pub fn unwrap(val: Capture) Unwrapped { - return switch (val.tag) { - .@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) }, - .runtime => .{ .runtime = @enumFromInt(val.idx) }, - .decl_val => .{ .decl_val = @enumFromInt(val.idx) }, - .decl_ref => .{ .decl_ref = @enumFromInt(val.idx) }, - }; - } - - pub const Unwrapped = union(enum) { - /// Index refers to the value. - @"comptime": Value, - /// Index refers to the type. - runtime: Value, - decl_val: DeclIndex, - decl_ref: DeclIndex, - }; - - pub const Slice = struct { - start: u32, - len: u32, - pub fn get(slice: Slice, ip: *const InternPool) []Capture { - return @ptrCast(ip.extra.items[slice.start..][0..slice.len]); - } - }; -}; - -const SimpleValue = enum { - true, - false, - void, - - pub fn key(self: SimpleValue) Key { - return .{ .simple_value = self }; - } -}; - -const SimpleType = enum { - @"opaque", - int32, - float, - string, - bool, - noreturn, - type, - numeric_literal, - - pub fn key(self: SimpleType) Key { - return .{ .simple_type = self }; - } -}; - -pub const Key = union(enum) { - comptime_float: f64, - int32: i32, - float: f64, - string: []const u8, - simple_value: SimpleValue, - simple_type: SimpleType, - type_function: TypeFunction, - func: Func, - - const Adapter = struct { - intern_pool: *const InternPool, - - pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool { - _ = b_void; - if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false; - return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool); - } - - pub fn hash(ctx: @This(), a: Key) u32 { - return a.hash32(ctx.intern_pool); - } - }; - - pub const TypeFunction = struct { - param_types: []Type, - return_type: Type, - - pub fn hash(self: TypeFunction, hasher: *Hash) void { - const t = tracer.trace(@src()); - defer t.end(); - - for (self.param_types) |param_type| { - std.hash.autoHash(hasher, param_type); - } - std.hash.autoHash(hasher, self.return_type); - } - - pub fn eql(a: TypeFunction, b: TypeFunction) bool { - return std.mem.eql(Type, a.param_types, b.param_types) and - a.return_type == b.return_type; - } - }; - - pub const Func = struct { - ty: Type, - owner_decl: DeclIndex, - teg_body_inst: TrackedInst.Index, - - pub fn hash(self: Func, hasher: *Hash) void { - std.hash.autoHash(hasher, self.ty); - std.hash.autoHash(hasher, self.owner_decl); - std.hash.autoHash(hasher, self.teg_body_inst); - } - }; - - pub fn hash32(key: Key, ip: *const InternPool) u32 { - return @truncate(key.hash64(ip)); - } - - pub fn hash64(key: Key, ip: *const InternPool) u64 { - const t = tracer.trace(@src()); - defer t.end(); - - _ = ip; // autofix - const asBytes = std.mem.asBytes; - const KeyTag = @typeInfo(Key).Union.tag_type.?; - const seed = @intFromEnum(@as(KeyTag, key)); - - return switch (key) { - inline .int32, - .simple_value, - .simple_type, - => |x| Hash.hash(seed, asBytes(&x)), - .string, - => |x| Hash.hash(seed, x), - .float, .comptime_float => |x| Hash.hash(seed, asBytes(&Float.assertPure(x))), - .type_function => |x| { - var hasher = Hash.init(seed); - x.hash(&hasher); - return hasher.final(); - }, - .func => |x| { - var hasher = Hash.init(seed); - x.hash(&hasher); - return hasher.final(); - }, - }; - } - - pub fn eql(a: Key, b: Key, ip: *const InternPool) bool { - const t = tracer.trace(@src()); - defer t.end(); - - _ = ip; // autofix - const KeyTag = @typeInfo(Key).Union.tag_type.?; - const a_tag: KeyTag = a; - const b_tag: KeyTag = b; - if (a_tag != b_tag) return false; - return switch (a) { - inline .int32, - .simple_value, - .simple_type, - => |x, tag| x == @field(b, @tagName(tag)), - .string, - => |x| std.mem.eql(u8, x, b.string), - .float => |x| Float.assertPure(x) == Float.assertPure(b.float), - .comptime_float => |x| Float.assertPure(x) == Float.assertPure(b.comptime_float), - .type_function => |x| x.eql(b.type_function), - .func => |a_info| { - const b_info = b.func; - if (a_info.owner_decl != b_info.owner_decl) - return false; - if (a_info.ty == b_info.ty) - return true; - - // TODO: zig has something extra here for inferred error sets - return false; - }, - }; - } -}; - -const EmbeddedNulls = enum { - no_embedded_nulls, - maybe_embedded_nulls, - - const OptionalNullTerminatedString = @compileError("TODO"); - const OptionalString = @compileError("TODO"); - - fn StringType(comptime embedded_nulls: EmbeddedNulls) type { - return switch (embedded_nulls) { - .no_embedded_nulls => NullTerminatedString, - .maybe_embedded_nulls => String, - }; - } - - fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type { - return switch (embedded_nulls) { - .no_embedded_nulls => OptionalNullTerminatedString, - .maybe_embedded_nulls => OptionalString, - }; - } -}; - -// implementations: - -/// To construct, instantiate `.{ .gpa = gpa }` then call this. -pub fn init(ip: *InternPool, gpa: Allocator) !void { - const t = tracer.trace(@src()); - defer t.end(); - - assert(ip.items.len == 0); - - // Reserve string index 0 for an empty string. - assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty); - - // So that we can use `catch unreachable` below. - try ip.items.ensureUnusedCapacity(gpa, static_keys.len); - try ip.map.ensureUnusedCapacity(gpa, static_keys.len); - try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len); - - // This inserts all the statically-known values into the intern pool in the - // order expected. - for (static_keys) |key| { - _ = ip.get(gpa, key) catch unreachable; - } - - if (std.debug.runtime_safety) { - // Sanity check. - assert(ip.indexToKey(.bool_true).simple_value == .true); - assert(ip.indexToKey(.bool_false).simple_value == .false); - } - - assert(ip.items.len == static_keys.len); -} - -pub fn deinit(ip: *InternPool, gpa: Allocator) void { - const t = tracer.trace(@src()); - defer t.end(); - - ip.map.deinit(gpa); - ip.items.deinit(gpa); - ip.extra.deinit(gpa); - // ip.limbs.deinit(gpa); - ip.string_bytes.deinit(gpa); - - var decls = ip.allocated_decls.iterator(0); - while (decls.next()) |decl| { - decl.deinit(gpa); - } - - ip.allocated_decls.deinit(gpa); - ip.decls_free_list.deinit(gpa); - - var namespaces = ip.allocated_namespaces.iterator(0); - while (namespaces.next()) |ns| { - ns.decls.deinit(gpa); - } - - ip.namespaces_free_list.deinit(gpa); - ip.allocated_namespaces.deinit(gpa); - - // for (ip.maps.items) |*map| map.deinit(gpa); - // ip.maps.deinit(gpa); - - ip.string_table.deinit(gpa); - - ip.tracked_insts.deinit(gpa); - - // ip.src_hash_deps.deinit(gpa); - // ip.decl_val_deps.deinit(gpa); - // ip.func_ies_deps.deinit(gpa); - // ip.namespace_deps.deinit(gpa); - // ip.namespace_name_deps.deinit(gpa); - - // ip.first_dependency.deinit(gpa); - - // ip.dep_entries.deinit(gpa); - // ip.free_dep_entries.deinit(gpa); - - ip.* = undefined; -} - -pub fn indexToKey(ip: *const InternPool, index: Value) Key { - assert(index != .none); - const item: Item = ip.items.get(@intFromEnum(index)); - const data = item.data; - return switch (item.tag) { - .removed => unreachable, - - .type_function => .{ - .type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)), - }, - - .simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) }, - .simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) }, - .int32 => .{ .int32 = data.decode(.int32, item.tag) }, - .float => .{ .float = data.decode(.float, item.tag).get(ip).decode() }, - .comptime_float => .{ .comptime_float = data.decode(.comptime_float, item.tag).get(ip).decode() }, - .string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) }, - .func => { - const func = data.decode(.func, item.tag).get(ip); - return .{ .func = .{ - .ty = func.ty, - .owner_decl = func.owner_decl, - .teg_body_inst = func.teg_body_inst, - } }; - }, - }; -} - -pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value { - const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)}); - defer t.end(); - - const adapter: Key.Adapter = .{ .intern_pool = ip }; - const gop = try ip.map.getOrPutAdapted(gpa, key, adapter); - if (gop.found_existing) return @enumFromInt(gop.index); - if (ip.items.len >= std.math.maxInt(u31) - 1) { - return error.TooManyValues; - } - - const t2 = tracer.traceFmt(@src(), "(insert)", .{}); - defer t2.end(); - - try ip.items.ensureUnusedCapacity(gpa, 1); - const item: Item = switch (key) { - inline .int32, - .simple_type, - .simple_value, - => |s, tag| Item.encode(@field(Tag, @tagName(tag)), s), - .string => |s| Item.encode( - .string, - try ip.extra.append( - gpa, - String.WithLen, - (try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len), - ), - ), - .float => |f| Item.encode( - .float, - try ip.extra.append(gpa, Float, Float.encode(f)), - ), - .comptime_float => |f| Item.encode( - .comptime_float, - try ip.extra.append(gpa, Float, Float.encode(f)), - ), - .func => |f| brk: { - const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{ - .analysis = .{ - .state = .none, - }, - .owner_decl = f.owner_decl, - .ty = f.ty, - .teg_body_inst = f.teg_body_inst, - }); - - break :brk Item.encode(.func, extra); - }, - .type_function => |f| brk: { - const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{ - .params_len = @intCast(f.param_types.len), - .return_type = f.return_type, - }, f.param_types.len); - defer trail.end(); - trail.writeSlice(f.param_types); - break :brk Item.encode(.type_function, extra); - }, - }; - ip.items.appendAssumeCapacity(item); - const index: Value = @enumFromInt(ip.items.len - 1); - if (std.debug.runtime_safety) { - assert(ip.indexToKey(index).eql(key, ip)); - } - return index; -} - -fn extraTypeFunction( - ip: *const InternPool, - extra_index: ExtraIndex(Tag.TypeFunction), -) Key.TypeFunction { - const type_function, var trail = extra_index.getTrail(ip); - const param_types = trail.readTypeSlice(type_function.params_len); - const result: Key.TypeFunction = .{ - .return_type = type_function.return_type, - .param_types = param_types.get(ip), - }; - return result; -} - -pub fn getOrPutString( - ip: *InternPool, - gpa: Allocator, - slice: []const u8, - comptime embedded_nulls: EmbeddedNulls, -) Allocator.Error!embedded_nulls.StringType() { - const t = tracer.trace(@src()); - defer t.end(); - - try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1); - ip.string_bytes.appendSliceAssumeCapacity(slice); - ip.string_bytes.appendAssumeCapacity(0); - return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls); -} - -fn getOrPutTrailingString( - ip: *InternPool, - gpa: Allocator, - len: usize, - comptime embedded_nulls: EmbeddedNulls, -) Allocator.Error!embedded_nulls.StringType() { - const t = tracer.trace(@src()); - defer t.end(); - - const string_bytes = &ip.string_bytes; - const str_index: u32 = @intCast(string_bytes.items.len - len); - if (len > 0 and string_bytes.getLast() == 0) { - _ = string_bytes.pop(); - } else { - try string_bytes.ensureUnusedCapacity(gpa, 1); - } - const key: []const u8 = string_bytes.items[str_index..]; - const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null; - switch (embedded_nulls) { - .no_embedded_nulls => assert(!has_embedded_null), - .maybe_embedded_nulls => if (has_embedded_null) { - string_bytes.appendAssumeCapacity(0); - return @enumFromInt(str_index); - }, - } - const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{ - .bytes = string_bytes, - }, std.hash_map.StringIndexContext{ - .bytes = string_bytes, - }); - if (gop.found_existing) { - string_bytes.shrinkRetainingCapacity(str_index); - return @enumFromInt(gop.key_ptr.*); - } else { - gop.key_ptr.* = str_index; - string_bytes.appendAssumeCapacity(0); - return @enumFromInt(str_index); - } -} - -pub fn trackInst(ip: *InternPool, gpa: Allocator, file: *Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index { - const t = tracer.trace(@src()); - defer t.end(); - - const key: TrackedInst = .{ - .path_digest = file.path_digest.?, - .inst = inst, - }; - const gop = try ip.tracked_insts.getOrPut(gpa, key); - return @enumFromInt(gop.index); -} - -pub fn declPtr(ip: *InternPool, index: DeclIndex) *Comp.Decl { - return ip.allocated_decls.at(@intFromEnum(index)); -} - -pub fn declPtrConst(ip: *const InternPool, index: DeclIndex) *const Comp.Decl { - return ip.allocated_decls.at(@intFromEnum(index)); -} - -pub fn namespacePtr(ip: *InternPool, index: NamespaceIndex) *Comp.Namespace { - return ip.allocated_namespaces.at(@intFromEnum(index)); -} - -pub fn createDecl( - ip: *InternPool, - gpa: Allocator, - initialization: Comp.Decl, -) Allocator.Error!DeclIndex { - const t = tracer.trace(@src()); - defer t.end(); - - if (ip.decls_free_list.popOrNull()) |index| { - ip.allocated_decls.at(@intFromEnum(index)).* = initialization; - return index; - } - const ptr = try ip.allocated_decls.addOne(gpa); - ptr.* = initialization; - return @enumFromInt(ip.allocated_decls.len - 1); -} - -pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void { - const t = tracer.trace(@src()); - defer t.end(); - - ip.declPtr(index).* = undefined; - ip.decls_free_list.append(gpa, index) catch { - // In order to keep `destroyDecl` a non-fallible function, we ignore memory - // allocation failures here, instead leaking the Decl until garbage collection. - }; -} - -pub fn createNamespace( - ip: *InternPool, - gpa: Allocator, - initialization: Comp.Namespace, -) Allocator.Error!NamespaceIndex { - const t = tracer.trace(@src()); - defer t.end(); - - if (ip.namespaces_free_list.popOrNull()) |index| { - ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization; - return index; - } - const ptr = try ip.allocated_namespaces.addOne(gpa); - ptr.* = initialization; - return @enumFromInt(ip.allocated_namespaces.len - 1); -} - -pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void { - const t = tracer.trace(@src()); - defer t.end(); - - ip.namespacePtr(index).* = undefined; - ip.namespaces_free_list.append(gpa, index) catch { - // In order to keep `destroyNamespace` a non-fallible function, we ignore memory - // allocation failures here, instead leaking the Namespace until garbage collection. - }; -} - -const InternPool = @This(); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Hash = std.hash.Wyhash; -const assert = std.debug.assert; - -const Comp = @import("Comp.zig"); -const Cache = @import("Cache.zig"); - -const Teg = @import("Teg.zig"); -const Air = @import("Air.zig"); - -const dod = @import("array.zig"); -const Float = dod.Float; - -const tracer = @import("tracer"); - -test InternPool { - const testing = std.testing; - std.testing.refAllDeclsRecursive(@This()); - - var ip_struct: InternPool = .{ .gpa = std.testing.allocator }; - const ip = &ip_struct; - defer ip.deinit(); - - const str = try ip.getOrPutString("hello", .no_embedded_nulls); - try testing.expectEqualStrings("hello", str.toSlice(ip)); - const str2 = try ip.getOrPutString("hello", .no_embedded_nulls); - try testing.expectEqual(str, str2); - - const int1 = try ip.get(.{ .int32 = 21483 }); - try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32); - const int2 = try ip.get(.{ .int32 = 21483 }); - try testing.expectEqual(int1, int2); -} diff --git a/src/compiler/JsLink.zig b/src/compiler/JsLink.zig deleted file mode 100644 index 99a557bcab8a283b579d45d9208b7cc340ad24c0..0000000000000000000000000000000000000000 --- a/src/compiler/JsLink.zig +++ /dev/null @@ -1,519 +0,0 @@ -/// This is the JavaScript backend's linker. -/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript -/// bundle out of them. -comp: *Comp, - -/// Value is an index into `link_strings` -decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{}, -link_strings: std.ArrayHashMapUnmanaged( - LinkString, - GlobalOwner, - LinkString.Context, - true, -) = .{}, -string_bytes: std.ArrayListUnmanaged(u8) = .{}, - -/// The first declaration is the function containing top-level code. -pub fn linkJsCode( - comp: *Comp, - decls: []const Decl.Index, - writer: anytype, -) !void { - var t = tracer.trace(@src()); - defer t.end(); - - // Arena allocator is used for the main flow - var link: JsLink = .{ .comp = comp }; - defer link.deinit(); - - // Temp allocator is used for extremely temporary string work. - // var temp_sfa = std.heap.stackFallback(4096, comp.gpa); - // const temp_alloc = temp_sfa.get(); - - // 1. Resolve all global identifiers. Function names must not shadow external - // names. We also need this list prepared so that local identifier names do not - // shadow functions or externals. - const root_decl = comp.intern_pool.declPtrConst(decls[0]); - assert(root_decl.kind == .named); - assert(root_decl.name == .empty); - - for (root_decl.codegen.?.extern_symbols) |extern_symbol| { - try link.reserveExternSymbol(extern_symbol); - } - - for (decls[1..]) |decl_index| { - assert(decl_index != decls[0]); - - const decl = comp.intern_pool.declPtrConst(decl_index); - - // todo: add all externs first - for (decl.codegen.?.extern_symbols) |extern_symbol| { - try link.reserveExternSymbol(extern_symbol); - } - - const name = decl.name.toSlice(&comp.intern_pool); - assert(name.len > 0); - - const gop = try link.link_strings.getOrPutContextAdapted( - comp.gpa, - name, - LinkString.SliceContext{ .link = &link }, - .{ .link = &link }, - ); - if (gop.found_existing) { - @panic("TODO: handle this shadowing case."); - } - const name_str = try link.appendLinkString(name); - gop.key_ptr.* = name_str; - gop.value_ptr.* = .decl; - try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index)); - } - - try writer.print("// Language Compiler v0.1.2. Alpha quality software.\n", .{}); - - // 2. Print every function decl - for (decls[1..]) |decl_index| { - try link.printDecl(decl_index, writer); - } - - // 3. Print the top-level code - if (root_decl.codegen.?.nodes.len > 1) { - try link.printDeclContents(&root_decl.codegen.?, writer); - } else { - try writer.print("// (program has no observable effects, thus an empty file)\n", .{}); - } -} - -fn deinit(link: *JsLink) void { - var t = tracer.trace(@src()); - defer t.end(); - - defer link.* = undefined; - link.decls.deinit(link.comp.gpa); - link.link_strings.deinit(link.comp.gpa); - link.string_bytes.deinit(link.comp.gpa); -} - -fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void { - var t = tracer.trace(@src()); - defer t.end(); - - const name = extern_symbol.toSlice(&link.comp.intern_pool); - const gop = try link.link_strings.getOrPutContextAdapted( - link.comp.gpa, - name, - LinkString.SliceContext{ .link = link }, - .{ .link = link }, - ); - - if (gop.found_existing) { - if (gop.value_ptr.* != .external) { - @panic("TODO: Rename Decl"); - } - } - gop.key_ptr.* = try link.appendLinkString(name); - gop.value_ptr.* = .external; -} - -fn appendLinkString(link: *JsLink, string: []const u8) !LinkString { - var t = tracer.trace(@src()); - defer t.end(); - - const start = link.string_bytes.items.len; - try link.string_bytes.appendSlice(link.comp.gpa, string); - return .{ .start = @intCast(start), .len = @intCast(string.len) }; -} - -fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 { - var t = tracer.trace(@src()); - defer t.end(); - - return (link.decls.get(decl) orelse if (std.debug.runtime_safety) { - std.debug.panic("Decl {d} ({}) is not present in link decl map", .{ - @intFromEnum(decl), - link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool), - }); - } else unreachable).toString(link).slice(link); -} - -fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - try writer.writeAll("let "); - try writer.writeAll(link.declIdentifier(decl_index)); - try writer.writeAll(" = "); - - const decl = link.comp.intern_pool.declPtrConst(decl_index); - const tree = &decl.codegen.?; - - try writer.writeAll("("); - for (0..tree.param_count) |id| { - try link.printLocal(@intCast(id), writer); - if (id != tree.param_count - 1) try writer.writeAll(", "); - } - try writer.writeAll(") => {\n"); - try link.printDeclContents(tree, writer); - try writer.writeAll("};\n"); -} - -fn printDeclContents(link: *JsLink, tree: *const JsTree, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - for (tree.mainStatements()) |stmt| { - try link.printExpr(tree, stmt, writer); - try writer.writeAll(";\n"); - } -} - -fn printBlockContents(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - assert(list.slice(tree).len > 0); - for (list.slice(tree)) |stmt| { - try link.printExpr(tree, stmt, writer); - switch (tree.nodeTag(stmt)) { - else => try writer.writeAll(";"), - .stmt_block, - .stmt_if, - => {}, - } - try writer.writeAll("\n"); - } -} - -fn printBlock(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - if (list.len != 1) { - try writer.writeAll("{"); - try link.printBlockContents(tree, list, writer); - try writer.writeAll("}"); - } else { - try link.printBlockContents(tree, list, writer); - } -} - -fn printExpr(link: *JsLink, tree: *const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void { - var t = tracer.trace(@src()); - defer t.end(); - - switch (tree.nodeTag(node)) { - .root => unreachable, - - .plain_value => { - const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node); - switch (plain_value.value.toKey(&link.comp.intern_pool)) { - // TODO: minification - .int32 => |val| try writer.print("{d}", .{val}), - // TODO: correct printing of many value types - .float => |val| try writer.print("{d}", .{val}), - .simple_value => |val| try writer.writeAll(switch (val) { - .true => "true", - .false => "false", - // TODO: make codegen decide from null and undefined - // instead of emitting void nodes. - .void => "undefined", - }), - // TODO: escapes - .string => |val| try writer.print("\"{s}\"", .{val}), - - .comptime_float => unreachable, // was not lowered to float - - else => unreachable, // this value is not a plain_value - } - }, - - .local => { - const ref: JsTree.LocalRef = tree.nodeData(.local, node); - try link.printLocal(ref.id, writer); - }, - - .global => { - const ref: JsTree.GlobalRef = tree.nodeData(.global, node); - try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool)); - }, - - .decl => { - const ref: JsTree.DeclRef = tree.nodeData(.decl, node); - try writer.writeAll(link.declIdentifier(ref.index)); - }, - - .stmt_break => { - const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node); - try writer.writeAll("break"); - if (true) { - try writer.writeAll(" "); - try link.printLabelIdentifier(stmt.label, writer); - } - }, - - .stmt_return => { - const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node); - try writer.writeAll("return"); - if (stmt.operand.unwrap()) |operand| { - try writer.writeAll(" "); - try link.printExpr(tree, operand, writer); - } - }, - - .stmt_declare_local => { - const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node); - try writer.writeAll("let "); - try link.printLocal(stmt.identifier_index, writer); - if (stmt.init.unwrap()) |init| { - try writer.writeAll(" = "); - try link.printExpr(tree, init, writer); - } - }, - - .stmt_if => { - const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node); - const data = stmt.data.get(tree); - try writer.writeAll("if ("); - try link.printExpr(tree, stmt.cond, writer); - try writer.writeAll(") "); - if (data.flags.yes_label) { - try link.printLabelIdentifier(data.flags.block_label, writer); - try writer.writeAll(":"); - } - try link.printBlock(tree, data.yes, writer); - if (data.no.len > 0) { - try writer.writeAll(" else "); - if (data.flags.no_label) { - try link.printLabelIdentifier(data.flags.block_label, writer); - try writer.writeAll(":"); - } - try link.printBlock(tree, data.no, writer); - } - }, - - .stmt_block => { - const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node); - try link.printLabelIdentifier(stmt.label, writer); - try writer.writeAll(": {\n"); - try link.printBlockContents(tree, stmt.content.get(tree), writer); - try writer.writeAll("}"); - }, - - inline .assign, - .add, - .sub, - .mul, - .div, - .eq, - .lt, - .gt, - .assign_add, - .assign_sub, - .assign_mul, - .assign_div, - => |tag| { - const bin: JsTree.Bin = tree.nodeData(tag, node); - const op: Operator = @field(Operator, @tagName(tag)); - try link.printExpr(tree, bin.lhs, writer); - try writer.writeAll(" "); - try writer.writeAll(comptime op.symbol()); - try writer.writeAll(" "); - try link.printExpr(tree, bin.rhs, writer); - }, - - inline .negate, - .bool_not, - .pre_dec, - .pre_inc, - => |tag| { - const un: JsTree.Un = tree.nodeData(tag, node); - const op: Operator = @field(Operator, @tagName(tag)); - try writer.writeAll(comptime op.symbol()); - try link.printExpr(tree, un.operand, writer); - }, - - .ternary => { - const ternary: JsTree.Ternary = tree.nodeData(.ternary, node); - const data = ternary.data.get(tree); - try writer.writeAll("("); - try link.printExpr(tree, ternary.cond, writer); - try writer.writeAll(" ? "); - try link.printExpr(tree, data.yes, writer); - try writer.writeAll(" : "); - try link.printExpr(tree, data.no, writer); - try writer.writeAll(")"); - }, - - .call => { - const call: JsTree.Call = tree.nodeData(.call, node); - const args = call.args.get(tree); - - try link.printExpr(tree, call.base, writer); - try writer.writeAll("("); - - for (args.slice(tree), 0..) |arg, i| { - if (i != 0) try writer.writeAll(", "); - try link.printExpr(tree, arg, writer); - } - - try writer.writeAll(")"); - }, - - .field => { - const field: JsTree.Field = tree.nodeData(.field, node); - - try link.printExpr(tree, field.base, writer); - try writer.writeAll("."); - try writer.writeAll(field.field.toSlice(&link.comp.intern_pool)); - }, - } -} - -fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - _ = link; // todo: be smarter about this - try writer.print("v{d}", .{local}); -} - -fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void { - var t = tracer.trace(@src()); - defer t.end(); - - _ = link; // todo: be smarter about this - try writer.print("label{d}", .{local}); -} - -const GlobalOwner = enum { - external, - decl, -}; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence -const Level = enum { - comma, - assignment, - conditional, - logical_or, - logical_and, - bitwise_or, - bitwise_xor, - bitwise_and, - equality, - relational, - bitshift, - additive, - multiplicative, - exponentiation, - prefix, - postfix, - new, - access_and_call, - member, -}; - -const Operator = enum { - assign, - assign_add, - assign_sub, - assign_mul, - assign_div, - add, - sub, - mul, - div, - eq, - lt, - gt, - negate, - bool_not, - pre_inc, - pre_dec, - - pub fn symbol(op: Operator) []const u8 { - return switch (op) { - .assign => "=", - .assign_add => "+=", - .assign_sub => "-=", - .assign_mul => "*=", - .assign_div => "/=", - .add => "+", - .sub => "-", - .mul => "*", - .div => "/", - .eq => "===", - .lt => "<", - .gt => ">", - - .pre_inc => "++", - .pre_dec => "++", - - .negate => "-", - .bool_not => "!", - }; - } -}; - -const LinkString = struct { - start: u32, - len: u32, - - fn slice(str: LinkString, link: *const JsLink) []const u8 { - return link.string_bytes.items[str.start..][0..str.len]; - } - - const Index = enum(u32) { - _, - - pub fn toString(index: Index, link: *const JsLink) LinkString { - return link.link_strings.keys()[@intFromEnum(index)]; - } - }; - - const Context = struct { - link: *const JsLink, - - pub fn hash(ctx: Context, str: LinkString) u32 { - // assert(str.start + str.len < ctx.link.string_bytes.items.len); - - return @truncate(Wyhash.hash(0, str.slice(ctx.link))); - } - - pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool { - assert(a.start + a.len < ctx.link.string_bytes.items.len); - assert(b.start + b.len < ctx.link.string_bytes.items.len); - return a.start == b.start; - } - }; - - const SliceContext = struct { - link: *const JsLink, - - pub fn hash(_: SliceContext, str: []const u8) u32 { - return @truncate(Wyhash.hash(0, str)); - } - - pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool { - // since the given string is not in string_bytes we do comparisons - return std.mem.eql(u8, a, b.slice(ctx.link)); - } - }; -}; - -const JsLink = @This(); -const std = @import("std"); -const assert = std.debug.assert; -const Wyhash = std.hash.Wyhash; -const log = std.log.scoped(.js_link); - -const Comp = @import("Comp.zig"); -const Decl = Comp.Decl; - -const JsTree = @import("JsTree.zig"); -const Node = JsTree.Node; - -const InternPool = @import("InternPool.zig"); -const String = InternPool.String; - -const tracer = @import("tracer"); diff --git a/src/compiler/JsNodeGen.zig b/src/compiler/JsNodeGen.zig deleted file mode 100644 index f3b801e05e8e7b37b38e3903f81bcc39a5e1381e..0000000000000000000000000000000000000000 --- a/src/compiler/JsNodeGen.zig +++ /dev/null @@ -1,700 +0,0 @@ -//! Air -> JsTree -gpa: Allocator, -arena: Allocator, -air: *const Air, -ip: *const InternPool, - -nodes: Node.List, -extra: dod.ExtraStore(.tfr, .@"var"), -extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void), -decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void), - -param_count: u32, -air_node_map: []MapEntry, -scratch: std.ArrayListUnmanaged(u32), -next_id: u32, -next_label: u32, - -const Error = error{ - OutOfMemory, -}; - -const MapEntry = union { - none: void, - node: Node.Index, - block: packed struct { - used: bool = false, - index: u31, - }, -}; - -pub fn init(gpa: Allocator, arena: Allocator, ip: *const InternPool, air: *const Air) !JsNodeGen { - const t = tracer.trace(@src()); - defer t.end(); - - const air_node_map = try arena.alloc(MapEntry, air.insts.len); - if (std.debug.runtime_safety) { - @memset(air_node_map, .{ .none = {} }); - } - const param_count = air.paramCount(); - return .{ - .gpa = gpa, - .arena = arena, - .air = air, - .ip = ip, - .param_count = param_count, - .nodes = .{}, - .extern_symbols = .{}, - .decl_deps = .{}, - .air_node_map = air_node_map, - .extra = .{}, - .scratch = .{}, - .next_id = param_count, - .next_label = 0, - }; -} - -pub fn deinit(c: *JsNodeGen) void { - const t = tracer.trace(@src()); - defer t.end(); - - defer c.* = undefined; - c.nodes.deinit(c.gpa); - c.extra.deinit(c.gpa); - c.extern_symbols.deinit(c.gpa); - c.decl_deps.deinit(c.gpa); - - // arena can deinit the rest -} - -pub fn finalize(c: *JsNodeGen) !JsTree { - const t = tracer.trace(@src()); - defer t.end(); - - // todo: errdefer will probably leave memory in a fully invalid state - try c.nodes.setCapacity(c.gpa, c.nodes.len); - - var extra = try c.extra.toOwnedStore(c.gpa); - errdefer extra.deinit(c.gpa); - - // todo: i dont like this dupe - const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys()); - errdefer c.gpa.free(extern_symbols); - - const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys()); - errdefer c.gpa.free(decl_deps); - - defer c.deinit(); - - return .{ - .param_count = c.param_count, - .nodes = c.nodes.toOwnedSlice(), - .extern_symbols = extern_symbols, - .decl_deps = decl_deps, - .extra = extra, - }; -} - -pub fn build(c: *JsNodeGen) !void { - const t = tracer.trace(@src()); - defer t.end(); - - assert(c.air.instTag(.main_block) == .block); - const statements = c.air.instData(.block, .main_block).slice(c.air); - - const scratch_top = c.scratch.items.len; - defer c.scratch.shrinkRetainingCapacity(scratch_top); - - _ = try c.add(.root, undefined); - - if (statements.len > 0) { - for (statements[0..c.param_count], 1..) |stmt, expected_id| { - assert(@intFromEnum(stmt) == expected_id); - assert(c.air.instTag(stmt) == .arg); - c.air_node_map[@intFromEnum(stmt)] = .{ - .node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }), - }; - } - - for (statements[c.param_count..]) |stmt| { - try c.codeGenInst(stmt); - } - } - - const items = c.scratch.items[scratch_top..]; - const top = c.extra.list.items.len; - try c.extra.list.appendSlice(c.gpa, items); - c.nodes.items(.data)[0] = EncodedData.encode(.root, .{ - .start = @intCast(top), - .len = @intCast(items.len), - }); -} - -fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void { - const t = tracer.traceFmt(@src(), "#{d} {s}", .{ @intFromEnum(inst), @tagName(c.air.instTag(inst)) }); - defer t.end(); - - switch (c.air.instTag(inst)) { - .arg => unreachable, // codegen must skip over these instructions. - - .block => try c.codeGenBlock(inst), - - .js_global_ref => { - const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst); - try c.extern_symbols.put(c.gpa, data, {}); - c.air_node_map[@intFromEnum(inst)] = .{ - .node = try c.add(.global, .{ .name = data }), - }; - }, - - .memory => { - const i = @intFromEnum(inst); - const data: Air.Memory = c.air.instData(.memory, inst); - const value: JsTree.Node.Index.Optional = if (data.init.isVoid()) - .none - else - (try c.fetchExpression(data.init)).toOptional(); - - try c.addStmt(try c.add(.stmt_declare_local, .{ - .identifier_index = c.next_id, - .init = value, - })); - const id = try c.add(.local, .{ .id = c.next_id }); - c.next_id += 1; - c.air_node_map[i] = .{ .node = id }; - }, - - inline .gt, - .lt, - .eq, - .add, - .sub, - .mul, - .div, - => |tag| { - const data: Air.Bin = c.air.instData(tag, inst); - const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{ - .lhs = try c.fetchExpression(data.lhs), - .rhs = try c.fetchExpression(data.rhs), - }); - try c.markExpression(inst, node); - }, - - .store => { - const data: Air.Store = c.air.instData(.store, inst); - - const target = try c.fetchInstExpression(data.memory); - - if (data.operand.kind == .index) { - const index = data.operand.decode().index; - switch (c.air.instTag(index)) { - inline .add, - .sub, - .mul, - .div, - => |tag| { - const inner_data: Air.Bin = c.air.instData(tag, index); - if (inner_data.lhs.eql(data.memory.toRef())) { - if ((tag == .add or tag == .sub) and - inner_data.rhs.kind == .value) - { - // if (inner_data.rhs.decode().value.toFloat(c.ip)) |f| { - // if (f == 1) { - // // convert this into ++ or -- - // const node = try c.add( - // if (tag == .add) - // .pre_inc - // else - // .pre_dec, - // .{ - // .operand = target, - // }, - // ); - // try c.markExpression(inst, node); - - // return; - // } - // } - } - - // convert into a compound assignment - const node = try c.add( - @field(JsTree.Tag, "assign_" ++ @tagName(tag)), - .{ - .lhs = target, - .rhs = try c.fetchExpression(inner_data.rhs), - }, - ); - try c.markExpression(inst, node); - - return; - } - }, - else => {}, - } - } - - const node = try c.add(.assign, .{ - .lhs = target, - .rhs = try c.fetchExpression(data.operand), - }); - try c.markExpression(inst, node); - }, - - inline .negate, - .bool_not, - => |tag| { - const data: Air.Un = c.air.instData(tag, inst); - const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{ - .operand = try c.fetchExpression(data.operand), - }); - try c.markExpression(inst, node); - }, - - .field => { - const data: Air.Field = c.air.instData(.field, inst); - const node = try c.add(.field, .{ - .base = try c.fetchExpression(data.base), - .field = data.field, - }); - try c.markExpression(inst, node); - }, - - .call => { - const data: Air.Call = c.air.instData(.call, inst); - const call_data = data.call_data.get(c.air); - - try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len); - const start = c.extra.list.items.len; - c.extra.list.items.len += call_data.args.len; - - const items = c.extra.list.items[start..][0..call_data.args.len]; - - for (call_data.args.slice(c.air), items) |arg, *dest| { - dest.* = @intFromEnum(try c.fetchExpression(arg)); - } - - const node = try c.add(.call, .{ - .base = try c.fetchExpression(data.base), - .args = try c.extra.append(c.gpa, JsTree.IndexList, .{ - .start = @intCast(start), - .len = call_data.args.len, - }), - }); - try c.markExpression(inst, node); - }, - - .@"if" => { - const data: Air.If = c.air.instData(.@"if", inst); - const extra = data.data.get(c.air); - var cond = try c.fetchExpression(data.cond); - - if (c.canLowerToExpression(extra.yes) and - c.canLowerToExpression(extra.no)) - { - const yes = try c.lowerToExpression(extra.yes); - const no = try c.lowerToExpression(extra.yes); - - const ternary = try c.add(.ternary, .{ - .cond = cond, - .data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{ - .yes = yes, - .no = no, - }), - }); - - try c.markExpression(inst, ternary); - } else { - const flags = c.air.instFlags(inst); - const uses = flags.uses; - - const target = if (uses != .none) - try c.addTemporary() - else - null; - - var yes = try c.codeGenBranch(extra.yes, target); - var no = try c.codeGenBranch(extra.no, target); - assert(!((yes == null) and (no == null))); // no "void else void" branches - - // when `yes` is void, swap the condition so that the `else` is omitted - if (yes == null) { - const temp_yes = yes; - yes = no; - no = temp_yes; - cond = try c.add(.bool_not, .{ - .operand = cond, - }); - } - - const main_expr = try c.add(.stmt_if, .{ - .cond = cond, - .data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{ - .yes = yes.?.list, - .no = if (no) |branch| - branch.list - else - .{ .start = 0, .len = 0 }, - .flags = .{ - .block_label = @intCast(c.next_label + 1), - .yes_label = yes.?.requires_label, - .no_label = if (no) |branch| branch.requires_label else false, - }, - }), - }); - - try c.addStmt(main_expr); - - if (target) |node| { - try c.markExpression(inst, node); - } - } - }, - - .@"break" => { - const brk: Air.Break = c.air.instData(.@"break", inst); - - const operand = try c.fetchExpression(brk.value); - - // break #0 -> return - if (brk.block_ref == .main_block) { - try c.addStmt(try c.add(.stmt_return, .{ - .operand = operand.toOptional(), - })); - } else { - const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block; - - const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air); - if (block_insts[block_insts.len - 1] != inst) { - block.used = true; - } - - const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]); - if (assignment.unwrap()) |target| { - try c.addStmt(try c.add(.assign, .{ - .lhs = target, - .rhs = operand, - })); - } - - try c.addStmt(try c.add(.stmt_break, .{ - .label = c.scratch.items[block.index + 1], - })); - } - }, - } -} - -fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void { - const t = tracer.trace(@src()); - defer t.end(); - - const flags: Air.Flags = c.air.instFlags(inst); - const i = @intFromEnum(inst); - switch (flags.uses) { - .none => { - try c.addStmt(node); - }, - .single => { - // TODO: this is not aware of side effect fences - c.air_node_map[i] = .{ .node = node }; - }, - .multi => { - const decl = try c.add(.stmt_declare_local, .{ - .identifier_index = c.next_id, - .init = node.toOptional(), - }); - try c.addStmt(decl); - const id = try c.add(.local, .{ .id = c.next_id }); - c.next_id += 1; - c.air_node_map[i] = .{ .node = id }; - }, - } -} - -fn fetchExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - return switch (ref.decode()) { - .value => |v| c.valueToExpression(v), - .index => |index| fetchInstExpression(c, index), - }; -} - -fn fetchInstExpression(c: *JsNodeGen, index: Air.Inst.Index) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - const data = c.air_node_map[@intFromEnum(index)]; - if (std.debug.runtime_safety) { - const flags: Air.Flags = c.air.instFlags(index); - switch (flags.uses) { - .none => unreachable, // incorrectly marked instruction as unused, - .single => {}, - .multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local), - } - } - return data.node; -} - -fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - const intern_tag = v.valueTag(c.ip); - switch (intern_tag) { - .removed => unreachable, // UAF - - .type_function, - .simple_type, - => unreachable, // Type values are not allowed here - - .int32, - .float, - .comptime_float, - .string, - .simple_value, - => return try c.add(.plain_value, .{ .value = v }), - - // funcs always point to decls - .func => { - const func = v.toKey(c.ip).func; - try c.decl_deps.put(c.gpa, func.owner_decl, {}); - return try c.add(.decl, .{ .index = func.owner_decl }); - }, - } -} - -fn addTemporary(c: *JsNodeGen) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - try c.addStmt(try c.add(.stmt_declare_local, .{ - .identifier_index = c.next_id, - .init = .none, - })); - const id = try c.add(.local, .{ .id = c.next_id }); - c.next_id += 1; - return id; -} - -const Branch = struct { - requires_label: bool, - list: JsTree.IndexList, -}; - -fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch { - const t = tracer.trace(@src()); - defer t.end(); - - const scratch_top = c.scratch.items.len; - defer c.scratch.shrinkRetainingCapacity(scratch_top); - - c.next_label += 1; - defer c.next_label -= 1; - const this_label = c.next_label; - - var requires_label = false; - var items = switch (inst.decode()) { - .value => |v| items: { - if (v == .void) - return null; - if (temporary) |temp| { - const node = try c.valueToExpression(v); - try c.addStmt(try c.add(.assign, .{ - .lhs = temp, - .rhs = node, - })); - } - break :items c.scratch.items[scratch_top..]; - }, - .index => |i| items: { - assert(c.air.instTag(i) == .block); - c.air_node_map[@intFromEnum(i)] = .{ - .block = .{ .index = @intCast(scratch_top) }, - }; - - try c.scratch.appendSlice(c.arena, &.{ - @intFromEnum(if (temporary) |temp| temp.toOptional() else .none), - this_label, - }); - - const block: Air.IndexList = c.air.instData(.block, i); - for (block.slice(c.air)) |item| { - try c.codeGenInst(item); - } - - requires_label = c.air_node_map[@intFromEnum(i)].block.used; - - break :items c.scratch.items[scratch_top + 2 ..]; - }, - }; - - const last_node = c.scratch.items[c.scratch.items.len - 1]; - if (c.nodes.items(.tag)[last_node] == .stmt_break and - c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label) - { - items.len -= 1; - c.nodes.len -= 1; - } - - const top = c.extra.list.items.len; - try c.extra.list.appendSlice(c.gpa, items); - return .{ - .requires_label = requires_label, - .list = .{ - .start = @intCast(top), - .len = @intCast(items.len), - }, - }; -} - -fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void { - const t = tracer.trace(@src()); - defer t.end(); - - const scratch_top = c.scratch.items.len; - errdefer c.scratch.shrinkRetainingCapacity(scratch_top); - - c.next_label += 1; - defer c.next_label -= 1; - - const next_label = c.next_label; - - try c.scratch.appendSlice(c.arena, &.{ - // @intFromEnum(if (temporary) |temp| temp.toOptional() else .none), - @intFromEnum(Node.Index.Optional.none), - next_label, - }); - - c.air_node_map[@intFromEnum(inst)] = .{ - .block = .{ .index = @intCast(scratch_top) }, - }; - const block: Air.IndexList = c.air.instData(.block, inst); - - for (block.slice(c.air)) |item| { - try c.codeGenInst(item); - } - - const items = c.scratch.items[scratch_top + 2 ..]; - const top = c.extra.list.items.len; - try c.extra.list.appendSlice(c.gpa, items); - c.scratch.shrinkRetainingCapacity(scratch_top); - try c.addStmt(try c.add(.stmt_block, .{ - .label = c.next_label, - .content = try c.extra.append(c.gpa, JsTree.IndexList, .{ - .start = @intCast(top), - .len = @intCast(items.len), - }), - })); -} - -// Ideally, we convert everything to an expression. This allows for much -// more compact code. When that isnt possible (`return`, `while`, etc), -// we detect that here and ensure to lower `if` to actual if statements, -// and so on. -fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool { - const t = tracer.trace(@src()); - defer t.end(); - - return switch (inst.decode()) { - .value => true, - .index => |index| switch (c.air.instTag(index)) { - .js_global_ref, - .call, - .arg, - .field, - .add, - .sub, - .mul, - .div, - .eq, - .lt, - .gt, - .negate, - .bool_not, - .store, - => true, - - // JavaScript variable declarations require statements. - // Later, this could be made true if we can hoist temporaries up. - .memory => false, - - // break/return must be a js statement - .@"break", - => false, - - .block => { - // TODO: blocks that are - // - all statements except last are lowerable - // - the last statement is lowerable OR a break of this block - return false; - }, - - .@"if" => { - const call = c.air.instData(.@"if", index); - const data = call.data.get(c.air); - - return c.canLowerToExpression(call.cond) and - c.canLowerToExpression(data.yes) and - c.canLowerToExpression(data.no); - }, - }, - }; -} - -fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - assert(c.canLowerToExpression(ref)); - return switch (ref.decode()) { - .value => |v| try c.valueToExpression(v), - .index => |index| { - try c.codeGenInst(index); - return c.fetchExpression(ref); - }, - }; -} - -fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index { - const t = tracer.trace(@src()); - defer t.end(); - - try c.nodes.append(c.gpa, .{ - .tag = tag, - .data = EncodedData.encode(tag, data), - }); - return @enumFromInt(c.nodes.len - 1); -} - -fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void { - const t = tracer.trace(@src()); - defer t.end(); - - try c.scratch.append(c.arena, @intFromEnum(stmt)); -} - -const JsNodeGen = @This(); - -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; -const NullTerminatedString = InternPool.NullTerminatedString; - -const Comp = @import("Comp.zig"); -const Decl = Comp.Decl; - -const JsTree = @import("JsTree.zig"); -const EncodedData = JsTree.EncodedData; -const Node = JsTree.Node; -const LocalIndex = JsTree.LocalIndex; - -const Air = @import("Air.zig"); - -const dod = @import("array.zig"); -const tracer = @import("tracer"); diff --git a/src/compiler/JsTree.zig b/src/compiler/JsTree.zig deleted file mode 100644 index 291b621802fe1957910128273fd77b223064ee14..0000000000000000000000000000000000000000 --- a/src/compiler/JsTree.zig +++ /dev/null @@ -1,257 +0,0 @@ -//! JavaScript Function Representation is the last representation before javascript -//! source bytes are created. One notable thing here is that all local identifiers -//! are numbered, instead of being named. The first `param_count` local identifiers -//! refer to parameters, and the rest refer to the respective block that defined them. - -param_count: u32, -nodes: Node.List.Slice, -extern_symbols: []String.WithLen, -decl_deps: []Decl.Index, -extra: ExtraStore, - -pub const LocalIndex = u32; -pub const ExtraStore = dod.ExtraStore(.tfr, .@"const"); -pub const EncodedData = dod.EncodedData(Tag, u64); -pub const ExtraIndex = ExtraStore.Index; - -pub const Tag = enum { - root, - /// Only allow simple value types: int, float, string, bool - plain_value, - /// Reference to a local identifier (give an VarId) - local, - /// Reference to a global identifier (give a NullTerminatedString) - global, - /// Reference to a declaration (contains a Decl.Index) - decl, - // bin ops - assign, - assign_add, - assign_sub, - assign_mul, - assign_div, - add, - sub, - mul, - div, - eq, - lt, - gt, - // exprs - call, - field, - ternary, - // prefix ops - negate, - bool_not, - pre_inc, - pre_dec, - // stmts - /// `let ... = ...` - stmt_declare_local, - /// `if (...) ... else ...` - stmt_if, - /// `break ... ...` - stmt_break, - /// `return ...` (break from block #0) - stmt_return, - /// `...: { ... }` - stmt_block, - - pub fn Data(comptime tag: Tag) type { - return switch (tag) { - .root => IndexList, - .plain_value => PlainValue, - .local => LocalRef, - .global => GlobalRef, - .decl => DeclRef, - .assign, - .assign_add, - .assign_sub, - .assign_mul, - .assign_div, - .add, - .sub, - .mul, - .div, - .eq, - .lt, - .gt, - => Bin, - .ternary => Ternary, - .negate, - .bool_not, - .pre_inc, - .pre_dec, - => Un, - .call => Call, - .field => Field, - .stmt_if => StmtIf, - .stmt_break => StmtBreak, - .stmt_return => StmtReturn, - .stmt_declare_local => StmtDeclareLocal, - .stmt_block => StmtBlock, - }; - } -}; - -pub const Node = struct { - tag: Tag, - data: EncodedData, - - pub const List = std.MultiArrayList(Node); - - pub const Index = enum(u32) { - root = 0, - _, - - pub fn toOptional(i: Index) Optional { - return @enumFromInt(@intFromEnum(i)); - } - - pub const Optional = enum(u32) { - none = std.math.maxInt(u32), - _, - - pub fn unwrap(oi: Optional) ?Index { - return if (oi == .none) - null - else - @enumFromInt(@intFromEnum(oi)); - } - }; - }; -}; - -pub const PlainValue = struct { - value: Value, - unused: Unused = .zero, -}; - -pub const LocalRef = struct { - id: LocalIndex, - unused: Unused = .zero, -}; - -pub const GlobalRef = struct { - name: InternPool.String.WithLen, -}; - -pub const DeclRef = struct { - index: Decl.Index, - unused: Unused = .zero, -}; - -pub const StmtDeclareLocal = struct { - identifier_index: u32, - init: Node.Index.Optional, -}; - -pub const Bin = struct { - lhs: Node.Index, - rhs: Node.Index, -}; - -pub const Un = struct { - operand: Node.Index, - unused: Unused = .zero, -}; - -pub const Call = struct { - base: Node.Index, - args: ExtraIndex(IndexList), -}; - -pub const Field = struct { - base: Node.Index, - field: NullTerminatedString, -}; - -pub const IndexList = struct { - start: u32, - len: u32, - - pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index { - return @ptrCast(tree.extra.list.items[list.start..][0..list.len]); - } -}; - -pub const StmtIf = struct { - cond: Node.Index, - data: ExtraIndex(Data), - - pub const Data = struct { - yes: IndexList, - no: IndexList, - flags: Flags, - }; - - pub const Flags = packed struct(u32) { - block_label: u16, - yes_label: bool, - no_label: bool, - _: u14 = 0, - }; -}; - -pub const Ternary = struct { - cond: Node.Index, - data: ExtraIndex(Data), - - pub const Data = struct { - yes: Node.Index, - no: Node.Index, - }; -}; - -pub const StmtBreak = struct { - label: u32, - unused: Unused = .zero, -}; - -pub const StmtReturn = struct { - operand: Node.Index.Optional, - unused: Unused = .zero, -}; - -pub const StmtBlock = struct { - label: u32, - content: ExtraIndex(IndexList), -}; - -pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void { - tree.nodes.deinit(gpa); - gpa.free(tree.decl_deps); - gpa.free(tree.extern_symbols); - tree.extra.deinit(gpa); -} - -pub fn nodeTag(tree: JsTree, index: Node.Index) Tag { - return tree.nodes.items(.tag)[@intFromEnum(index)]; -} - -pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() { - return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index)); -} - -pub fn mainStatements(tree: *const JsTree) []const Node.Index { - return tree.nodeData(.root, .root).slice(tree); -} - -const JsTree = @This(); - -const std = @import("std"); -const assert = std.debug.assert; - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; - -const NullTerminatedString = InternPool.NullTerminatedString; -const String = InternPool.String; - -const Comp = @import("Comp.zig"); -const Decl = Comp.Decl; - -const dod = @import("array.zig"); - -const Unused = @import("Ast.zig").Unused; diff --git a/src/compiler/Parse.zig b/src/compiler/Parse.zig deleted file mode 100644 index 423adae31bfa917eb9cc976a3a43296e5b1300ab..0000000000000000000000000000000000000000 --- a/src/compiler/Parse.zig +++ /dev/null @@ -1,1146 +0,0 @@ -//! Parser takes a Token list and creates a Node list. It does not own the source code or -//! token data, however transferring into an `Ast` does move token list ownership. -gpa: Allocator, -source: [:0]const u8, -token_tags: []const Token.Tag, -token_starts: []const ByteOffset, - -pos: TokenIndex = 0, -nodes: Node.List = .{}, -extra: ArrayListUnmanaged(u32) = .{}, -scratch: ArrayListUnmanaged(NodeIndex) = .{}, - -ast_error: ?AstError.WithTrace = null, - -depth: u32 = 0, - -/// Instead of implementing a non-recursive parser, the stack size is limited by this constant. -const depth_limit = 4096; - -pub fn parse(gpa: std.mem.Allocator, source: [:0]const u8, ast_error: *AstError.WithTrace) Parse.Error!Ast { - var err_byte: ByteOffset = 0; - var tokens = Tokenizer.tokenize( - source, - gpa, - &err_byte, - ) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => { - ast_error.* = .{ - .err = .{ .tokenization = .{ - .err = err, - .byte = err_byte, - } }, - .trace = Report.DebugStoredTrace.from(@errorReturnTrace()), - }; - return error.ParseFail; - }, - }; - errdefer tokens.deinit(gpa); - - var parser = Parse.init(gpa, source, tokens); - errdefer parser.deinit(); - - try parser.nodes.ensureUnusedCapacity( - gpa, - @max(1, @divFloor(tokens.len, 4)), - ); - - parser.parseRoot() catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseFail => { - ast_error.* = parser.ast_error.?; - return error.ParseFail; - }, - }; - - return try parser.finalize(tokens); -} - -fn init(gpa: Allocator, source: [:0]const u8, tokens: Token.List.Slice) Parse { - return Parse{ - .gpa = gpa, - .source = source, - .token_tags = tokens.items(.tag), - .token_starts = tokens.items(.offset), - }; -} - -/// This transfers the parse result into an Ast -fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast { - const extra = try p.extra.toOwnedSlice(p.gpa); - defer p.* = undefined; - assert(token_slice.items(.tag).ptr == p.token_tags.ptr); - assert(token_slice.items(.offset).ptr == p.token_starts.ptr); - p.scratch.deinit(p.gpa); - - const ast: Ast = .{ - .utf8_source = p.source, - .tokens = token_slice, - .nodes = p.nodes.toOwnedSlice(), - .extra = extra, - }; - - return ast; -} - -/// This discards all parser work -fn deinit(p: *Parse) void { - defer p.* = undefined; - p.nodes.deinit(p.gpa); - p.extra.deinit(p.gpa); - p.scratch.deinit(p.gpa); -} - -pub const Error = error{ - OutOfMemory, - ParseFail, -}; - -pub const ExpectedContext = enum { - param, - argument, - statement, - expression, - else_clause, - - pub fn description(tag: ExpectedContext) []const u8 { - return switch (tag) { - .param => "a parameter", - .argument => "an argument", - .statement => "a statement", - .expression => "an expression", - .else_clause => "an else clause", - }; - } -}; - -/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos` -/// When printing these, token list and Ast is not actually available, so full tokens -/// are stored in the errors. -pub const AstError = union(enum) { - /// Failure during tokenization - tokenization: struct { - err: Tokenizer.TokenizeError, - byte: ByteOffset, - }, - /// Unexpected data, expected a token. - unexpected_token: struct { - expected: Token.Tag, - got: Token, - }, - /// Unexpected data, expected - unexpected_context: struct { - expected: ExpectedContext, - got: Token, - }, - /// Went past depth maximum - depth_limit: Token, - - pub fn toReport(err: AstError, comp: *Comp, file: *Comp.File, trace: Report.DebugStoredTrace) !Report { - return (Report.Err{ .ast = err }).toReport(comp, file, trace); - } - - pub fn build(err: AstError, ctx: *Report.MsgContext) !void { - switch (err) { - .tokenization => |data| { - // TODO: consider making tokenizer emit AstError more directly, - // so that it can just annotate start and ends - try ctx.msg(switch (data.err) { - error.UnterminatedString => .{ .codepoint_eol = data.byte }, - else => .{ .codepoint = data.byte }, - }, "{s}", .{switch (data.err) { - error.OutOfMemory => unreachable, - error.InvalidSyntax => "unexpected token", - error.TrailingNumericSeparator => "trailing numberic separator", - error.UnterminatedString => "unterminated string literal", - error.InvalidEscapeSequence => "invalid escape sequence", - error.InvalidEscapeCodepoint => "invalid escape codepoint", - error.InvalidUtf8 => "file is not valid utf8", - error.GitConflict => "git merge conflict encountered", - }}); - }, - .unexpected_token => |data| { - try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{ - data.expected.description(), - data.got.tag.description(), - }); - }, - .unexpected_context => |data| { - try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{ - data.expected.description(), - data.got.tag.description(), - }); - }, - .depth_limit => |data| { - try ctx.msg(.{ .token_stored = data }, "syntax tree is nested too deeply (limit: {d})", .{depth_limit}); - }, - } - } - - pub const WithTrace = struct { - err: AstError, - trace: Report.DebugStoredTrace, - - pub fn toReport(err: WithTrace, comp: *Comp, file: *Comp.File) !Report { - return err.err.toReport(comp, file, err.trace); - } - }; -}; - -pub fn parseRoot(p: *Parse) Error!void { - const trace = t.trace(@src()); - defer trace.end(); - - assert(p.nodes.len == 0); - assert(p.extra.items.len == 0); - assert(p.pos == 0); - assert(p.nodes.capacity > 0); - - p.nodes.appendAssumeCapacity(.{ - .tag = .root, - .main_token = 0, - .data = undefined, - }); - - const list = try p.parseModuleMembers(); - p.nodes.items(.data)[0] = .{ - .list = list, - }; -} - -fn parseModuleMembers(p: *Parse) Error!SubList { - p.depth += 1; - defer p.depth -= 1; - if (p.depth > depth_limit) { - return p.fail(.{ .depth_limit = p.fullToken(p.pos) }); - } - - const trace = t.trace(@src()); - defer trace.end(); - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - // skip module doc comment - while (p.eatToken(.module_doc_comment)) |_| {} - - while (true) switch (p.currentTag()) { - .kw_let, - .kw_var, - .kw_fn, - .kw_pub, - => try p.scratch.append(p.gpa, try p.parseModuleDecl()), - - .kw_struct, - .kw_enum, - .kw_trait, - .kw_impl, - => |tag| std.debug.panic("TODO: parse {s}", .{@tagName(tag)}), - - .eof => break, - - else => { - if (!p.isExpressionStart()) { - return p.failUnexpectedToken(.statement); - } - - try p.scratch.append(p.gpa, try p.parseExpression(true)); - _ = try p.expect(.semi); - }, - }; - - return p.scratchToSubList(scratch_top); -} - -fn parseModuleDecl(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - _ = p.eatToken(.kw_pub); - - return switch (p.currentTag()) { - .kw_var, .kw_let => try p.parseVarDecl(), - - .kw_fn => try p.parseFnDecl(), - - else => unreachable, - }; -} - -fn parseFnDecl(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.kw_fn); - - _ = try p.expect(.id); - - _ = try p.expect(.start_paren); - - const params = try p.parseParamList(); - - const return_type = if (p.eatToken(.minus_lt)) |_| - if (!p.isExpressionStart()) - return p.failUnexpectedToken(.expression) - else - try p.parseExpression(false) - else - null; - - try p.expectWithoutAdvance(.start_brace); - const body = try p.parseBlock(); - - const extra = try p.addExtra(Ast.FnDecl, .{ - .params = params, - .return_type = optionalNodeIndex(return_type), - }); - - return try p.addNode(.{ - .tag = .fn_decl, - .main_token = main_token, - .data = .{ .fn_decl = .{ - .body = body, - .extra = extra, - } }, - }); -} - -fn parseParamList(p: *Parse) Error!SubList { - const trace = t.trace(@src()); - defer trace.end(); - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - while (true) { - const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top}); - defer trace_param.end(); - - if (p.currentTag() == .end_paren) break; - - try p.scratch.append(p.gpa, try p.parseParamDecl()); - - switch (p.currentTag()) { - .comma => { - _ = p.nextToken(); - }, - .end_paren => break, - else => return p.failUnexpectedToken(.param), - } - } - - _ = p.assertToken(.end_paren); - - return p.scratchToSubList(scratch_top); -} - -fn parseParamDecl(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.id); - - _ = try p.expect(.colon); - - const ty = try p.parseExpression(false); - - return try p.addNode(.{ - .tag = .param_decl, - .main_token = main_token, - .data = .{ .param_decl = .{ - .type = ty, - } }, - }); -} - -fn parseBlock(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.start_brace); - - const tag, const list = try p.parseBlockContents(); - assert(p.lastTag() == .end_brace); - - return try p.addNode(.{ - .tag = tag, - .main_token = main_token, - .data = .{ - .list = list, - }, - }); -} - -fn parseLabelExpression(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.id); - _ = try p.expect(.colon); - const expr = try p.parseExpression(false); - return try p.addNode(.{ - .tag = .label, - .main_token = main_token, - .data = .{ .label = .{ - .expr = expr, - } }, - }); -} - -fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } { - const trace = t.trace(@src()); - defer trace.end(); - - assert(p.lastTag() == .start_brace); - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - const tag: Node.Tag = while (true) switch (p.currentTag()) { - .kw_let, - .kw_var, - => try p.scratch.append(p.gpa, try p.parseVarDecl()), - - .end_brace => break .block_void, - - else => { - if (!p.isExpressionStart()) { - return p.failUnexpectedToken(.statement); - } - - const first_tag = p.currentTag(); - - try p.scratch.append(p.gpa, try p.parseExpression(true)); - - switch (first_tag) { - // some control flow and blocks are allowed to not have a - // trailing semicolons. this is a stylistic choice. - .kw_if, - .start_brace, - => { - // ending a block here should also count as an implicit return, - // even though voiding it with a semicolon is not allowed. - if (p.currentTag() == .end_brace) break .block_implicit_return; - }, - - // TODO: make kw_break and kw_return REQUIRE a semicolon, as - // they are noreturn statements, doing implicit break on - // them makes no sense - - else => { - if (p.eatToken(.semi) == null) { - // this must be last statement in block if it isnt, - // the p.expect(.end_brace) later will fail. - break .block_implicit_return; - } - }, - } - }, - }; - - _ = try p.expect(.end_brace); - - return .{ tag, try p.scratchToSubList(scratch_top) }; -} - -/// VarDecl <- (kw_pub)? (kw_var | kw_let) id (colon expr)? equal expr semi -fn parseVarDecl(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const kind = p.currentTag(); - assert(kind == .kw_var or kind == .kw_let); - - const main_token = p.nextToken(); - - _ = try p.expect(.id); - - const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) |_| - optionalNodeIndex(try p.parseExpression(false)) - else - .none; - - _ = try p.expect(.equal); - - const data_init = try p.parseExpression(true); - - _ = try p.expect(.semi); - - return try p.addNode(.{ - .tag = .var_decl, - .main_token = main_token, - .data = .{ .var_decl = .{ - .type = data_type, - .init = data_init, - } }, - }); -} - -fn isExpressionStart(p: *Parse) bool { - return switch (p.currentTag()) { - .id, - .number_int, - .number_float, - .string, - .builtin_id, - .kw_return, - .kw_break, - .kw_if, - .start_paren, - .start_brace, - .bang, - .tilde, - .minus, - => true, - else => false, - }; -} - -const Operator = enum(u8) { - add = @intFromEnum(Token.Tag.plus), - sub = @intFromEnum(Token.Tag.minus), - mul = @intFromEnum(Token.Tag.star), - div = @intFromEnum(Token.Tag.slash), - rem = @intFromEnum(Token.Tag.percent), - pow = @intFromEnum(Token.Tag.star_star), - @"and" = @intFromEnum(Token.Tag.and_and), - @"or" = @intFromEnum(Token.Tag.pipe_pipe), - bit_and = @intFromEnum(Token.Tag.@"and"), - bit_or = @intFromEnum(Token.Tag.pipe), - bit_xor = @intFromEnum(Token.Tag.caret), - shl = @intFromEnum(Token.Tag.lt_lt), - shr = @intFromEnum(Token.Tag.gt_gt), - eq = @intFromEnum(Token.Tag.equal_equal), - ne = @intFromEnum(Token.Tag.bang_equal), - lt = @intFromEnum(Token.Tag.lt), - le = @intFromEnum(Token.Tag.lt_equal), - gt = @intFromEnum(Token.Tag.gt), - ge = @intFromEnum(Token.Tag.gt_equal), - // Following operators are assignment operators - assign = @intFromEnum(Token.Tag.equal), - add_assign = @intFromEnum(Token.Tag.plus_equal), - sub_assign = @intFromEnum(Token.Tag.minus_equal), - mul_assign = @intFromEnum(Token.Tag.star_equal), - div_assign = @intFromEnum(Token.Tag.slash_equal), - rem_assign = @intFromEnum(Token.Tag.percent_equal), - pow_assign = @intFromEnum(Token.Tag.star_star_equal), - and_assign = @intFromEnum(Token.Tag.and_and_equal), - or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal), - bit_and_assign = @intFromEnum(Token.Tag.and_equal), - bit_or_assign = @intFromEnum(Token.Tag.pipe_equal), - bit_xor_assign = @intFromEnum(Token.Tag.caret_equal), - shl_assign = @intFromEnum(Token.Tag.lt_lt_equal), - shr_assign = @intFromEnum(Token.Tag.gt_gt_equal), - - const Direction = enum { ltr, rtl }; - - /// Only the binary operator levels are actually used, - /// but all are listed for reference. - const Level = enum { - /// e. e[] e() - accessor, - /// new - new, - /// ++e, --e, !e, ~e, +e, -e, typeof, void, delete - prefix, - /// ** - exponentiation, - /// *, /, % - multiplicative, - /// +, - - additive, - /// <<, >>, >>> - bitwise_shift, - /// <, <=, >, >= - relational, - /// ==, != - equality, - /// & - bit_and, - /// ^ - bit_xor, - /// | - bit_or, - /// && - logical_and, - /// ||, ?? - logical_or, - /// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |= - assign, - }; - - pub fn direction(op: Operator) Direction { - return switch (op) { - .pow => .rtl, - - .assign, - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => .rtl, - - else => .ltr, - }; - } - - pub fn level(op: Operator) Level { - return switch (op) { - .pow => .exponentiation, - .mul, .div, .rem => .multiplicative, - .add, .sub => .additive, - .shl, .shr => .bitwise_shift, - .lt, .le, .gt, .ge => .relational, - .eq, .ne => .equality, - .@"and" => .logical_and, - .@"or" => .logical_or, - .bit_and => .bit_and, - .bit_xor => .bit_xor, - .bit_or => .bit_or, - .assign, - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => .assign, - }; - } - - const lut = brk: { - const token_tags = std.enums.values(Token.Tag); - const op_tags = std.enums.values(Operator); - - var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len; - for (op_tags) |op| { - is_operator_lut[@intFromEnum(op)] = true; - } - - var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len; - for (op_tags) |op| { - ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op)); - } - - break :brk struct { - const is_operator = is_operator_lut; - const ast_tag = ast_tag_lut; - }; - }; - - pub fn fromToken(tag: Token.Tag) ?Operator { - return if (lut.is_operator[@intFromEnum(tag)]) - @enumFromInt(@intFromEnum(tag)) - else - null; - } - - pub fn astTag(op: Operator) Node.Tag { - const tag = lut.ast_tag[@intFromEnum(op)]; - assert(tag != .root); // invalid operator or lut construction - return tag; - } - - pub fn isAssign(op: Operator) bool { - return switch (op) { - .assign, - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => true, - else => false, - }; - } -}; - -fn parseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - p.depth += 1; - defer p.depth -= 1; - if (p.depth > depth_limit) { - return p.fail(.{ .depth_limit = p.fullToken(p.pos) }); - } - - assert(p.isExpressionStart()); - - // These unary operators are greedy, so handle them first. - switch (p.currentTag()) { - .kw_return => return p.parseReturn(), - .kw_break => return p.parseBreak(), - else => {}, - } - - var expr = try p.parseExpressionPart(); - - while (Operator.fromToken(p.currentTag())) |op| { - if (!allow_assign_operators and op.isAssign()) break; - expr = try p.parseBinarySuffix(expr, op, allow_assign_operators); - } - - return expr; -} - -fn parseExpressionPart(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - var expr = switch (p.currentTag()) { - // Literals - .id => if (p.nextTag() == .colon) - try p.parseLabelExpression() - else - try p.blankNode(.id), - .number_int => try p.blankNode(.number_int), - .number_float => try p.blankNode(.number_float), - .string => try p.blankNode(.string), - - // Complex expressions - .builtin_id => try p.parseBuiltinCall(), - .kw_return => try p.parseReturn(), - .kw_break => try p.parseBreak(), - .kw_if => try p.parseIf(), - .start_paren => try p.parseGroup(), - .start_brace => try p.parseBlock(), - - // Unary operators - inline .bang, .tilde, .minus => |tag| unary: { - const main_token = p.nextToken(); - const expr = try p.parseExpressionPart(); - - break :unary try p.addNode(.{ - .tag = switch (tag) { - .bang => .bool_not, - .tilde => .bit_not, - .minus => .negate, - else => comptime unreachable, - }, - .main_token = main_token, - .data = .{ .un = .{ - .expr = expr, - } }, - }); - }, - - else => return p.failUnexpectedToken(.expression), - }; - - // parse field, calls, and index expressions - while (true) { - const trace2 = t.traceFmt(@src(), "(field/call)", .{}); - defer trace2.end(); - - switch (p.currentTag()) { - .dot => { - const main_token = p.nextToken(); - const field = try p.expect(.id); - - expr = try p.addNode(.{ - .tag = .field, - .main_token = main_token, - .data = .{ .field = .{ - .base = expr, - .field_token = field, - } }, - }); - }, - .start_paren => { - const main_token = p.nextToken(); - const args = try p.parseCallArgs(); - - const extra = try p.addExtra(Ast.Call, .{ - .args = args, - }); - - expr = try p.addNode(.{ - .tag = .call, - .main_token = main_token, - .data = .{ .call = .{ - .base = expr, - .extra = extra, - } }, - }); - }, - .start_bracket => { - const main_token = p.nextToken(); - const index = try p.parseExpression(false); - - expr = try p.addNode(.{ - .tag = .index, - .main_token = main_token, - .data = .{ .bin = .{ - .lhs = expr, - .rhs = index, - } }, - }); - }, - - else => break, - } - } - - return expr; -} - -fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator, allow_assign_operators: bool) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.nextToken(); - const level = op.level(); - var rhs = try p.parseExpressionPart(); - - while (Operator.fromToken(p.currentTag())) |next_op| { - if (!allow_assign_operators and next_op.isAssign()) break; - - const new_level = next_op.level(); - const use = switch (op.direction()) { - .ltr => @intFromEnum(new_level) < @intFromEnum(level), - .rtl => @intFromEnum(new_level) <= @intFromEnum(level), - }; - if (!use) break; - - rhs = try p.parseBinarySuffix(rhs, next_op, allow_assign_operators); - } - - return try p.addNode(.{ - .tag = op.astTag(), - .main_token = main_token, - .data = .{ .bin = .{ - .lhs = lhs, - .rhs = rhs, - } }, - }); -} - -fn parseReturn(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.kw_return); - - const expr = if (p.isExpressionStart()) - try p.parseExpression(false) - else - null; - - return try p.addNode(.{ - .tag = .@"return", - .main_token = main_token, - .data = .{ .@"return" = .{ - .expr = optionalNodeIndex(expr), - } }, - }); -} - -fn parseBreak(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.kw_break); - - const label = if (p.eatToken(.colon)) |_| - try p.expect(.id) - else - null; - - const expr = if (p.isExpressionStart()) - try p.parseExpression(false) - else - null; - - return try p.addNode(.{ - .tag = .@"break", - .main_token = main_token, - .data = .{ .@"break" = .{ - .label = optionalTokenIndex(label), - .value = optionalNodeIndex(expr), - } }, - }); -} - -fn parseBuiltinCall(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.builtin_id); - - _ = try p.expect(.start_paren); - - const args = try p.parseCallArgs(); - - return try p.addNode(.{ - .tag = .builtin_call, - .main_token = main_token, - .data = .{ .list = args }, - }); -} - -fn parseCallArgs(p: *Parse) Error!SubList { - const trace = t.trace(@src()); - defer trace.end(); - - const scratch_top = p.scratch.items.len; - defer p.scratch.shrinkRetainingCapacity(scratch_top); - - while (p.isExpressionStart()) { - try p.scratch.append(p.gpa, try p.parseExpression(false)); - - switch (p.currentTag()) { - .end_paren => { - break; - }, - .comma => { - _ = p.nextToken(); - if (p.currentTag() == .end_paren) - break; - }, - else => return p.failUnexpectedToken(.argument), - } - } - - _ = try p.expect(.end_paren); - - return p.scratchToSubList(scratch_top); -} - -fn parseIf(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.kw_if); - - const cond = try p.parseExpression(false); - - _ = try p.expectWithoutAdvance(.start_brace); - const yes = try p.parseBlock(); - - const no = if (p.eatToken(.kw_else)) |_| - try p.parseElseValue() - else - null; - - const extra = try p.addExtra(Ast.If, .{ - .yes = yes, - .no = optionalNodeIndex(no), - }); - - return try p.addNode(.{ - .tag = .@"if", - .main_token = main_token, - .data = .{ .@"if" = .{ - .cond = cond, - .extra = extra, - } }, - }); -} - -fn parseElseValue(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - assert(p.lastTag() == .kw_else); - - return switch (p.currentTag()) { - .start_brace => p.parseBlock(), - .kw_if => p.parseIf(), - .id => p.parseLabelExpression(), - - else => return p.failUnexpectedToken(.else_clause), - }; -} - -fn parseGroup(p: *Parse) Error!NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const main_token = p.assertToken(.start_paren); - - const expr = try p.parseExpression(false); - - _ = try p.expect(.end_paren); - - return try p.addNode(.{ - .tag = .group, - .main_token = main_token, - .data = .{ .un = .{ - .expr = expr, - } }, - }); -} - -// helpers - -/// Append a node and return it's index. -fn addNode(p: *Parse, node: Node) !NodeIndex { - const trace = t.trace(@src()); - defer trace.end(); - - try p.nodes.append(p.gpa, node); - // @intCast is always safe because 2^32 tokens alwayss produce less than - // 2^32 nodes, and there is a 2^32 limit on source file bytes. - return @intCast(p.nodes.len - 1); -} - -fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex { - const trace = t.trace(@src()); - defer trace.end(); - - const extra_top = p.extra.items.len; - const bytes = std.mem.asBytes(&extra); - try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes)); - return @intCast(extra_top); -} - -fn fail(p: *Parse, ast_error: AstError) Error { - p.ast_error = .{ - .err = ast_error, - .trace = Report.DebugStoredTrace.capture(@returnAddress()), - }; - return Error.ParseFail; -} - -fn failUnexpectedToken(p: *Parse, kind: ExpectedContext) Error { - return p.fail(.{ .unexpected_context = .{ - .got = p.fullToken(p.pos), - .expected = kind, - } }); -} - -fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex { - const main_token = p.assertToken(tag); - return try p.addNode(.{ - .tag = @field(Node.Tag, @tagName(tag)), - .main_token = main_token, - .data = .{ .none = .{} }, - }); -} - -fn currentTag(p: *Parse) Token.Tag { - return p.token_tags[p.pos]; -} - -fn lastTag(p: *Parse) Token.Tag { - return p.token_tags[p.pos - 1]; -} - -fn nextTag(p: *Parse) Token.Tag { - return p.token_tags[p.pos + 1]; -} - -fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex { - return if (p.token_tags[p.pos] == tag) p.nextToken() else null; -} - -fn expect(p: *Parse, tag: Token.Tag) !TokenIndex { - if (p.currentTag() != tag) { - return p.fail(.{ .unexpected_token = .{ - .got = p.fullToken(p.pos), - .expected = tag, - } }); - } - return p.nextToken(); -} - -fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void { - if (p.currentTag() != tag) { - return p.fail(.{ .unexpected_token = .{ - .got = p.fullToken(p.pos), - .expected = tag, - } }); - } -} - -fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex { - const token = p.nextToken(); - assert(p.token_tags[token] == tag); - return token; -} - -fn nextToken(p: *Parse) TokenIndex { - const result = p.pos; - p.pos += 1; - return result; -} - -fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool { - return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null; -} - -fn fullToken(p: *Parse, token: TokenIndex) Token { - return Token{ - .tag = p.token_tags[token], - .offset = p.token_starts[token], - }; -} - -fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList { - const trace = t.trace(@src()); - defer trace.end(); - - const items = p.scratch.items[scratch_top..]; - const extra_top = p.extra.items.len; - try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]); - return .{ - .start = @intCast(extra_top), - .len = @intCast(items.len), - }; -} - -const Parse = @This(); - -const std = @import("std"); -const assert = std.debug.assert; -const mem = std.mem; -const Allocator = mem.Allocator; -const ArrayListUnmanaged = std.ArrayListUnmanaged; - -const Tokenizer = @import("Tokenizer.zig"); -const Token = Tokenizer.Token; -const ByteOffset = Tokenizer.ByteOffset; - -const Ast = @import("Ast.zig"); -const Node = Ast.Node; -const Data = Node.Data; -const SubList = Ast.SubList; -const NodeIndex = Ast.NodeIndex; -const TokenIndex = Ast.TokenIndex; -const ExtraIndex = Ast.ExtraIndex; -const OptionalNodeIndex = Ast.OptionalNodeIndex; -const OptionalTokenIndex = Ast.OptionalTokenIndex; -const OptionalExtraIndex = Ast.OptionalExtraIndex; -const optionalTokenIndex = OptionalTokenIndex.init; -const optionalNodeIndex = OptionalNodeIndex.init; -const optionalExtraIndex = OptionalExtraIndex.init; - -const t = @import("tracer"); - -const Report = @import("Report.zig"); -const Comp = @import("Comp.zig"); diff --git a/src/compiler/Report.zig b/src/compiler/Report.zig deleted file mode 100644 index 814a0f3510f5d342912441a6b39b4a66df3cd735..0000000000000000000000000000000000000000 --- a/src/compiler/Report.zig +++ /dev/null @@ -1,864 +0,0 @@ -//! A Report is a list of errors and warnings for a part of a compilation. -//! -//! `Err/Warn` structs are unions of every possible message, the logic -//! for printing them is all done here. -//! -//! Across the compiler, `Report.Builder`s' `.err` and `.warn` methods -//! can add messages by ID. -errors: []Item = &.{}, -warnings: []Item = &.{}, -string_bytes: []u8 = &.{}, - -pub var debug_show_report_trace: if (builtin.mode == .Debug) bool else void = - if (builtin.mode == .Debug) false; - -pub const Item = struct { - /// What is this diagnostic - data: union { @"error": Err, warn: Warn }, - /// What file was it in - url: String, - - debug_stack_trace: DebugStoredTrace, - - /// Type is never stored, inferred from which list it is in. - pub const Type = enum { @"error", warn }; -}; - -pub const Err = union(enum) { - pub const FieldEnum = std.meta.FieldEnum(Err); - - /// Constructs a `Report` with just a single error. - pub fn toReport(err: Err, comp: *Comp, file: *File, trace: DebugStoredTrace) !Report { - const string_bytes = try comp.gpa.dupe(u8, file.file_path); - errdefer comp.gpa.free(string_bytes); - - const errors = try comp.gpa.dupe(Item, &.{.{ - .data = .{ .@"error" = err }, - .url = .{ .start = 0, .len = @intCast(string_bytes.len) }, - .debug_stack_trace = trace, - }}); - errdefer comp.gpa.free(errors); - - return .{ .errors = errors, .string_bytes = string_bytes }; - } - - ast: @import("Parse.zig").AstError, - - unknown_identifier: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{ - ctx.ast.basicNodeSlice(data.node), - }); - } - }, - invalid_builtin: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken( - ctx.ast.nodeToken(data.node), - "unknown built-in function \"{s}\"", - .{ctx.ast.basicNodeSlice(data.node)}, - ); - } - }, - duplicate_function: struct { - node: NodeIndex, - prev: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ - ctx.ast.fullFnDecl(data.node).id, - }); - try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{}); - // TODO: be able to store a full list of other declarations - // and add them here as "and here" and so on - } - }, - duplicate_param: struct { - node: NodeIndex, - prev: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ - ctx.ast.fullParamDecl(data.node).id, - }); - try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{}); - // TODO: be able to store a full list of other declarations - // and add them here as "and here" and so on - } - }, - invalid_return: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgNode( - data.node, - "return cannot be used outside of a function body", - .{}, - ); - } - }, - unresolved_label: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msg( - .{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 }, - "unknown block label \"{s}\"", - .{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)}, - ); - } - }, - shadow_builtin_variable: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken( - ctx.ast.nodeToken(data.node) + 1, - "declaration shadows reserved built-in \"{s}\"", - .{ctx.ast.fullVarDecl(data.node).id}, - ); - } - }, - comptime_float_to_int: struct { - val: f64, - src: LazyNode, - - fn build(data: @This(), ctx: *MsgContext) !void { - const node = try data.src.get(ctx); - try ctx.msgNode(node, "numeric literal \"{d}\" cannot be coerced to \"int\"", .{data.val}); - } - }, - coersion: struct { - from: Type, - to: Type, - src: LazyNode, - - fn build(data: @This(), ctx: *MsgContext) !void { - const node = try data.src.get(ctx); - try ctx.msgNode(node, "expected \"{s}\", got \"{s}\"", .{ - data.to.displayName(&ctx.comp.intern_pool), - data.from.displayName(&ctx.comp.intern_pool), - }); - } - }, - call_non_function: struct { - call: LazyNode, - ty: Type, - - fn build(data: @This(), ctx: *MsgContext) !void { - const node = try data.call.get(ctx); - try ctx.msgNode(node, "\"{s}\" is not callable", .{ - data.ty.displayName(&ctx.comp.intern_pool), - }); - } - }, - call_mismatch_args: struct { - call: LazyNode, - actual: u32, - expected: u32, - - fn build(data: @This(), ctx: *MsgContext) !void { - const node = try data.call.get(ctx); - try ctx.msgNode(node, "function expected {d} arguments, recieved {d}", .{ - data.expected, - data.actual, - }); - } - }, - assign_builtin: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgNode( - data.node, - "cannot re-assign built-in \"{s}\"", - .{ctx.ast.basicNodeSlice(data.node)}, - ); - } - }, - invalid_assignment_symbol: struct { - node: NodeIndex, - decl: NodeIndex, - kind: UnusedKind, - - fn build(data: @This(), ctx: *MsgContext) !void { - const id = switch (ctx.ast.nodeTag(data.decl)) { - .param_decl => ctx.ast.fullParamDecl(data.decl).id, - .fn_decl => ctx.ast.fullFnDecl(data.decl).id, - .var_decl => ctx.ast.fullVarDecl(data.decl).id, - else => unreachable, // not a valid symbol node - }; - - try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{ - @tagName(data.kind), - id, - }); - try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{}); - } - }, - invalid_assignment_target: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgNode( - data.node, - "invalid assignment target", - .{}, - ); - } - }, -}; - -pub const Warn = union(enum) { - pub const FieldEnum = std.meta.FieldEnum(Warn); - - unused: struct { - node: NodeIndex, - kind: UnusedKind, - - fn build(data: @This(), ctx: *MsgContext) !void { - const id = switch (ctx.ast.nodeTag(data.node)) { - .param_decl => ctx.ast.fullParamDecl(data.node).id, - .fn_decl => ctx.ast.fullFnDecl(data.node).id, - .var_decl => ctx.ast.fullVarDecl(data.node).id, - else => unreachable, // not a valid symbol node - }; - - try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{ - @tagName(data.kind), - id, - }); - - try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id}); - } - }, - not_mutated_var: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - const id = switch (ctx.ast.nodeTag(data.node)) { - .var_decl => ctx.ast.fullVarDecl(data.node).id, - else => unreachable, // not a valid variable symbol node - }; - - try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{ - id, - }); - } - }, - invalid_discard: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.nodeToken(data.node), "\"let\" is not required for a discard", .{}); - } - }, - discard_builtin: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.nodeToken(data.node), "discarding a built-in symbol is meaningless", .{}); - } - }, - unnecessary_discard: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.nodeToken(data.node), "useless discard", .{}); - } - }, - dangling_identifier: struct { - node: NodeIndex, - - fn build(data: @This(), ctx: *MsgContext) !void { - try ctx.msgToken(ctx.ast.nodeToken(data.node), "dangling identifier does nothing", .{}); - } - }, - numeric_int_var_ambiguity: struct { - node: LazyNode, - - fn build(data: @This(), ctx: *MsgContext) !void { - const node = try data.node.get(ctx); - try ctx.msgNode(node, "constant integer in a \"var\" has an unknown type, assuming \"float\"", .{}); - try ctx.msgAlone("add \": float\" or \": int\" after the variable name to set the type", .{}); - } - }, -}; - -/// A variant of std.builtin.StackTrace that stores it's data within itself -/// instead of being a pointer. This allows capturing the stack trace and -/// saving it for later printing. -pub const StoredTrace = struct { - data: [32]usize, - index: usize, - - pub const empty = .{ - .data = .{0} ** 32, - .index = 0, - }; - - fn trace(stored: *StoredTrace) std.builtin.StackTrace { - return .{ - .index = stored.index, - .instruction_addresses = &stored.data, - }; - } - - pub fn capture(begin: ?usize) StoredTrace { - var stored: StoredTrace = undefined; - var frame = stored.trace(); - std.debug.captureStackTrace(begin orelse @returnAddress(), &frame); - stored.index = frame.index; - return stored; - } - - pub fn from(stack_trace: ?*std.builtin.StackTrace) StoredTrace { - if (stack_trace) |stack| { - var data: [32]usize = undefined; - @memset(&data, 0); - const items = @min(stack.instruction_addresses.len, 32); - @memcpy(data[0..items], stack.instruction_addresses[0..items]); - return .{ - .data = data, - .index = @min(items, stack.index), - }; - } else { - return empty; - } - } -}; - -pub const DebugStoredTrace = if (builtin.mode == .Debug) - StoredTrace -else - struct { - pub fn capture(_: ?usize) @This() { - return .{}; - } - - pub fn from(_: ?*std.builtin.StackTrace) @This() { - return .{}; - } - }; - -pub const UnusedKind = enum { - param, - constant, - variable, - function, -}; - -pub const String = struct { - start: u32, - len: u32, - - pub fn slice(str: String, report: Report) []const u8 { - return report.string_bytes[str.start..][0..str.len]; - } -}; - -const Prefix = enum { - @"error", - warn, - note, - - fn color(prefix: Prefix) std.io.tty.Color { - return switch (prefix) { - .@"error" => .red, - .warn => .yellow, - .note => .blue, - }; - } -}; - -pub fn deinit(report: Report, gpa: Allocator) void { - gpa.free(report.errors); - gpa.free(report.warnings); - gpa.free(report.string_bytes); -} - -pub fn dumpToWriter(report: Report, comp: *Comp, writer: anytype, tty: std.io.tty.Config) !void { - // try writer.print("{d} errors, {d} warnings\n", .{ report.errors.len, report.warnings.len }); - for (report.errors) |item| { - try report.formatItem(item, .@"error", comp, writer, tty); - } - for (report.warnings) |item| { - try report.formatItem(item, .warn, comp, writer, tty); - } -} - -pub fn formatItem( - report: Report, - item: Item, - item_type: Item.Type, - comp: *Comp, - writer: anytype, - tty: std.io.tty.Config, -) !void { - var arena = std.heap.ArenaAllocator.init(comp.gpa); - defer arena.deinit(); - var sfa = std.heap.stackFallback(16384, arena.allocator()); - const alloc = sfa.get(); - - const file = try comp.getFile(item.url.slice(report)); - - const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast); - const ast_data = if (need_load_ast) try file.getAst(comp) else undefined; - - var ctx: MsgContext = .{ - .arena = alloc, - .comp = comp, - .file = file, - .ast = &ast_data, - .utf8_source = (try file.getSource(comp.gpa)).code, - .components = ArrayListUnmanaged(Component).initCapacity(alloc, 16) catch - unreachable, // stack-fallback allocator has enough space. - }; - - switch (item_type) { - inline else => |tag| switch (@field(item.data, @tagName(tag))) { - inline else => |payload| try payload.build(&ctx), - }, - } - - assert(ctx.components.items.len > 0); - - for (ctx.components.items, 1..) |component, i| { - const prefix: Prefix = if (i == 1) - switch (item_type) { - inline else => |tag| @field(Prefix, @tagName(tag)), - } - else - .note; - - const is_last = i == ctx.components.items.len; - - if (i != 1) { - try tty.setColor(writer, .dim); - try writer.writeAll(" │ "); - try tty.setColor(writer, .reset); - } - - try tty.setColor(writer, prefix.color()); - try tty.setColor(writer, .bold); - try writer.writeAll(@tagName(prefix)); - try tty.setColor(writer, .reset); - try tty.setColor(writer, .dim); - try writer.writeAll(": "); - try tty.setColor(writer, .reset); - try writer.writeAll(component.message_utf8); - try writer.writeAll("\n"); - - const utf8_source = ctx.utf8_source; - - const at = component.location orelse { - if (is_last) { - try tty.setColor(writer, .dim); - try writer.print(" {s}", .{"┴"}); - try writer.writeAll("\n"); - try tty.setColor(writer, .reset); - } - continue; - }; - - const line_before = utf8_source[at.loc.line_start..at.byte]; - const trimmed_before = std.mem.trimLeft(u8, line_before, " \t\r"); - const line_after = utf8_source[at.byte + at.slice.len .. at.loc.line_end + 1]; - const width = std.fmt.count("{d}", .{at.loc.line.oneBased()}); - - try tty.setColor(writer, .dim); - try writer.writeByteNTimes(' ', width); - try writer.print("{s}{s}:{d}:{d}\n", .{ - if (i == 1) "┬ " else "├ ", - file.file_path, - at.loc.line.oneBased(), - at.loc.column.oneBased(), - }); - try writer.print("{d}", .{at.loc.line.oneBased()}); - try writer.print("{s} ", .{"│"}); - try tty.setColor(writer, .reset); - try writer.writeAll(trimmed_before); - try tty.setColor(writer, prefix.color()); - try tty.setColor(writer, .bold); - try writer.writeAll(at.slice); - try tty.setColor(writer, .reset); - try writer.writeAll(line_after); - try tty.setColor(writer, .dim); - try writer.writeAll(if (is_last) " ╰" else " ├"); - try writer.writeBytesNTimes( - "─", - width + - (at.byte - at.loc.line_start) - - (line_before.len - trimmed_before.len), - ); - try tty.setColor(writer, .reset); - try tty.setColor(writer, prefix.color()); - try tty.setColor(writer, .bold); - try writer.writeBytesNTimes("━", at.slice.len); - try tty.setColor(writer, .reset); - try writer.writeAll("\n"); - } - - if (builtin.mode == .Debug and debug_show_report_trace) { - var trace = item.debug_stack_trace; - std.debug.dumpStackTrace(trace.trace()); - } -} - -pub fn dumpToStdErr(report: Report, comp: *Comp) !void { - const stderr = std.io.getStdErr(); - const tty = std.io.tty.detectConfig(stderr); - - if (false or - // Windows API uses it's own API calls, so a buffered writer - // cannot be used :/ - (builtin.os.tag == .windows and tty == .windows_api) or - // Debug stack trace printer writes directly to standard err, - // so we need to be unbuffered for that. - (builtin.mode == .Debug and debug_show_report_trace)) - { - try dumpToWriter(report, comp, stderr.writer(), tty); - } else { - var buf = std.io.bufferedWriter(stderr.writer()); - const writer = buf.writer(); - try dumpToWriter(report, comp, writer, tty); - try buf.flush(); - } -} - -pub const Builder = struct { - gpa: Allocator, - current_file: *Comp.File, - errors: ArrayListUnmanaged(Item) = .{}, - warnings: ArrayListUnmanaged(Item) = .{}, - string_bytes: ArrayListUnmanaged(u8) = .{}, - current_file_str: ?String = null, - - pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void { - try b.errors.append(b.gpa, .{ - .data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) }, - .url = try b.currentFileString(), - .debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{}, - }); - } - - pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void { - try b.warnings.append(b.gpa, .{ - .data = .{ .warn = @unionInit(Warn, @tagName(kind), data) }, - .url = try b.currentFileString(), - .debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{}, - }); - } - - pub fn addString(b: *Builder, str: []const u8) !String { - const top = b.string_bytes.items.len; - try b.string_bytes.appendSlice(b.gpa, str); - return .{ .start = @intCast(top), .len = @intCast(str.len) }; - } - - pub fn currentFileString(b: *Builder) !String { - return b.current_file_str orelse { - b.current_file_str = try b.addString(b.current_file.file_path); - return b.current_file_str.?; - }; - } - - pub fn deinit(b: *Builder) void { - b.errors.deinit(b.gpa); - b.warnings.deinit(b.gpa); - b.string_bytes.deinit(b.gpa); - } - - pub fn finalize(b: *Builder) !Report { - defer b.* = undefined; - errdefer b.deinit(); - - const errors = try b.errors.toOwnedSlice(b.gpa); - errdefer b.gpa.free(errors); - - const warnings = try b.warnings.toOwnedSlice(b.gpa); - errdefer b.gpa.free(warnings); - - const string_bytes = try b.string_bytes.toOwnedSlice(b.gpa); - errdefer b.gpa.free(string_bytes); - - return .{ - .errors = errors, - .warnings = warnings, - .string_bytes = string_bytes, - }; - } -}; - -pub const SliceLoc = struct { - loc: Location, - byte: ByteOffset, - slice: []const u8, -}; - -const Component = struct { - message_utf8: []const u8, - location: ?SliceLoc, -}; - -pub const Highlight = union(enum) { - // No source view - alone, - // Highlight one token - token: TokenIndex, - // Highlight one token without referring to the Ast - token_stored: Tokenizer.Token, - // Highlight two tokens - two_tokens: TokenIndex, - // Highlight a Node - node: NodeIndex, - // Highlight a codepoint - codepoint: ByteOffset, - // Highlight from the start of a codepoint to '\n' - codepoint_eol: ByteOffset, - - pub fn loc(highlight: Highlight, ast: *const Ast) ?SliceLoc { - return switch (highlight) { - .alone => null, - inline .node, - .token, - .two_tokens, - => |payload, tag| loc: { - const token_index: TokenIndex = switch (tag) { - .node => ast.nodeToken(payload), - .token, .two_tokens => payload, - else => comptime unreachable, - }; - - const location = ast.tokenLocation(0, token_index); - const token = ast.tokens.get(token_index); - - break :loc .{ - .loc = location, - .slice = token.slice(ast.utf8_source), - .byte = token.offset, - }; - }, - - .token_stored => |token| loc: { - const location = Tokenizer.tokenLocation(ast.utf8_source, 0, token.offset); - - break :loc .{ - .loc = location, - .slice = token.slice(ast.utf8_source), - .byte = token.offset, - }; - }, - inline .codepoint, - .codepoint_eol, - => |b, tag| loc: { - const len = switch (tag) { - .codepoint => std.unicode.utf8ByteSequenceLength(ast.utf8_source[b]) catch 1, - .codepoint_eol => if (std.mem.indexOfScalar(u8, ast.utf8_source[b..], '\n')) |n| - n - else - (ast.utf8_source.len - b), - else => comptime unreachable, - }; - break :loc .{ - .byte = b, - .slice = ast.utf8_source[b..][0..len], - .loc = .{ - // TODO: - .column = .start, - .line = .start, - .line_start = 0, - .line_end = @intCast(ast.utf8_source.len), - }, - }; - }, - }; - } -}; - -/// This serves as a small API to make creating messages easier. -pub const MsgContext = struct { - file: *File, - comp: *Comp, - ast: *const Ast, - utf8_source: [:0]const u8, - - /// This arena is is a stack fallback with 16kb, then - /// falling back to an arena allocator. This should - /// make the cost of arbitrarily creating `Component` - /// nodes extremely lightweight. - arena: Allocator, - - /// First component is "error" or "warn" - /// Trailing components are "note" - components: ArrayListUnmanaged(Component), - - pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void { - const text = try std.fmt.allocPrint(m.arena, fmt, args); - try m.components.append(m.arena, .{ - .message_utf8 = text, - .location = highlight.loc(m.ast), - }); - } - - pub fn msgNode(m: *MsgContext, node: NodeIndex, comptime fmt: []const u8, args: anytype) !void { - return m.msg(.{ .node = node }, fmt, args); - } - - pub fn msgToken(m: *MsgContext, token: TokenIndex, comptime fmt: []const u8, args: anytype) !void { - return m.msg(.{ .token = token }, fmt, args); - } - - pub fn msgAlone(m: *MsgContext, comptime fmt: []const u8, args: anytype) !void { - return m.msg(.alone, fmt, args); - } -}; - -pub const LazyNode = struct { - decl: Comp.Decl.Index, - inst: Teg.Inst.Index, - operand: Operand, - - pub fn get(lazy: LazyNode, ctx: *MsgContext) !NodeIndex { - const decl = ctx.comp.intern_pool.declPtrConst(lazy.decl); - const ns = ctx.comp.intern_pool.namespacePtr(decl.src_namespace); - - // TODO: This might change if we have errors like - // - function arguments not matching at - // - function declared - // And they are both in different files. - // To resolve this, `Component` needs to store which file it is in. - assert(ns.file_scope == ctx.file); - - if (ns.parent.unwrap() != null) { - @panic("TODO: LazyNode.get when the decl is in a nested namespace"); - } - - const teg = &(ctx.file.teg orelse - // this will be required once incremental compilation hits - @panic("TODO: teg was unloaded")); - - const decl_inst = decl.teg_decl_index.unwrap() orelse - unreachable; // file was not analyzed, yet teg loaded? - - const teg_decl = teg.instData(.declaration, decl_inst.resolve(&ctx.comp.intern_pool)); - - const node_offset = teg.instNodeOffset(lazy.inst); - - const base_node: NodeIndex = @intCast(teg_decl.src_node + node_offset); - if (lazy.operand == .main_node) - return base_node; - - const data = ctx.ast.nodeData(base_node); - - const node_tag = ctx.ast.nodeTag(base_node); - - switch (teg.instTag(lazy.inst)) { - .as => switch (node_tag) { - .var_decl => { - switch (lazy.operand) { - .rhs => return data.var_decl.init, - else => {}, - } - }, - else => {}, - }, - .call => switch (node_tag) { - .call => switch (lazy.operand) { - .operand => return data.call.base, - else => |op| return ctx.ast.extraData(data.call.extra, Ast.Call) - .args.slice(ctx.ast)[op.getArgId()], - }, - else => {}, - }, - .@"break" => switch (node_tag) { - .@"return" => return data.@"return".expr.unwrap().?, - .@"break" => return data.@"break".value.unwrap().?, - .block_implicit_return => { - const statements = ctx.ast.nodeStatements(base_node); - return statements[statements.len - 1]; - }, - else => return base_node, - }, - .memory => switch (node_tag) { - .var_decl => switch (lazy.operand) { - .operand => return data.var_decl.init, - else => {}, - }, - else => {}, - }, - .store => { - assert(node_tag.isAssignBin()); - switch (lazy.operand) { - .lhs => return data.bin.lhs, - .rhs => return data.bin.rhs, - else => unreachable, - } - }, - else => {}, - } - - if (!std.debug.runtime_safety) unreachable; - - std.debug.panic("TODO: operand lookup: .{s} + .{s} + {}", .{ - @tagName(teg.instTag(lazy.inst)), - @tagName(ctx.ast.nodeTag(base_node)), - lazy.operand, - }); - } -}; - -pub const Operand = enum(u32) { - /// Point to the main node of an instruction - main_node = 0, - /// Base of a call, field, or operand to a unary instruction - operand = 1, - /// Return type of a function declaration or lambda node - return_type = 2, - /// Left hand side of a binary instruction - lhs = 4, - /// Right hand side of a binary instruction - rhs = 5, - - /// One of: - /// - An argument of a .call node - /// - A parameter of a .fn_decl node - pub fn argument(id: u32) Operand { - return @enumFromInt(id + 5); - } - - fn getArgId(op: Operand) u32 { - return @intFromEnum(op) - 5; - } -}; - -const Report = @This(); -const std = @import("std"); -const ArrayListUnmanaged = std.ArrayListUnmanaged; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; - -const builtin = @import("builtin"); - -const Tokenizer = @import("Tokenizer.zig"); -const ByteOffset = Tokenizer.ByteOffset; - -const Ast = @import("Ast.zig"); -const TokenIndex = Ast.TokenIndex; -const NodeIndex = Ast.NodeIndex; -const Location = Ast.Location; - -const Teg = @import("Teg.zig"); - -const Comp = @import("Comp.zig"); -const File = Comp.File; - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; -const Type = InternPool.Type; - -const util = @import("util.zig"); diff --git a/src/compiler/Sema.zig b/src/compiler/Sema.zig deleted file mode 100644 index cae7f9f6fd9a8663acf6d0f2c55f06954ec7e15f..0000000000000000000000000000000000000000 --- a/src/compiler/Sema.zig +++ /dev/null @@ -1,999 +0,0 @@ -/// absolutely stealing zig on this one haha -/// This struct is created per declaration and chills - -// inputs: -comp: *Comp, -gpa: Allocator, -arena: Allocator, -code: Teg, -ip: *InternPool, -decl: *Decl, -decl_index: Decl.Index, -namespace_index: Namespace.Index, - -// temp data: -inst_map: InstMap = undefined, -/// scratch memory. can be used in any place you might temporarily allocate an arraylist -/// this is most notably used to store block contents before the block is finalized -scratch: std.ArrayListUnmanaged(u32) = .{}, -params: []Type = undefined, - -// outputs -report: Report.Builder = undefined, -air_instructions: std.MultiArrayList(Air.Inst) = .{}, -air_extra: dod.ExtraStore(.air, .@"var") = .{}, - -temp_result_map: std.AutoArrayHashMapUnmanaged(Teg.Inst.Index, Type) = .{}, - -pub const Error = error{ - /// Analysis has failed, see error list - AnalysisFail, - // This analysis is waiting on other dependencies to finish - // Interrupt, - - // TODO: add handling of this - TooManyValues, - HasNulls, - - OutOfMemory, -} || Comp.File.GetTegError; - -/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything. -/// This is fine because all instructions are going to be adacent in a single declaration. -const InstMap = struct { - items: []Air.Inst.Ref.Optional, - start: Teg.Inst.Index, - - pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap { - const decl = code.declData(inst_start); - const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count); - @memset(insts, .none); - return .{ - .items = insts, - .start = inst_start, - }; - } - - pub fn deinit(map: *InstMap, allocator: Allocator) void { - allocator.free(map.items); - map.* = undefined; - } - - pub fn get(map: *const InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref { - const source_i = @intFromEnum(teg_code); - const start_i = @intFromEnum(map.start); - assert(source_i >= start_i and source_i <= (start_i + map.items.len)); - return map.items[source_i - start_i].unwrap(); - } - - pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void { - if (assertWithMessage(map.get(from) == null)) { - std.debug.panic("%{d} is already mapped (to #{d})", .{ - @intFromEnum(from), - @intFromEnum(map.get(from).?.decode().index), - }); - } - const source_i = @intFromEnum(from); - const start_i = @intFromEnum(map.start); - map.items[source_i - start_i] = to.toOptional(); - } -}; - -pub fn init(s: *Sema) !void { - const t = tracer.trace(@src()); - defer t.end(); - - s.report = .{ - .gpa = s.gpa, - .current_file = s.ip.namespacePtr(s.namespace_index).file_scope, - }; - - try s.comp.current_compile_decls.append(s.gpa, s.decl_index); - - s.inst_map = try InstMap.init( - s.arena, - &s.code, - s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip), - ); -} - -pub fn deinit(s: *Sema) void { - const t = tracer.trace(@src()); - defer t.end(); - - const report = s.report.finalize() catch @panic("OOM"); - report.dumpToStdErr(s.comp) catch @panic("Error"); - report.deinit(s.gpa); - - defer s.* = undefined; - s.inst_map.deinit(s.arena); - s.scratch.deinit(s.arena); - s.air_extra.deinit(s.gpa); - s.air_instructions.deinit(s.gpa); - s.temp_result_map.deinit(s.arena); -} - -pub fn finalize(s: *Sema) !void { - const air: Air = s.tempAir(); - - if (s.comp.options.dump_ir) { - std.debug.print("=" ** 20, .{}); - std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)}); - air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {}; - } - - try s.comp.codeGenDecl(s.decl, &air); -} - -// --- analyze functions --- - -/// This is the entrypoint to semantic analysis: It all starts at a declaration. -pub fn analyzeDeclaration(s: *Sema) Error!Value { - const t = tracer.trace(@src()); - defer t.end(); - - // TODO: turn this entire function into a nonrecursive state machine, so - // that analysis can pause and continue at any point. for example, when a - // dependency on another declaration is required, but that declaration has - // not been resolved yet. or, we are waiting on a file read to happen - - const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start); - assert(decl.kind == .function); - - s.params = try s.gpa.alloc(Type, fn_decl.params.len); - defer s.gpa.free(s.params); - - for (fn_decl.params.slice(&s.code), 0..) |param, i| { - s.params[i] = try s.resolveType( - s.inst_map.start.op(Report.Operand.argument(@intCast(i))), - param, - ); - } - - const return_type = try s.resolveType( - s.inst_map.start.op(.return_type), - fn_decl.return_type, - ); - - const root_block = fn_decl.body; - switch (root_block.decode()) { - .value => |value| { - const coerced = try s.coerceValue( - s.inst_map.start.op(.operand), - value, - return_type, - ); - const block = try s.addBlock(); - _ = try s.addInstRef(.@"break", .{ - .block_ref = block.block_index, - .value = coerced.toAir(), - }); - }, - .index => |index| { - try s.temp_result_map.putNoClobber(s.arena, index, return_type); - _ = try s.analyzeBlock(index); - }, - } - - const func_prototype = (try s.ip.get(s.gpa, .{ - .type_function = .{ - .param_types = s.params, - .return_type = return_type, - }, - })).toTypeUnsafe(s.ip); - const value = try s.ip.get(s.gpa, .{ - .func = .{ - .ty = func_prototype, - .owner_decl = s.decl_index, - .teg_body_inst = s.decl.teg_decl_index.unwrap().?, - }, - }); - s.decl.val = value; - return value; -} - -pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { - const t = tracer.trace(@src()); - defer t.end(); - - const scratch_top = s.scratch.items.len; - defer s.scratch.shrinkRetainingCapacity(scratch_top); - - const block = try s.addBlock(); - try s.inst_map.add(in, block.block_index.toRef()); - - var last: Air.Inst.Ref = undefined; - const instructions = s.code.blockInstructions(in); - assert(instructions.len != 0); - for (instructions) |inst| { - last = try s.analyzeCode(inst); - } - - if (scratch_top + 1 == s.scratch.items.len) { - const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1]; - if (single_instruction == .@"break") { - const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1] - .decode(.@"break", single_instruction); - if (data.block_ref == block.block_index) { - // This is a block that resolved to just a single value ref with - // no side effects. This means we can just lower this to the ref - // itself, and give codegen an easier time. - - // Delete the block and the single instruction - // Break does not use any extra data, so this call removes both items - s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index)); - - return data.value; - } - } - } - - return (try block.finish(s, scratch_top)).toRef(); -} - -fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref { - const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) }); - defer t.end(); - - const out = try s.analyzeCodeInner(in); - - // Block refs are added automatically - if (s.code.instTag(in) == .block) { - assert(s.inst_map.get(in) != null); - } else { - try s.inst_map.add(in, out); - } - - return out; -} - -fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { - switch (s.code.instTag(in)) { - .string => { - const string: Teg.SlicedString = s.code.instPayload(.string, in); - return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) }); - }, - .number => { - const number: dod.Float = s.code.instPayload(.number, in); - return s.makeValueAir(.{ .comptime_float = number.decode() }); - }, - .js => { - const operand_ref = s.code.instData(.js, in).operand; - const operand = try s.resolveRefComptimeType(in.op(.operand), operand_ref, .string); - return s.addInstRef(.js_global_ref, operand.asString(s.ip)); - }, - .field => { - const field: Teg.Inst.Bin = s.code.instPayload(.field, in); - const base = try s.resolveRefRuntime(field.lhs); - - const rhs = try s.resolveRefComptimeType(in.op(.rhs), field.rhs, .string); - const string = rhs.asString(s.ip); - - const base_type = s.typeOfRef(base); - switch (base_type) { - .@"opaque" => { - return s.addInstRef(.field, .{ - .base = base, - .field = try string.tryToNullTerminatedString(s.ip, s.gpa), - }); - }, - else => |tag| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}), - } - }, - .load_decl => { - const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in); - const decl_name = try s.ip.getOrPutString( - s.gpa, - request.string.slice(s.code.string_bytes), - .no_embedded_nulls, - ); - const decl_index = s.lookupDecl(decl_name) orelse { - std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)}); - return error.AnalysisFail; - }; - const resolved = try s.resolveDecl(decl_index); - return resolved.toAir(); - }, - .param => { - const param: Teg.Inst.DataNode = s.code.instData(.param, in); - return s.addInstRef(.arg, .{ - .type = s.params[param.data], - .arg_number = param.data, - }); - }, - inline .add, - .sub, - .mul, - .div, - .lt, - .gt, - .eq, - => |tag| { - const bin: Teg.Inst.Bin = s.code.instPayload(tag, in); - - const lhs = try s.resolveRefRuntime(bin.lhs); - const rhs = try s.resolveRefRuntime(bin.rhs); - - if (lhs.kind == .value and rhs.kind == .value) { - return (try s.comptimeBin( - @field(BinaryOp, @tagName(tag)), - lhs.decode().value, - rhs.decode().value, - )); - } - - const left_type = s.typeOfRef(lhs); - _ = left_type; // autofix - const right_type = s.typeOfRef(rhs); - _ = right_type; // autofix - - // TODO: validate type for operation - - return s.addInstRef(tag.airTag(), .{ - .lhs = lhs, - .rhs = rhs, - }); - }, - inline .negate, .bool_not => |tag| { - const un: Teg.Inst.RefNode = s.code.instData(tag, in); - - const operand = try s.resolveRefRuntime(un.operand); - - const operand_type = s.typeOfRef(operand); - _ = operand_type; // autofix - - // TODO: validate type for operation - - return s.addInstRef(tag.airTag(), .{ - .operand = operand, - }); - }, - .@"break" => { - const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in); - const operand = try s.resolveRefRuntime(brk.operand); - const block_ref = s.inst_map.get(brk.block).?.decode().index; // block ref is resolved by teg - - const coerced = if (s.temp_result_map.get(brk.block)) |ty| - try s.coerceRef(in.op(.operand), operand, ty) - else - operand; - - return s.addInstRef(.@"break", .{ - .block_ref = block_ref, - .value = coerced, - }); - }, - .call => { - const call: Teg.Inst.Call = s.code.instPayload(.call, in); - const base = try s.resolveRefRuntime(call.base); - const base_type = s.typeOfRef(base); - - const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len); - defer s.arena.free(scratch); - if (base_type != .@"opaque") { - const proto = base_type.toFunctionPrototype(s.ip) orelse { - return s.fail(.call_non_function, .{ - .call = s.nodeLoc(in, .operand), - .ty = base_type, - }); - }; - - if (proto.param_types.len != call.args.len) { - // TODO: also attempt to coerce some of the params to reveal more errors. - return s.fail(.call_mismatch_args, .{ - .call = s.nodeLoc(in, .operand), - .actual = @intCast(call.args.len), - .expected = @intCast(proto.param_types.len), - }); - } - - // validate function args - for (proto.param_types, call.args.slice(&s.code), 0..) |param, arg, i| { - const ref = try s.resolveRefRuntime(arg); - const coerced = try s.coerceRef( - in.op(Report.Operand.argument(@intCast(i))), - ref, - param, - ); - scratch[i] = coerced; - } - } else { - // just copy the args in without checking - for (call.args.slice(&s.code), 0..) |arg, i| { - const ref = try s.resolveRefRuntime(arg); - scratch[i] = ref; - } - } - - const start = s.air_extra.list.items.len; - try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch)); - - return s.addInstRef(.call, .{ - .base = base, - .call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{ - .args = .{ - .start = @intCast(start), - .len = @intCast(scratch.len), - }, - }), - }); - }, - .ensure_void => { - const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false); - const t = s.typeOfRef(operand); - _ = t; - // if (!t.isEnsureVoid(s.ip)) { - // std.debug.print("oh no, we are discarding {} on accident\n", .{ - // t.fmt(s.ip), - // }); - // } - return InternPool.Value.void.toAir(); - }, - .discard => { - const operand = try s.resolveRefRuntimeExtra(s.code.instData(.discard, in).operand, false); - const t = s.typeOfRef(operand); - if (t == .void) { - std.debug.print("oh no, we have a useless _ = void;\n", .{}); - } - return InternPool.Value.void.toAir(); - }, - .@"if" => { - const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code); - - const cond = try s.resolveRefRuntime(data.cond); - - // try s.validateType(s.typeOfRef(cond), .bool_type); - - // If known at comptime, only analyze the taken path - if (cond.kind == .value) { - @panic("TODO: comptime IF"); - } - - // Emit a runtime branch - const yes_block = try s.analyzeRefToBlock(data.yes); - const no_block = try s.analyzeRefToBlock(data.no); - - return try s.addInstRef(.@"if", .{ - .cond = cond, - .data = try s.air_extra.append(s.gpa, Air.If.Data, .{ - .yes = yes_block, - .no = no_block, - .union_ty = s.unionType(s.typeOfRef(yes_block), s.typeOfRef(no_block)), - }), - }); - }, - .block => { - const block = try s.analyzeBlock(in); - switch (block.decode()) { - .index => |t| if (s.tempAir().instTag(t) == .block) { - try s.scratch.append(s.gpa, @intFromEnum(t)); - }, - else => {}, - } - return block; - }, - .memory => { - const data = s.code.instData(.memory, in); - - var initial_value = try s.resolveRefRuntime(data.operand); - - switch (initial_value.decode()) { - .value => |v| switch (v.toKey(s.ip)) { - .comptime_float => |val| { - try s.report.warn(.numeric_int_var_ambiguity, .{ - .node = s.nodeLoc(in, .operand), - }); - initial_value = (try s.makeValue(.{ .float = val })).toAir(); - }, - else => {}, - }, - else => {}, - } - - return try s.addInstRef(.memory, .{ - .init = initial_value, - .ty = s.typeOfRef(initial_value), - }); - }, - .store => { - const data: Teg.Inst.Store = s.code.instPayload(.store, in); - assert(s.code.instTag(data.memory) == .memory); // corrupt store instruction - - const location = s.mapTeg(data.memory.toRef()).decode().index; - s.markInst(location); - const operand = try s.resolveRefRuntime(data.operand); - const operand_coerced = try s.coerceRef( - in.op(.rhs), - operand, - s.typeOfRef(location.toRef()), - ); - return try s.addInstRef(.store, .{ - .memory = location, - .operand = operand_coerced, - }); - }, - .as => { - const as: Teg.Inst.As = s.code.instPayload(.as, in); - const coerce_to = try s.resolveRefComptimeType(in.op(.lhs), as.type, .type); - const operand = try s.resolveRefRuntime(as.operand); - return try s.coerceRef(in.op(.rhs), operand, coerce_to.toType(s.ip).?); - }, - else => |tag| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}), - } -} - -fn markInst(s: *Sema, inst: Air.Inst.Index) void { - const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)]; - flags.uses = flags.uses.increment(); -} - -fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) Error!Air.Inst.Ref { - const t = tracer.trace(@src()); - defer t.end(); - - const scratch_top = s.scratch.items.len; - defer s.scratch.shrinkRetainingCapacity(scratch_top); - - const visited = try s.resolveRefRuntime(in); - - if (scratch_top == s.scratch.items.len and visited.kind == .value) { - return visited; - } - - if (visited.kind == .index and - s.air_instructions.items(.tag)[@intFromEnum(visited.decode().index)] == .block) - { - return visited; - } - - const block = try s.addBlock(); - _ = try s.addInstRef(.@"break", .{ - .block_ref = block.block_index, - .value = visited, - }); - return (try block.finish(s, scratch_top)).toRef(); -} - -fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) Error!Air.Inst.Ref { - const index = ref.toIndex() orelse { - return ref.toValue().?.toAir(); - }; - - if (s.inst_map.get(index)) |air| { - if (mark) { - switch (air.decode()) { - .index => |i| s.markInst(i), - else => {}, - } - } - return air; - } - - return s.analyzeCode(index); -} - -fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) Error!Air.Inst.Ref { - return resolveRefRuntimeExtra(s, ref, true); -} - -fn resolveRefComptimeAny(s: *Sema, ref: Teg.Inst.Ref) !Value { - const index = switch (ref.decode()) { - .value => |value| return value, - .index => |index| index, - }; - - if (s.inst_map.get(index)) |air| { - switch (air.decode()) { - .value => |value| return value, - else => {}, - } - } - - switch (s.code.instTag(index)) { - .string => { - const string: Teg.SlicedString = s.code.instPayload(.string, index); - return s.makeValue(.{ .string = string.slice(s.code.string_bytes) }); - }, - - else => |tag| std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}), - } -} - -fn resolveRefComptimeType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref, ty: Type) !Value { - const any = try s.resolveRefComptimeAny(ref); - return try s.coerceValue(src, any, ty); -} - -fn resolveType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref) !Type { - return (try s.resolveRefComptimeType(src, ref, .type)).toTypeUnsafe(s.ip); -} - -/// Coerce `value` runtime ref to `coerce_type`, and emit an error at `teg_ref`'s location -fn coerceRef(s: *Sema, src: InstOp, value: Air.Inst.Ref, to: Type) !Air.Inst.Ref { - const from = s.typeOfRef(value); - if (from == to) { - return value; - } - const index = switch (value.decode()) { - .value => |v| return (try s.coerceValue(src, v, to)).toAir(), - .index => |index| index, - }; - switch (to) { - .@"opaque" => { - // - }, - .bool => { - // - }, - .int32 => switch (from) { - .int32 => unreachable, // same type handled - .numeric_literal => unreachable, // must should comptime known - else => {}, - }, - .float => switch (from) { - .float => unreachable, // same type handled - .int32 => return index.toRef(), - else => {}, - }, - .string => { - // - }, - .noreturn => { - // - }, - .type => { - // - }, - .numeric_literal => { - @panic("TODO: error message for not known at compile time"); - }, - .void => { - // - }, - else => {}, - } - return s.coersionError(src, from, to); -} - -fn coerceValue(s: *Sema, src: InstOp, value: Value, to: Type) !Value { - const from = value.typeOf(s.ip); - if (from == to) { - return value; - } - switch (to) { - .@"opaque" => { - // - }, - .bool => { - // - }, - .int32 => switch (from) { - .int32 => unreachable, // same type handled - .numeric_literal, .float => switch (value.toKey(s.ip)) { - .comptime_float, .float => |val| { - if (!math.isFinite(val) or - val < math.minInt(i32) or - val > math.maxInt(i32) or - (@rem(val, 1) != 0)) - { - return s.fail(.comptime_float_to_int, .{ - .val = val, - .src = s.lazyNode(src), - }); - } - - return try s.makeValue(.{ - .int32 = @intFromFloat(val), - }); - }, - else => unreachable, // not a numeric literal - }, - else => return s.coersionError(src, from, to), - }, - .float => switch (from) { - .float => unreachable, // same type handled - .numeric_literal => if (value.toFloat(s.ip)) |val| - return try s.makeValue(.{ - .float = val, - }), - else => {}, - }, - .string => { - // - }, - .noreturn => { - // - }, - .type => { - if (value == .void) return .void; - }, - .numeric_literal => {}, - .void => { - if (value == .void) return .void; - }, - else => {}, - } - return s.coersionError(src, from, to); -} - -fn resolveDecl(s: *Sema, decl_index: Decl.Index) Error!Value { - const decl = s.ip.declPtr(decl_index); - switch (decl.analysis) { - .unreferenced => { - decl.analysis = .in_progress; - - // TODO: make this short-circuit the current analysis, move this analyze - // job into a "deferred" list, and then make this thread analyze something else - - var arena = std.heap.ArenaAllocator.init(s.gpa); - defer arena.deinit(); - - var todo: Report = undefined; - var sema: Sema = .{ - .comp = s.comp, - .gpa = s.gpa, - .ip = s.ip, - .arena = arena.allocator(), - .code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.comp, &todo), - .decl = decl, - .decl_index = decl_index, - .namespace_index = decl.src_namespace, - }; - - try sema.init(); - defer sema.deinit(); - const value = try sema.analyzeDeclaration(); - try sema.finalize(); - - decl.analysis = .complete; - - return value; - }, - else => { - return decl.val; - }, - } -} - -fn unionType(s: *Sema, a: Type, b: Type) Type { - if (a == b) { - return a; - } - - if (a == .noreturn) - return b; - - if (b == .noreturn) - return a; - - std.debug.panic( - "TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}", - .{ - a.dump(s.ip), - b.dump(s.ip), - }, - ); - // return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } }); -} - -const BinaryOp = enum { - add, - sub, - mul, - div, - lt, - gt, - eq, -}; - -fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref { - switch (op) { - .add => { - const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add"); - const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add"); - return (try s.ip.get(s.gpa, .{ - .float = l + r, - })).toAir(); - }, - inline .sub, .mul, .div => |tag| { - const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); - const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); - - const result = switch (tag) { - .sub => l - r, - .mul => l * r, - .div => l / r, - else => comptime unreachable, - }; - - return (try s.ip.get(s.gpa, .{ - .float = result, - })).toAir(); - }, - inline .lt, - .gt, - => |tag| { - const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); - const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); - const result = switch (tag) { - .lt => l < r, - .gt => l > r, - else => comptime unreachable, - }; - return @as( - InternPool.Value, - if (result) .bool_true else .bool_false, - ).toAir(); - }, - .eq, - => { - const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq"); - const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq"); - const result = l == r; - return @as( - InternPool.Value, - if (result) .bool_true else .bool_false, - ).toAir(); - }, - } -} - -/// --- helper --- -const IncompleteBlock = struct { - block_index: Air.Inst.Index, - - pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index { - s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top)); - return block.block_index; - } -}; - -fn addBlock(s: *Sema) !IncompleteBlock { - try s.air_instructions.append(s.gpa, .{ - .tag = .block, - .data = undefined, - .flags = .{}, - }); - return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) }; -} - -fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref { - try s.air_instructions.append(s.gpa, .{ - .tag = tag, - .data = Air.EncodedData.encode(tag, data), - .flags = .{}, - }); - const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1); - try s.scratch.append(s.arena, @intFromEnum(index)); - return index.toRef(); -} - -fn setInstData( - s: *Sema, - comptime tag: Air.Inst.Tag, - index: Air.Inst.Index, - data: tag.Data(), -) void { - const i = @intFromEnum(index); - assert(s.air_instructions.items(.tag)[i] == tag); - s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data); -} - -fn makeValue(s: *Sema, key: InternPool.Key) !Value { - return try s.ip.get(s.gpa, key); -} - -fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref { - return (try s.makeValue(key)).toAir(); -} - -fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref { - if (inst.toValue()) |interned| { - return interned.toAir(); - } - const index = inst.toIndex().?; - return s.inst_map.get(index) orelse { - if (std.debug.runtime_safety) { - std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)}); - } - unreachable; - }; -} - -fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList { - const items = s.scratch.items[scratch_top..]; - const extra_top = s.air_extra.list.items.len; - try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]); - return .{ - .start = @intCast(extra_top), - .len = @intCast(items.len), - }; -} - -fn tempAir(s: *const Sema) Air { - return .{ - .insts = s.air_instructions.slice(), - .extra = s.air_extra.constant(), - }; -} - -fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Type { - return s.tempAir().typeOf(ref, s.ip); -} - -fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex { - var nsi = s.namespace_index; - while (true) { - const ns = s.ip.namespacePtr(nsi); - if (ns.lookupDirect(s.comp, name)) |decl_id| { - return decl_id; - } - nsi = ns.parent.unwrap() orelse - unreachable; // transform ensures all identifiers get resolved - } -} - -fn nodeLoc(s: *const Sema, in: Teg.Inst.Index, op: Report.Operand) Report.LazyNode { - return s.lazyNode(in.op(op)); -} - -fn addWarn(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { - try s.report.warn(kind, data); -} - -fn addError(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { - try s.report.err(kind, data); -} - -fn fail(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) Error { - try s.report.err(kind, data); - return Error.AnalysisFail; -} - -fn coersionError(s: *Sema, src: InstOp, from: Type, to: Type) Error { - return s.fail(.coersion, .{ - .from = from, - .to = to, - .src = s.lazyNode(src), - }); -} - -fn lazyNode(s: *const Sema, src: InstOp) Report.LazyNode { - return src.lazyNode(s.decl_index); -} - -const Sema = @This(); - -const std = @import("std"); -const mem = std.mem; -const math = std.math; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const log = std.log.scoped(.sema); - -inline fn assertWithMessage(cond: bool) bool { - if (std.debug.runtime_safety) { - return !cond; - } else if (!cond) { - unreachable; // assertion failure - } -} - -const Comp = @import("Comp.zig"); -const File = Comp.File; -const Decl = Comp.Decl; -const Namespace = Comp.Namespace; - -const Teg = @import("Teg.zig"); -const InstOp = Teg.InstOp; - -const Air = @import("Air.zig"); -const Report = @import("Report.zig"); -const Ast = @import("Ast.zig"); -const dod = @import("array.zig"); -const tracer = @import("tracer"); - -const InternPool = @import("InternPool.zig"); -const Value = InternPool.Value; -const Type = InternPool.Type; -const NullTerminatedString = InternPool.NullTerminatedString; diff --git a/src/compiler/Teg.Dump.zig b/src/compiler/Teg.Dump.zig deleted file mode 100644 index e50cd07496835e9fabe3f26ca134788c4f1918ab..0000000000000000000000000000000000000000 --- a/src/compiler/Teg.Dump.zig +++ /dev/null @@ -1,314 +0,0 @@ -//! This implements a debug formatter for Teg -teg: Teg, - -const State = struct { - arena: std.mem.Allocator, - teg: *const Teg, - indent: usize = 0, - tty: std.io.tty.Config, - visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{}, - - fn newline(state: State, writer: anytype) !void { - try writer.writeByte('\n'); - if (state.indent > 0) - try writer.writeByteNTimes(' ', state.indent * 2); - } - - fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void { - const teg = s.teg; - const tty = s.tty; - - try writer.print("%{} = ", .{@intFromEnum(inst)}); - - const tag = s.teg.instTag(inst); - - try tty.setColor(writer, .bright_black); - const node_offset = teg.instNodeOffset(inst); - try writer.print("{c}{d} ", .{ - @as(u8, if (node_offset >= 0) '+' else '-'), - @abs(node_offset), - }); - try tty.setColor(writer, .reset); - - try tty.setColor(writer, .cyan); - try writer.print("{s}", .{@tagName(tag)}); - try tty.setColor(writer, .reset); - - if ((try s.visited.fetchPut(s.arena, inst, {})) != null) { - try tty.setColor(writer, .red); - try writer.writeAll(" [error: recursive tree]"); - try tty.setColor(writer, .reset); - return; - } - - switch (tag) { - .module => { - try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}'); - }, - .declaration => { - const decl_info = teg.declData(inst); - try writer.print(" ({s}) ", .{@tagName(decl_info.kind)}); - - const name = decl_info.name.slice(teg.string_bytes); - if (name.len == 0) { - try tty.setColor(writer, .bold); - try writer.writeAll(""); - } else { - try tty.setColor(writer, .magenta); - try writer.writeAll(name); - } - try tty.setColor(writer, .reset); - - try writer.writeAll(" {"); - s.indent += 1; - try s.newline(writer); - - switch (decl_info.kind) { - .function => { - const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1"; - try writer.writeAll("params: "); - try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']'); - try s.newline(writer); - try writer.writeAll("return_type: "); - try s.dumpRef(writer, fn_data.return_type); - try s.newline(writer); - try writer.writeAll("body: "); - try s.dumpRef(writer, fn_data.body); - }, - } - s.indent -= 1; - try s.newline(writer); - try writer.writeAll("}"); - }, - .param => { - const data = teg.instData(.param, inst); - try writer.writeAll("("); - try tty.setColor(writer, .yellow); - try writer.print("{d}", .{data.data}); - try tty.setColor(writer, .reset); - try writer.writeAll(")"); - }, - .block => { - const block = teg.instData(.block, inst); - const index_list = block.extra_index.get(teg); - try s.dumpInstList(writer, index_list.slice(teg), '[', ']'); - }, - .number => { - const number = teg.instData(.number, inst); - const float = number.extra_index.get(teg).decode(); - try tty.setColor(writer, .yellow); - try writer.print(" {d}", .{float}); - try tty.setColor(writer, .reset); - }, - inline .field, - .add, - .sub, - .mul, - .div, - .rem, - .pow, - .@"and", - .@"or", - .bit_and, - .bit_or, - .bit_xor, - .shl, - .shr, - .eq, - .ne, - .lt, - .le, - .gt, - .ge, - => |comptime_tag| { - const data = teg.instData(comptime_tag, inst).extra_index.get(teg); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.lhs); - try writer.writeAll(", "); - try s.dumpRefMarker(writer, data.rhs); - try writer.writeAll(")"); - }, - .as => { - const data = teg.instData(.as, inst).extra_index.get(teg); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.type); - try writer.writeAll(", "); - try s.dumpRefMarker(writer, data.operand); - try writer.writeAll(")"); - }, - .@"break" => { - const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.block.toRef()); - try writer.writeAll(", "); - try s.dumpRefMarker(writer, data.operand); - try writer.writeAll(")"); - }, - .store => { - const data: Teg.Inst.Store = teg.instData(.store, inst).extra_index.get(teg); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.memory.toRef()); - try writer.writeAll(", "); - try s.dumpRefMarker(writer, data.operand); - try writer.writeAll(")"); - }, - .string => { - const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg); - try tty.setColor(writer, .green); - try writer.writeAll(" \""); - try writer.writeAll(data.slice(teg.string_bytes)); - try writer.writeAll("\""); - try tty.setColor(writer, .reset); - }, - inline .memory, - .discard, - .ensure_void, - .ensure_bool, - .negate, - .bool_not, - .bit_not, - .js, - .import, - .fetch, - .compile_error, - => |comptime_tag| { - const data = teg.instData(comptime_tag, inst); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.operand); - try writer.writeAll(")"); - }, - .call => { - const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg); - try writer.writeAll("("); - try s.dumpRefMarker(writer, call.base); - try writer.writeAll(", "); - try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']'); - try writer.writeAll(")"); - }, - .load_decl => { - const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst); - - try tty.setColor(writer, .red); - try writer.print(" {s}", .{call.string.slice(teg.string_bytes)}); - try tty.setColor(writer, .reset); - }, - .@"if" => { - const data: Teg.Inst.If = teg.instPayload(.@"if", inst); - try writer.writeAll("("); - try s.dumpRefMarker(writer, data.cond); - try writer.writeAll(") {"); - s.indent += 1; - try s.newline(writer); - try writer.writeAll("yes: "); - try s.dumpRef(writer, data.yes); - try s.newline(writer); - try writer.writeAll("no: "); - try s.dumpRef(writer, data.no); - s.indent -= 1; - try s.newline(writer); - try writer.writeAll("}"); - }, - } - } - - fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void { - if (ref.toIndex()) |index| { - try s.dumpInst(writer, index); - } else { - try s.tty.setColor(writer, .blue); - try writer.print("%{s}", .{@tagName(ref)}); - try s.tty.setColor(writer, .reset); - } - } - - fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void { - if (ref.toIndex()) |index| { - try s.tty.setColor(writer, .blue); - try writer.print("%{d}", .{@intFromEnum(index)}); - try s.tty.setColor(writer, .reset); - } else { - try s.tty.setColor(writer, .blue); - try writer.print("%{s}", .{@tagName(ref)}); - try s.tty.setColor(writer, .reset); - } - } - - fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - if (list.len > 0) { - state.indent += 1; - - for (list) |inst| { - try state.newline(writer); - try state.dumpInst(writer, inst); - } - - state.indent -= 1; - try state.newline(writer); - } - try writer.print("{c}", .{r}); - } - - fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - if (list.len > 0) { - state.indent += 1; - - for (list) |ref| { - try state.newline(writer); - try state.dumpRef(writer, ref); - } - - state.indent -= 1; - - try state.newline(writer); - } - try writer.print("{c}", .{r}); - } - - fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void { - try writer.print(" {c}", .{l}); - if (list.len > 0) { - state.indent += 1; - - for (list) |ref| { - try state.newline(writer); - try state.dumpRefMarker(writer, ref); - } - - state.indent -= 1; - - try state.newline(writer); - } - try writer.print("{c}", .{r}); - } -}; - -pub fn format( - dump: @This(), - comptime unused_fmt: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, -) !void { - comptime assert(unused_fmt.len == 0); - - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const teg = dump.teg; - var state = State{ - .teg = &teg, - .tty = detectTTYFromWriter(writer), - .arena = arena.allocator(), - }; - try state.dumpInst(writer, .main_module_inst); - try writer.writeAll("\n"); - assert(state.indent == 0); -} - -const std = @import("std"); -const assert = std.debug.assert; - -const Teg = @import("Teg.zig"); - -const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/compiler/Teg.zig b/src/compiler/Teg.zig deleted file mode 100644 index 5b76ced3658ab57f233eca18e6396439a6a21573..0000000000000000000000000000000000000000 --- a/src/compiler/Teg.zig +++ /dev/null @@ -1,486 +0,0 @@ -//! TODO: rename this to something else. Teg is not a graph - -insts: Inst.List.Slice, -string_bytes: []const u8, -extra: ExtraStore = .{}, -report: Report, - -pub const ExtraStore = dod.ExtraStore(.teg, .@"const"); -pub const EncodedData = dod.EncodedData(Inst.Tag, u64); -pub const ExtraIndex = ExtraStore.Index; -pub const Dump = @import("Teg.Dump.zig"); - -pub const Inst = struct { - pub const List = std.MultiArrayList(Inst); - - tag: Tag, - data: EncodedData, - - // todo: fully document every thing - pub const Tag = enum { - module, - declaration, - - block, - /// Creates a memory cell in the current scope. - memory, - /// Performs a type coersion, such as in variable declarations. - /// Emits a sema error if the type conversion fails. - /// `let x: i32 = 1` - as, - load_decl, - /// Load a parameter - param, - store, - discard, - ensure_void, - ensure_bool, - /// lowered from `break` or `return` - /// ends a block with a resulting value - @"break", - number, - string, - - @"if", - - add, - sub, - mul, - div, - rem, - pow, - @"and", - @"or", - bit_and, - bit_or, - bit_xor, - shl, - shr, - eq, - ne, - lt, - le, - gt, - ge, - - negate, - bool_not, - bit_not, - field, - call, - - /// Implements the `@js` builtin - js, - /// Implements the `@import` builtin - import, - /// Implements the `@fetch` builtin - fetch, - /// Implements the `@compileError` builtin - compile_error, - - pub fn Data(comptime tag: Tag) type { - return switch (tag) { - .module => PlNode(IndexList), - .declaration => PlNode(Decl), - .block => PlNode(IndexList), - .as => PlNode(As), - .@"break" => PlNode(Break), - .number => PlNode(Float), - .string => PlNode(SlicedString), - .call => PlNode(Call), - .@"if" => PlNode(If), - .load_decl => StrNode, - .param => DataNode, - .store => PlNode(Store), - - // Single instruction operand - .memory, - .discard, - .ensure_void, - .ensure_bool, - .negate, - .bool_not, - .bit_not, - .js, - .import, - .fetch, - .compile_error, - => RefNode, - - // Two instruction operands - .field, - .add, - .sub, - .mul, - .div, - .rem, - .pow, - .@"and", - .@"or", - .bit_and, - .bit_or, - .bit_xor, - .shl, - .shr, - .eq, - .ne, - .lt, - .le, - .gt, - .ge, - => PlNode(Bin), - }; - } - - pub fn airTag(comptime tag: Tag) Air.Inst.Tag { - return @field(Air.Inst.Tag, @tagName(tag)); - } - }; - - /// TODO: make enum? idk - const NodeOffset = i32; - - pub fn PlNode(comptime Payload: type) type { - return struct { - pub const T = Payload; - - src_node: NodeOffset, - extra_index: ExtraIndex(Payload), - }; - } - - pub const RefNode = struct { - src_node: NodeOffset, - operand: Inst.Ref, - }; - - pub const StrNode = struct { - src_node: NodeOffset, - string: NullTerminatedString, - }; - - pub const DataNode = struct { - src_node: NodeOffset, - data: u32, - }; - - /// The position of a Teg instruction within the `Teg` instructions array. - pub const Index = enum(u32) { - /// Teg is structured so that the outermost "main" struct of any file - /// is always at index 0. - main_module_inst = 0, - _, - - pub const static_len = @intFromEnum(Ref.empty_string) + 1; - - pub fn toRef(i: Index) Inst.Ref { - return @enumFromInt(static_len + @intFromEnum(i)); - } - - pub fn toOptional(i: Index) OptionalIndex { - return @enumFromInt(@intFromEnum(i)); - } - - pub fn op(i: Index, operand: Report.Operand) InstOp { - return .{ .index = i, .op = operand }; - } - }; - - pub const OptionalIndex = enum(u32) { - /// Teg is structured so that the outermost "main" module of any file - /// is always at index 0. - main_module_inst = 0, - ref_start_index = Index.static_len, - none = std.math.maxInt(u32), - _, - - pub fn unwrap(oi: OptionalIndex) ?Index { - return if (oi == .none) null else @enumFromInt(@intFromEnum(oi)); - } - }; - - /// A reference to instruction, or to an InternPool index, or neither. - /// - /// If the integer tag value is < InternPool.static_len, then it - /// corresponds to an InternPool index. Otherwise, this refers to a ZIR - /// instruction. - /// - /// The tag type is specified so that it is safe to bitcast between `[]u32` - /// and `[]Ref`. - pub const Ref = enum(u32) { - opaque_type, - bool_type, - int32_type, - float_type, - string_type, - noreturn_type, - type_type, - numeric_literal_type, - - void, - bool_true, - bool_false, - int32_zero, - float_zero, - nan, - empty_string, - standard_library, - - /// This Ref does not correspond to any instruction or constant - /// value and may instead be used as a sentinel to indicate null. - none = std.math.maxInt(u32), - - _, - - pub const Decoded = union(enum) { - value: InternPool.Value, - index: Teg.Inst.Index, - }; - - pub fn decode(ref: Ref) Decoded { - const ref_int = @intFromEnum(ref); - return if (ref_int >= Index.static_len) - .{ .index = @enumFromInt(ref_int - Index.static_len) } - else - .{ .value = @enumFromInt(ref_int) }; - } - - pub fn toIndex(ref: Ref) ?Index { - assert(ref != .none); - const ref_int = @intFromEnum(ref); - if (ref_int >= Index.static_len) { - return @enumFromInt(ref_int - Index.static_len); - } else { - return null; - } - } - - pub fn toIndexAllowNone(inst: Ref) ?Index { - if (inst == .none) return null; - return toIndex(inst); - } - - pub fn toValue(inst: Ref) ?InternPool.Value { - const ref_int = @intFromEnum(inst); - - // todo: is this > or >= - if (ref_int > Index.static_len) - return null; - - return @enumFromInt(ref_int); - } - }; - - pub const Bin = struct { - lhs: Inst.Ref, - rhs: Inst.Ref, - }; - - pub const Store = struct { - memory: Inst.Index, - operand: Inst.Ref, - }; - - pub const As = struct { - type: Inst.Ref, - operand: Inst.Ref, - }; - - pub const Break = struct { - block: Inst.Index, - operand: Inst.Ref, - }; - - pub const Call = struct { - base: Inst.Ref, - args: RefList, - }; - - pub const If = struct { - cond: Inst.Ref, - yes: Inst.Ref, - no: Inst.Ref, - }; - - /// This will contain trailing data depending on the .kind - pub const Decl = struct { - name: NullTerminatedString, - hash: Cache.BinDigest, - kind: Kind, - instruction_count: u32, - - pub const Kind = enum { - function, - // future: struct, let, var, etc - }; - }; - - /// Trailng data for function declarations - pub const DeclFn = struct { - params: RefList, - return_type: Ref, - body: Ref, - }; -}; - -pub const InstOp = struct { - index: Inst.Index, - op: Report.Operand, - - pub fn lazyNode(inst: InstOp, decl: Comp.Decl.Index) LazyNode { - return .{ - .inst = inst.index, - .operand = inst.op, - .decl = decl, - }; - } -}; - -pub const NullTerminatedString = enum(u32) { - empty = 0, - _, - - pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 { - if (str == .empty) return ""; - const sub = string_bytes[@intFromEnum(str)..]; - const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable; - - return sub[0..i :0]; - } -}; - -pub const SlicedString = struct { - start: u32, - len: u32, - - pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 { - return string_bytes[str.start..][0..str.len]; - } -}; - -pub const RefList = struct { - start: u32, - len: u32, - - pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref { - return @ptrCast(teg.extra.list.items[self.start..][0..self.len]); - } -}; - -pub const IndexList = struct { - start: u32, - len: u32, - - pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index { - return @ptrCast(teg.extra.list.items[self.start..][0..self.len]); - } -}; - -pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void { - gpa.free(teg.string_bytes); - teg.extra.deinit(gpa); - teg.insts.deinit(gpa); - teg.report.deinit(gpa); - teg.* = undefined; -} - -pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag { - return teg.insts.items(.tag)[@intFromEnum(index)]; -} - -pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData { - return teg.insts.items(.data)[@intFromEnum(index)]; -} - -pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() { - return teg.instEncodedData(index).decode(tag, teg.instTag(index)); -} - -pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T { - return teg.instData(tag, index).extra_index.get(teg); -} - -pub fn instNodeOffset(teg: *const Teg, index: Inst.Index) i32 { - switch (teg.instTag(index)) { - inline else => |tag| { - return teg.instData(tag, index).src_node; - }, - } -} - -pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index { - return teg.instData(.block, index).extra_index.get(teg).slice(teg); -} - -pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } { - const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg); - const decl_fn = trail.read(&teg.extra, Inst.DeclFn); - return .{ decl, decl_fn }; -} - -pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl { - return teg.instData(.declaration, index).extra_index.get(teg); -} - -pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index { - const module = teg.instData(.module, module_index); - const module_data: IndexList = module.extra_index.get(teg); - const slice = module_data.slice(teg); - if (std.debug.runtime_safety) { - for (slice) |item| { - if (teg.instTag(item) != .declaration) { - std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))}); - unreachable; - } - } - } - return slice; -} - -pub fn rootFunction(teg: *const Teg) Inst.Index { - const decls = teg.moduleDeclarations(.main_module_inst); - const root_fn = decls[decls.len - 1]; - assert(teg.declName(root_fn).len == 0); - return root_fn; -} - -pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 { - const decl = teg.instData(.declaration, decl_index); - const decl_data: Inst.Decl = decl.extra_index.get(teg); - return decl_data.name.slice(teg.string_bytes); -} - -pub fn dumpToWriter(teg: Teg, writer: anytype) !void { - try writer.print("{}", .{Dump{ .teg = teg }}); -} - -const Teg = @This(); - -const std = @import("std"); -const mem = std.mem; -const assert = std.debug.assert; - -const Cache = @import("Cache.zig"); - -const Ast = @import("Ast.zig"); -const Unused = Ast.Unused; -const NodeIndex = Ast.NodeIndex; -const TokenIndex = Ast.TokenIndex; -const OptionalNodeIndex = Ast.OptionalNodeIndex; -const OptionalTokenIndex = Ast.OptionalTokenIndex; -const OptionalExtraIndex = Ast.OptionalExtraIndex; -const optionalTokenIndex = OptionalTokenIndex.init; -const optionalNodeIndex = OptionalNodeIndex.init; -const optionalExtraIndex = OptionalExtraIndex.init; - -const dod = @import("array.zig"); -const Float = dod.Float; - -const InternPool = @import("InternPool.zig"); - -const Air = @import("Air.zig"); - -const Comp = @import("Comp.zig"); - -const Report = @import("Report.zig"); -const LazyNode = Report.LazyNode; diff --git a/src/compiler/Tokenizer.zig b/src/compiler/Tokenizer.zig deleted file mode 100644 index 9afadd8b6918d36cab142ccf359eb25584502c2a..0000000000000000000000000000000000000000 --- a/src/compiler/Tokenizer.zig +++ /dev/null @@ -1,995 +0,0 @@ -//! The tokenizer is extremely simple structure that, given a view of source -//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit -//! offset to where that token is located. -//! -//! Retriving payloads like string/numeric values, identifier names, etc is -//! done separately, either in Parse or Transform, when the respective data -//! is actually needed. Tokenization does guarante that such data is valid. -code: [:0]const u8, -pos: ByteOffset = 0, - -/// Tokenize source code into a list of tokens. -/// When an error occurs, the position of the error is returned in `error_pos`. -/// Caller owns returned slice. -pub fn tokenize( - source_code: [:0]const u8, - gpa: mem.Allocator, - error_pos: *ByteOffset, -) TokenizeError!Token.List.Slice { - const trace = T.trace(@src()); - defer trace.end(); - - var tokens = Token.List{}; - errdefer tokens.deinit(gpa); - - // pre-allocate some space based on a very rough guess - try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8)); - - var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 }; - errdefer error_pos.* = tokenizer.pos; - - tokenizer.skipWhitespace(); - while (try tokenizer.next()) |token| { - try tokens.append(gpa, token); - tokenizer.skipWhitespace(); - } - - try tokens.append(gpa, .{ - .tag = .eof, - .offset = @intCast(source_code.len), - }); - - return tokens.toOwnedSlice(); -} - -pub const ByteOffset = u32; - -pub const Token = struct { - tag: Tag, - offset: ByteOffset, - - pub const List = std.MultiArrayList(Token); - - pub const Tag = enum { - id, - builtin_id, - number_int, - number_float, - string, - multiline_string, - doc_comment, - module_doc_comment, - - // sort-lines: start - kw_assert, - kw_break, - kw_comptime, - kw_defer, - kw_else, - kw_enum, - kw_fn, - kw_infer, - kw_for, - kw_while, - kw_if, - kw_impl, - kw_json, - kw_let, - kw_match, - kw_module, - kw_pub, - kw_pure, - kw_return, - kw_struct, - kw_trait, - kw_var, - // sort-lines: end - - // sort-lines: start - @"and", - and_and, - and_and_equal, - and_equal, - bang, - bang_equal, - caret, - caret_equal, - colon, - comma, - dot, - end_brace, - end_bracket, - end_paren, - equal, - equal_equal, - equal_lt, - gt, - gt_equal, - gt_gt, - gt_gt_equal, - gt_gt_gt, - gt_gt_gt_equal, - lt, - lt_equal, - lt_lt, - lt_lt_equal, - minus, - minus_equal, - minus_lt, - percent, - percent_equal, - pipe, - pipe_equal, - pipe_pipe, - pipe_pipe_equal, - plus, - plus_equal, - semi, - slash, - slash_equal, - star, - star_equal, - star_star, - star_star_equal, - start_brace, - start_bracket, - start_paren, - tilde, - tilde_equal, - // sort-lines: end - - /// Represents a tokenization error. Instead of reporting an error during - /// tokenization, parsing will begin, and an error will be reported when - /// this token is hit. That way, the error can tell you what it expected. - invalid, - /// End of file - /// Used so that parser does not need to do out of bounds checks - eof, - - /// Most tokens have statically known lexeme, such as "struct" or "+=" - pub fn lexeme(tag: Tag) ?[]const u8 { - return switch (tag) { - .invalid => unreachable, - - // do not have a static lexeme - .eof, - .id, - .builtin_id, - .number_int, - .number_float, - .string, - .multiline_string, - .doc_comment, - .module_doc_comment, - => null, - - .@"and" => "&", - .and_and => "&&", - .bang => "!", - .bang_equal => "!=", - .equal_lt => "=<", - .caret => "^", - .end_brace => "}", - .end_bracket => "]", - .end_paren => ")", - .colon => ":", - .comma => ",", - .dot => ".", - .equal => "=", - .equal_equal => "==", - .gt => ">", - .gt_equal => ">=", - .gt_gt => ">>", - .gt_gt_gt => ">>>", - .lt => "<", - .lt_equal => "<=", - .lt_lt => "<<", - .minus => "-", - .start_brace => "{", - .start_bracket => "[", - .start_paren => "(", - .pipe => "|", - .pipe_pipe => "||", - .percent => "%", - .plus => "+", - .minus_lt => "->", - .semi => ";", - .slash => "/", - .star => "*", - .star_star => "**", - .tilde => "~", - .slash_equal => "/=", - .percent_equal => "%=", - .tilde_equal => "~=", - .caret_equal => "^=", - .and_equal => "&=", - .pipe_equal => "|=", - .lt_lt_equal => "<<=", - .gt_gt_equal => ">>=", - .gt_gt_gt_equal => ">>>=", - .plus_equal => "+=", - .minus_equal => "-=", - .star_equal => "*=", - .star_star_equal => "**=", - .pipe_pipe_equal => "||=", - .and_and_equal => "&&=", - - inline else => |keyword| comptime len: { - if (!mem.startsWith(u8, @tagName(keyword), "kw_")) { - @compileError("Token tag is not a keyword: " ++ @tagName(keyword)); - } - break :len @tagName(keyword)[3..]; - }, - }; - } - - pub fn description(tag: Tag) []const u8 { - return switch (tag) { - .invalid => unreachable, - .eof => "end of file", - .id => "an identifier", - .builtin_id => "a builtin identifier", - .number_int => "a numeric literal", - .number_float => "a numeric literal", - .string => "a string literal", - .multiline_string => "a multiline string literal", - .doc_comment => "a documentation comment", - .module_doc_comment => "a module documentation comment", - inline else => |t| comptime "\"" ++ t.lexeme().? ++ "\"", - }; - } - - /// Most tokens have a static length, such as "struct" or "+=" - pub fn staticLen(tag: Tag) ?u32 { - return if (tag.lexeme()) |str| @intCast(str.len) else null; - } - }; - - /// Instead of storing the length of a token, we can recompute this - /// information on demand, as it is not often needed. - /// - /// If you are computing the length of something that is knowable - /// without the source code, use token.tag.staticLen() instead. - pub fn len(token: Token, utf8_source: [:0]const u8) u32 { - return token.tag.staticLen() orelse { - var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset }; - assert( - (tokenizer.next() catch - unreachable // data corruption - orelse unreachable // data corruption - ).tag == token.tag, - ); - return tokenizer.pos - token.offset; - }; - } - - pub fn slice(token: Token, utf8_source: [:0]const u8) []const u8 { - return token.tag.lexeme() orelse - utf8_source[token.offset..][0..token.len(utf8_source)]; - } - - const keywords = std.ComptimeStringMap(Tag, keywords: { - const fields = std.enums.values(Tag); - const KV = struct { []const u8, Tag }; - var kws: [fields.len]KV = undefined; - var i = 0; - for (fields) |field| { - if (mem.startsWith(u8, @tagName(field), "kw_")) { - kws[i] = .{ @tagName(field)[3..], field }; - i += 1; - } - } - break :keywords kws[0..i]; - }); -}; - -/// When any of these error occurs, the tokenizer's position is set to -/// the start of the offending token. -pub const NextTokenError = error{ - /// Generic token syntax error - InvalidSyntax, - /// A trailing '_' was seen in a numeric literal. - TrailingNumericSeparator, - /// Newline or EOF encountered while parsing a string literal. - UnterminatedString, - /// A string escape sequence was invalid. - InvalidEscapeSequence, - /// The codepoint represented by a unicode escape sequence - /// is not a valid unicode codepoint. - InvalidEscapeCodepoint, - /// There are unresolved git conflicts in the source code. - GitConflict, - /// The source code is not valid UTF-8. - InvalidUtf8, -}; - -pub const TokenizeError = mem.Allocator.Error || NextTokenError; - -/// Next assumes the position is already at the start of a token. -/// You must call `skipWhitespace` before calling this. -pub fn next(t: *Tokenizer) !?Token { - while (true) { - const trace = T.trace(@src()); - defer trace.end(); - - switch (t.byte()) { - ' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace - - 'a'...'z', 'A'...'Z', '_', '$' => { - const start = t.pos; - t.pos += 1; - return try t.tokenizeIdentifier(start); - }, - - '@' => switch (t.nextByte()) { - '"' => { - const start = t.pos; - t.pos += 1; - try t.consumeString(); - return .{ .tag = .id, .offset = start }; - }, - else => { - const start = t.pos; - t.pos += 1; - return try t.tokenizeIdentifier(start); - }, - }, - - '0'...'9' => return try t.tokenizeNumber(), - - '.' => switch (t.nextByte()) { - '0'...'9' => return try t.tokenizeNumber(), - else => return t.simpleToken(.dot), - }, - - '/' => switch (t.nextByte()) { - '=' => return t.simpleToken(.slash_equal), - '/' => { - if ((t.byteAt(2) orelse 0 == '/') and - (t.byteAt(3) orelse 0 != '/')) - { - const start = t.pos; - t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len); - return .{ .tag = .doc_comment, .offset = start }; - } - - if (t.byteAt(2) orelse 0 == '!') { - const start = t.pos; - t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); - return .{ .tag = .module_doc_comment, .offset = start }; - } - - // skip the comment - t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); - t.skipWhitespace(); - continue; - }, - else => return t.simpleToken(.slash), - }, - - '\\' => switch (t.nextByte()) { - '\\' => { - const start = t.pos; - t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); - return .{ .tag = .multiline_string, .offset = start }; - }, - else => return t.maybeUTF8Error(error.InvalidSyntax), - }, - - '"' => return try t.tokenizeString(), - - '&' => switch (t.nextByte()) { - '&' => return t.simpleToken(.and_and), - '=' => return t.simpleToken(.and_equal), - else => return t.simpleToken(.@"and"), - }, - '|' => switch (t.nextByte()) { - '|' => return t.simpleToken(.pipe_pipe), - '=' => return t.simpleToken(.pipe_equal), - else => return t.simpleToken(.pipe), - }, - '^' => switch (t.nextByte()) { - '^' => return t.simpleToken(.caret), - '=' => return t.simpleToken(.caret_equal), - else => return t.simpleToken(.caret), - }, - '!' => switch (t.nextByte()) { - '=' => return t.simpleToken(.equal_equal), - else => return t.simpleToken(.bang_equal), - }, - '=' => switch (t.nextByte()) { - '=' => return t.simpleToken(.equal_equal), - else => return t.simpleToken(.equal), - }, - '>' => switch (t.nextByte()) { - '=' => return t.simpleToken(.gt_equal), - '>' => switch (t.byteAt(2) orelse 0) { - '>' => return t.simpleToken(.gt_gt_gt), - else => return t.simpleToken(.gt_gt), - }, - else => return t.simpleToken(.gt), - }, - '<' => switch (t.nextByte()) { - '=' => return t.simpleToken(.lt_equal), - '<' => switch (t.byteAt(2) orelse 0) { - '=' => return t.simpleToken(.lt_lt_equal), - else => { - // Git merge conflicts are <<<<<<< - const start = t.pos; - t.pos += 2; - var remain: u3 = 4; - while (t.byte() == '<') { - t.pos += 1; - remain -= 1; - if (remain == 0) { - return error.GitConflict; - } - } - t.pos = start; - return t.simpleToken(.lt_lt); - }, - }, - else => return t.simpleToken(.lt), - }, - '+' => switch (t.nextByte()) { - '=' => return t.simpleToken(.plus_equal), - else => return t.simpleToken(.plus), - }, - '-' => switch (t.nextByte()) { - '=' => return t.simpleToken(.minus_equal), - '>' => return t.simpleToken(.minus_lt), - else => return t.simpleToken(.minus), - }, - '*' => switch (t.nextByte()) { - '*' => return t.simpleToken(.star_star), - '=' => return t.simpleToken(.star_equal), - else => return t.simpleToken(.star), - }, - '%' => switch (t.nextByte()) { - '=' => return t.simpleToken(.percent_equal), - else => return t.simpleToken(.percent), - }, - '~' => switch (t.nextByte()) { - '=' => return t.simpleToken(.tilde_equal), - else => return t.simpleToken(.tilde), - }, - - ';' => return t.simpleToken(.semi), - ':' => return t.simpleToken(.colon), - '(' => return t.simpleToken(.start_paren), - ')' => return t.simpleToken(.end_paren), - '[' => return t.simpleToken(.start_bracket), - ']' => return t.simpleToken(.end_bracket), - '{' => return t.simpleToken(.start_brace), - '}' => return t.simpleToken(.end_brace), - ',' => return t.simpleToken(.comma), - - 0x80...0xFF => { - // need to decode utf-8 - const byte_len, const code_point = try t.codePoint(); - - if (unicode_props.isXidStart(code_point)) { - const start = t.pos; - t.pos += byte_len; - return try t.tokenizeIdentifier(start); - } - - return error.InvalidSyntax; - }, - - else => return error.InvalidSyntax, - - 0 => { - if (t.pos == t.code.len) return null; - return error.InvalidSyntax; - }, - } - } -} - -/// Source code is null terminated, so accessing this is safe -fn byte(t: Tokenizer) u8 { - return t.code[t.pos]; -} - -/// Source code is null terminated. -fn nextByte(t: Tokenizer) u8 { - assert(t.byte() < 0x80); - return t.code[t.pos + 1]; -} - -/// Remember that using this only makes sense if you know for certain that all -/// previous bytes are ASCII, otherwise you may hit UB -fn byteAt(t: Tokenizer, comptime offset: u32) ?u8 { - if (offset == 1) @compileError("use nextByte()"); - - // assert that the previous bytes are ASCII - // current byte is allowed to be a continuation byte - for (t.code[t.pos .. t.pos + offset]) |b| { - assert(b < 0x80); - } - return t.code[t.pos + offset]; -} - -fn byteAtRt(t: Tokenizer, offset: u32) ?u8 { - // assert that the previous bytes are ASCII - // current byte is allowed to be a continuation byte - for (t.code[t.pos .. t.pos + offset]) |b| { - assert(b < 0x80); - } - return t.code[t.pos + offset]; -} - -fn codePoint(t: Tokenizer) !struct { u3, u21 } { - const b = t.byte(); // must check the first byte manually - assert(b >= 0x80); // must be a continuation byte - - const byte_len = unicode.utf8ByteSequenceLength(b) catch - return error.InvalidUtf8; - assert(byte_len > 1); - - const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch - return error.InvalidUtf8; - - return .{ byte_len, code_point }; -} - -fn isAtIdentifierStart(t: *Tokenizer) !bool { - switch (t.byte()) { - 'a'...'z', 'A'...'Z', '_', '$' => return true, - else => return false, - 0x80...0xFF => { - _, const code_point = try t.codePoint(); - return unicode_props.isXidStart(code_point); - }, - } -} - -fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token { - const trace = T.trace(@src()); - defer trace.end(); - - while (true) { - switch (t.byte()) { - 'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => { - t.pos += 1; - }, - 0x80...0xFF => { - const byte_len, const code_point = try t.codePoint(); - - t.pos += byte_len; - - if (!unicode_props.isXidContinue(code_point)) { - break; - } - }, - else => break, - } - } - - return .{ - .tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse - if (t.code[start_byte] == '@') - .builtin_id - else - .id, - .offset = start_byte, - }; -} - -/// Numbers have the following rules: -/// - May start with a `.`, but cannot end with `.` -/// - May contain leading zeros, but transform pass will warn. -/// - May contain a `_` separator, cannot have two in a row. -/// - May contain `e` or `E` for exponents. -/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers. -/// The token is tagged as `number_int` or `number_float` depending -/// on if it contained a `.` This is to help later re-parses. -// -// TODO: is it a zig compiler bug that `?` is required here? -fn tokenizeNumber(t: *Tokenizer) !?Token { - const trace = T.trace(@src()); - defer trace.end(); - - // Caller ensures the following invariant: - switch (t.byte()) { - '.' => switch (t.nextByte()) { - '0'...'9' => {}, - else => unreachable, - }, - '0'...'9' => {}, - else => unreachable, - } - - // Parse Hex, Binary, Octal numbers - if (t.byte() == '0') { - switch (t.nextByte()) { - 'x', 'X' => { - t.pos += 2; - try t.consumeIntegerRadix(.hex); - }, - 'b', 'B' => { - t.pos += 2; - try t.consumeIntegerRadix(.binary); - }, - 'o', 'O' => { - t.pos += 2; - try t.consumeIntegerRadix(.octal); - }, - else => {}, - } - } - - const start = t.pos; - var tag: Token.Tag = .number_int; - - if (t.byte() == '.') { - t.pos += 1; - tag = .number_float; - try t.consumeInteger(); - } else { - try t.consumeInteger(); - - // decimal point notation - switch (t.byte()) { - '.' => switch (t.nextByte()) { - '0'...'9' => { - t.pos += 1; - tag = .number_float; - try t.consumeInteger(); - }, - else => { - // we have not advanced the tokenizer past the `.` - assert(tag == .number_int); - return .{ .tag = tag, .offset = start }; - }, - }, - else => {}, - } - } - - switch (t.byte()) { - 'e', 'E' => { - tag = .number_float; - t.pos += 1; - - switch (t.byte()) { - '+', '-' => { - t.pos += 1; - switch (t.byte()) { - '0'...'9' => {}, - else => return t.maybeUTF8Error(error.InvalidSyntax), - } - }, - '0'...'9' => {}, - else => return t.maybeUTF8Error(error.InvalidSyntax), - } - - try t.consumeInteger(); - - // decimal point notation - switch (t.byte()) { - '.' => switch (t.nextByte()) { - '0'...'9' => { - t.pos += 1; - tag = .number_float; - try t.consumeInteger(); - }, - else => {}, - }, - else => {}, - } - }, - else => {}, - } - - // after numbers must be a valid word break the above returns are right - // before parsing `.` so those can be unchecked. - if (try t.isAtIdentifierStart()) { - return error.InvalidSyntax; - } - - return .{ - .tag = tag, - .offset = start, - }; -} - -fn maybeConsumeInteger(t: *Tokenizer) !?void { - const trace = T.trace(@src()); - defer trace.end(); - - switch (t.byte()) { - '0'...'9' => { - t.pos += 1; - }, - else => return null, - } - t.consumeInteger(); -} - -/// Requires the first byte to be a digit -fn consumeInteger(t: *Tokenizer) !void { - const trace = T.trace(@src()); - defer trace.end(); - - switch (t.byte()) { - '0'...'9' => { - t.pos += 1; - }, - else => unreachable, - } - - while (true) { - switch (t.byte()) { - '0'...'9' => { - t.pos += 1; - }, - '_' => { - t.pos += 1; - switch (t.byte()) { - '0'...'9' => { - t.pos += 1; - }, - else => return t.maybeUTF8Error(error.TrailingNumericSeparator), - } - }, - else => break, - } - } -} - -const Radix = enum { hex, binary, octal }; - -fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void { - const trace = T.trace(@src()); - defer trace.end(); - - const digits = switch (radix) { - .hex => "0123456789abcdefABCDEF", - .binary => "01", - .octal => "01234567", - }; - - first: { - const b = t.byte(); - inline for (digits) |digit| { - if (b == digit) { - break :first; - } - } - return t.maybeUTF8Error(error.InvalidSyntax); - } - - t.pos += 1; - - while (true) { - brk: { - const b = t.byte(); - inline for (digits) |digit| { - if (b == digit) { - break :brk; - } - } - return; - } - t.pos += 1; - } -} - -fn tokenizeString(t: *Tokenizer) !Token { - assert(t.byte() == '"'); - const start = t.pos; - try t.consumeString(); - return .{ .tag = .string, .offset = start }; -} - -fn consumeString(t: *Tokenizer) !void { - const trace = T.trace(@src()); - defer trace.end(); - - const start = t.pos; - assert(t.byte() == '"'); - - t.pos += 1; - - while (true) { - switch (t.byte()) { - '"' => { - t.pos += 1; - break; - }, - '\\' => { - switch (t.nextByte()) { - '\n' => { - t.pos = start; - return error.UnterminatedString; - }, - '\\' => t.pos += 2, - '"' => t.pos += 2, - 'n' => t.pos += 2, - 'r' => t.pos += 2, - 't' => t.pos += 2, - '0' => switch (t.byteAt(2) orelse { - t.pos = start; - return error.UnterminatedString; - }) { - // '\0xFF' hex escape - 'x' => {}, - // '\0' is special cased - else => t.pos += 2, - }, - 'u' => { - const esc_start = t.pos; - t.consumeStringUnicodeEscape() catch |err| switch (err) { - error.InvalidEscapeSequence => { - if (t.byte() == '\n') - return error.UnterminatedString; - t.pos = esc_start; - return err; - }, - error.InvalidEscapeCodepoint => { - t.pos = esc_start; - return err; - }, - // error.UnterminatedString => return err, - }; - }, - else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1), - } - }, - '\n' => { - t.pos = start; - return error.UnterminatedString; - }, - else => { - t.pos += 1; - }, - 0x80...0xFF => { - const byte_len, _ = try t.codePoint(); - t.pos += byte_len; - }, - } - } -} - -fn consumeStringUnicodeEscape(t: *Tokenizer) !void { - const trace = T.trace(@src()); - defer trace.end(); - - assert(t.byte() == '\\'); - assert(t.nextByte() == 'u'); - t.pos += 2; - - if (t.byte() != '{') - return error.InvalidEscapeSequence; - t.pos += 1; - - if (t.byte() == '0') { - if (t.nextByte() == 'x') { - t.pos += 2; - } - } - - const start = t.pos; - t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence; - - const slice = t.code[start..t.pos]; - - if (t.byte() != '}') - return error.InvalidEscapeSequence; - - const code_point = std.fmt.parseInt(u21, slice, 16) catch |err| switch (err) { - error.Overflow => return error.InvalidEscapeCodepoint, - error.InvalidCharacter => unreachable, - }; - _ = std.unicode.utf8CodepointSequenceLength(code_point) catch |err| switch (err) { - error.CodepointTooLarge => return error.InvalidEscapeCodepoint, - }; - - t.pos += 1; -} - -fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token { - const start = t.pos; - t.pos += comptime (tag.staticLen() orelse unreachable); - return .{ .tag = tag, .offset = start }; -} - -/// We are about to return an error, such as error.TrailingNumericSeparator. -/// However, the character we ran into might actually be invalid UTF-8. -/// This utility just checks for that to make sure we dont mask error.InvalidUtf8 -fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{ - .ErrorSet = &[_]std.builtin.Type.Error{ - .{ .name = "InvalidUtf8" }, - .{ .name = @errorName(real_err) }, - }, -}) { - switch (t.byte()) { - 0x80...0xFF => { - _ = try t.codePoint(); - return real_err; - }, - else => return real_err, - } -} - -fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{ - .ErrorSet = &[_]std.builtin.Type.Error{ - .{ .name = "InvalidUtf8" }, - .{ .name = @errorName(real_err) }, - }, -}) { - switch (t.byteAtRt(at) orelse return real_err) { - 0x80...0xFF => { - _ = try t.codePoint(); - return real_err; - }, - else => return real_err, - } -} - -pub fn skipWhitespace(t: *Tokenizer) void { - const trace = T.trace(@src()); - defer trace.end(); - - while (true) { - switch (t.byte()) { - ' ', '\t', '\n', '\r' => { - t.pos += 1; - }, - else => break, - } - } -} - -pub fn tokenLocation( - utf8_source: []const u8, - start_offset: ByteOffset, - token_start: ByteOffset, -) Location { - var loc: Location = .{ - .line = .start, - .column = .start, - .line_start = start_offset, - .line_end = @intCast(utf8_source.len), - }; - - // Scan to by line until we go past the token start - while (std.mem.indexOfScalarPos(u8, utf8_source, loc.line_start, '\n')) |i| { - if (i >= token_start) { - break; // Went past - } - loc.line = loc.line.add(1); - loc.line_start = @intCast(i + 1); - } - - const offset = loc.line_start; - for (utf8_source[offset..], 0..) |c, i| { - if (i + offset == token_start) { - loc.line_end = @intCast(i + offset); - while (loc.line_end < utf8_source.len and utf8_source[loc.line_end] != '\n') { - loc.line_end += 1; - } - return loc; - } - if (c == '\n') { - loc.line = loc.line.add(1); - loc.column = .start; - loc.line_start = @intCast(i + 1); - } else { - loc.column = loc.column.add(1); - } - } - - return loc; -} - -const Tokenizer = @This(); -const std = @import("std"); -const unicode = std.unicode; -const assert = std.debug.assert; -const mem = std.mem; -const unicode_props = @import("unicode_props.zig"); -const T = @import("tracer"); -const Location = @import("Ast.zig").Location; diff --git a/src/compiler/Transform.zig b/src/compiler/Transform.zig deleted file mode 100644 index 9652b3edc67059de5b4f81652cff4a753db1f0fb..0000000000000000000000000000000000000000 --- a/src/compiler/Transform.zig +++ /dev/null @@ -1,1479 +0,0 @@ -//! Transform pass -gpa: Allocator, -ast: Ast, -insts: Inst.List = .{}, -string_bytes: std.ArrayListUnmanaged(u8) = .{}, -extra: ExtraStore = .{}, - -scratch: std.ArrayListUnmanaged(u32) = .{}, - -// TODO: refactor this away -// The new home for this memory should be pre-allocated on scratch, -// but then remove the elements, moving scratch items on top of this -// backwards a few. -call_args: std.ArrayListUnmanaged(Inst.Ref) = .{}, -scopes: std.ArrayListUnmanaged(Scope) = .{}, - -report: Report.Builder, - -pub const Error = error{ - OutOfMemory, - TransformFail, -}; - -pub fn transform( - ast: Ast, - gpa: Allocator, - file: *Comp.File, - fail_report: *Report, -) Error!Teg { - const trace = tracer.trace(@src()); - defer trace.end(); - - var t: Transform = .{ - .ast = ast, - .gpa = gpa, - .report = .{ .gpa = gpa, .current_file = file }, - }; - errdefer t.deinit(); - - try t.string_bytes.append(t.gpa, 0); - - const module = t.transformModule(0) catch |err| switch (err) { - error.OutOfMemory => return err, - }; - - const report = try t.report.finalize(); - if (report.errors.len > 0) { - fail_report.* = report; - return error.TransformFail; - } - - assert(@intFromEnum(module) == 0); - - try t.insts.setCapacity(t.gpa, t.insts.len); - - defer t.deinit(); - return Teg{ - .insts = t.insts.toOwnedSlice(), - .string_bytes = try t.string_bytes.toOwnedSlice(t.gpa), - .extra = try t.extra.toOwnedStore(t.gpa), - .report = report, - }; -} - -pub fn deinit(t: *Transform) void { - t.scratch.deinit(t.gpa); - t.call_args.deinit(t.gpa); - assert(t.scopes.items.len == 0); - t.scopes.deinit(t.gpa); - t.string_bytes.deinit(t.gpa); - t.insts.deinit(t.gpa); - t.extra.deinit(t.gpa); - t.* = undefined; -} - -const Scope = struct { - symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{}, - - const Symbol = struct { - node: NodeIndex, - data: Symbol.Data, - // TODO: optimize this structure - used: bool = false, - mutated: bool = false, - discarded: ?NodeIndex = null, - - const Data = union(enum) { - declaration, - param, - mutable: Inst.Index, - immutable: Inst.Ref, - }; - - pub fn addWarnings(symbol: Symbol, t: *Transform) !void { - if (!symbol.mutated and symbol.data == .mutable) { - try t.report.warn(.not_mutated_var, .{ - .node = symbol.node, - }); - } - if (!symbol.used) { - try t.report.warn(.unused, .{ - .node = symbol.node, - .kind = switch (symbol.data) { - .declaration => .function, - .param => .param, - .mutable => .variable, - .immutable => switch (t.ast.nodeTag(symbol.node)) { - .param_decl => .param, - else => .constant, - }, - }, - }); - } else if (symbol.discarded) |discard| { - try t.report.warn(.unnecessary_discard, .{ - .node = discard, - }); - } - } - }; - - fn addWarnings(s: Scope, t: *Transform) !void { - for (s.symbols.values()) |value| { - try value.addWarnings(t); - } - } - - fn deinit(s: *Scope, gpa: Allocator) void { - s.symbols.deinit(gpa); - s.* = undefined; - } -}; - -const ScratchType = enum { - ref, - index, -}; - -const BlockContext = struct { - node_base: i32, - return_target: ?*?Inst.Index = null, - break_target: ?*BreakTarget = null, - - /// This struct is always stored on the stack; the idea is to pass - /// mutable pointers to it. When `break :brk` is encountered - const BreakTarget = struct { - label: ?[]const u8, - /// This allows `continue` as well as implicit break targetting. - is_loop: bool, - /// If `null`, this label was unused. - block_index: ?Inst.Index = null, - parent: ?*BreakTarget = null, - - fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget { - return .{ - .parent = parent, - .label = label, - .is_loop = is_loop, - }; - } - - fn getInst(target: *BreakTarget, t: *Transform) !Inst.Index { - return target.block_index orelse { - const inst = try t.addRootInst(.{ - .tag = .block, - .data = undefined, - }); - target.block_index = inst; - return inst; - }; - } - }; - - fn init(root_node: NodeIndex) BlockContext { - return .{ - .node_base = @intCast(root_node), - }; - } - - fn nodeOffset(self: BlockContext, node: NodeIndex) i32 { - return @as(i32, @intCast(node)) - self.node_base; - } - - fn declOffset(self: BlockContext, node: NodeIndex) BlockContext { - return .{ - .node_base = self.nodeOffset(node), - }; - } - - fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext { - return .{ - .node_base = self.node_base, - .return_target = return_target, - }; - } - - fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext { - return .{ - .node_base = self.node_base, - .return_target = self.return_target, - .break_target = break_target, - }; - } - - fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index { - var break_target = self.break_target; - while (break_target) |target| : (break_target = target.parent) { - if (target.label) |target_label| { - if (mem.eql(u8, label, target_label)) { - return try target.getInst(t); - } - } - } - return null; - } -}; - -const Liveness = enum { alive, dead }; - -const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{ - .{ "void", .void }, - .{ "true", .bool_true }, - .{ "false", .bool_false }, - .{ "int", .int32_type }, - .{ "float", .float_type }, - .{ "string", .string_type }, - // .{ "bigint", .bigint_type }, - .{ "bool", .bool_type }, - .{ "noreturn", .noreturn_type }, - .{ "type", .type_type }, - .{ "std", .standard_library }, -}); - -fn transformModule(t: *Transform, module_node: NodeIndex) Allocator.Error!Inst.Index { - const trace = tracer.trace(@src()); - defer trace.end(); - - const module = try t.addRootInst(.{ - .tag = .module, - .data = undefined, - }); - - const decl = BlockContext.init(module_node); - - try t.pushScope(); - defer t.popScope(); - - const statements = t.ast.nodeStatements(module_node); - for (statements) |statement| { - try t.previsitModuleStatement(statement); - } - - const s = t.scope(); - - const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1); - defer t.gpa.free(temp); - - { - // prevent concurrent access - s.symbols.lockPointers(); - defer s.symbols.unlockPointers(); - for (s.symbols.values(), 0..) |value, i| { - temp[i] = try t.transformFunction(decl, value.node); - } - } - - const top_level_fn = try t.addRootInst(.{ - .tag = .declaration, - .data = undefined, - }); - - temp[temp.len - 1] = top_level_fn; - - const declarations: IndexList = .{ - .start = @intCast(t.extra.list.items.len), - .len = @intCast(temp.len), - }; - try t.extra.list.appendSlice(t.gpa, @ptrCast(temp)); - - const block = try t.addRootInst(.{ - .tag = .block, - .data = undefined, - }); - - { - const scratch_top = t.scratch.items.len; - defer t.scratch.shrinkRetainingCapacity(scratch_top); - - for (statements) |statement| { - try t.transformModuleStatement(decl, statement); - } - - t.getInstData(block).* = EncodedData.encode(.block, .{ - .src_node = decl.nodeOffset(module_node), - .extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)), - }); - } - - const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{ - .kind = .function, - .name = NullTerminatedString.empty, - .hash = .{0} ** 16, - .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn), - }, dod.extraSizeOf(Inst.DeclFn)); - - trail.writeStruct(Inst.DeclFn, .{ - .params = .{ .start = 0, .len = 0 }, - .return_type = Inst.Ref.void, - .body = block.toRef(), - }); - - t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{ - .src_node = decl.nodeOffset(module_node), - .extra_index = decl_extra, - }); - - t.getInstData(module).* = EncodedData.encode(.module, .{ - .src_node = 0, - .extra_index = try t.extra.append(t.gpa, IndexList, declarations), - }); - - try t.scope().addWarnings(t); - - return module; -} - -fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void { - const trace = tracer.trace(@src()); - defer trace.end(); - - const tag = t.ast.nodes.items(.tag)[node]; - switch (tag) { - .fn_decl => { - const fn_decl = t.ast.fullFnDecl(node); - const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id); - if (gop.found_existing) { - try t.report.err(.duplicate_function, .{ - .node = node, - .prev = gop.value_ptr.node, - }); - return; - } - gop.value_ptr.* = .{ - .node = node, - .data = .declaration, - }; - }, - else => {}, - } -} - -fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void { - const trace = tracer.trace(@src()); - defer trace.end(); - - const tag = t.ast.nodes.items(.tag)[node]; - switch (tag) { - .root, - .param_decl, - => unreachable, - - .fn_decl => {}, // functions analyzed separately - - else => _ = try t.transformBlockStatement(decl, node, false), - } -} - -fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) Allocator.Error!Inst.Index { - const trace = tracer.trace(@src()); - defer trace.end(); - - const decl = outer_decl.declOffset(node); - const fn_decl = t.ast.fullFnDecl(node); - - const func = try t.addRootInst(.{ - .tag = .declaration, - .data = undefined, - }); - - try t.pushScope(); - defer t.popScope(); - - const param_list = brk: { - const scratch_top = t.scratch.items.len; - defer t.scratch.shrinkRetainingCapacity(scratch_top); - - for (fn_decl.params) |param| { - try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param)); - } - - break :brk try t.scratchToRefList(scratch_top); - }; - - const return_type = if (fn_decl.return_type) |return_type_node| - try t.transformExpressionToBlock(decl, return_type_node, false) - else - Inst.Ref.void; - - const body = try t.transformExpressionToBlock(decl, fn_decl.body, true); - - const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{ - .kind = .function, - .name = try t.addNullTerminatedString(fn_decl.id), - .hash = .{0} ** 16, - .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func), - }); - - _ = try t.extra.append(t.gpa, Inst.DeclFn, .{ - .params = param_list, - .return_type = return_type, - .body = body, - }); - - t.getInstData(func).* = EncodedData.encode(.declaration, .{ - .src_node = outer_decl.nodeOffset(node), - .extra_index = decl_extra, - }); - - try t.scope().addWarnings(t); - - return func; -} - -const TransformFunctionCtx = struct { - param_list: RefList, -}; - -/// Converts expressions into blocks with a break. This is important -/// for places where a block is not allowed, such as a parameter list. -/// This is also used for the function body, even though it is a block. -/// Simple refs turn into just the ref -fn transformExpressionToBlock( - t: *Transform, - decl: BlockContext, - node: NodeIndex, - comptime is_function: bool, -) Allocator.Error!Inst.Ref { - const trace = tracer.trace(@src()); - defer trace.end(); - - const node_tag = t.ast.nodes.items(.tag)[node]; - switch (node_tag) { - .id => { - // Lower builtin ids without using a block - const id = t.ast.basicNodeSlice(node); - if (BuiltInIds.get(id)) |ref| { - return ref; - } - }, - - else => {}, - } - - var block: ?Inst.Index = null; - - const scratch_top = t.scratch.items.len; - defer t.scratch.shrinkRetainingCapacity(scratch_top); - - if (is_function) { - const s = t.scopes.getLast(); - for (s.symbols.values(), 0..) |*param, i| { - assert(param.data == .param); - const ref = try t.addInstRef(.{ - .tag = .param, - .data = EncodedData.encode(.param, .{ - .src_node = decl.nodeOffset(node), - .data = @intCast(i), - }), - }); - param.data = .{ .immutable = ref }; - } - } - - const expr = try t.transformExpression( - if (is_function) - decl.withReturn(&block) - else - decl, - node, - ); - - // if there are no new instructions, we dont have to create a block context - if (scratch_top == t.scratch.items.len) { - const result = expr orelse unreachable; // null means we added break or return - - // TODO: evaluate if `if` blocks would be fine in sema with instruction refs - // with this extra if check here, we filter out non-blocks. - if (result.toIndex() == null) { - return result; - } - } - - if (expr) |result| { - if (result.toIndex()) |index| { - if (t.insts.items(.tag)[@intFromEnum(index)] == .block) { - return result; - } - } - } - - const ret_block = if (block) |b| - b - else - try t.addRootInst(.{ - .tag = .block, - .data = undefined, - }); - - if (expr) |result| { - _ = try t.addInstRef(.{ - .tag = .@"break", - .data = EncodedData.encode(.@"break", .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ - .block = ret_block, - .operand = result, - }), - }), - }); - } - - const content = try t.scratchToIndexList(scratch_top); - - t.getInstData(ret_block).* = EncodedData.encode(.block, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, IndexList, content), - }); - - return ret_block.toRef(); -} - -fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Allocator.Error!Inst.Ref { - const trace = tracer.trace(@src()); - defer trace.end(); - - const param_decl = t.ast.fullParamDecl(param); - const value = try t.transformExpressionToBlock(decl, param_decl.type, false); - - // add `sym` into the scope so that the next param's type - // can use this param's value, if known at compile time. - const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id); - if (sym.found_existing) { - try t.report.err(.duplicate_param, .{ - .node = param, - .prev = sym.value_ptr.node, - }); - return .void; - } - - sym.value_ptr.* = .{ - .node = param, - .data = .param, - }; - - return value; -} - -const Builtin = enum { - js, - import, - fetch, - compile_error, - - const map = std.ComptimeStringMap(Builtin, .{ - .{ "js", .js }, - .{ "import", .import }, - .{ "fetch", .fetch }, - .{ "compileError", .compile_error }, - }); - - fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref { - const trace = tracer.trace(@src()); - defer trace.end(); - - if (args.len != 1) { - // try t.report.err(.invalid_builtin_args, .{ .id = "js" }); - return .void; - } - - const source = try t.transformExpression(decl, args[0]) orelse return null; - - return try t.addInstRef(.{ - .tag = .js, - .data = EncodedData.encode(.js, .{ - .src_node = decl.nodeOffset(args[0]), - .operand = source, - }), - }); - } -}; - -/// This is the primary transformation function: converting an AST expression into instructions. -/// `null` represents a `return` or `break`, aka this branch of code is dead. -fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Ref { - const node_tag = t.ast.nodes.items(.tag)[node]; - - const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); - defer trace.end(); - - switch (node_tag) { - .root, - .param_decl, - .var_decl, - .fn_decl, - => unreachable, // not an expression - - .id => { - const id = t.ast.basicNodeSlice(node); - if (BuiltInIds.get(id)) |ref| { - return ref; - } - - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); - } - - const lookup = t.lookupId(id) orelse { - try t.report.err(.unknown_identifier, .{ .node = node }); - return .void; - }; - - lookup.used = true; - - return switch (lookup.data) { - .declaration => try t.addInstRef(.{ - .tag = .load_decl, - .data = EncodedData.encode(.load_decl, .{ - .src_node = decl.nodeOffset(node), - .string = try t.addNullTerminatedString(id), - }), - }), - .mutable => |idx| idx.toRef(), - .immutable => |ref| ref, - .param => unreachable, //unresolved param - }; - }, - - inline .number_int, .number_float => |tag| { - const text = t.ast.basicNodeSlice(node); - const value = switch (tag) { - .number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"), - .number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"), - else => comptime unreachable, - }; - return try t.addInstRef(.{ - .tag = .number, - .data = EncodedData.encode(.number, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)), - }), - }); - }, - - .string => { - const str = t.ast.basicNodeSlice(node); - const sliced = try t.addSlicedString(str[1 .. str.len - 1]); - return try t.addInstRef(.{ - .tag = .string, - .data = EncodedData.encode(.string, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, SlicedString, sliced), - }), - }); - }, - - .assign => { - const ast_bin = t.ast.nodeData(node).bin; - const lhs = try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse - return .void; - const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse - return null; - const assign_inst = try t.addInstRef(.{ - .tag = .store, - .data = EncodedData.encode(.store, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Store, .{ - .memory = lhs, - .operand = rhs, - }), - }), - }); - return assign_inst; - }, - - inline .add, - .sub, - .mul, - .div, - .rem, - .pow, - .@"and", - .@"or", - .bit_and, - .bit_or, - .bit_xor, - .shl, - .shr, - .eq, - .ne, - .lt, - .le, - .gt, - .ge, - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => |tag| { - // This code path implements binary operations, as well as the - // compound compute + assign operation. - const is_assign = comptime tag.isAssignBin(); - const teg_tag = @field( - Teg.Inst.Tag, - if (is_assign) - @tagName(tag)[0 .. @tagName(tag).len - "_assign".len] - else - @tagName(tag), - ); - const ast_bin = t.ast.nodeData(node).bin; - const lhs = if (is_assign) - try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse - return .void - else - try t.transformExpression(decl, ast_bin.lhs) orelse - return null; - const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null; - - const compute_inst = try t.addInstRef(.{ - .tag = teg_tag, - .data = EncodedData.encode(teg_tag, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{ - .lhs = if (is_assign) lhs.toRef() else lhs, - .rhs = rhs, - }), - }), - }); - - if (is_assign) { - const assign_inst = try t.addInstRef(.{ - .tag = .store, - .data = EncodedData.encode(.store, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Store, .{ - .memory = lhs, - .operand = compute_inst, - }), - }), - }); - return assign_inst; - } - - return compute_inst; - }, - - inline .negate, - .bool_not, - .bit_not, - => |tag| { - const ast_un = t.ast.nodeData(node).un; - const teg_tag = @field(Teg.Inst.Tag, @tagName(tag)); - - const operand = try t.transformExpression(decl, ast_un.expr) orelse - return null; - - return try t.addInstRef(.{ - .tag = teg_tag, - .data = EncodedData.encode(teg_tag, .{ - .src_node = decl.nodeOffset(node), - .operand = operand, - }), - }); - }, - - .label => { - const expr = t.ast.nodeData(node).label.expr; - const label = t.ast.tokenSlice(t.ast.nodeToken(node)); - - var target: BlockContext.BreakTarget = .{ - .parent = decl.break_target, - .label = label, - .is_loop = false, - }; - const inner_ctx = decl.withBreak(&target); - - const scratch_top = t.scratch.items.len; - - const inner_expr = try t.transformExpression(inner_ctx, expr); - - // if there are no new instructions, we dont have to create a block context - if (scratch_top == t.scratch.items.len) - return inner_expr orelse unreachable; // null means we added break or return - - if (target.block_index) |block| { - if (inner_expr) |result| { - _ = try t.addInstRef(.{ - .tag = .@"break", - .data = EncodedData.encode(.@"break", .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ - .block = block, - .operand = result, - }), - }), - }); - } - - const content = try t.scratchToIndexList(scratch_top); - t.scratch.shrinkRetainingCapacity(scratch_top); - - t.getInstData(block).* = EncodedData.encode(.block, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, IndexList, content), - }); - - try t.addScratch(Inst.Index, block); - - return block.toRef(); - } else { - std.debug.print("Warn, unused label {s}", .{label}); - } - - return inner_expr; - }, - - inline .block_void, - .block_implicit_return, - => |tag| { - const statements = t.ast.nodeStatements(node); - - var last: Inst.Ref = .void; - for (statements, 1..) |statement, i| { - const is_last = i == statements.len; - last = try t.transformBlockStatement( - decl, - statement, - tag == .block_implicit_return and is_last, - ) orelse { - return null; - }; - } - - return if (tag == .block_implicit_return) last else .void; - }, - - .builtin_call, - => { - const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse { - try t.report.err(.invalid_builtin, .{ .node = node }); - return .void; - }; - - const base = t.ast.nodeData(node).list; - const args = base.slice(&t.ast); - - return switch (builtin) { - .js => Builtin.builtin_js(t, decl, args), - // .import => Builtin.import(t, decl, args), - // .fetch => Builtin.fetch(t, decl, args), - // .compile_error => Builtin.compile_error(t, decl, args), - else => @panic("TODO"), - }; - }, - - .call => { - const call = t.ast.nodeData(node).call; - const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast); - - const base = try t.transformExpression(decl, call.base) orelse return null; - - const args_list: RefList = brk: { - const args_top = t.call_args.items.len; - defer t.call_args.shrinkRetainingCapacity(args_top); - - for (args) |arg| { - try t.call_args.append( - t.gpa, - try t.transformExpression(decl, arg) orelse - return null, - ); - } - - const items = t.call_args.items[args_top..]; - const extra_top = t.extra.list.items.len; - try t.extra.list.appendSlice(t.gpa, @ptrCast(items)); - - break :brk .{ - .start = @intCast(extra_top), - .len = @intCast(items.len), - }; - }; - - const call_inst = try t.addInstRef(.{ - .tag = .call, - .data = EncodedData.encode(.call, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Call, .{ - .base = base, - .args = args_list, - }), - }), - }); - - return call_inst; - }, - - .field => { - const field = t.ast.nodeData(node).field; - - const base = try t.transformExpression(decl, field.base) orelse return null; - - // TODO: handle identifier escapes - const id = t.ast.tokenSlice(field.field_token); - const prop_name_str = try t.addSlicedString(id); - - const string_ref = try t.addInstRef(.{ - .tag = .string, - .data = EncodedData.encode(.string, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str), - }), - }); - - return try t.addInstRef(.{ - .tag = .field, - .data = EncodedData.encode(.field, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{ - .lhs = base, - .rhs = string_ref, - }), - }), - }); - }, - - .@"if" => { - const if_node = t.ast.nodeData(node).@"if"; - const if_data = t.ast.extraData(if_node.extra, Ast.If); - - const cond = try t.transformExpression(decl, if_node.cond) orelse - return null; - - const yes = try t.transformExpressionToBlock( - decl, - if_data.yes, - false, - ); - const no = if (if_data.no.unwrap()) |no| - try t.transformExpressionToBlock( - decl, - no, - false, - ) - else - Inst.Ref.void; - - if (yes == .void and no == .void) { - // when both branches do nothing, we can omit the branch, - // however we still need to type-check the condition operand - _ = try t.addInstRef(.{ - .tag = .ensure_bool, - .data = EncodedData.encode(.ensure_bool, .{ - .src_node = decl.nodeOffset(node), - .operand = cond, - }), - }); - return .void; - } - - return try t.addInstRef(.{ - .tag = .@"if", - .data = EncodedData.encode(.@"if", .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.If, .{ - .cond = cond, - .yes = yes, - .no = no, - }), - }), - }); - }, - - .@"return" => { - const ret_node = t.ast.nodeData(node).@"return"; - - const ret_block_ptr = decl.return_target orelse { - try t.report.err(.invalid_return, .{ .node = node }); - return .void; - }; - - const ret_block = if (ret_block_ptr.*) |block| - block - else append_block: { - const block = try t.addRootInst(.{ - .tag = .block, - .data = undefined, - }); - ret_block_ptr.* = block; - break :append_block block; - }; - - const operand: Inst.Ref = if (ret_node.expr.unwrap()) |expr| - try t.transformExpression(decl.withReturn(null), expr) orelse - return null - else - .void; - - _ = try t.addInstRef(.{ - .tag = .@"break", - .data = EncodedData.encode(.@"break", .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ - .block = ret_block, - .operand = operand, - }), - }), - }); - - return null; // control flow ends - }, - - .@"break" => { - const brk_node = t.ast.nodeData(node).@"break"; - const label = t.ast.tokenSlice( - brk_node.label.unwrap() orelse - @panic("TODO: unlabelled breaks"), - ); - - const block = try decl.findAndMarkBreak(t, label) orelse { - try t.report.err(.unresolved_label, .{ .node = node }); - return null; - }; - - const operand: Inst.Ref = if (brk_node.value.unwrap()) |expr| - try t.transformExpression(decl.withReturn(null), expr) orelse - return null - else - .void; - - _ = try t.addInstRef(.{ - .tag = .@"break", - .data = EncodedData.encode(.@"break", .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ - .block = block, - .operand = operand, - }), - }), - }); - - return null; // control flow ends - }, - - else => |tag| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}), - } -} - -fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Index { - const node_tag = t.ast.nodes.items(.tag)[node]; - - const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); - defer trace.end(); - - switch (node_tag) { - .root, - .param_decl, - .var_decl, - .fn_decl, - => unreachable, // not an expression - - .id => { - const id = t.ast.basicNodeSlice(node); - assert(!mem.eql(u8, id, "_")); // discards are not assignment targets - - if (BuiltInIds.has(id)) { - try t.report.err(.assign_builtin, .{ .node = node }); - return null; - } - - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); - } - - const lookup = t.lookupId(id) orelse { - try t.report.err(.unknown_identifier, .{ .node = node }); - return null; - }; - - return switch (lookup.data) { - .mutable => |inst| { - lookup.mutated = true; - return inst; - }, - else => { - lookup.used = true; - - try t.report.err(.invalid_assignment_symbol, .{ - .node = node, - .decl = lookup.node, - .kind = switch (lookup.data) { - .mutable => unreachable, - .declaration => .function, - .param => .param, - .immutable => switch (t.ast.nodeTag(lookup.node)) { - .param_decl => .param, - else => .constant, - }, - }, - }); - return null; - }, - }; - }, - - // TODO: allow field expressions, but these need to emit a different store node - - else => { - _ = try t.transformExpression(decl, node); - try t.report.err(.invalid_assignment_target, .{ - .node = node, - }); - return null; - }, - } -} - -fn transformBlockStatement( - t: *Transform, - decl: BlockContext, - node: NodeIndex, - is_implicit_return_expression: bool, -) !?Inst.Ref { - const node_tag = t.ast.nodes.items(.tag)[node]; - - const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); - defer trace.end(); - - switch (node_tag) { - .root => unreachable, // cannot appear in a block - .param_decl => unreachable, // cannot appear in a block - - .var_decl => { - const var_decl = t.ast.fullVarDecl(node); - - if (mem.eql(u8, var_decl.id, "_")) { - try t.report.warn(.invalid_discard, .{ .node = node }); - - const ref = switch (t.ast.nodeTag(var_decl.init)) { - .id => { - const id = t.ast.basicNodeSlice(node); - if (BuiltInIds.has(id)) { - try t.report.warn(.discard_builtin, .{ .node = node }); - return .void; - } - - if (mem.startsWith(u8, id, "@")) { - @panic("TODO: string identifiers"); - } - - const lookup = t.lookupId(id) orelse { - try t.report.err(.unknown_identifier, .{ .node = node }); - return .void; - }; - - lookup.discarded = node; - - return .void; - }, - else => try t.transformExpression(decl, var_decl.init) orelse - return null, - }; - - return try t.addInstRef(.{ - .tag = .discard, - .data = EncodedData.encode(.discard, .{ - .src_node = decl.nodeOffset(node), - .operand = ref, - }), - }); - } - - if (BuiltInIds.has(var_decl.id)) { - try t.report.err(.shadow_builtin_variable, .{ .node = node }); - return .void; - } - - const type_ref = if (var_decl.type) |data_type| - try t.transformExpression(decl, data_type) - else - null; - const data_ref = try t.transformExpression(decl, var_decl.init) orelse - return null; - const data_coerced_ref = if (type_ref) |ref| - try t.addInstRef(.{ - .tag = .as, - .data = EncodedData.encode(.as, .{ - .src_node = decl.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, Inst.As, .{ - .type = ref, - .operand = data_ref, - }), - }), - }) - else - data_ref; - - const mem_ref = if (var_decl.is_const) - data_coerced_ref - else - try t.addInstRef(.{ - .tag = .memory, - .data = EncodedData.encode(.memory, .{ - .src_node = decl.nodeOffset(node), - .operand = data_coerced_ref, - }), - }); - - const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id); - - // if re-declaring, apply unused warnings now - if (gop.found_existing) { - try gop.value_ptr.addWarnings(t); - } - - gop.value_ptr.* = .{ - .node = node, - .data = switch (var_decl.is_const) { - true => .{ .immutable = mem_ref }, - false => .{ .mutable = mem_ref.toIndex().? }, - }, - }; - - return .void; - }, - - .fn_decl, - => |tag| { - std.debug.panic("TODO: block statement {s}", .{@tagName(tag)}); - }, - - .assign, - => { - const bin = t.ast.nodeData(node).bin; - const lhs = bin.lhs; - - if (t.ast.nodeTag(lhs) == .id and - mem.eql(u8, t.ast.basicNodeSlice(lhs), "_")) - { - // discard, instead of appending an .ensure_void check, - // we will add a .discard instruction instead. - // this will emit a warning if given void. - const result = try t.transformExpression(decl, bin.rhs) orelse - return null; - return try t.addInstRef(.{ - .tag = .discard, - .data = EncodedData.encode(.discard, .{ - .src_node = decl.nodeOffset(node), - .operand = result, - }), - }); - } else { - // process this assignment as usual, do not append .ensure_void - return try t.transformExpression(decl, node); - } - }, - - .add_assign, - .sub_assign, - .mul_assign, - .div_assign, - .rem_assign, - .pow_assign, - .and_assign, - .or_assign, - .bit_and_assign, - .bit_or_assign, - .bit_xor_assign, - .shl_assign, - .shr_assign, - => { - // do not append a .ensure_void check - return try t.transformExpression(decl, node); - }, - - else => { - const result = try t.transformExpression(decl, node) orelse - return null; - - if (!is_implicit_return_expression and result != .void) { - if (t.ast.nodeTag(node) == .id) { - try t.report.warn(.dangling_identifier, .{ - .node = node, - }); - return result; - } - - return try t.addInstRef(.{ - .tag = .ensure_void, - .data = EncodedData.encode(.ensure_void, .{ - .src_node = decl.nodeOffset(node), - .operand = result, - }), - }); - } - - return result; - }, - } -} - -fn addRootInst(t: *Transform, inst: Inst) !Inst.Index { - try t.insts.append(t.gpa, inst); - return @enumFromInt(t.insts.len - 1); -} - -fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref { - try t.scratch.ensureUnusedCapacity(t.gpa, 1); - try t.insts.append(t.gpa, inst); - const index: Inst.Index = @enumFromInt(t.insts.len - 1); - t.scratch.appendAssumeCapacity(@intFromEnum(index)); - return index.toRef(); -} - -fn pushScope(t: *Transform) !void { - try t.scopes.append(t.gpa, .{}); -} - -fn popScope(t: *Transform) void { - t.scope().deinit(t.gpa); - t.scopes.items[t.scopes.items.len - 1] = undefined; - t.scopes.items.len -= 1; -} - -fn scope(t: *Transform) *Scope { - return &t.scopes.items[t.scopes.items.len - 1]; -} - -fn getInstData(t: *Transform, index: Inst.Index) *EncodedData { - return &t.insts.items(.data)[@intFromEnum(index)]; -} - -fn addScratch(t: *Transform, comptime T: type, value: T) !void { - try t.scratch.append(t.gpa, @intFromEnum(value)); -} - -fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList { - const items = t.scratch.items[scratch_top..]; - const extra_top = t.extra.list.items.len; - try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]); - return .{ - .start = @intCast(extra_top), - .len = @intCast(items.len), - }; -} - -fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList { - const items = t.scratch.items[scratch_top..]; - const extra_top = t.extra.list.items.len; - try t.extra.list.appendSlice(t.gpa, items); - return .{ - .start = @intCast(extra_top), - .len = @intCast(items.len), - }; -} - -fn lookupId(t: *Transform, id: []const u8) ?*Scope.Symbol { - var i = t.scopes.items.len; - while (i > 0) { - i -= 1; - if (t.scopes.items[i].symbols.getPtr(id)) |sym| { - return sym; - } - } - return null; -} - -fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString { - if (str.len == 0) { - return .empty; - } - // loop through strings trying to find an existing match, and dedupe memory - var pos: usize = 0; - while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) { - const start_index = i - str.len; - if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) { - const string: NullTerminatedString = @enumFromInt(pos); - assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); - return string; - } - } - - const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len); - try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1); - t.string_bytes.appendSliceAssumeCapacity(str); - t.string_bytes.appendAssumeCapacity(0); - assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); - return string; -} - -fn addSlicedString(t: *Transform, str: []const u8) !SlicedString { - if (str.len == 0) { - return .{ .start = 0, .len = 0 }; - } - - // loop through strings trying to find an existing match, and dedupe memory - var pos: usize = 0; - while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) { - const start_index = i - str.len; - if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) { - const string: SlicedString = .{ - .start = @intCast(start_index), - .len = @intCast(str.len), - }; - assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); - return string; - } - } - - const string: SlicedString = .{ - .start = @intCast(t.string_bytes.items.len), - .len = @intCast(str.len), - }; - try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len); - t.string_bytes.appendSliceAssumeCapacity(str); - assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); - return string; -} - -const Transform = @This(); -const std = @import("std"); -const mem = std.mem; -const Allocator = mem.Allocator; -const assert = std.debug.assert; - -const Teg = @import("Teg.zig"); -const Inst = Teg.Inst; -const NullTerminatedString = Teg.NullTerminatedString; -const SlicedString = Teg.SlicedString; -const RefList = Teg.RefList; -const IndexList = Teg.IndexList; -const EncodedData = Teg.EncodedData; -const ExtraIndex = Teg.ExtraIndex; - -const dod = @import("array.zig"); -const ExtraStore = dod.ExtraStore(.teg, .@"var"); -const Float = dod.Float; - -const Tokenizer = @import("Tokenizer.zig"); -const Token = Tokenizer.Token; -const ByteOffset = Tokenizer.ByteOffset; - -const Ast = @import("Ast.zig"); -const Unused = Ast.Unused; -const Node = Ast.Node; -const Data = Node.Data; -const SubList = Ast.SubList; -const NodeIndex = Ast.NodeIndex; -const TokenIndex = Ast.TokenIndex; -const OptionalNodeIndex = Ast.OptionalNodeIndex; -const OptionalTokenIndex = Ast.OptionalTokenIndex; -const OptionalExtraIndex = Ast.OptionalExtraIndex; -const optionalTokenIndex = OptionalTokenIndex.init; -const optionalNodeIndex = OptionalNodeIndex.init; -const optionalExtraIndex = OptionalExtraIndex.init; - -const InternPool = @import("InternPool.zig"); - -const Report = @import("Report.zig"); -const Comp = @import("Comp.zig"); - -const tracer = @import("tracer"); diff --git a/src/compiler/array.zig b/src/compiler/array.zig deleted file mode 100644 index 57ccac606608a1b511c3f0f54871592143edb8a1..0000000000000000000000000000000000000000 --- a/src/compiler/array.zig +++ /dev/null @@ -1,390 +0,0 @@ -const extra = @This(); - -pub const Mutability = enum { @"const", @"var" }; - -fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void { - const fields = std.meta.fields(@TypeOf(payload)); - var i = index; - inline for (fields) |field| { - const value = @field(payload, field.name); - switch (field.type) { - u32 => items[i] = value, - i32 => items[i] = @bitCast(value), - - [16]u8 => { - const u32s: [4]u32 = @bitCast(value); - items[i] = u32s[0]; - items[i + 1] = u32s[1]; - items[i + 2] = u32s[2]; - items[i + 3] = u32s[3]; - }, - - else => brk: { - const info = @typeInfo(field.type); - if (info == .Enum) { - items[i] = @intFromEnum(value); - break :brk; - } - - const struct_data = info.Struct; - if (struct_data.layout == .@"packed") { - items[i] = @bitCast(value); - break :brk; - } - - @call(.always_inline, setAsU32, .{ items, field.type, i, value }); - }, - } - i += extraSizeOf(field.type); - } -} - -fn getAsU32(items: []const u32, comptime T: type, index: usize) T { - const fields = std.meta.fields(T); - var i = index; - var result: T = undefined; - inline for (fields) |field| { - @field(result, field.name) = switch (field.type) { - u32 => items[i], - i32 => @bitCast(items[i]), - - [16]u8 => @bitCast(items[i..][0..4].*), - - else => brk: { - const info = @typeInfo(field.type); - if (info == .Enum) { - break :brk @enumFromInt(items[i]); - } - - const struct_data = info.Struct; - if (struct_data.layout == .@"packed") { - break :brk @bitCast(items[i]); - } - - break :brk @call( - .always_inline, - getAsU32, - .{ items, field.type, i }, - ); - }, - }; - i += extraSizeOf(field.type); - } - return result; -} - -pub const StoreKind = enum { - intern_pool, - teg, - air, - tfr, - - pub fn resolveStoreConst( - comptime kind: StoreKind, - owner: anytype, - ) *const kind.ResolvedStore(@TypeOf(owner)) { - return &owner.extra; - } - - pub fn resolveStoreMut( - comptime kind: StoreKind, - owner: anytype, - ) *kind.ResolvedStore(@TypeOf(owner)) { - return &owner.extra; - } - - pub fn ResolvedStore( - comptime kind: StoreKind, - comptime Owner: type, - ) type { - const ptr = @typeInfo(Owner).Pointer; - assert(ptr.size == .One); - const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) |field| { - if (std.mem.eql(u8, field.name, "extra")) { - break :brk field.type.mutability; - } - } else unreachable; - return ExtraStore(kind, mutability); - } -}; - -fn ExtraWrap(comptime kind: StoreKind) type { - return struct { - pub fn Index(comptime T: type) type { - return enum(u32) { - _, - - // Reads the data from `store.extra` and returns it. - pub fn get(index: @This(), owner: anytype) T { - return kind.resolveStoreConst(owner).get(T, @intFromEnum(index)); - } - - /// Returns data with a trailing reader to read data stored immediately after - /// the main data structure. What is stored should be documented on `T` - pub fn getTrail(index: @This(), owner: anytype) struct { - T, - kind.ResolvedStore(@TypeOf(owner)).TrailReader, - } { - const advance = comptime extraSizeOf(T); - return .{ - index.get(owner), - .{ .index = @intFromEnum(index) + advance }, - }; - } - }; - } - }; -} - -pub fn EncodedData(comptime Tag: type, backing_type: type) type { - return enum(backing_type) { - _, - - pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() { - assert(proof == tag); // incorrect memory access - const Data = tag.Data(); - const info = @typeInfo(Data); - if (info == .Enum) { - return @enumFromInt(@intFromEnum(data)); - } - if (info == .Struct) { - assert(backing_type == u64); - var fields: [2]u32 = @bitCast(@intFromEnum(data)); - return getAsU32(&fields, Data, 0); - } - return @bitCast(@intFromEnum(data)); - } - - pub fn encode(comptime tag: Tag, data: tag.Data()) @This() { - const Data = tag.Data(); - const info = @typeInfo(Data); - if (info == .Enum) { - return @enumFromInt(@intFromEnum(data)); - } - if (info == .Struct) { - assert(backing_type == u64); - var fields: [2]u32 = undefined; - setAsU32(&fields, Data, 0, data); - return @enumFromInt(@as(backing_type, @bitCast(fields))); - } - return @enumFromInt(@as(backing_type, @bitCast(data))); - } - }; -} - -/// The number of 'u32's this type consumes in `ip.extra`. -pub fn extraSizeOf(comptime T: type) usize { - return switch (T) { - u32, - i32, - => return 1, - - [16]u8 => return 4, - - else => { - const info = @typeInfo(T); - if (info == .Enum) { - comptime assert(@sizeOf(info.Enum.tag_type) <= 32); - return 1; - } - - const struct_data = info.Struct; - if (struct_data.layout == .@"packed") { - comptime assert(struct_data.backing_integer == u32); - return 1; - } - - const fields = std.meta.fields(T); - var size: usize = 0; - inline for (fields) |field| { - size += extraSizeOf(field.type); - } - return size; - }, - }; -} - -pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type { - return struct { - list: List = .{}, - - pub const kind = set_kind; - pub const mutability = set_mutability; - - const Store = @This(); - - const index_holder = ExtraWrap(kind); - pub const Index = index_holder.Index; - - pub const List = switch (mutability) { - .@"const" => struct { - items: []const u32 = &.{}, - - pub fn deinit(slice: *@This(), gpa: Allocator) void { - gpa.free(slice.items); - } - }, - .@"var" => std.ArrayListUnmanaged(u32), - }; - - pub fn deinit(store: *Store, gpa: Allocator) void { - store.list.deinit(gpa); - store.* = undefined; - } - - pub const TrailReader = struct { - index: usize, - - pub fn read(trail: *TrailReader, ip: *const Store, comptime T: type) T { - defer trail.index += comptime extraSizeOf(T); - return ip.get(T, trail.index); - } - - pub fn readValueSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Value.Slice { - defer trail.index += len; - return .{ - .start = @intCast(trail.index), - .len = len, - }; - } - - pub fn readTypeSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Type.Slice { - defer trail.index += len; - return .{ - .start = @intCast(trail.index), - .len = len, - }; - } - }; - - pub const TrailWriter = struct { - store: *Store, - leftover: usize, - - pub fn init(store: *Store, len: usize) TrailWriter { - return .{ .store = store, .leftover = len }; - } - - pub fn end(w: *TrailWriter) void { - assert(w.leftover == 0); - } - - pub fn writeU32(w: *TrailWriter, value: u32) void { - assert(w.leftover > 0); - w.ip.extra.appendAssumeCapacity(value); - w.leftover -= 1; - } - - pub fn writeSlice(w: *TrailWriter, slice: anytype) void { - const Child = std.meta.Child(@TypeOf(slice)); - const child = @typeInfo(Child); - comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32)); - assert(w.leftover >= slice.len); - w.store.list.appendSliceAssumeCapacity(@ptrCast(slice)); - w.leftover -= slice.len; - } - - pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void { - _ = appendAssumeCapacity(w.store, T, data); - } - }; - - pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) { - comptime assert(mutability == .@"var"); - try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload))); - return store.appendAssumeCapacity(T, payload); - } - - pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) { - comptime assert(mutability == .@"var"); - const extra_index: u32 = @intCast(ip.list.items.len); - ip.list.items.len += extraSizeOf(@TypeOf(payload)); - ip.set(T, extra_index, payload); - return @enumFromInt(extra_index); - } - - pub fn appendTrailing( - store: *Store, - gpa: Allocator, - comptime T: type, - payload: T, - trailing_len: usize, - ) Allocator.Error!struct { Index(T), TrailWriter } { - comptime assert(mutability == .@"var"); - try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len); - const extra_index: u32 = @intCast(store.list.items.len); - store.list.items.len += extraSizeOf(@TypeOf(payload)); - store.set(T, extra_index, payload); - return .{ - @enumFromInt(extra_index), - TrailWriter.init(store, trailing_len), - }; - } - - fn set(store: *Store, comptime T: type, index: usize, payload: T) void { - comptime assert(mutability == .@"var"); - setAsU32(store.list.items, T, index, payload); - } - - fn get(store: *const Store, comptime T: type, index: usize) T { - return getAsU32(store.list.items, T, index); - } - - pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") { - comptime assert(mutability == .@"var"); - return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } }; - } - - pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") { - return .{ .list = .{ .items = store.list.items } }; - } - }; -} - -pub const Float = struct { - low: u32, - high: u32, - - pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 }; - - pub fn purifyNaN(f: f64) f64 { - if (std.math.isNan(f)) { - return comptime Float.pure_nan.decode(); - } - return f; - } - - pub fn assertPure(f: f64) f64 { - if (std.math.isNan(f)) { - assert(Float.pure_nan.eql(encode(f))); - } - return f; - } - - pub fn encode(decoded: f64) Float { - if (std.math.isNan(decoded)) { - return pure_nan; - } - const as_bytes: [2]u32 = @bitCast(decoded); - return .{ - .low = as_bytes[0], - .high = as_bytes[1], - }; - } - - pub fn decode(encoded: Float) f64 { - const as_bytes: [2]u32 = .{ - encoded.low, - encoded.high, - }; - return @bitCast(as_bytes); - } - - pub fn eql(a: Float, b: Float) bool { - return a.low == b.low and a.high == b.high; - } -}; - -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; diff --git a/src/compiler/main.zig b/src/compiler/main.zig deleted file mode 100644 index ef8964f9cb66b4fdcee80059967ce6fcbb9b0a1d..0000000000000000000000000000000000000000 --- a/src/compiler/main.zig +++ /dev/null @@ -1,146 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const Tokenizer = @import("Tokenizer.zig"); -const Parse = @import("Parse.zig"); -const Transform = @import("Transform.zig"); -const Comp = @import("Comp.zig"); -const unicode_props = @import("unicode_props.zig"); - -const tracer = @import("tracer"); - -pub const tracer_impl = @field(tracer, @import("build_options").tracing); - -pub const Report = @import("Report.zig"); - -pub fn main() !void { - try tracer.init(); - defer tracer.deinit(); - try tracer.init_thread(null); - defer tracer.deinit_thread(); - - try unicode_props.init(); - - if (builtin.mode == .Debug) { - std.debug.print("\r\x1b[K", .{}); - } - - const use_gpa = builtin.mode == .Debug; - - var gpa: if (use_gpa) std.heap.GeneralPurposeAllocator(.{ - .stack_trace_frames = 100, - }) else struct {} = .{}; - defer if (use_gpa) { - _ = gpa.deinit(); - }; - const alloc = if (use_gpa) - gpa.allocator() - else - std.heap.c_allocator; - - var maybe_file_path: ?[]const u8 = null; - defer if (maybe_file_path) |path| alloc.free(path); - - var cwd_buf: [std.fs.max_path_bytes]u8 = undefined; - const cwd = try std.posix.getcwd(&cwd_buf); - - var exe_name: []const u8 = undefined; - var dump_ir = false; - - { - var args = try std.process.argsWithAllocator(alloc); - defer args.deinit(); - - exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli")); - errdefer alloc.free(exe_name); - - while (args.next()) |arg| { - if (std.mem.startsWith(u8, arg, "-")) { - if (builtin.mode == .Debug and std.mem.eql(u8, arg, "--report-trace")) { - Report.debug_show_report_trace = true; - continue; - } - if (std.mem.eql(u8, arg, "--dump-ir")) { - dump_ir = true; - continue; - } - std.debug.print("error: unknown flag '{s}'\n", .{arg}); - maybe_file_path = null; - break; - } - - if (maybe_file_path == null) { - maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg }); - } else { - std.debug.print("error: multiple files specified\n", .{}); - maybe_file_path = null; - break; - } - } - } - defer alloc.free(exe_name); - - const file_path = maybe_file_path orelse { - std.debug.print("Usage: {s} [flags] \n", .{exe_name}); - { - std.debug.print( - // | 60 chars wrap point| - \\ - \\Flags: - \\ --dump-ir Dump intermediate representations. - \\ - , .{}); - } - if (builtin.mode == .Debug) { - std.debug.print( - \\ - \\Debug Mode Flags: - \\ --report-trace Print stack traces per error. Very noisy - \\ - , .{}); - } - std.process.exit(1); - }; - - var comp: Comp = .{ - .gpa = tracer.tracingAllocator(&alloc), - .options = .{ - .dump_ir = dump_ir, - }, - }; - try comp.init(); - defer comp.deinit(); - const main_file = try comp.getFile(file_path); - - var fail_report: Report = undefined; - const teg = main_file.getTeg(&comp, &fail_report) catch |err| switch (err) { - error.TransformFail => { - std.debug.print("Transform failed:\n", .{}); - try fail_report.dumpToStdErr(&comp); - std.process.exit(1); - }, - error.ParseFail => { - // Already printed. - std.process.exit(1); - }, - else => return err, - }; - - if (comp.options.dump_ir) { - const stderr = std.io.getStdErr().writer(); - try teg.dumpToWriter(stderr); - } - - comp.semaEntryPoint(main_file) catch |err| switch (err) { - error.AnalysisFail => std.process.exit(1), - else => return err, - }; - - const output = std.io.getStdOut().writer(); - var bufw = std.io.bufferedWriter(output); - const writer = bufw.writer(); - try comp.link(writer); - try bufw.flush(); - - std.process.cleanExit(); -} diff --git a/src/compiler/unicode_props.zig b/src/compiler/unicode_props.zig deleted file mode 100644 index f7e2f98f9686de6527a169bb37435880a7a890e9..0000000000000000000000000000000000000000 --- a/src/compiler/unicode_props.zig +++ /dev/null @@ -1,103 +0,0 @@ -//! Wrapper around the 'zg' PropsData library. -const std = @import("std"); -const PropsData = @import("zg:PropsData"); -const T = @import("tracer"); - -var props: ?PropsData = undefined; -var props_memory: []u8 = undefined; - -const raw_allocator = if (@import("builtin").target.isWasm()) - std.heap.wasm_allocator -else - std.heap.page_allocator; - -pub fn init() !void { - if (props != null) return; - - const trace = T.trace(@src()); - defer trace.end(); - - // We know exactly how much this uses, so let's preallocate these pages and use a FBA. - const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856; - - if (allocation_size == 0) { - var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator); - props = try PropsData.init(logging_alloc.allocator()); - std.log.info("please write the above numbers into unicode.zig to prealloc this", .{}); - std.process.exit(1); - } - - props_memory = try raw_allocator.alloc(u8, allocation_size); - errdefer std.heap.page_allocator.free(props_memory); - - var fba = std.heap.FixedBufferAllocator.init(props_memory); - props = try PropsData.init(fba.allocator()); - std.debug.assert(fba.end_index == allocation_size); -} - -pub fn deinit() void { - std.debug.assert(props != null); - raw_allocator.free(props_memory); - props = null; - props_memory = undefined; -} - -/// True if `cp` is a mathematical symbol. -pub inline fn isMath(cp: u21) bool { - return props.?.isMath(cp); -} - -/// True if `cp` is an alphabetic character. -pub inline fn isAlphabetic(cp: u21) bool { - return props.?.isAlphabetic(cp); -} - -/// True if `cp` is a valid identifier start character. -pub inline fn isIdStart(cp: u21) bool { - return props.?.isIdStart(cp); -} - -/// True if `cp` is a valid identifier continuation character. -pub inline fn isIdContinue(cp: u21) bool { - return props.?.isIdContinue(cp); -} - -/// True if `cp` is a valid extended identifier start character. -pub inline fn isXidStart(cp: u21) bool { - return props.?.isXidStart(cp); -} - -/// True if `cp` is a valid extended identifier continuation character. -pub inline fn isXidContinue(cp: u21) bool { - return props.?.isXidContinue(cp); -} - -/// True if `cp` is a whitespace character. -pub inline fn isWhitespace(cp: u21) bool { - return props.?.isWhitespace(cp); -} - -/// True if `cp` is a hexadecimal digit. -pub inline fn isHexDigit(cp: u21) bool { - return props.?.isHexDigit(cp); -} - -/// True if `cp` is a diacritic mark. -pub inline fn isDiacritic(cp: u21) bool { - return props.?.isDiacritic(cp); -} - -/// True if `cp` is numeric. -pub inline fn isNumeric(cp: u21) bool { - return props.?.isNumeric(cp); -} - -/// True if `cp` is a digit. -pub inline fn isDigit(cp: u21) bool { - return props.?.isDigit(cp); -} - -/// True if `cp` is decimal. -pub inline fn isDecimal(cp: u21) bool { - return props.?.isDecimal(cp); -} diff --git a/src/compiler/util.zig b/src/compiler/util.zig deleted file mode 100644 index 6e244c52b8da94449206f1cf1ce0fcedd0fc369b..0000000000000000000000000000000000000000 --- a/src/compiler/util.zig +++ /dev/null @@ -1,56 +0,0 @@ -pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config { - if (@hasField(@TypeOf(writer), "unbuffered_writer")) { - return detectTTYFromWriter(writer.unbuffered_writer); - } - if (@TypeOf(writer) == std.fs.File.Writer) { - return std.io.tty.detectConfig(writer.context); - } - if (@TypeOf(writer) == std.io.AnyWriter) { - const fileWriteFn = comptime (std.fs.File{ .handle = undefined }) - .writer().any().writeFn; - if (writer.writeFn == fileWriteFn) { - const file: *const std.fs.File = @alignCast(@ptrCast(writer.context)); - return std.io.tty.detectConfig(file.*); - } - } - return .no_color; -} - -pub fn Ordinal(Int: type) type { - comptime assert(@typeInfo(Int).Int.signedness == .unsigned); - return enum(Int) { - start = 0, - _, - - pub fn fromZeroBased(int: Int) @This() { - assert(int != std.math.maxInt(Int)); - return @enumFromInt(int); - } - pub fn fromOneBased(int: Int) @This() { - assert(int > 0); - return @enumFromInt(int - 1); - } - pub fn zeroBased(ord: @This()) Int { - return @intFromEnum(ord); - } - pub fn oneBased(ord: @This()) Int { - return @intFromEnum(ord) + 1; - } - - pub fn add(ord: @This(), inc: Int) @This() { - return fromZeroBased(ord.zeroBased() + inc); - } - - pub fn format( - _: @This(), - comptime _: []const u8, - _: std.fmt.FormatOptions, - _: anytype, - ) !void { - @compileError("Do not format `Ordinal` directly. Call oneBased or zeroBased to specify."); - } - }; -} - -const std = @import("std"); -const assert = std.debug.assert; diff --git a/src/core/Air.Dump.zig b/src/core/Air.Dump.zig new file mode 100644 index 0000000000000000000000000000000000000000..51d7af7eaf89f4021e3893f0202526b803c38115 --- /dev/null +++ b/src/core/Air.Dump.zig @@ -0,0 +1,316 @@ +//! This implements a debug formatter for Teg +air: Air, +ip: *const InternPool, + +pub const State = struct { + arena: std.mem.Allocator, + air: *const Air, + ip: *const InternPool, + indent: usize = 0, + tty: std.io.tty.Config, + visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{}, + + fn newline(state: State, writer: anytype) !void { + try writer.writeByte('\n'); + if (state.indent > 0) + try writer.writeByteNTimes(' ', state.indent * 2); + } + + fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void { + const air = s.air; + const tty = s.tty; + const tag = s.air.instTag(inst); + + try tty.setColor(w, switch (s.air.instFlags(inst).uses) { + .none => .bright_black, + .multi => .red, + .single => .magenta, + }); + try w.writeByte(switch (s.air.instFlags(inst).uses) { + .none => 'X', + .multi => 'M', + .single => 'S', + }); + try tty.setColor(w, .cyan); + + try w.print(" #{} = ", .{ + @intFromEnum(inst), + }); + + try tty.setColor(w, .cyan); + try w.print("{s}", .{@tagName(tag)}); + try tty.setColor(w, .reset); + + if ((try s.visited.fetchPut(s.arena, inst, {})) != null) { + try tty.setColor(w, .red); + try w.writeAll(" [error: recursive tree]"); + try tty.setColor(w, .reset); + return; + } + + switch (tag) { + .block => { + const index_list: Air.IndexList = air.instData(.block, inst); + const insts = index_list.slice(air); + try s.dumpInstList(w, insts, '{', '}'); + }, + .js_global_ref => { + const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst); + try tty.setColor(w, .magenta); + try w.print(" {s}", .{name.toSlice(s.ip)}); + try tty.setColor(w, .reset); + }, + .arg => { + const arg: Air.Arg = air.instData(.arg, inst); + try w.writeAll(" "); + try tty.setColor(w, .yellow); + try w.print("{d}", .{arg.arg_number}); + try tty.setColor(w, .reset); + try w.writeAll(": "); + try s.dumpValue(w, arg.type.value()); + }, + .@"break" => { + const brk: Air.Break = air.instData(.@"break", inst); + try w.writeAll("("); + try s.dumpInstRef(w, brk.block_ref); + try w.writeAll(", "); + try s.dumpRef(w, brk.value); + try w.writeAll(")"); + }, + .store => { + const store: Air.Store = air.instData(.store, inst); + try w.writeAll("("); + try s.dumpInstRef(w, store.memory); + try w.writeAll(", "); + try s.dumpRef(w, store.operand); + try w.writeAll(")"); + }, + inline .add, + .mul, + .sub, + .div, + .lt, + .gt, + .eq, + => |comptime_tag| { + const bin: Air.Bin = air.instData(comptime_tag, inst); + try w.writeAll("("); + try s.dumpRef(w, bin.lhs); + try w.writeAll(", "); + try s.dumpRef(w, bin.rhs); + try w.writeAll(")"); + }, + inline .negate, + .bool_not, + => |comptime_tag| { + const bin: Air.Un = air.instData(comptime_tag, inst); + try w.writeAll("("); + try s.dumpRef(w, bin.operand); + try w.writeAll(")"); + }, + .memory => { + const memory: Air.Memory = air.instData(.memory, inst); + try w.writeAll("("); + try s.dumpRef(w, memory.init); + try w.writeAll(", type: "); + try s.dumpValue(w, memory.ty.value()); + try w.writeAll(")"); + }, + .field => { + const field: Air.Field = air.instData(.field, inst); + try w.writeAll("("); + try s.dumpRef(w, field.base); + try w.writeAll(", "); + try tty.setColor(w, .magenta); + try w.print("{s}", .{field.field.toSlice(s.ip)}); + try tty.setColor(w, .reset); + try w.writeAll(")"); + }, + .call => { + const call: Air.Call = air.instData(.call, inst); + const call_data = call.call_data.get(air); + try w.writeAll("("); + try s.dumpRef(w, call.base); + try w.writeAll(", "); + try s.dumpRefList(w, call_data.args.slice(air), '[', ']'); + try w.writeAll(")"); + }, + .@"if" => { + const if_inst: Air.If = air.instData(.@"if", inst); + const if_data: Air.If.Data = if_inst.data.get(air); + try w.writeAll("("); + try s.dumpRef(w, if_inst.cond); + try w.writeAll(", {"); + s.indent += 1; + try s.newline(w); + try w.writeAll("yes: "); + try s.dumpRefRecurse(w, if_data.yes); + try s.newline(w); + try w.writeAll("no: "); + try s.dumpRefRecurse(w, if_data.no); + s.indent -= 1; + try s.newline(w); + try w.writeAll("}"); + }, + } + } + + fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void { + switch (ref.decode()) { + .index => |i| try s.dumpInstRef(w, i), + .value => |v| try s.dumpValue(w, v), + } + } + + fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void { + switch (ref.decode()) { + .index => |i| try s.dumpInst(w, i), + .value => |v| try s.dumpValue(w, v), + } + } + + fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void { + try s.tty.setColor(w, .red); + try w.print("#{d}", .{@intFromEnum(inst)}); + try s.tty.setColor(w, .reset); + } + + pub fn dumpValue(s: *State, w: anytype, val: Value) !void { + const tty = s.tty; + switch (val.toKey(s.ip)) { + .simple_type => |t| { + try tty.setColor(w, .blue); + try w.print("{s}", .{@tagName(t)}); + try tty.setColor(w, .reset); + }, + .simple_value => |t| { + try tty.setColor(w, .yellow); + try w.print("{s}", .{@tagName(t)}); + try tty.setColor(w, .reset); + }, + .type_function => |ty| { + try tty.setColor(w, .blue); + try w.writeAll("fn("); + try tty.setColor(w, .reset); + for (ty.param_types, 0..) |param, i| { + if (i != 0) { + try tty.setColor(w, .blue); + try w.writeAll(","); + try tty.setColor(w, .reset); + } + try s.dumpValue(w, param.value()); + } + try tty.setColor(w, .blue); + try w.writeAll(") -> "); + try tty.setColor(w, .reset); + try s.dumpValue(w, ty.return_type.value()); + }, + inline .float, .int32, .comptime_float => |data, tag| { + try tty.setColor(w, .bright_black); + try w.writeAll("<"); + try tty.setColor(w, .blue); + try w.writeAll(@tagName(tag)); + try tty.setColor(w, .reset); + try tty.setColor(w, .bright_black); + try w.writeAll(", "); + try tty.setColor(w, .yellow); + try w.print("{d}", .{data}); + try tty.setColor(w, .bright_black); + try w.writeAll(">"); + try tty.setColor(w, .reset); + }, + .string => |data| { + try w.writeAll("<"); + try tty.setColor(w, .blue); + try w.writeAll("string"); + try tty.setColor(w, .reset); + try w.writeAll(", "); + try tty.setColor(w, .green); + try w.writeAll("\""); + try w.writeAll(data); + try w.writeAll("\""); + try tty.setColor(w, .reset); + try w.writeAll(">"); + }, + .func => |func| { + try tty.setColor(w, .bright_black); + try w.writeAll("<"); + try s.dumpValue(w, func.ty.value()); + try tty.setColor(w, .bright_black); + try w.writeAll(", "); + const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip); + try tty.setColor(w, .bright_green); + try w.print("decl '{s}'", .{decl_name}); + try tty.setColor(w, .bright_black); + try w.writeAll(">"); + try tty.setColor(w, .reset); + }, + } + } + + fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + if (list.len > 0) { + state.indent += 1; + + for (list) |inst| { + try state.newline(writer); + try state.dumpInst(writer, inst); + } + + state.indent -= 1; + try state.newline(writer); + } + try writer.print("{c}", .{r}); + } + + fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + if (list.len > 0) { + state.indent += 1; + + for (list) |inst| { + try state.newline(writer); + try state.dumpRef(writer, inst); + } + + state.indent -= 1; + try state.newline(writer); + } + try writer.print("{c}", .{r}); + } +}; + +pub fn format( + dump: @This(), + comptime unused_fmt: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) !void { + comptime assert(unused_fmt.len == 0); + + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + + const air = dump.air; + var state = State{ + .air = &air, + .ip = dump.ip, + .tty = detectTTYFromWriter(writer), + .arena = arena.allocator(), + }; + try state.dumpInst(writer, .main_block); + try writer.writeAll("\n"); + assert(state.indent == 0); +} + +const std = @import("std"); +const assert = std.debug.assert; + +const Air = @import("Air.zig"); +const Inst = Air.Inst; + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; + +const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/core/Air.zig b/src/core/Air.zig new file mode 100644 index 0000000000000000000000000000000000000000..4252ffa290e9295ce7318681d943d70aad6298ed --- /dev/null +++ b/src/core/Air.zig @@ -0,0 +1,320 @@ +insts: Inst.List.Slice, +extra: ExtraStore = .{}, + +pub const ExtraStore = dod.ExtraStore(.air, .@"const"); +pub const EncodedData = dod.EncodedData(Inst.Tag, u64); +pub const ExtraIndex = ExtraStore.Index; + +pub const Inst = struct { + tag: Tag, + data: EncodedData, + flags: Flags, + + pub const List = std.MultiArrayList(Inst); + + pub const Tag = enum { + block, + js_global_ref, + call, + @"break", + @"if", + arg, + field, + + memory, + store, + + // bin + add, + sub, + mul, + div, + + eq, + + lt, + gt, + + // un + negate, + bool_not, + + pub fn Data(comptime tag: Tag) type { + return switch (tag) { + .block => IndexList, + .js_global_ref => InternPool.String.WithLen, + .@"break" => Break, + .@"if" => If, + .call => Call, + .arg => Arg, + .field => Field, + .memory => Memory, + .store => Store, + .add, + .sub, + .mul, + .div, + .lt, + .gt, + .eq, + => Bin, + .negate, .bool_not => Un, + }; + } + }; + + /// reference to either a value or another instruction in this air + pub const Ref = packed struct(u32) { + kind: Kind, + data: u31, + + pub const Kind = enum(u1) { value, index }; + + pub const Decoded = union(enum) { + value: Value, + index: Index, + }; + + pub fn decode(ref: Ref) Decoded { + return switch (ref.kind) { + .value => .{ .value = @enumFromInt(ref.data) }, + .index => .{ .index = @enumFromInt(ref.data) }, + }; + } + + pub fn toOptional(ref: ?Ref) Optional { + return if (ref) |r| @enumFromInt(@as(u32, @bitCast(r))) else .none; + } + + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(opt: Optional) ?Ref { + return if (opt != .none) @bitCast(@intFromEnum(opt)) else null; + } + }; + + pub fn isVoid(ref: Ref) bool { + return ref.kind == .value and + ref.decode().value == .void; + } + + pub fn eql(ref: Ref, other: Ref) bool { + return @as(u32, @bitCast(ref)) == @as(u32, @bitCast(other)); + } + }; + + /// reference to another instruction in this air + /// note that these cannot point to code in other functions, such makes no sense + pub const Index = enum(u32) { + main_block, + _, + + pub fn toRef(index: Index) Ref { + return .{ + .kind = .index, + .data = @intCast(@intFromEnum(index)), + }; + } + + pub fn toOptional(index: Index) Optional { + return @enumFromInt(@intFromEnum(index)); + } + + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: Optional) ?Index { + return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null; + } + }; + }; +}; + +pub const Flags = packed struct { + uses: Uses = .none, + + pub const Uses = enum(u2) { + none, + single, + multi, + + pub fn increment(uses: Uses) Uses { + return switch (uses) { + .none => .single, + .single, .multi => .multi, + }; + } + }; +}; + +pub const IndexList = struct { + start: u32, + len: u32, + + pub fn slice(self: IndexList, air: *const Air) []const Inst.Index { + return @ptrCast(air.extra.list.items[self.start..][0..self.len]); + } +}; + +pub const RefList = struct { + start: u32, + len: u32, + + pub fn slice(self: RefList, air: *const Air) []const Inst.Ref { + return @ptrCast(air.extra.list.items[self.start..][0..self.len]); + } +}; + +pub const Break = struct { + block_ref: Inst.Index, + value: Inst.Ref, +}; + +pub const Bin = struct { + lhs: Inst.Ref, + rhs: Inst.Ref, +}; + +pub const Store = struct { + memory: Inst.Index, + operand: Inst.Ref, +}; + +pub const Un = struct { + operand: Inst.Ref, + unused: Unused = .zero, +}; + +pub const Field = struct { + base: Inst.Ref, + field: InternPool.NullTerminatedString, +}; + +pub const Call = struct { + base: Inst.Ref, + call_data: ExtraIndex(Data), + + pub const Data = struct { + args: RefList, + }; +}; + +pub const Arg = struct { + type: Type, + arg_number: u32, +}; + +pub const If = struct { + cond: Inst.Ref, + data: ExtraIndex(Data), + + pub const Data = struct { + yes: Inst.Ref, + no: Inst.Ref, + union_ty: Type, + }; +}; + +pub const Memory = struct { + init: Inst.Ref, + ty: Type, +}; + +pub fn deinit(air: *Air, gpa: std.mem.Allocator) void { + defer air.* = undefined; + air.insts.deinit(gpa); + air.extra.deinit(gpa); +} + +pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag { + return air.insts.items(.tag)[@intFromEnum(index)]; +} + +pub fn instFlags(air: *const Air, index: Inst.Index) Flags { + return air.insts.items(.flags)[@intFromEnum(index)]; +} + +pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData { + return air.insts.items(.data)[@intFromEnum(index)]; +} + +pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() { + return air.instEncodedData(index).decode(tag, air.instTag(index)); +} + +pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T { + return air.instData(tag, index).extra_index.get(air); +} + +pub fn typeOf(air: *const Air, ref: Inst.Ref, ip: *InternPool) InternPool.Type { + return switch (ref.decode()) { + .value => |v| v.typeOf(ip), + .index => |i| air.typeOfIndex(i, ip), + }; +} + +pub fn typeOfIndex(air: *const Air, index: Inst.Index, ip: *InternPool) InternPool.Type { + return switch (air.instTag(index)) { + .js_global_ref => .@"opaque", + + .block => { + const items = air.instData(.block, index).slice(air); + return air.typeOfIndex(items[items.len - 1], ip); + }, + + inline .add, .sub, .mul, .div => |t| air.typeOf(air.instData(t, index).lhs, ip), + + .lt, .gt, .eq, .bool_not => .bool, + + .arg => air.instData(.arg, index).type, + + .negate => air.typeOf(air.instData(.negate, index).operand, ip), + + .@"break" => .noreturn, + + // todo: + .field => .@"opaque", + + .call => { + const base_type = air.typeOf(air.instData(.call, index).base, ip); + if (base_type == .@"opaque") return .@"opaque"; + return base_type.value().toKey(ip).type_function.return_type; + }, + + .memory => air.instData(.memory, index).ty, + + .store => air.instData(.memory, air.instData(.store, index).memory).ty, + + .@"if" => air.instData(.@"if", index).data.get(air).union_ty, + }; +} + +pub fn paramCount(air: Air) u32 { + for (air.insts.items(.tag)[1..], 0..) |tag, i| { + if (tag != .arg) return @intCast(i); + } + return @intCast(air.insts.len); +} + +pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void { + try writer.print("{}", .{Dump{ .air = air, .ip = ip }}); +} + +pub const Dump = @import("Air.Dump.zig"); + +const Air = @This(); +const std = @import("std"); +const mem = std.mem; +const assert = std.debug.assert; + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; +const Type = InternPool.Type; + +const Teg = @import("Teg.zig"); +const dod = @import("array.zig"); + +const Unused = enum(u32) { zero = 0 }; diff --git a/src/core/Ast.Dump.zig b/src/core/Ast.Dump.zig new file mode 100644 index 0000000000000000000000000000000000000000..33186d4e05d9b4ffaa86a9542af1b0698d9ef82c --- /dev/null +++ b/src/core/Ast.Dump.zig @@ -0,0 +1,300 @@ +//! A flat array is not very useful for debugging, so this structure implements +//! a debug formatter for Ast. +ast: Ast, + +const State = struct { + ast: *const Ast, + indent: usize = 0, + node_tags: []const Node.Tag, + node_data: []const Node.Data, + node_tokens: []const TokenIndex, + tty: std.io.tty.Config, + + fn newline(state: State, writer: anytype) !void { + try writer.writeByte('\n'); + if (state.indent > 0) + try writer.writeByteNTimes(' ', state.indent * 2); + } +}; + +pub fn format( + dump: Dump, + comptime unused_fmt: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) !void { + comptime assert(unused_fmt.len == 0); + const ast = dump.ast; + var state = State{ + .ast = &ast, + .node_tags = ast.nodes.items(.tag), + .node_data = ast.nodes.items(.data), + .node_tokens = ast.nodes.items(.main_token), + .tty = detectTTYFromWriter(writer), + }; + try dumpNode(writer, &state, 0); + try writer.writeAll("\n"); + assert(state.indent == 0); +} + +fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void { + const tag = state.node_tags[node]; + const data = state.node_data[node]; + const data_tag = Node.data_tags[@intFromEnum(tag)]; + const main_token = state.node_tokens[node]; + const tty = state.tty; + + try state.tty.setColor(writer, .bright_black); + try writer.print("{d}", .{node}); + try state.tty.setColor(writer, .reset); + try writer.writeAll(" "); + + try tty.setColor(writer, .cyan); + try writer.print("{s}", .{@tagName(tag)}); + try tty.setColor(writer, .reset); + + switch (tag) { + .builtin_call => { + try tty.setColor(writer, .red); + try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}); + try tty.setColor(writer, .reset); + }, + else => {}, + } + + switch (data_tag) { + .none => { + switch (tag) { + .id => try tty.setColor(writer, .red), + .string => try tty.setColor(writer, .green), + .number_int, .number_float => try tty.setColor(writer, .yellow), + else => {}, + } + switch (tag) { + .id, + .string, + .number_int, + .number_float, + => try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}), + + else => unreachable, + } + try tty.setColor(writer, .reset); + }, + .fn_decl => { + try writer.writeAll(" "); + const fn_decl = state.ast.fullFnDecl(node); + if (fn_decl.public) { + try writer.writeAll("[pub] "); + } + try tty.setColor(writer, .magenta); + try writer.writeAll(fn_decl.id); + try tty.setColor(writer, .reset); + try writer.writeAll(" {"); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try writer.writeAll("params:"); + try dumpNodeList(writer, state, fn_decl.params, '[', ']'); + if (fn_decl.return_type) |return_type| { + try state.newline(writer); + try writer.writeAll("return_type: "); + try dumpNode(writer, state, return_type); + } + try state.newline(writer); + try writer.writeAll("body: "); + try dumpNode(writer, state, fn_decl.body); + } + try state.newline(writer); + try writer.writeAll("}"); + }, + .list => { + const list = data.list.slice(state.ast); + switch (tag) { + .root, + .block_void, + .block_implicit_return, + => try dumpNodeList(writer, state, list, '{', '}'), + .builtin_call, + => try dumpNodeList(writer, state, list, '(', ')'), + else => unreachable, + } + }, + .var_decl => { + try writer.writeAll(" "); + const var_decl = state.ast.fullVarDecl(node); + if (var_decl.public) { + try writer.writeAll("[pub] "); + } + try writer.writeAll(switch (var_decl.is_const) { + true => "constant", + false => "mutable", + }); + try tty.setColor(writer, .magenta); + try writer.print(" {s}", .{var_decl.id}); + try tty.setColor(writer, .reset); + var indent = false; + if (var_decl.type) |type_node| { + state.indent += 1; + indent = true; + try state.newline(writer); + try writer.writeAll(": "); + try dumpNode(writer, state, type_node); + try writer.writeAll(", "); + } + try writer.writeAll(" = "); + try dumpNode(writer, state, var_decl.init); + if (indent) { + state.indent -= 1; + } + }, + .param_decl => { + try writer.writeAll(" "); + const param_decl = state.ast.fullParamDecl(node); + try tty.setColor(writer, .magenta); + try writer.writeAll(param_decl.id); + try tty.setColor(writer, .reset); + try writer.writeAll(": "); + try dumpNode(writer, state, param_decl.type); + }, + .bin => { + try writer.writeAll(" ("); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try dumpNode(writer, state, data.bin.lhs); + try state.newline(writer); + try dumpNode(writer, state, data.bin.rhs); + } + try state.newline(writer); + try writer.writeAll(")"); + }, + .un => { + try writer.writeAll(" ("); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try dumpNode(writer, state, data.un.expr); + } + try state.newline(writer); + try writer.writeAll(")"); + }, + .label => { + const label_text = state.ast.tokenSlice(main_token); + try writer.print(" {s}:", .{label_text}); + + try writer.writeAll(" ("); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try dumpNode(writer, state, data.label.expr); + } + try state.newline(writer); + try writer.writeAll(")"); + }, + .field => { + try writer.writeAll(" ("); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try dumpNode(writer, state, data.field.base); + } + try state.newline(writer); + try writer.writeAll(")"); + }, + .@"return" => { + if (data.@"return".expr.unwrap()) |expr| { + try writer.writeAll(" ("); + try dumpNode(writer, state, expr); + try writer.writeAll(")"); + } + }, + .@"break" => { + const label = data.@"break".label; + const value = data.@"break".value; + if (label != .none or value != .none) { + // try writer.writeAll("("); + // try writer.writeAll(")"); + if (label.unwrap()) |l| { + const label_text = state.ast.tokenSlice(l); + try writer.print(" :{s}", .{label_text}); + } + if (value.unwrap()) |expr| { + try writer.writeAll(" ("); + try dumpNode(writer, state, expr); + try writer.writeAll(")"); + } + } + }, + .call => { + const base = data.call.base; + const extra = state.ast.extraData(data.call.extra, Call); + const arg_list = extra.args.slice(state.ast); + + try writer.writeAll("("); + try writer.writeAll("base: "); + try dumpNode(writer, state, base); + try writer.writeAll(", "); + try writer.writeAll("arg:"); + try dumpNodeList(writer, state, arg_list, '[', ']'); + try writer.writeAll(")"); + }, + .@"if" => { + const branch = data.@"if"; + const if_data = state.ast.extraData(branch.extra, Ast.If); + + try writer.writeAll(" ("); + try dumpNode(writer, state, branch.cond); + try writer.writeAll(") {"); + { + state.indent += 1; + defer state.indent -= 1; + try state.newline(writer); + try writer.writeAll("yes: "); + try dumpNode(writer, state, if_data.yes); + try state.newline(writer); + if (if_data.no.unwrap()) |no| { + try writer.writeAll("no: "); + try dumpNode(writer, state, no); + } else { + try writer.writeAll("no: [none]"); + } + } + try state.newline(writer); + try writer.writeAll("}"); + }, + } +} + +fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + { + state.indent += 1; + defer state.indent -= 1; + + for (list) |node| { + try state.newline(writer); + try dumpNode(writer, state, node); + } + } + try state.newline(writer); + try writer.print("{c}", .{r}); +} + +const Dump = @This(); + +const std = @import("std"); +const assert = std.debug.assert; + +const Ast = @import("Ast.zig"); +const NodeIndex = Ast.NodeIndex; +const Node = Ast.Node; +const TokenIndex = Ast.TokenIndex; +const Call = Ast.Call; + +const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/core/Ast.zig b/src/core/Ast.zig new file mode 100644 index 0000000000000000000000000000000000000000..e11c6d8fa3ed6be8c5d0d1be924fcfe2707e062f --- /dev/null +++ b/src/core/Ast.zig @@ -0,0 +1,678 @@ +//! The Abstract Syntax Tree preserves most details of the source code, +//! as it is used for formatting. Before semantic analysis, this +//! structure is normalized into `Teg`, which removes constructs like +//! grouping, unneded let bindings, and other transforms. +//! +//! `ast.nodes` contains nodes stored as three parts: +//! - tag: the type of the node +//! - main_token: index to it's primary token, depends on the tag +//! - data: two u32 values, depends on the tag +//! +//! Many nodes encode all their metadata with just `node.data`, using +//! facts about the language grammar to be able to infer data without +//! storing it. +//! +//! For example, `var_decl` has the following encoding: +//! - `main_token` is either a `kw_var` or `kw_let` +//! - the token after `main_token` is always the identifier +//! - `data.var_decl` contains `NodeIndex`es pointing at the type +//! and the initializer, which are both expressions. +//! - If the keyword before `main_token` is `kw_pub` it is a public variable. +//! +//! To make decoding these easier, many nodes contain a "full" variant, +//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`. +//! +//! For nodes that must store more than 64 bits of payload data, an "extra" +//! struct is stored out-of-line in the `extra` field. These can be read +//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use +//! the `full` structures with their convenience functions. +//! +//! Documentation comments are not stored anywhere, but you can find them +//! by iterating backwards from a declaration and seeing if a `.doc_comment` +//! is present. This is provided by `ast.declDocComment(id)`. +//! +//! The `Ast` owns token and node data, but not the source code. + +/// Externally owned source code. +utf8_source: [:0]const u8, + +tokens: Token.List.Slice, +nodes: Node.List.Slice, +extra: []const u32, + +pub const Node = struct { + tag: Tag, + main_token: TokenIndex, + data: Data, + + pub const List = std.MultiArrayList(Node); + + pub const Tag = enum { + /// Read with `ast.rootNodes()` + /// data is .list (containing stmt slice) + /// TODO: rename to `module` + root, + /// main_token is the { + /// data is .list (containing stmt slice) + /// + /// This guarantees that the block either empty, or the last statement + /// ends with a semicolon. + block_void, + /// main_token is the { + /// data is .list (containing stmt slice) + /// + /// This guarantees that the block is not empty and that the last + /// statement is an expression statement without a trailing semicolon. + block_implicit_return, + /// Variable or constant declaration + /// `var x: type = init;` + /// `let y = x;` + /// main_token is the keyword + /// data is .var_decl + var_decl, + /// Function declaration + /// `fn foo(x: i32) -> i32 { rhs }` + /// Params and return type are stored in extra data + fn_decl, + /// 'name: type' declaration + /// Main token is the identifier + param_decl, + /// An identifier. main_token is the identifier. + id, + /// Main token is the data. May not fit in i32 range, + /// but will contain no dots or e's. + number_int, + /// Main token is data. Will contain '.' and/or 'e'. + number_float, + /// A string literal + string, + /// A multi-line string literal + /// data is .token_list of the tokens, which is guaranteed to be contiguous + multiline_string, + /// Call to a function + /// `foo(x, y);` + /// main_token is the `(` + /// data is .call + call, + /// Call to a builtin function + /// `@raw("hello");` + /// data is .list (containing arg slice) + builtin_call, + /// `if cond { ... } else ...` + /// main_token is the `if` + /// data is .@"if" + @"if", + /// `if let id = ... { ... } else ...` + /// main_token is the `if` + /// data is .@"if" + // if_let_nonvoid, + /// `id: ...` + /// data is un + label, + /// `expr.field` + /// main_token is the `.` + /// data is .field + field, + /// `lhs[rhs]` + /// main_token is the `[` + /// data is .bin + index, + /// `return expr;` + /// main_token is the `return` + /// data is .@"return" + @"return", + /// `break :brk expr;` + /// main_token is the `break` + /// data is .@"break" + @"break", + /// `(expr)` + /// main_token is the `(` + /// data is .un + group, + /// `lhs + rhs`. main_token is the operator. + add, + /// `lhs - rhs`. main_token is the operator. + sub, + /// `lhs * rhs`. main_token is the operator. + mul, + /// `lhs / rhs`. main_token is the operator. + div, + /// `lhs % rhs`. main_token is the operator. + rem, + /// `lhs ** rhs`. main_token is the operator. + pow, + /// `lhs && rhs`. main_token is the operator. + @"and", + /// `lhs || rhs`. main_token is the operator. + @"or", + /// `lhs & rhs`. main_token is the operator. + bit_and, + /// `lhs | rhs`. main_token is the operator. + bit_or, + /// `lhs ^ rhs`. main_token is the operator. + bit_xor, + /// `lhs << rhs`. main_token is the operator. + shl, + /// `lhs >> rhs`. main_token is the operator. + shr, + /// `lhs == rhs`. main_token is the operator. + eq, + /// `lhs != rhs`. main_token is the operator. + ne, + /// `lhs < rhs`. main_token is the operator. + lt, + /// `lhs <= rhs`. main_token is the operator. + le, + /// `lhs > rhs`. main_token is the operator. + gt, + /// `lhs >= rhs`. main_token is the operator. + ge, + /// `lhs = rhs`. main_token is the operator. + assign, + /// `lhs += rhs`. main_token is the operator. + add_assign, + /// `lhs -= rhs`. main_token is the operator. + sub_assign, + /// `lhs *= rhs`. main_token is the operator. + mul_assign, + /// `lhs /= rhs`. main_token is the operator. + div_assign, + /// `lhs %= rhs`. main_token is the operator. + rem_assign, + /// `lhs **= rhs`. main_token is the operator. + pow_assign, + /// `lhs &&= rhs`. main_token is the operator. + and_assign, + /// `lhs ||= rhs`. main_token is the operator. + or_assign, + /// `lhs &= rhs`. main_token is the operator. + bit_and_assign, + /// `lhs |= rhs`. main_token is the operator. + bit_or_assign, + /// `lhs ^= rhs`. main_token is the operator. + bit_xor_assign, + /// `lhs <<= rhs`. main_token is the operator. + shl_assign, + /// `lhs >>= rhs`. main_token is the operator. + shr_assign, + /// `!expr`. main_token is the operator. + bool_not, + /// `-expr`. main_token is the operator. + negate, + /// `~expr`. main_token is the operator. + bit_not, + /// `comptime expr`. main_token is the operator. + @"comptime", + /// `pure expr`. main_token is the operator. + // pure, + /// `assert expr`. main_token is the operator. + // assert, + + pub fn isAssignBin(op: Tag) bool { + return switch (op) { + .assign, + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => true, + .add, + .sub, + .mul, + .div, + .rem, + .pow, + .@"and", + .@"or", + .bit_and, + .bit_or, + .bit_xor, + .shl, + .shr, + .eq, + .ne, + .lt, + .le, + .gt, + .ge, + => false, + else => unreachable, // not a binary operator + }; + } + + pub fn isBasic(op: Tag) bool { + return switch (op) { + .id, + .string, + .number_int, + .number_float, + .builtin_call, + => true, + else => false, + }; + } + }; + + pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{ + .root = .list, + .block_void = .list, + .block_implicit_return = .list, + .var_decl = .var_decl, + .fn_decl = .fn_decl, + .param_decl = .param_decl, + .id = .none, + .number_int = .none, + .number_float = .none, + .string = .none, + .multiline_string = .list, + .call = .call, + .builtin_call = .list, + .@"if" = .@"if", + .field = .field, + .index = .bin, + .@"return" = .@"return", + .@"break" = .@"break", + .group = .un, + .add = .bin, + .sub = .bin, + .mul = .bin, + .div = .bin, + .rem = .bin, + .pow = .bin, + .@"and" = .bin, + .@"or" = .bin, + .bit_and = .bin, + .bit_or = .bin, + .bit_xor = .bin, + .shl = .bin, + .shr = .bin, + .eq = .bin, + .ne = .bin, + .lt = .bin, + .le = .bin, + .gt = .bin, + .ge = .bin, + .assign = .bin, + .add_assign = .bin, + .sub_assign = .bin, + .mul_assign = .bin, + .div_assign = .bin, + .rem_assign = .bin, + .pow_assign = .bin, + .and_assign = .bin, + .or_assign = .bin, + .bit_and_assign = .bin, + .bit_or_assign = .bin, + .bit_xor_assign = .bin, + .shl_assign = .bin, + .shr_assign = .bin, + .bool_not = .un, + .negate = .un, + .bit_not = .un, + .@"comptime" = .un, + .label = .label, + }); + + pub const Data = union { + none: struct { Unused = .zero, Unused = .zero }, + fn_decl: struct { + // A `FnDecl` + extra: ExtraIndex, + body: NodeIndex, + }, + /// either is a node list or a token list, depending on the node's tag + list: SubList, + var_decl: struct { + type: OptionalNodeIndex, + init: NodeIndex, + }, + param_decl: struct { + type: NodeIndex, + unused: Unused = .zero, + }, + bin: struct { + lhs: NodeIndex, + rhs: NodeIndex, + }, + field: struct { + base: NodeIndex, + field_token: TokenIndex, + }, + un: struct { + expr: NodeIndex, + unused: Unused = .zero, + }, + label: struct { + expr: NodeIndex, + unused: Unused = .zero, + }, + @"return": struct { + expr: OptionalNodeIndex, + unused: Unused = .zero, + }, + @"break": struct { + label: OptionalTokenIndex, + value: OptionalNodeIndex, + }, + call: struct { + base: NodeIndex, + /// A `Call` + extra: ExtraIndex, + }, + @"if": struct { + cond: NodeIndex, + /// A `If` + extra: ExtraIndex, + }, + + const FieldEnum = std.meta.FieldEnum(Data); + }; +}; + +pub const Error = error{ OutOfMemory, ParseFail }; + +pub const Location = struct { + line: Ordinal(u32), + column: Ordinal(u32), + // Byte offset of the start of the line + line_start: ByteOffset, + // Byte offset of the end of the line + line_end: ByteOffset, +}; + +pub const Span = struct { + start: u32, + end: u32, + main: u32, +}; + +// This can be removed when the following issue is implemented: +// See: https://github.com/ziglang/zig/issues/104 +// See: https://github.com/ziglang/zig/issues/3806 +pub const OptionalNodeIndex = enum(u32) { + none = 0, + _, + + pub fn init(idx: ?NodeIndex) OptionalNodeIndex { + assert(idx != 0); + return @enumFromInt(idx orelse return .none); + } + + pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex { + return switch (idx) { + .none => null, + _ => @intFromEnum(idx), + }; + } +}; + +pub const OptionalTokenIndex = enum(u32) { + none = 0, + _, + + pub fn init(idx: ?TokenIndex) OptionalTokenIndex { + assert(idx != 0); + return @enumFromInt(idx orelse return .none); + } + + pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex { + return switch (idx) { + .none => null, + _ => @intFromEnum(idx), + }; + } +}; + +pub const OptionalExtraIndex = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn init(idx: ?ExtraIndex) OptionalExtraIndex { + assert(idx != std.math.maxInt(u32)); + return @enumFromInt(idx orelse return .none); + } + + pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex { + return switch (idx) { + .none => null, + _ => @intFromEnum(idx), + }; + } +}; + +pub const SubList = struct { + /// Either NodeIndex or TokenIndex, depending on where this came from. + start: u32, + len: u32, + + pub fn slice(self: SubList, ast: *const Ast) []const u32 { + return ast.extra[self.start..][0..self.len]; + } +}; + +pub const FnDecl = struct { + params: SubList, + return_type: OptionalNodeIndex, +}; + +pub const Call = struct { + args: SubList, +}; + +pub const If = struct { + /// either .block_void or .block_implicit_return + yes: NodeIndex, + /// if it exists, it is either + /// .block_void, .block_implicit_return, or .@"if" + no: OptionalNodeIndex, +}; + +pub const full = struct { + pub const FnDecl = struct { + start_token: TokenIndex, + public: bool, + id: []const u8, + params: []const NodeIndex, + return_type: ?NodeIndex, + body: NodeIndex, + }; + + pub const VarDecl = struct { + start_token: TokenIndex, + public: bool, + is_const: bool, + id: []const u8, + type: ?NodeIndex, + init: NodeIndex, + }; + + pub const ParamDecl = struct { + start_token: TokenIndex, + id: []const u8, + type: NodeIndex, + }; +}; + +pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void { + ast.tokens.deinit(gpa); + ast.nodes.deinit(gpa); + gpa.free(ast.extra); + ast.* = undefined; +} + +pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag { + return ast.nodes.items(.tag)[node]; +} + +pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data { + return ast.nodes.items(.data)[node]; +} + +pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex { + return ast.nodes.items(.main_token)[node]; +} + +pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl { + const tok_tags = ast.tokens.items(.tag); + + const main_token = ast.nodes.items(.main_token)[node]; + const public = tok_tags[main_token -| 1] == .kw_pub; + assert(tok_tags[main_token] == .kw_fn); + assert(tok_tags[main_token + 1] == .id); + const id = ast.tokenSlice(main_token + 1); + + const data = ast.nodes.items(.data)[node].fn_decl; + const extra = ast.extraData(data.extra, FnDecl); + + return .{ + .start_token = if (public) main_token - 1 else main_token, + .public = public, + .id = id, + .body = data.body, + .params = extra.params.slice(&ast), + .return_type = extra.return_type.unwrap(), + }; +} + +pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl { + const tok_tags = ast.tokens.items(.tag); + + const main_token = ast.nodes.items(.main_token)[node]; + const public = tok_tags[main_token -| 1] == .kw_pub; + const is_const = switch (tok_tags[main_token]) { + .kw_let => true, + .kw_var => false, + else => |tag| { + if (std.debug.runtime_safety) { + std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)}); + } + unreachable; + }, + }; + assert(tok_tags[main_token + 1] == .id); + const id = ast.tokenSlice(main_token + 1); + const data = ast.nodes.items(.data)[node].var_decl; + return .{ + .start_token = if (public) main_token - 1 else main_token, + .public = public, + .is_const = is_const, + .id = id, + .type = data.type.unwrap(), + .init = data.init, + }; +} + +pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl { + const tok_tags = ast.tokens.items(.tag); + + const main_token = ast.nodes.items(.main_token)[node]; + assert(tok_tags[main_token] == .id); + const id = ast.tokenSlice(main_token); + const data = ast.nodes.items(.data)[node].param_decl; + return .{ + .start_token = main_token, + .id = id, + .type = data.type, + }; +} + +pub fn idToken(ast: Ast, node: NodeIndex) TokenIndex { + return switch (ast.nodeTag(node)) { + .param_decl, + .id, + .label, + => ast.nodeToken(node), + .fn_decl, + .var_decl, + => ast.nodeToken(node) + 1, + else => unreachable, // does not have an id + }; +} + +pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex { + const data = ast.nodes.items(.data)[node].list; + return data.slice(ast); +} + +pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location { + const token_start = self.tokens.items(.offset)[token_index]; + return Tokenizer.tokenLocation(self.utf8_source, start_offset, token_start); +} + +pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 { + return ast.tokens.get(token_index).slice(ast.utf8_source); +} + +pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 { + assert(ast.nodes.items(.tag)[node].isBasic()); + const token = ast.nodes.items(.main_token)[node]; + return ast.tokenSlice(token); +} + +pub fn fnDeclIdToken(tree: Ast, node: NodeIndex) TokenIndex { + assert(tree.nodeTag(node) == .fn_decl); + return tree.nodeToken(node) + 1; +} + +pub fn extraData(tree: Ast, index: usize, comptime T: type) T { + const fields = std.meta.fields(T); + var result: T = undefined; + var i: usize = 0; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + NodeIndex => tree.extra[index + i], + SubList => SubList{ + .start = tree.extra[index + i], + .len = tree.extra[index + i + 1], + }, + OptionalNodeIndex => @enumFromInt(tree.extra[index + i]), + OptionalTokenIndex => @enumFromInt(tree.extra[index + i]), + OptionalExtraIndex => @enumFromInt(tree.extra[index + i]), + else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)), + }; + i += switch (field.type) { + NodeIndex, + OptionalNodeIndex, + OptionalTokenIndex, + OptionalExtraIndex, + => 1, + SubList => 2, + else => comptime unreachable, + }; + } + return result; +} + +pub fn dumpToWriter(ast: Ast, writer: anytype) !void { + try writer.print("{}", .{Dump{ .ast = ast }}); +} + +pub const Dump = @import("Ast.Dump.zig"); + +pub const Unused = enum(u32) { zero = 0 }; +pub const TokenIndex = u32; +pub const ExtraIndex = u32; +pub const NodeIndex = u32; + +const Ast = @This(); +const std = @import("std"); +const assert = std.debug.assert; + +const Tokenizer = @import("Tokenizer.zig"); +const Token = Tokenizer.Token; +const TokenList = Tokenizer.Token.List; +const ByteOffset = Tokenizer.ByteOffset; + +const Report = @import("Report.zig"); + +const util = @import("util.zig"); +const Ordinal = util.Ordinal; diff --git a/src/core/Cache.zig b/src/core/Cache.zig new file mode 100644 index 0000000000000000000000000000000000000000..49a4a49d428577d564b2d7495a65c21c68539943 --- /dev/null +++ b/src/core/Cache.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 +pub const bin_digest_len = 16; +pub const hex_digest_len = bin_digest_len * 2; +pub const BinDigest = [bin_digest_len]u8; +pub const HexDigest = [hex_digest_len]u8; + +pub const HashHelper = std.Build.Cache.HashHelper; diff --git a/src/core/Comp.zig b/src/core/Comp.zig new file mode 100644 index 0000000000000000000000000000000000000000..9c9791273104a5940a38ae8a40328ba6785112a9 --- /dev/null +++ b/src/core/Comp.zig @@ -0,0 +1,443 @@ +//! Compilation + +/// General purpose allocator +gpa: Allocator, +/// Stores all Type and Value objects. +intern_pool: InternPool = .{}, +/// All files in the compilation +files: std.StringArrayHashMapUnmanaged(*File) = .{}, + +/// # of sema instructions in this compilation cycle +analyzed_instructions: usize = 0, + +options: Options = .{}, + +// TODO: consider computing this dynamicly per-link +current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{}, + +pub fn init(comp: *Comp) !void { + try comp.intern_pool.init(comp.gpa); +} + +pub fn deinit(comp: *Comp) void { + comp.intern_pool.deinit(comp.gpa); + for (comp.files.values()) |file| { + file.deinit(comp.gpa); + } + comp.files.deinit(comp.gpa); + comp.current_compile_decls.deinit(comp.gpa); +} + +pub fn getFile(comp: *Comp, absolute_file_path: []const u8) !*File { + assert(std.fs.path.isAbsolute(absolute_file_path)); + const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path); + if (gop.found_existing) { + return gop.value_ptr.*; + } + const file = try comp.gpa.create(File); + file.* = .{ + .file_path = absolute_file_path, + .path_digest = null, + .root_module = .none, + .status = .never_loader, + .source = null, + .ast = null, + .teg = null, + }; + gop.value_ptr.* = file; + return file; +} + +pub const Source = struct { + code: [:0]const u8, + stat: Stat, +}; + +pub const File = struct { + /// TODO: change this to a URL + file_path: []const u8, + + root_module: Decl.Index.Optional, + status: enum { + never_loader, + /// A failure occurred during loading the file. Retrying might succeed. + retryable_failure, + /// Errors during parse + parse_failure, + /// Errors during transform + transform_failure, + /// Teg is valid + success, + }, + + source: ?Source, + ast: ?Ast, + teg: ?Teg, + + path_digest: ?Cache.BinDigest, + + const GetSourceError = + std.fs.File.OpenError || + std.fs.File.ReadError || + std.fs.File.StatError || + Allocator.Error || error{ + FileTooBig, + UnexpectedEndOfFile, + }; + + pub fn getSource(file: *File, gpa: Allocator) GetSourceError!Source { + if (file.source) |source| { + return source; + } + + // Keep track of inode, file size, mtime, hash so we can detect which files + // have been modified when an incremental update is requested. + var f = try std.fs.openFileAbsolute(file.file_path, .{}); + defer f.close(); + + const stat = try f.stat(); + + if (stat.size > std.math.maxInt(u32)) + return error.FileTooBig; + + const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); + errdefer gpa.free(source); + const amt = try f.readAll(source); + if (amt != stat.size) + return error.UnexpectedEndOfFile; + + // Here we do not modify stat fields because this function is the one + // used for error reporting. We need to keep the stat fields stale so that + // transformFile can know to regenerate Teg. + + const loaded_source = Source{ + .code = source, + .stat = .{ + .size = stat.size, + .inode = stat.inode, + .mtime = stat.mtime, + }, + }; + file.source = loaded_source; + return loaded_source; + } + + pub const GetAstError = GetSourceError || Parse.Error; + + pub fn getAst(file: *File, comp: *Comp) GetAstError!Ast { + if (file.ast) |ast| { + return ast; + } + + var err_data: Parse.AstError.WithTrace = undefined; + const ast = Parse.parse( + comp.gpa, + (try file.getSource(comp.gpa)).code, + &err_data, + ) catch |err| switch (err) { + error.OutOfMemory => return err, + error.ParseFail => { + // TODO: let this merge with other report + var report = try err_data.toReport(comp, file); + report.dumpToStdErr(comp) catch {}; + report.deinit(comp.gpa); + return err; + }, + }; + + if (comp.options.dump_ir) { + ast.dumpToWriter(std.io.getStdErr().writer()) catch { + // Simplify the error set. + }; + } + + file.ast = ast; + return ast; + } + + pub const GetTegError = GetAstError || Transform.Error; + + pub fn getTeg(file: *File, comp: *Comp, fail_report: *Report) GetTegError!Teg { + if (file.teg) |teg| { + return teg; + } + + const bin_digest = hash: { + var path_hash: Cache.HashHelper = .{}; + path_hash.addBytes("0.1.0"); + path_hash.addBytes(file.file_path); + var bin: Cache.BinDigest = undefined; + path_hash.hasher.final(&bin); + break :hash bin; + }; + file.path_digest = bin_digest; + + const teg = try Transform.transform( + try file.getAst(comp), + comp.gpa, + file, + fail_report, + ); + + file.teg = teg; + return teg; + } + + pub fn unloadSource(file: *File, gpa: Allocator) void { + if (file.source) |source| { + gpa.free(source.code); + file.source = null; + } + } + + pub fn unloadAst(file: *File, gpa: Allocator) void { + if (file.ast) |*ast| { + ast.deinit(gpa); + file.ast = null; + } + } + + pub fn unloadTeg(file: *File, gpa: Allocator) void { + if (file.teg) |*teg| { + teg.deinit(gpa); + file.teg = null; + } + } + + pub fn deinit(file: *File, gpa: Allocator) void { + file.unloadTeg(gpa); + file.unloadAst(gpa); + file.unloadSource(gpa); + gpa.destroy(file); + } +}; + +pub const Decl = struct { + name: InternPool.NullTerminatedString, + val: InternPool.Value, + /// The direct parent namespace of the Decl. In the case of the Decl + /// corresponding to a file, this is the namespace of the struct, since + /// there is no parent. + src_namespace: Namespace.Index, + + /// The AST node index of this declaration. + /// Must be recomputed when the corresponding source file is modified. + // src_node: Ast.NodeIndex, + /// Line number corresponding to `src_node`. Stored separately so that source files + /// do not need to be loaded into memory in order to compute debug line numbers. + /// This value is absolute. + // src_line: u32, + /// Index of the ZIR `declaration` instruction from which this `Decl` was created. + /// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`. + teg_decl_index: InternPool.TrackedInst.Index.Optional, + + /// Represents the "shallow" analysis status. For example, for decls that are functions, + /// the function type is analyzed with this set to `in_progress`, however, the semantic + /// analysis of the function body is performed with this value set to `success`. Functions + /// have their own analysis status field. + analysis: enum { + /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore + /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced. + unreferenced, + /// Semantic analysis for this Decl is running right now. + /// This state detects dependency loops. + in_progress, + /// The file corresponding to this Decl had a parse error or ZIR error. + /// There will be a corresponding ErrorMsg in Zcu.failed_files. + file_failure, + /// This Decl might be OK but it depends on another one which did not + /// successfully complete semantic analysis. + dependency_failure, + /// Semantic analysis failure. + /// There will be a corresponding ErrorMsg in Zcu.failed_decls. + sema_failure, + /// There will be a corresponding ErrorMsg in Zcu.failed_decls. + codegen_failure, + /// Sematic analysis and constant value codegen of this Decl has + /// succeeded. However, the Decl may be outdated due to an in-progress + /// update. Note that for a function, this does not mean codegen of the + /// function body succeded: that state is indicated by the function's + /// `analysis` field. + complete, + }, + + codegen: ?JsTree = null, + + is_pub: bool, + + /// What kind of a declaration is this. + kind: Kind, + + pub fn deinit(decl: *Decl, gpa: Allocator) void { + defer decl.* = undefined; + if (decl.codegen) |*cg| cg.deinit(gpa); + } + + pub const Kind = enum { + named, + }; + + pub const Index = InternPool.DeclIndex; +}; + +pub const Namespace = struct { + parent: Index.Optional, + file_scope: *File, + /// Direct children of the namespace. + /// Declaration order is preserved via entry order. + /// These are only declarations named directly by the AST; anonymous + /// declarations are not stored here. + decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{}, + + pub const DeclContext = struct { + comp: *Comp, + + pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 { + const decl = ctx.comp.intern_pool.declPtr(decl_index); + return std.hash.uint32(@intFromEnum(decl.name)); + } + + pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool { + _ = b_index; + const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index); + const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index); + return a_decl.name == b_decl.name; + } + }; + + pub const DeclNameAdapter = struct { + comp: *Comp, + + pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 { + _ = self; + return std.hash.uint32(@intFromEnum(s)); + } + + pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool { + _ = b_index; + return a == self.comp.intern_pool.declPtr(b_decl_index).name; + } + }; + + pub fn lookupDirect(ns: *Namespace, comp: *Comp, name: InternPool.NullTerminatedString) ?Decl.Index { + return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp }); + } + + pub const Index = InternPool.NamespaceIndex; +}; + +/// Index a module in a Teg. +/// When a File's Teg is loaded, we call this on the root module. +/// submodules are analyzed lazily. +pub fn indexTegModule(comp: *Comp, file: *File, module_index: Teg.Inst.Index) !Namespace.Index { + const teg = file.teg.?; + + const ns = try comp.intern_pool.createNamespace(comp.gpa, .{ + .parent = .none, + .file_scope = file, + }); + const ns_ptr = comp.intern_pool.namespacePtr(ns); + errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns); + + for (teg.moduleDeclarations(module_index)) |decl_index| { + const name = try comp.intern_pool.getOrPutString( + comp.gpa, + teg.declName(decl_index), + .no_embedded_nulls, + ); + const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index); + const decl = try comp.intern_pool.createDecl(comp.gpa, .{ + .name = name, + .val = undefined, + .src_namespace = ns, + .teg_decl_index = tracked_inst.toOptional(), + .analysis = .unreferenced, + .is_pub = false, // todo: + .kind = .named, + }); + try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp }); + } + + return ns; +} + +pub fn semaEntryPoint(comp: *Comp, file: *File) !void { + var report: Report = undefined; + const teg = try file.getTeg(comp, &report); + + // TODO, move this elsewhere. we should print all errors and warnings at once + if (teg.report.warnings.len > 0) { + try teg.report.dumpToStdErr(comp); + } + + const ns = try comp.indexTegModule(file, .main_module_inst); + const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse + @panic("no top level code?"); + const decl = comp.intern_pool.declPtr(decl_index); + + assert(comp.current_compile_decls.items.len == 0); + + var arena = std.heap.ArenaAllocator.init(comp.gpa); + defer arena.deinit(); + + var sema: Sema = .{ + .comp = comp, + .gpa = comp.gpa, + .code = teg, + .arena = arena.allocator(), + .ip = &comp.intern_pool, + .decl = decl, + .decl_index = decl_index, + .namespace_index = ns, + }; + try sema.init(); + defer sema.deinit(); + _ = try sema.analyzeDeclaration(); + try sema.finalize(); +} + +pub fn codeGenDecl(c: *Comp, decl: *Decl, air: *const Air) !void { + const t = tracer.trace(@src()); + defer t.end(); + + var arena_state = std.heap.ArenaAllocator.init(c.gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air); + errdefer cg.deinit(); + try cg.build(); + decl.codegen = try cg.finalize(); +} + +pub fn link(c: *Comp, writer: anytype) !void { + try JsLink.linkJsCode(c, c.current_compile_decls.items, writer); +} + +const Options = struct { + dump_ir: bool = false, +}; + +const Comp = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Stat = std.Build.Cache.File.Stat; +const log = std.log.scoped(.comp); + +const InternPool = @import("InternPool.zig"); + +const Ast = @import("Ast.zig"); +const Air = @import("Air.zig"); +const Teg = @import("Teg.zig"); +const Cache = @import("Cache.zig"); +const Sema = @import("Sema.zig"); +const Transform = @import("Transform.zig"); +const JsNodeGen = @import("JsNodeGen.zig"); +const JsTree = @import("JsTree.zig"); +const JsLink = @import("JsLink.zig"); +const Report = @import("Report.zig"); +const Parse = @import("Parse.zig"); + +const tracer = @import("tracer"); diff --git a/src/core/InternPool.zig b/src/core/InternPool.zig new file mode 100644 index 0000000000000000000000000000000000000000..7863bf6debc6688cc2765fa2bd99bfef73f3294d --- /dev/null +++ b/src/core/InternPool.zig @@ -0,0 +1,1196 @@ +//! Large parts of this file are based on the Zig compiler's InternPool. +//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig + +/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are +/// constructed lazily. +map: std.AutoArrayHashMapUnmanaged(void, void) = .{}, +items: std.MultiArrayList(Item) = .{}, +extra: ExtraStore = .{}, + +/// In order to store references to strings in fewer bytes, we copy all +/// string bytes into here. String bytes can be null. It is up to whomever +/// is referencing the data here whether they want to store both index and length, +/// thus allowing null bytes, or store only index, and use null-termination. The +/// `string_bytes` array is agnostic to either usage. +string_bytes: std.ArrayListUnmanaged(u8) = .{}, + +/// Used for finding the index inside `string_bytes`. +string_table: std.HashMapUnmanaged( + u32, + void, + std.hash_map.StringIndexContext, + std.hash_map.default_max_load_percentage, +) = .{}, + +/// An index into `tracked_insts` gives a reference to a single ZIR instruction which +/// persists across incremental updates. +tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{}, + +/// Rather than allocating Decl objects with an Allocator, we instead allocate +/// them with this SegmentedList. This provides four advantages: +/// * Stable memory so that one thread can access a Decl object while another +/// thread allocates additional Decl objects from this list. +/// * It allows us to use u32 indexes to reference Decl objects rather than +/// pointers, saving memory in Type, Value, and dependency sets. +/// * Using integers to reference Decl objects rather than pointers makes +/// serialization trivial. +/// * It provides a unique integer to be used for anonymous symbol names, avoiding +/// multi-threaded contention on an atomic counter. +allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{}, +/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack. +decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{}, + +/// Same pattern as with `allocated_decls`. +allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{}, +/// Same pattern as with `decls_free_list`. +namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{}, + +/// How many items in the InternPool are statically known. +/// This is specified with an integer literal and a corresponding comptime +/// assert below to break an unfortunate and arguably incorrect dependency loop +/// when compiling. +pub const static_len = Teg.Inst.Index.static_len; +comptime { + // @compileLog(static_len, static_keys.len); + assert(static_len == static_keys.len); +} + +pub const Item = struct { + tag: Tag, + data: EncodedData, + + pub fn encode(comptime tag: Tag, data: tag.Data()) Item { + return .{ .tag = tag, .data = EncodedData.encode(tag, data) }; + } +}; + +pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var"); +pub const EncodedData = dod.EncodedData(Tag, u32); +pub const ExtraIndex = ExtraStore.Index; + +pub const Tag = enum(u8) { + /// This special tag represents a value which was removed from this pool via + /// `InternPool.remove`. The item remains allocated to preserve indices, but + /// lookups will consider it not equal to any other item, and all queries + /// assert not this tag. `data` is unused. + removed, + + /// A function type + /// data is an extra index into a `FuncType` + type_function, + /// A type that can be represented with only an enum tag. + /// data is SimpleType enum value. + simple_type, + + // --- before this are types --- + // --- past this are values --- + + /// A value that can be represented with only an enum tag. + /// data is SimpleValue enum value. + simple_value, + /// A numeric literal that can be represented as a float. + /// Integers greater than 2^53 known at compile time are currently not supported. + /// data is an extra index into a `Float` + comptime_float, + /// An int32 value + /// data is i32 + int32, + /// A float value + /// data is an extra index into a `Float` + float, + /// A function + /// data is an extra index into a `FuncDecl` + func, + /// A string + /// data is an extra index into a `String` + string, + + pub fn Data(comptime tag: Tag) type { + return switch (tag) { + .removed, + => unreachable, // does not have a data + + .simple_type => SimpleType, + .simple_value => SimpleValue, + .int32 => i32, + + else => ExtraIndex(tag.Payload()), + }; + } + + /// When `data` is an extra index, this is the type of the payload. + pub fn Payload(comptime tag: Tag) type { + return switch (tag) { + .removed, + .simple_type, + .simple_value, + .int32, + => unreachable, // does not have a payload + .float, .comptime_float => Float, + .func => FuncDecl, + .type_function => TypeFunction, + .string => String.WithLen, + }; + } + + /// Trailing: N/A + pub const FuncDecl = struct { + analysis: FuncAnalysis, + owner_decl: DeclIndex, + ty: Type, + teg_body_inst: TrackedInst.Index, + }; + + /// Trailing: + /// 0. param_type: Index for each params_len + pub const TypeFunction = struct { + params_len: u32, + return_type: Type, + }; +}; + +pub const Value = enum(u32) { + pub const first_type: Value = .opaque_type; + pub const last_type: Value = .noreturn_type; + pub const first_value: Value = .void; + pub const last_value: Value = .empty_string; + + opaque_type, + bool_type, + int32_type, + float_type, + string_type, + noreturn_type, + type_type, + numeric_literal_type, + + void, + bool_true, + bool_false, + int32_zero, + float_zero, + nan, + empty_string, + standard_library, + + none = std.math.maxInt(u32), + _, + + /// An array of `Index` existing within the `extra` array. + /// This type exists to provide a struct with lifetime that is + /// not invalidated when items are added to the `InternPool`. + pub const Slice = struct { + start: u32, + len: u32, + + pub fn get(slice: Slice, ip: *const InternPool) []Value { + return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]); + } + }; + + /// Used for a map of `Index` values to the index within a list of `Index` values. + const Adapter = struct { + indexes: []const Value, + + pub fn eql(ctx: @This(), a: Value, b_void: void, b_map_index: usize) bool { + _ = b_void; + return a == ctx.indexes[b_map_index]; + } + + pub fn hash(ctx: @This(), a: Value) u32 { + _ = ctx; + return std.hash.uint32(@intFromEnum(a)); + } + }; + + pub fn valueTag(value: Value, ip: *const InternPool) Tag { + return ip.items.items(.tag)[@intFromEnum(value)]; + } + + pub fn valueData(value: Value, ip: *const InternPool, comptime tag: Tag) tag.Data() { + return ip.items.items(.data)[@intFromEnum(value)].decode(tag, value.valueTag(ip)); + } + + pub fn toAir(value: Value) Air.Inst.Ref { + return .{ .kind = .value, .data = @intCast(@intFromEnum(value)) }; + } + + pub fn toKey(value: Value, ip: *const InternPool) Key { + return ip.indexToKey(value); + } + + pub fn dump(value: Value, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) { + return .{ .data = .{ .val = value, .ip = ip } }; + } + + fn formatValue( + ctx: struct { val: Value, ip: *const InternPool }, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) !void { + // TODO: move the formatter out of Air.Dump + var state: Air.Dump.State = undefined; + state.ip = ctx.ip; + state.tty = @import("util.zig").detectTTYFromWriter(writer); + try Air.Dump.State.dumpValue(&state, writer, ctx.val); + } + + pub fn typeOf(value: Value, ip: *const InternPool) Type { + return switch (value.valueTag(ip)) { + .removed => unreachable, // UAF + + .type_function => .type, + .simple_type => .type, + + .simple_value => switch (value.valueData(ip, .simple_value)) { + .true => .bool, + .false => .bool, + .void => .void, + }, + + .comptime_float => .numeric_literal, + .int32 => .int32, + .float => .float, + .string => .string, + .func => value.toKey(ip).func.ty, + }; + } + + /// Try to re-interpret this `Value` as a `Type`. + /// Returns `null` if the value is not a type. + pub fn toType(value: Value, ip: *const InternPool) ?Type { + return switch (value.valueTag(ip)) { + .removed => unreachable, // UAF + + .type_function, + .simple_type, + => @enumFromInt(@intFromEnum(value)), + + // exception: void is a value AND a type + .simple_value => if (value == .void) .void else null, + + else => null, + }; + } + + pub fn toTypeUnsafe(value: Value, proof_ip: *const InternPool) Type { + if (std.debug.runtime_safety) { + return value.toType(proof_ip).?; + } + return @enumFromInt(@intFromEnum(value)); + } + + pub fn asString(val: Value, ip: *const InternPool) InternPool.String.WithLen { + return val.valueData(ip, .string).get(ip); + } + + pub fn toFloat(val: Value, ip: *const InternPool) ?f64 { + return switch (val.valueTag(ip)) { + inline .comptime_float, .float => |tag| val.valueData(ip, tag).get(ip).decode(), + else => null, + }; + } +}; + +pub const Type = enum(u32) { + @"opaque" = @intFromEnum(Value.opaque_type), + bool = @intFromEnum(Value.bool_type), + int32 = @intFromEnum(Value.int32_type), + float = @intFromEnum(Value.float_type), + string = @intFromEnum(Value.string_type), + noreturn = @intFromEnum(Value.noreturn_type), + type = @intFromEnum(Value.type_type), + numeric_literal = @intFromEnum(Value.numeric_literal_type), + void = @intFromEnum(Value.void), + _, + + // Restricted version of `InternPool.Tag` + pub const Tag = enum(u32) { + type_function = @intFromEnum(InternPool.Tag.type_function), + simple_type = @intFromEnum(InternPool.Tag.simple_type), + }; + + /// An array of `Index` existing within the `extra` array. + /// This type exists to provide a struct with lifetime that is + /// not invalidated when items are added to the `InternPool`. + pub const Slice = struct { + start: u32, + len: u32, + + pub fn get(slice: Slice, ip: *const InternPool) []Type { + return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]); + } + }; + + pub fn value(ty: Type) Value { + return @enumFromInt(@intFromEnum(ty)); + } + + pub fn tag(ty: Type, ip: *const InternPool) Type.Tag { + return @enumFromInt(@intFromEnum(ty.value().valueTag(ip))); + } + + pub fn toFunctionPrototype(ty: Type, ip: *const InternPool) ?Key.TypeFunction { + if (ty.tag(ip) != .type_function) return null; + return ip.indexToKey(ty.value()).type_function; + } + + pub fn displayName(ty: Type, ip: *const InternPool) std.fmt.Formatter(formatDisplayName) { + return .{ .data = .{ .type = ty, .ip = ip } }; + } + + pub fn dump(ty: Type, ip: *const InternPool) std.fmt.Formatter(Value.formatValue) { + return ty.value().dump(ip); + } + + fn formatDisplayName( + ctx: struct { type: Type, ip: *const InternPool }, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) !void { + const ty = ctx.type; + const ip = ctx.ip; + + switch (ty) { + .@"opaque" => return try writer.writeAll("opaque"), + .bool => return try writer.writeAll("bool"), + .int32 => return try writer.writeAll("int"), + .float => return try writer.writeAll("float"), + .string => return try writer.writeAll("string"), + .noreturn => return try writer.writeAll("noreturn"), + .type => return try writer.writeAll("type"), + .numeric_literal => return try writer.writeAll("numeric literal"), + .void => return try writer.writeAll("void"), + else => {}, + } + + switch (ty.tag(ip)) { + .simple_type => unreachable, // all simple types are already printed + .type_function => { + const type_function = ty.toFunctionPrototype(ip).?; + try writer.writeAll("fn("); + for (type_function.param_types, 0..) |param, i| { + if (i != 0) try writer.writeAll(", "); + try writer.print("{}", .{param.displayName(ip)}); + } + try writer.writeAll(") -> "); + try writer.print("{}", .{type_function.return_type.displayName(ip)}); + }, + } + } +}; + +pub const static_keys = [_]Key{ + SimpleType.@"opaque".key(), + SimpleType.bool.key(), + SimpleType.int32.key(), + SimpleType.float.key(), + SimpleType.string.key(), + SimpleType.noreturn.key(), + SimpleType.type.key(), + SimpleType.numeric_literal.key(), + + SimpleValue.void.key(), + SimpleValue.true.key(), + SimpleValue.false.key(), + .{ .int32 = 0 }, + .{ .float = 0.0 }, + .{ .float = Float.pure_nan.decode() }, + .{ .string = "" }, +}; + +/// State that is mutable during semantic analysis. This data is not used for +/// equality or hashing. +pub const FuncAnalysis = packed struct(u32) { + state: State, + + _: u24 = 0, + + pub const State = enum(u8) { + /// This function has not yet undergone analysis, because we have not + /// seen a potential runtime call. It may be analyzed in future. + none, + /// Analysis for this function has been queued, but not yet completed. + queued, + /// This function intentionally only has ZIR generated because it is marked + /// inline, which means no runtime version of the function will be generated. + inline_only, + in_progress, + /// There will be a corresponding ErrorMsg in Module.failed_decls + sema_failure, + /// This function might be OK but it depends on another Decl which did not + /// successfully complete semantic analysis. + dependency_failure, + /// There will be a corresponding ErrorMsg in Module.failed_decls. + /// Indicates that semantic analysis succeeded, but code generation for + /// this function failed. + codegen_failure, + /// Semantic analysis and code generation of this function succeeded. + success, + }; +}; + +/// An index into `string_bytes`. +pub const String = enum(u32) { + /// An empty string. + empty = 0, + _, + + pub const WithLen = struct { + string: String, + len: u32, + + pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 { + return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)]; + } + + pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString { + return string.string.tryToNullTerminatedString(string.len, ip, gpa); + } + }; + + pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 { + return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)]; + } + + pub fn withLen(string: String, len: u64) WithLen { + return .{ .string = string, .len = @intCast(len) }; + } + + pub fn at(string: String, index: u64, ip: *const InternPool) u8 { + return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)]; + } + + pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString { + assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null); + assert(string.at(len, ip) == 0); + return @enumFromInt(@intFromEnum(string)); + } + + pub fn tryToNullTerminatedString( + string: String, + len: u64, + ip: *InternPool, + gpa: Allocator, + ) !NullTerminatedString { + const slice = string.toSlice(len, ip); + if (std.mem.indexOfScalar(u8, slice, 0)) |_| { + return error.HasNulls; + } + if (string.at(len, ip) == 0) { + return @enumFromInt(@intFromEnum(string)); + } + return ip.getOrPutString(gpa, slice, .no_embedded_nulls); + } +}; + +pub const NullTerminatedString = enum(u32) { + empty = 0, + _, + + pub fn toString(self: NullTerminatedString) String { + return @enumFromInt(@intFromEnum(self)); + } + + // pub fn toOptional(self: NullTerminatedString) Optional{ + // return @enumFromInt(@intFromEnum(self)); + // } + + pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { + const slice = ip.string_bytes.items[@intFromEnum(string)..]; + return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0]; + } + + pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { + return @intCast(string.toSlice(ip).len); + } + + pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { + return std.mem.eql(u8, string.toSlice(ip), slice); + } + + const Adapter = struct { + strings: []const NullTerminatedString, + + pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool { + _ = b_void; + return a == ctx.strings[b_map_index]; + } + + pub fn hash(ctx: @This(), a: NullTerminatedString) u32 { + _ = ctx; + return std.hash.uint32(@intFromEnum(a)); + } + }; + + const FormatData = struct { + string: NullTerminatedString, + ip: *const InternPool, + }; + fn format( + data: FormatData, + comptime specifier: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + const slice = data.string.toSlice(data.ip); + if (comptime std.mem.eql(u8, specifier, "")) { + try writer.writeAll(slice); + } else if (comptime std.mem.eql(u8, specifier, "i")) { + try writer.print("{p}", .{std.zig.fmtId(slice)}); + } else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'"); + } + + pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) { + return .{ .data = .{ .string = self, .ip = ip } }; + } +}; + +pub const TrackedInst = extern struct { + path_digest: Cache.BinDigest, + inst: Teg.Inst.Index, + + comptime { + // The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`. + assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index)); + } + + pub const Index = enum(u32) { + _, + pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index { + return ip.tracked_insts.keys()[@intFromEnum(i)].inst; + } + pub fn toOptional(i: TrackedInst.Index) Optional { + return @enumFromInt(@intFromEnum(i)); + } + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + pub fn unwrap(opt: Optional) ?TrackedInst.Index { + return switch (opt) { + .none => null, + _ => @enumFromInt(@intFromEnum(opt)), + }; + } + }; + }; +}; + +pub const DeclIndex = enum(u32) { + _, + + pub fn toOptional(i: DeclIndex) Optional { + return @enumFromInt(@intFromEnum(i)); + } + + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn init(oi: ?DeclIndex) Optional { + return @enumFromInt(@intFromEnum(oi orelse return .none)); + } + + pub fn unwrap(oi: Optional) ?DeclIndex { + if (oi == .none) return null; + return @enumFromInt(@intFromEnum(oi)); + } + }; +}; + +pub const NamespaceIndex = enum(u32) { + _, + + pub fn toOptional(i: DeclIndex) Optional { + return @enumFromInt(@intFromEnum(i)); + } + + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn init(oi: ?NamespaceIndex) Optional { + return @enumFromInt(@intFromEnum(oi orelse return .none)); + } + + pub fn unwrap(oi: Optional) ?NamespaceIndex { + if (oi == .none) return null; + return @enumFromInt(@intFromEnum(oi)); + } + }; +}; + +/// A single value captured in the closure of a namespace type. This is not a plain +/// `Index` because we must differentiate between the following cases: +/// * runtime-known value (where we store the type) +/// * comptime-known value (where we store the value) +/// * decl val (so that we can analyze the value lazily) +/// * decl ref (so that we can analyze the reference lazily) +pub const Capture = packed struct(u32) { + tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref }, + idx: u30, + + pub fn wrap(val: Unwrapped) Capture { + return switch (val) { + .@"comptime" => |i| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) }, + .runtime => |i| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) }, + .decl_val => |i| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) }, + .decl_ref => |i| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) }, + }; + } + pub fn unwrap(val: Capture) Unwrapped { + return switch (val.tag) { + .@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) }, + .runtime => .{ .runtime = @enumFromInt(val.idx) }, + .decl_val => .{ .decl_val = @enumFromInt(val.idx) }, + .decl_ref => .{ .decl_ref = @enumFromInt(val.idx) }, + }; + } + + pub const Unwrapped = union(enum) { + /// Index refers to the value. + @"comptime": Value, + /// Index refers to the type. + runtime: Value, + decl_val: DeclIndex, + decl_ref: DeclIndex, + }; + + pub const Slice = struct { + start: u32, + len: u32, + pub fn get(slice: Slice, ip: *const InternPool) []Capture { + return @ptrCast(ip.extra.items[slice.start..][0..slice.len]); + } + }; +}; + +const SimpleValue = enum { + true, + false, + void, + + pub fn key(self: SimpleValue) Key { + return .{ .simple_value = self }; + } +}; + +const SimpleType = enum { + @"opaque", + int32, + float, + string, + bool, + noreturn, + type, + numeric_literal, + + pub fn key(self: SimpleType) Key { + return .{ .simple_type = self }; + } +}; + +pub const Key = union(enum) { + comptime_float: f64, + int32: i32, + float: f64, + string: []const u8, + simple_value: SimpleValue, + simple_type: SimpleType, + type_function: TypeFunction, + func: Func, + + const Adapter = struct { + intern_pool: *const InternPool, + + pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool { + _ = b_void; + if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false; + return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool); + } + + pub fn hash(ctx: @This(), a: Key) u32 { + return a.hash32(ctx.intern_pool); + } + }; + + pub const TypeFunction = struct { + param_types: []Type, + return_type: Type, + + pub fn hash(self: TypeFunction, hasher: *Hash) void { + const t = tracer.trace(@src()); + defer t.end(); + + for (self.param_types) |param_type| { + std.hash.autoHash(hasher, param_type); + } + std.hash.autoHash(hasher, self.return_type); + } + + pub fn eql(a: TypeFunction, b: TypeFunction) bool { + return std.mem.eql(Type, a.param_types, b.param_types) and + a.return_type == b.return_type; + } + }; + + pub const Func = struct { + ty: Type, + owner_decl: DeclIndex, + teg_body_inst: TrackedInst.Index, + + pub fn hash(self: Func, hasher: *Hash) void { + std.hash.autoHash(hasher, self.ty); + std.hash.autoHash(hasher, self.owner_decl); + std.hash.autoHash(hasher, self.teg_body_inst); + } + }; + + pub fn hash32(key: Key, ip: *const InternPool) u32 { + return @truncate(key.hash64(ip)); + } + + pub fn hash64(key: Key, ip: *const InternPool) u64 { + const t = tracer.trace(@src()); + defer t.end(); + + _ = ip; // autofix + const asBytes = std.mem.asBytes; + const KeyTag = @typeInfo(Key).Union.tag_type.?; + const seed = @intFromEnum(@as(KeyTag, key)); + + return switch (key) { + inline .int32, + .simple_value, + .simple_type, + => |x| Hash.hash(seed, asBytes(&x)), + .string, + => |x| Hash.hash(seed, x), + .float, .comptime_float => |x| Hash.hash(seed, asBytes(&Float.assertPure(x))), + .type_function => |x| { + var hasher = Hash.init(seed); + x.hash(&hasher); + return hasher.final(); + }, + .func => |x| { + var hasher = Hash.init(seed); + x.hash(&hasher); + return hasher.final(); + }, + }; + } + + pub fn eql(a: Key, b: Key, ip: *const InternPool) bool { + const t = tracer.trace(@src()); + defer t.end(); + + _ = ip; // autofix + const KeyTag = @typeInfo(Key).Union.tag_type.?; + const a_tag: KeyTag = a; + const b_tag: KeyTag = b; + if (a_tag != b_tag) return false; + return switch (a) { + inline .int32, + .simple_value, + .simple_type, + => |x, tag| x == @field(b, @tagName(tag)), + .string, + => |x| std.mem.eql(u8, x, b.string), + .float => |x| Float.assertPure(x) == Float.assertPure(b.float), + .comptime_float => |x| Float.assertPure(x) == Float.assertPure(b.comptime_float), + .type_function => |x| x.eql(b.type_function), + .func => |a_info| { + const b_info = b.func; + if (a_info.owner_decl != b_info.owner_decl) + return false; + if (a_info.ty == b_info.ty) + return true; + + // TODO: zig has something extra here for inferred error sets + return false; + }, + }; + } +}; + +const EmbeddedNulls = enum { + no_embedded_nulls, + maybe_embedded_nulls, + + const OptionalNullTerminatedString = @compileError("TODO"); + const OptionalString = @compileError("TODO"); + + fn StringType(comptime embedded_nulls: EmbeddedNulls) type { + return switch (embedded_nulls) { + .no_embedded_nulls => NullTerminatedString, + .maybe_embedded_nulls => String, + }; + } + + fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type { + return switch (embedded_nulls) { + .no_embedded_nulls => OptionalNullTerminatedString, + .maybe_embedded_nulls => OptionalString, + }; + } +}; + +// implementations: + +/// To construct, instantiate `.{ .gpa = gpa }` then call this. +pub fn init(ip: *InternPool, gpa: Allocator) !void { + const t = tracer.trace(@src()); + defer t.end(); + + assert(ip.items.len == 0); + + // Reserve string index 0 for an empty string. + assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty); + + // So that we can use `catch unreachable` below. + try ip.items.ensureUnusedCapacity(gpa, static_keys.len); + try ip.map.ensureUnusedCapacity(gpa, static_keys.len); + try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len); + + // This inserts all the statically-known values into the intern pool in the + // order expected. + for (static_keys) |key| { + _ = ip.get(gpa, key) catch unreachable; + } + + if (std.debug.runtime_safety) { + // Sanity check. + assert(ip.indexToKey(.bool_true).simple_value == .true); + assert(ip.indexToKey(.bool_false).simple_value == .false); + } + + assert(ip.items.len == static_keys.len); +} + +pub fn deinit(ip: *InternPool, gpa: Allocator) void { + const t = tracer.trace(@src()); + defer t.end(); + + ip.map.deinit(gpa); + ip.items.deinit(gpa); + ip.extra.deinit(gpa); + // ip.limbs.deinit(gpa); + ip.string_bytes.deinit(gpa); + + var decls = ip.allocated_decls.iterator(0); + while (decls.next()) |decl| { + decl.deinit(gpa); + } + + ip.allocated_decls.deinit(gpa); + ip.decls_free_list.deinit(gpa); + + var namespaces = ip.allocated_namespaces.iterator(0); + while (namespaces.next()) |ns| { + ns.decls.deinit(gpa); + } + + ip.namespaces_free_list.deinit(gpa); + ip.allocated_namespaces.deinit(gpa); + + // for (ip.maps.items) |*map| map.deinit(gpa); + // ip.maps.deinit(gpa); + + ip.string_table.deinit(gpa); + + ip.tracked_insts.deinit(gpa); + + // ip.src_hash_deps.deinit(gpa); + // ip.decl_val_deps.deinit(gpa); + // ip.func_ies_deps.deinit(gpa); + // ip.namespace_deps.deinit(gpa); + // ip.namespace_name_deps.deinit(gpa); + + // ip.first_dependency.deinit(gpa); + + // ip.dep_entries.deinit(gpa); + // ip.free_dep_entries.deinit(gpa); + + ip.* = undefined; +} + +pub fn indexToKey(ip: *const InternPool, index: Value) Key { + assert(index != .none); + const item: Item = ip.items.get(@intFromEnum(index)); + const data = item.data; + return switch (item.tag) { + .removed => unreachable, + + .type_function => .{ + .type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)), + }, + + .simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) }, + .simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) }, + .int32 => .{ .int32 = data.decode(.int32, item.tag) }, + .float => .{ .float = data.decode(.float, item.tag).get(ip).decode() }, + .comptime_float => .{ .comptime_float = data.decode(.comptime_float, item.tag).get(ip).decode() }, + .string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) }, + .func => { + const func = data.decode(.func, item.tag).get(ip); + return .{ .func = .{ + .ty = func.ty, + .owner_decl = func.owner_decl, + .teg_body_inst = func.teg_body_inst, + } }; + }, + }; +} + +pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Value { + const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)}); + defer t.end(); + + const adapter: Key.Adapter = .{ .intern_pool = ip }; + const gop = try ip.map.getOrPutAdapted(gpa, key, adapter); + if (gop.found_existing) return @enumFromInt(gop.index); + if (ip.items.len >= std.math.maxInt(u31) - 1) { + return error.TooManyValues; + } + + const t2 = tracer.traceFmt(@src(), "(insert)", .{}); + defer t2.end(); + + try ip.items.ensureUnusedCapacity(gpa, 1); + const item: Item = switch (key) { + inline .int32, + .simple_type, + .simple_value, + => |s, tag| Item.encode(@field(Tag, @tagName(tag)), s), + .string => |s| Item.encode( + .string, + try ip.extra.append( + gpa, + String.WithLen, + (try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len), + ), + ), + .float => |f| Item.encode( + .float, + try ip.extra.append(gpa, Float, Float.encode(f)), + ), + .comptime_float => |f| Item.encode( + .comptime_float, + try ip.extra.append(gpa, Float, Float.encode(f)), + ), + .func => |f| brk: { + const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{ + .analysis = .{ + .state = .none, + }, + .owner_decl = f.owner_decl, + .ty = f.ty, + .teg_body_inst = f.teg_body_inst, + }); + + break :brk Item.encode(.func, extra); + }, + .type_function => |f| brk: { + const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{ + .params_len = @intCast(f.param_types.len), + .return_type = f.return_type, + }, f.param_types.len); + defer trail.end(); + trail.writeSlice(f.param_types); + break :brk Item.encode(.type_function, extra); + }, + }; + ip.items.appendAssumeCapacity(item); + const index: Value = @enumFromInt(ip.items.len - 1); + if (std.debug.runtime_safety) { + assert(ip.indexToKey(index).eql(key, ip)); + } + return index; +} + +fn extraTypeFunction( + ip: *const InternPool, + extra_index: ExtraIndex(Tag.TypeFunction), +) Key.TypeFunction { + const type_function, var trail = extra_index.getTrail(ip); + const param_types = trail.readTypeSlice(type_function.params_len); + const result: Key.TypeFunction = .{ + .return_type = type_function.return_type, + .param_types = param_types.get(ip), + }; + return result; +} + +pub fn getOrPutString( + ip: *InternPool, + gpa: Allocator, + slice: []const u8, + comptime embedded_nulls: EmbeddedNulls, +) Allocator.Error!embedded_nulls.StringType() { + const t = tracer.trace(@src()); + defer t.end(); + + try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1); + ip.string_bytes.appendSliceAssumeCapacity(slice); + ip.string_bytes.appendAssumeCapacity(0); + return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls); +} + +fn getOrPutTrailingString( + ip: *InternPool, + gpa: Allocator, + len: usize, + comptime embedded_nulls: EmbeddedNulls, +) Allocator.Error!embedded_nulls.StringType() { + const t = tracer.trace(@src()); + defer t.end(); + + const string_bytes = &ip.string_bytes; + const str_index: u32 = @intCast(string_bytes.items.len - len); + if (len > 0 and string_bytes.getLast() == 0) { + _ = string_bytes.pop(); + } else { + try string_bytes.ensureUnusedCapacity(gpa, 1); + } + const key: []const u8 = string_bytes.items[str_index..]; + const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null; + switch (embedded_nulls) { + .no_embedded_nulls => assert(!has_embedded_null), + .maybe_embedded_nulls => if (has_embedded_null) { + string_bytes.appendAssumeCapacity(0); + return @enumFromInt(str_index); + }, + } + const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{ + .bytes = string_bytes, + }, std.hash_map.StringIndexContext{ + .bytes = string_bytes, + }); + if (gop.found_existing) { + string_bytes.shrinkRetainingCapacity(str_index); + return @enumFromInt(gop.key_ptr.*); + } else { + gop.key_ptr.* = str_index; + string_bytes.appendAssumeCapacity(0); + return @enumFromInt(str_index); + } +} + +pub fn trackInst(ip: *InternPool, gpa: Allocator, file: *Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index { + const t = tracer.trace(@src()); + defer t.end(); + + const key: TrackedInst = .{ + .path_digest = file.path_digest.?, + .inst = inst, + }; + const gop = try ip.tracked_insts.getOrPut(gpa, key); + return @enumFromInt(gop.index); +} + +pub fn declPtr(ip: *InternPool, index: DeclIndex) *Comp.Decl { + return ip.allocated_decls.at(@intFromEnum(index)); +} + +pub fn declPtrConst(ip: *const InternPool, index: DeclIndex) *const Comp.Decl { + return ip.allocated_decls.at(@intFromEnum(index)); +} + +pub fn namespacePtr(ip: *InternPool, index: NamespaceIndex) *Comp.Namespace { + return ip.allocated_namespaces.at(@intFromEnum(index)); +} + +pub fn createDecl( + ip: *InternPool, + gpa: Allocator, + initialization: Comp.Decl, +) Allocator.Error!DeclIndex { + const t = tracer.trace(@src()); + defer t.end(); + + if (ip.decls_free_list.popOrNull()) |index| { + ip.allocated_decls.at(@intFromEnum(index)).* = initialization; + return index; + } + const ptr = try ip.allocated_decls.addOne(gpa); + ptr.* = initialization; + return @enumFromInt(ip.allocated_decls.len - 1); +} + +pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void { + const t = tracer.trace(@src()); + defer t.end(); + + ip.declPtr(index).* = undefined; + ip.decls_free_list.append(gpa, index) catch { + // In order to keep `destroyDecl` a non-fallible function, we ignore memory + // allocation failures here, instead leaking the Decl until garbage collection. + }; +} + +pub fn createNamespace( + ip: *InternPool, + gpa: Allocator, + initialization: Comp.Namespace, +) Allocator.Error!NamespaceIndex { + const t = tracer.trace(@src()); + defer t.end(); + + if (ip.namespaces_free_list.popOrNull()) |index| { + ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization; + return index; + } + const ptr = try ip.allocated_namespaces.addOne(gpa); + ptr.* = initialization; + return @enumFromInt(ip.allocated_namespaces.len - 1); +} + +pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void { + const t = tracer.trace(@src()); + defer t.end(); + + ip.namespacePtr(index).* = undefined; + ip.namespaces_free_list.append(gpa, index) catch { + // In order to keep `destroyNamespace` a non-fallible function, we ignore memory + // allocation failures here, instead leaking the Namespace until garbage collection. + }; +} + +const InternPool = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Hash = std.hash.Wyhash; +const assert = std.debug.assert; + +const Comp = @import("Comp.zig"); +const Cache = @import("Cache.zig"); + +const Teg = @import("Teg.zig"); +const Air = @import("Air.zig"); + +const dod = @import("array.zig"); +const Float = dod.Float; + +const tracer = @import("tracer"); + +test InternPool { + const testing = std.testing; + std.testing.refAllDeclsRecursive(@This()); + + var ip_struct: InternPool = .{ .gpa = std.testing.allocator }; + const ip = &ip_struct; + defer ip.deinit(); + + const str = try ip.getOrPutString("hello", .no_embedded_nulls); + try testing.expectEqualStrings("hello", str.toSlice(ip)); + const str2 = try ip.getOrPutString("hello", .no_embedded_nulls); + try testing.expectEqual(str, str2); + + const int1 = try ip.get(.{ .int32 = 21483 }); + try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32); + const int2 = try ip.get(.{ .int32 = 21483 }); + try testing.expectEqual(int1, int2); +} diff --git a/src/core/JsLink.zig b/src/core/JsLink.zig new file mode 100644 index 0000000000000000000000000000000000000000..99a557bcab8a283b579d45d9208b7cc340ad24c0 --- /dev/null +++ b/src/core/JsLink.zig @@ -0,0 +1,519 @@ +/// This is the JavaScript backend's linker. +/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript +/// bundle out of them. +comp: *Comp, + +/// Value is an index into `link_strings` +decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{}, +link_strings: std.ArrayHashMapUnmanaged( + LinkString, + GlobalOwner, + LinkString.Context, + true, +) = .{}, +string_bytes: std.ArrayListUnmanaged(u8) = .{}, + +/// The first declaration is the function containing top-level code. +pub fn linkJsCode( + comp: *Comp, + decls: []const Decl.Index, + writer: anytype, +) !void { + var t = tracer.trace(@src()); + defer t.end(); + + // Arena allocator is used for the main flow + var link: JsLink = .{ .comp = comp }; + defer link.deinit(); + + // Temp allocator is used for extremely temporary string work. + // var temp_sfa = std.heap.stackFallback(4096, comp.gpa); + // const temp_alloc = temp_sfa.get(); + + // 1. Resolve all global identifiers. Function names must not shadow external + // names. We also need this list prepared so that local identifier names do not + // shadow functions or externals. + const root_decl = comp.intern_pool.declPtrConst(decls[0]); + assert(root_decl.kind == .named); + assert(root_decl.name == .empty); + + for (root_decl.codegen.?.extern_symbols) |extern_symbol| { + try link.reserveExternSymbol(extern_symbol); + } + + for (decls[1..]) |decl_index| { + assert(decl_index != decls[0]); + + const decl = comp.intern_pool.declPtrConst(decl_index); + + // todo: add all externs first + for (decl.codegen.?.extern_symbols) |extern_symbol| { + try link.reserveExternSymbol(extern_symbol); + } + + const name = decl.name.toSlice(&comp.intern_pool); + assert(name.len > 0); + + const gop = try link.link_strings.getOrPutContextAdapted( + comp.gpa, + name, + LinkString.SliceContext{ .link = &link }, + .{ .link = &link }, + ); + if (gop.found_existing) { + @panic("TODO: handle this shadowing case."); + } + const name_str = try link.appendLinkString(name); + gop.key_ptr.* = name_str; + gop.value_ptr.* = .decl; + try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index)); + } + + try writer.print("// Language Compiler v0.1.2. Alpha quality software.\n", .{}); + + // 2. Print every function decl + for (decls[1..]) |decl_index| { + try link.printDecl(decl_index, writer); + } + + // 3. Print the top-level code + if (root_decl.codegen.?.nodes.len > 1) { + try link.printDeclContents(&root_decl.codegen.?, writer); + } else { + try writer.print("// (program has no observable effects, thus an empty file)\n", .{}); + } +} + +fn deinit(link: *JsLink) void { + var t = tracer.trace(@src()); + defer t.end(); + + defer link.* = undefined; + link.decls.deinit(link.comp.gpa); + link.link_strings.deinit(link.comp.gpa); + link.string_bytes.deinit(link.comp.gpa); +} + +fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void { + var t = tracer.trace(@src()); + defer t.end(); + + const name = extern_symbol.toSlice(&link.comp.intern_pool); + const gop = try link.link_strings.getOrPutContextAdapted( + link.comp.gpa, + name, + LinkString.SliceContext{ .link = link }, + .{ .link = link }, + ); + + if (gop.found_existing) { + if (gop.value_ptr.* != .external) { + @panic("TODO: Rename Decl"); + } + } + gop.key_ptr.* = try link.appendLinkString(name); + gop.value_ptr.* = .external; +} + +fn appendLinkString(link: *JsLink, string: []const u8) !LinkString { + var t = tracer.trace(@src()); + defer t.end(); + + const start = link.string_bytes.items.len; + try link.string_bytes.appendSlice(link.comp.gpa, string); + return .{ .start = @intCast(start), .len = @intCast(string.len) }; +} + +fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 { + var t = tracer.trace(@src()); + defer t.end(); + + return (link.decls.get(decl) orelse if (std.debug.runtime_safety) { + std.debug.panic("Decl {d} ({}) is not present in link decl map", .{ + @intFromEnum(decl), + link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool), + }); + } else unreachable).toString(link).slice(link); +} + +fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + try writer.writeAll("let "); + try writer.writeAll(link.declIdentifier(decl_index)); + try writer.writeAll(" = "); + + const decl = link.comp.intern_pool.declPtrConst(decl_index); + const tree = &decl.codegen.?; + + try writer.writeAll("("); + for (0..tree.param_count) |id| { + try link.printLocal(@intCast(id), writer); + if (id != tree.param_count - 1) try writer.writeAll(", "); + } + try writer.writeAll(") => {\n"); + try link.printDeclContents(tree, writer); + try writer.writeAll("};\n"); +} + +fn printDeclContents(link: *JsLink, tree: *const JsTree, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + for (tree.mainStatements()) |stmt| { + try link.printExpr(tree, stmt, writer); + try writer.writeAll(";\n"); + } +} + +fn printBlockContents(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + assert(list.slice(tree).len > 0); + for (list.slice(tree)) |stmt| { + try link.printExpr(tree, stmt, writer); + switch (tree.nodeTag(stmt)) { + else => try writer.writeAll(";"), + .stmt_block, + .stmt_if, + => {}, + } + try writer.writeAll("\n"); + } +} + +fn printBlock(link: *JsLink, tree: *const JsTree, list: JsTree.IndexList, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + if (list.len != 1) { + try writer.writeAll("{"); + try link.printBlockContents(tree, list, writer); + try writer.writeAll("}"); + } else { + try link.printBlockContents(tree, list, writer); + } +} + +fn printExpr(link: *JsLink, tree: *const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void { + var t = tracer.trace(@src()); + defer t.end(); + + switch (tree.nodeTag(node)) { + .root => unreachable, + + .plain_value => { + const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node); + switch (plain_value.value.toKey(&link.comp.intern_pool)) { + // TODO: minification + .int32 => |val| try writer.print("{d}", .{val}), + // TODO: correct printing of many value types + .float => |val| try writer.print("{d}", .{val}), + .simple_value => |val| try writer.writeAll(switch (val) { + .true => "true", + .false => "false", + // TODO: make codegen decide from null and undefined + // instead of emitting void nodes. + .void => "undefined", + }), + // TODO: escapes + .string => |val| try writer.print("\"{s}\"", .{val}), + + .comptime_float => unreachable, // was not lowered to float + + else => unreachable, // this value is not a plain_value + } + }, + + .local => { + const ref: JsTree.LocalRef = tree.nodeData(.local, node); + try link.printLocal(ref.id, writer); + }, + + .global => { + const ref: JsTree.GlobalRef = tree.nodeData(.global, node); + try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool)); + }, + + .decl => { + const ref: JsTree.DeclRef = tree.nodeData(.decl, node); + try writer.writeAll(link.declIdentifier(ref.index)); + }, + + .stmt_break => { + const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node); + try writer.writeAll("break"); + if (true) { + try writer.writeAll(" "); + try link.printLabelIdentifier(stmt.label, writer); + } + }, + + .stmt_return => { + const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node); + try writer.writeAll("return"); + if (stmt.operand.unwrap()) |operand| { + try writer.writeAll(" "); + try link.printExpr(tree, operand, writer); + } + }, + + .stmt_declare_local => { + const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node); + try writer.writeAll("let "); + try link.printLocal(stmt.identifier_index, writer); + if (stmt.init.unwrap()) |init| { + try writer.writeAll(" = "); + try link.printExpr(tree, init, writer); + } + }, + + .stmt_if => { + const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node); + const data = stmt.data.get(tree); + try writer.writeAll("if ("); + try link.printExpr(tree, stmt.cond, writer); + try writer.writeAll(") "); + if (data.flags.yes_label) { + try link.printLabelIdentifier(data.flags.block_label, writer); + try writer.writeAll(":"); + } + try link.printBlock(tree, data.yes, writer); + if (data.no.len > 0) { + try writer.writeAll(" else "); + if (data.flags.no_label) { + try link.printLabelIdentifier(data.flags.block_label, writer); + try writer.writeAll(":"); + } + try link.printBlock(tree, data.no, writer); + } + }, + + .stmt_block => { + const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node); + try link.printLabelIdentifier(stmt.label, writer); + try writer.writeAll(": {\n"); + try link.printBlockContents(tree, stmt.content.get(tree), writer); + try writer.writeAll("}"); + }, + + inline .assign, + .add, + .sub, + .mul, + .div, + .eq, + .lt, + .gt, + .assign_add, + .assign_sub, + .assign_mul, + .assign_div, + => |tag| { + const bin: JsTree.Bin = tree.nodeData(tag, node); + const op: Operator = @field(Operator, @tagName(tag)); + try link.printExpr(tree, bin.lhs, writer); + try writer.writeAll(" "); + try writer.writeAll(comptime op.symbol()); + try writer.writeAll(" "); + try link.printExpr(tree, bin.rhs, writer); + }, + + inline .negate, + .bool_not, + .pre_dec, + .pre_inc, + => |tag| { + const un: JsTree.Un = tree.nodeData(tag, node); + const op: Operator = @field(Operator, @tagName(tag)); + try writer.writeAll(comptime op.symbol()); + try link.printExpr(tree, un.operand, writer); + }, + + .ternary => { + const ternary: JsTree.Ternary = tree.nodeData(.ternary, node); + const data = ternary.data.get(tree); + try writer.writeAll("("); + try link.printExpr(tree, ternary.cond, writer); + try writer.writeAll(" ? "); + try link.printExpr(tree, data.yes, writer); + try writer.writeAll(" : "); + try link.printExpr(tree, data.no, writer); + try writer.writeAll(")"); + }, + + .call => { + const call: JsTree.Call = tree.nodeData(.call, node); + const args = call.args.get(tree); + + try link.printExpr(tree, call.base, writer); + try writer.writeAll("("); + + for (args.slice(tree), 0..) |arg, i| { + if (i != 0) try writer.writeAll(", "); + try link.printExpr(tree, arg, writer); + } + + try writer.writeAll(")"); + }, + + .field => { + const field: JsTree.Field = tree.nodeData(.field, node); + + try link.printExpr(tree, field.base, writer); + try writer.writeAll("."); + try writer.writeAll(field.field.toSlice(&link.comp.intern_pool)); + }, + } +} + +fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + _ = link; // todo: be smarter about this + try writer.print("v{d}", .{local}); +} + +fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void { + var t = tracer.trace(@src()); + defer t.end(); + + _ = link; // todo: be smarter about this + try writer.print("label{d}", .{local}); +} + +const GlobalOwner = enum { + external, + decl, +}; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence +const Level = enum { + comma, + assignment, + conditional, + logical_or, + logical_and, + bitwise_or, + bitwise_xor, + bitwise_and, + equality, + relational, + bitshift, + additive, + multiplicative, + exponentiation, + prefix, + postfix, + new, + access_and_call, + member, +}; + +const Operator = enum { + assign, + assign_add, + assign_sub, + assign_mul, + assign_div, + add, + sub, + mul, + div, + eq, + lt, + gt, + negate, + bool_not, + pre_inc, + pre_dec, + + pub fn symbol(op: Operator) []const u8 { + return switch (op) { + .assign => "=", + .assign_add => "+=", + .assign_sub => "-=", + .assign_mul => "*=", + .assign_div => "/=", + .add => "+", + .sub => "-", + .mul => "*", + .div => "/", + .eq => "===", + .lt => "<", + .gt => ">", + + .pre_inc => "++", + .pre_dec => "++", + + .negate => "-", + .bool_not => "!", + }; + } +}; + +const LinkString = struct { + start: u32, + len: u32, + + fn slice(str: LinkString, link: *const JsLink) []const u8 { + return link.string_bytes.items[str.start..][0..str.len]; + } + + const Index = enum(u32) { + _, + + pub fn toString(index: Index, link: *const JsLink) LinkString { + return link.link_strings.keys()[@intFromEnum(index)]; + } + }; + + const Context = struct { + link: *const JsLink, + + pub fn hash(ctx: Context, str: LinkString) u32 { + // assert(str.start + str.len < ctx.link.string_bytes.items.len); + + return @truncate(Wyhash.hash(0, str.slice(ctx.link))); + } + + pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool { + assert(a.start + a.len < ctx.link.string_bytes.items.len); + assert(b.start + b.len < ctx.link.string_bytes.items.len); + return a.start == b.start; + } + }; + + const SliceContext = struct { + link: *const JsLink, + + pub fn hash(_: SliceContext, str: []const u8) u32 { + return @truncate(Wyhash.hash(0, str)); + } + + pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool { + // since the given string is not in string_bytes we do comparisons + return std.mem.eql(u8, a, b.slice(ctx.link)); + } + }; +}; + +const JsLink = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const Wyhash = std.hash.Wyhash; +const log = std.log.scoped(.js_link); + +const Comp = @import("Comp.zig"); +const Decl = Comp.Decl; + +const JsTree = @import("JsTree.zig"); +const Node = JsTree.Node; + +const InternPool = @import("InternPool.zig"); +const String = InternPool.String; + +const tracer = @import("tracer"); diff --git a/src/core/JsNodeGen.zig b/src/core/JsNodeGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..f3b801e05e8e7b37b38e3903f81bcc39a5e1381e --- /dev/null +++ b/src/core/JsNodeGen.zig @@ -0,0 +1,700 @@ +//! Air -> JsTree +gpa: Allocator, +arena: Allocator, +air: *const Air, +ip: *const InternPool, + +nodes: Node.List, +extra: dod.ExtraStore(.tfr, .@"var"), +extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void), +decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void), + +param_count: u32, +air_node_map: []MapEntry, +scratch: std.ArrayListUnmanaged(u32), +next_id: u32, +next_label: u32, + +const Error = error{ + OutOfMemory, +}; + +const MapEntry = union { + none: void, + node: Node.Index, + block: packed struct { + used: bool = false, + index: u31, + }, +}; + +pub fn init(gpa: Allocator, arena: Allocator, ip: *const InternPool, air: *const Air) !JsNodeGen { + const t = tracer.trace(@src()); + defer t.end(); + + const air_node_map = try arena.alloc(MapEntry, air.insts.len); + if (std.debug.runtime_safety) { + @memset(air_node_map, .{ .none = {} }); + } + const param_count = air.paramCount(); + return .{ + .gpa = gpa, + .arena = arena, + .air = air, + .ip = ip, + .param_count = param_count, + .nodes = .{}, + .extern_symbols = .{}, + .decl_deps = .{}, + .air_node_map = air_node_map, + .extra = .{}, + .scratch = .{}, + .next_id = param_count, + .next_label = 0, + }; +} + +pub fn deinit(c: *JsNodeGen) void { + const t = tracer.trace(@src()); + defer t.end(); + + defer c.* = undefined; + c.nodes.deinit(c.gpa); + c.extra.deinit(c.gpa); + c.extern_symbols.deinit(c.gpa); + c.decl_deps.deinit(c.gpa); + + // arena can deinit the rest +} + +pub fn finalize(c: *JsNodeGen) !JsTree { + const t = tracer.trace(@src()); + defer t.end(); + + // todo: errdefer will probably leave memory in a fully invalid state + try c.nodes.setCapacity(c.gpa, c.nodes.len); + + var extra = try c.extra.toOwnedStore(c.gpa); + errdefer extra.deinit(c.gpa); + + // todo: i dont like this dupe + const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys()); + errdefer c.gpa.free(extern_symbols); + + const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys()); + errdefer c.gpa.free(decl_deps); + + defer c.deinit(); + + return .{ + .param_count = c.param_count, + .nodes = c.nodes.toOwnedSlice(), + .extern_symbols = extern_symbols, + .decl_deps = decl_deps, + .extra = extra, + }; +} + +pub fn build(c: *JsNodeGen) !void { + const t = tracer.trace(@src()); + defer t.end(); + + assert(c.air.instTag(.main_block) == .block); + const statements = c.air.instData(.block, .main_block).slice(c.air); + + const scratch_top = c.scratch.items.len; + defer c.scratch.shrinkRetainingCapacity(scratch_top); + + _ = try c.add(.root, undefined); + + if (statements.len > 0) { + for (statements[0..c.param_count], 1..) |stmt, expected_id| { + assert(@intFromEnum(stmt) == expected_id); + assert(c.air.instTag(stmt) == .arg); + c.air_node_map[@intFromEnum(stmt)] = .{ + .node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }), + }; + } + + for (statements[c.param_count..]) |stmt| { + try c.codeGenInst(stmt); + } + } + + const items = c.scratch.items[scratch_top..]; + const top = c.extra.list.items.len; + try c.extra.list.appendSlice(c.gpa, items); + c.nodes.items(.data)[0] = EncodedData.encode(.root, .{ + .start = @intCast(top), + .len = @intCast(items.len), + }); +} + +fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void { + const t = tracer.traceFmt(@src(), "#{d} {s}", .{ @intFromEnum(inst), @tagName(c.air.instTag(inst)) }); + defer t.end(); + + switch (c.air.instTag(inst)) { + .arg => unreachable, // codegen must skip over these instructions. + + .block => try c.codeGenBlock(inst), + + .js_global_ref => { + const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst); + try c.extern_symbols.put(c.gpa, data, {}); + c.air_node_map[@intFromEnum(inst)] = .{ + .node = try c.add(.global, .{ .name = data }), + }; + }, + + .memory => { + const i = @intFromEnum(inst); + const data: Air.Memory = c.air.instData(.memory, inst); + const value: JsTree.Node.Index.Optional = if (data.init.isVoid()) + .none + else + (try c.fetchExpression(data.init)).toOptional(); + + try c.addStmt(try c.add(.stmt_declare_local, .{ + .identifier_index = c.next_id, + .init = value, + })); + const id = try c.add(.local, .{ .id = c.next_id }); + c.next_id += 1; + c.air_node_map[i] = .{ .node = id }; + }, + + inline .gt, + .lt, + .eq, + .add, + .sub, + .mul, + .div, + => |tag| { + const data: Air.Bin = c.air.instData(tag, inst); + const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{ + .lhs = try c.fetchExpression(data.lhs), + .rhs = try c.fetchExpression(data.rhs), + }); + try c.markExpression(inst, node); + }, + + .store => { + const data: Air.Store = c.air.instData(.store, inst); + + const target = try c.fetchInstExpression(data.memory); + + if (data.operand.kind == .index) { + const index = data.operand.decode().index; + switch (c.air.instTag(index)) { + inline .add, + .sub, + .mul, + .div, + => |tag| { + const inner_data: Air.Bin = c.air.instData(tag, index); + if (inner_data.lhs.eql(data.memory.toRef())) { + if ((tag == .add or tag == .sub) and + inner_data.rhs.kind == .value) + { + // if (inner_data.rhs.decode().value.toFloat(c.ip)) |f| { + // if (f == 1) { + // // convert this into ++ or -- + // const node = try c.add( + // if (tag == .add) + // .pre_inc + // else + // .pre_dec, + // .{ + // .operand = target, + // }, + // ); + // try c.markExpression(inst, node); + + // return; + // } + // } + } + + // convert into a compound assignment + const node = try c.add( + @field(JsTree.Tag, "assign_" ++ @tagName(tag)), + .{ + .lhs = target, + .rhs = try c.fetchExpression(inner_data.rhs), + }, + ); + try c.markExpression(inst, node); + + return; + } + }, + else => {}, + } + } + + const node = try c.add(.assign, .{ + .lhs = target, + .rhs = try c.fetchExpression(data.operand), + }); + try c.markExpression(inst, node); + }, + + inline .negate, + .bool_not, + => |tag| { + const data: Air.Un = c.air.instData(tag, inst); + const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{ + .operand = try c.fetchExpression(data.operand), + }); + try c.markExpression(inst, node); + }, + + .field => { + const data: Air.Field = c.air.instData(.field, inst); + const node = try c.add(.field, .{ + .base = try c.fetchExpression(data.base), + .field = data.field, + }); + try c.markExpression(inst, node); + }, + + .call => { + const data: Air.Call = c.air.instData(.call, inst); + const call_data = data.call_data.get(c.air); + + try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len); + const start = c.extra.list.items.len; + c.extra.list.items.len += call_data.args.len; + + const items = c.extra.list.items[start..][0..call_data.args.len]; + + for (call_data.args.slice(c.air), items) |arg, *dest| { + dest.* = @intFromEnum(try c.fetchExpression(arg)); + } + + const node = try c.add(.call, .{ + .base = try c.fetchExpression(data.base), + .args = try c.extra.append(c.gpa, JsTree.IndexList, .{ + .start = @intCast(start), + .len = call_data.args.len, + }), + }); + try c.markExpression(inst, node); + }, + + .@"if" => { + const data: Air.If = c.air.instData(.@"if", inst); + const extra = data.data.get(c.air); + var cond = try c.fetchExpression(data.cond); + + if (c.canLowerToExpression(extra.yes) and + c.canLowerToExpression(extra.no)) + { + const yes = try c.lowerToExpression(extra.yes); + const no = try c.lowerToExpression(extra.yes); + + const ternary = try c.add(.ternary, .{ + .cond = cond, + .data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{ + .yes = yes, + .no = no, + }), + }); + + try c.markExpression(inst, ternary); + } else { + const flags = c.air.instFlags(inst); + const uses = flags.uses; + + const target = if (uses != .none) + try c.addTemporary() + else + null; + + var yes = try c.codeGenBranch(extra.yes, target); + var no = try c.codeGenBranch(extra.no, target); + assert(!((yes == null) and (no == null))); // no "void else void" branches + + // when `yes` is void, swap the condition so that the `else` is omitted + if (yes == null) { + const temp_yes = yes; + yes = no; + no = temp_yes; + cond = try c.add(.bool_not, .{ + .operand = cond, + }); + } + + const main_expr = try c.add(.stmt_if, .{ + .cond = cond, + .data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{ + .yes = yes.?.list, + .no = if (no) |branch| + branch.list + else + .{ .start = 0, .len = 0 }, + .flags = .{ + .block_label = @intCast(c.next_label + 1), + .yes_label = yes.?.requires_label, + .no_label = if (no) |branch| branch.requires_label else false, + }, + }), + }); + + try c.addStmt(main_expr); + + if (target) |node| { + try c.markExpression(inst, node); + } + } + }, + + .@"break" => { + const brk: Air.Break = c.air.instData(.@"break", inst); + + const operand = try c.fetchExpression(brk.value); + + // break #0 -> return + if (brk.block_ref == .main_block) { + try c.addStmt(try c.add(.stmt_return, .{ + .operand = operand.toOptional(), + })); + } else { + const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block; + + const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air); + if (block_insts[block_insts.len - 1] != inst) { + block.used = true; + } + + const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]); + if (assignment.unwrap()) |target| { + try c.addStmt(try c.add(.assign, .{ + .lhs = target, + .rhs = operand, + })); + } + + try c.addStmt(try c.add(.stmt_break, .{ + .label = c.scratch.items[block.index + 1], + })); + } + }, + } +} + +fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void { + const t = tracer.trace(@src()); + defer t.end(); + + const flags: Air.Flags = c.air.instFlags(inst); + const i = @intFromEnum(inst); + switch (flags.uses) { + .none => { + try c.addStmt(node); + }, + .single => { + // TODO: this is not aware of side effect fences + c.air_node_map[i] = .{ .node = node }; + }, + .multi => { + const decl = try c.add(.stmt_declare_local, .{ + .identifier_index = c.next_id, + .init = node.toOptional(), + }); + try c.addStmt(decl); + const id = try c.add(.local, .{ .id = c.next_id }); + c.next_id += 1; + c.air_node_map[i] = .{ .node = id }; + }, + } +} + +fn fetchExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + return switch (ref.decode()) { + .value => |v| c.valueToExpression(v), + .index => |index| fetchInstExpression(c, index), + }; +} + +fn fetchInstExpression(c: *JsNodeGen, index: Air.Inst.Index) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + const data = c.air_node_map[@intFromEnum(index)]; + if (std.debug.runtime_safety) { + const flags: Air.Flags = c.air.instFlags(index); + switch (flags.uses) { + .none => unreachable, // incorrectly marked instruction as unused, + .single => {}, + .multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local), + } + } + return data.node; +} + +fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + const intern_tag = v.valueTag(c.ip); + switch (intern_tag) { + .removed => unreachable, // UAF + + .type_function, + .simple_type, + => unreachable, // Type values are not allowed here + + .int32, + .float, + .comptime_float, + .string, + .simple_value, + => return try c.add(.plain_value, .{ .value = v }), + + // funcs always point to decls + .func => { + const func = v.toKey(c.ip).func; + try c.decl_deps.put(c.gpa, func.owner_decl, {}); + return try c.add(.decl, .{ .index = func.owner_decl }); + }, + } +} + +fn addTemporary(c: *JsNodeGen) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + try c.addStmt(try c.add(.stmt_declare_local, .{ + .identifier_index = c.next_id, + .init = .none, + })); + const id = try c.add(.local, .{ .id = c.next_id }); + c.next_id += 1; + return id; +} + +const Branch = struct { + requires_label: bool, + list: JsTree.IndexList, +}; + +fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch { + const t = tracer.trace(@src()); + defer t.end(); + + const scratch_top = c.scratch.items.len; + defer c.scratch.shrinkRetainingCapacity(scratch_top); + + c.next_label += 1; + defer c.next_label -= 1; + const this_label = c.next_label; + + var requires_label = false; + var items = switch (inst.decode()) { + .value => |v| items: { + if (v == .void) + return null; + if (temporary) |temp| { + const node = try c.valueToExpression(v); + try c.addStmt(try c.add(.assign, .{ + .lhs = temp, + .rhs = node, + })); + } + break :items c.scratch.items[scratch_top..]; + }, + .index => |i| items: { + assert(c.air.instTag(i) == .block); + c.air_node_map[@intFromEnum(i)] = .{ + .block = .{ .index = @intCast(scratch_top) }, + }; + + try c.scratch.appendSlice(c.arena, &.{ + @intFromEnum(if (temporary) |temp| temp.toOptional() else .none), + this_label, + }); + + const block: Air.IndexList = c.air.instData(.block, i); + for (block.slice(c.air)) |item| { + try c.codeGenInst(item); + } + + requires_label = c.air_node_map[@intFromEnum(i)].block.used; + + break :items c.scratch.items[scratch_top + 2 ..]; + }, + }; + + const last_node = c.scratch.items[c.scratch.items.len - 1]; + if (c.nodes.items(.tag)[last_node] == .stmt_break and + c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label) + { + items.len -= 1; + c.nodes.len -= 1; + } + + const top = c.extra.list.items.len; + try c.extra.list.appendSlice(c.gpa, items); + return .{ + .requires_label = requires_label, + .list = .{ + .start = @intCast(top), + .len = @intCast(items.len), + }, + }; +} + +fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void { + const t = tracer.trace(@src()); + defer t.end(); + + const scratch_top = c.scratch.items.len; + errdefer c.scratch.shrinkRetainingCapacity(scratch_top); + + c.next_label += 1; + defer c.next_label -= 1; + + const next_label = c.next_label; + + try c.scratch.appendSlice(c.arena, &.{ + // @intFromEnum(if (temporary) |temp| temp.toOptional() else .none), + @intFromEnum(Node.Index.Optional.none), + next_label, + }); + + c.air_node_map[@intFromEnum(inst)] = .{ + .block = .{ .index = @intCast(scratch_top) }, + }; + const block: Air.IndexList = c.air.instData(.block, inst); + + for (block.slice(c.air)) |item| { + try c.codeGenInst(item); + } + + const items = c.scratch.items[scratch_top + 2 ..]; + const top = c.extra.list.items.len; + try c.extra.list.appendSlice(c.gpa, items); + c.scratch.shrinkRetainingCapacity(scratch_top); + try c.addStmt(try c.add(.stmt_block, .{ + .label = c.next_label, + .content = try c.extra.append(c.gpa, JsTree.IndexList, .{ + .start = @intCast(top), + .len = @intCast(items.len), + }), + })); +} + +// Ideally, we convert everything to an expression. This allows for much +// more compact code. When that isnt possible (`return`, `while`, etc), +// we detect that here and ensure to lower `if` to actual if statements, +// and so on. +fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool { + const t = tracer.trace(@src()); + defer t.end(); + + return switch (inst.decode()) { + .value => true, + .index => |index| switch (c.air.instTag(index)) { + .js_global_ref, + .call, + .arg, + .field, + .add, + .sub, + .mul, + .div, + .eq, + .lt, + .gt, + .negate, + .bool_not, + .store, + => true, + + // JavaScript variable declarations require statements. + // Later, this could be made true if we can hoist temporaries up. + .memory => false, + + // break/return must be a js statement + .@"break", + => false, + + .block => { + // TODO: blocks that are + // - all statements except last are lowerable + // - the last statement is lowerable OR a break of this block + return false; + }, + + .@"if" => { + const call = c.air.instData(.@"if", index); + const data = call.data.get(c.air); + + return c.canLowerToExpression(call.cond) and + c.canLowerToExpression(data.yes) and + c.canLowerToExpression(data.no); + }, + }, + }; +} + +fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + assert(c.canLowerToExpression(ref)); + return switch (ref.decode()) { + .value => |v| try c.valueToExpression(v), + .index => |index| { + try c.codeGenInst(index); + return c.fetchExpression(ref); + }, + }; +} + +fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index { + const t = tracer.trace(@src()); + defer t.end(); + + try c.nodes.append(c.gpa, .{ + .tag = tag, + .data = EncodedData.encode(tag, data), + }); + return @enumFromInt(c.nodes.len - 1); +} + +fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void { + const t = tracer.trace(@src()); + defer t.end(); + + try c.scratch.append(c.arena, @intFromEnum(stmt)); +} + +const JsNodeGen = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; +const NullTerminatedString = InternPool.NullTerminatedString; + +const Comp = @import("Comp.zig"); +const Decl = Comp.Decl; + +const JsTree = @import("JsTree.zig"); +const EncodedData = JsTree.EncodedData; +const Node = JsTree.Node; +const LocalIndex = JsTree.LocalIndex; + +const Air = @import("Air.zig"); + +const dod = @import("array.zig"); +const tracer = @import("tracer"); diff --git a/src/core/JsTree.zig b/src/core/JsTree.zig new file mode 100644 index 0000000000000000000000000000000000000000..291b621802fe1957910128273fd77b223064ee14 --- /dev/null +++ b/src/core/JsTree.zig @@ -0,0 +1,257 @@ +//! JavaScript Function Representation is the last representation before javascript +//! source bytes are created. One notable thing here is that all local identifiers +//! are numbered, instead of being named. The first `param_count` local identifiers +//! refer to parameters, and the rest refer to the respective block that defined them. + +param_count: u32, +nodes: Node.List.Slice, +extern_symbols: []String.WithLen, +decl_deps: []Decl.Index, +extra: ExtraStore, + +pub const LocalIndex = u32; +pub const ExtraStore = dod.ExtraStore(.tfr, .@"const"); +pub const EncodedData = dod.EncodedData(Tag, u64); +pub const ExtraIndex = ExtraStore.Index; + +pub const Tag = enum { + root, + /// Only allow simple value types: int, float, string, bool + plain_value, + /// Reference to a local identifier (give an VarId) + local, + /// Reference to a global identifier (give a NullTerminatedString) + global, + /// Reference to a declaration (contains a Decl.Index) + decl, + // bin ops + assign, + assign_add, + assign_sub, + assign_mul, + assign_div, + add, + sub, + mul, + div, + eq, + lt, + gt, + // exprs + call, + field, + ternary, + // prefix ops + negate, + bool_not, + pre_inc, + pre_dec, + // stmts + /// `let ... = ...` + stmt_declare_local, + /// `if (...) ... else ...` + stmt_if, + /// `break ... ...` + stmt_break, + /// `return ...` (break from block #0) + stmt_return, + /// `...: { ... }` + stmt_block, + + pub fn Data(comptime tag: Tag) type { + return switch (tag) { + .root => IndexList, + .plain_value => PlainValue, + .local => LocalRef, + .global => GlobalRef, + .decl => DeclRef, + .assign, + .assign_add, + .assign_sub, + .assign_mul, + .assign_div, + .add, + .sub, + .mul, + .div, + .eq, + .lt, + .gt, + => Bin, + .ternary => Ternary, + .negate, + .bool_not, + .pre_inc, + .pre_dec, + => Un, + .call => Call, + .field => Field, + .stmt_if => StmtIf, + .stmt_break => StmtBreak, + .stmt_return => StmtReturn, + .stmt_declare_local => StmtDeclareLocal, + .stmt_block => StmtBlock, + }; + } +}; + +pub const Node = struct { + tag: Tag, + data: EncodedData, + + pub const List = std.MultiArrayList(Node); + + pub const Index = enum(u32) { + root = 0, + _, + + pub fn toOptional(i: Index) Optional { + return @enumFromInt(@intFromEnum(i)); + } + + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: Optional) ?Index { + return if (oi == .none) + null + else + @enumFromInt(@intFromEnum(oi)); + } + }; + }; +}; + +pub const PlainValue = struct { + value: Value, + unused: Unused = .zero, +}; + +pub const LocalRef = struct { + id: LocalIndex, + unused: Unused = .zero, +}; + +pub const GlobalRef = struct { + name: InternPool.String.WithLen, +}; + +pub const DeclRef = struct { + index: Decl.Index, + unused: Unused = .zero, +}; + +pub const StmtDeclareLocal = struct { + identifier_index: u32, + init: Node.Index.Optional, +}; + +pub const Bin = struct { + lhs: Node.Index, + rhs: Node.Index, +}; + +pub const Un = struct { + operand: Node.Index, + unused: Unused = .zero, +}; + +pub const Call = struct { + base: Node.Index, + args: ExtraIndex(IndexList), +}; + +pub const Field = struct { + base: Node.Index, + field: NullTerminatedString, +}; + +pub const IndexList = struct { + start: u32, + len: u32, + + pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index { + return @ptrCast(tree.extra.list.items[list.start..][0..list.len]); + } +}; + +pub const StmtIf = struct { + cond: Node.Index, + data: ExtraIndex(Data), + + pub const Data = struct { + yes: IndexList, + no: IndexList, + flags: Flags, + }; + + pub const Flags = packed struct(u32) { + block_label: u16, + yes_label: bool, + no_label: bool, + _: u14 = 0, + }; +}; + +pub const Ternary = struct { + cond: Node.Index, + data: ExtraIndex(Data), + + pub const Data = struct { + yes: Node.Index, + no: Node.Index, + }; +}; + +pub const StmtBreak = struct { + label: u32, + unused: Unused = .zero, +}; + +pub const StmtReturn = struct { + operand: Node.Index.Optional, + unused: Unused = .zero, +}; + +pub const StmtBlock = struct { + label: u32, + content: ExtraIndex(IndexList), +}; + +pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void { + tree.nodes.deinit(gpa); + gpa.free(tree.decl_deps); + gpa.free(tree.extern_symbols); + tree.extra.deinit(gpa); +} + +pub fn nodeTag(tree: JsTree, index: Node.Index) Tag { + return tree.nodes.items(.tag)[@intFromEnum(index)]; +} + +pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() { + return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index)); +} + +pub fn mainStatements(tree: *const JsTree) []const Node.Index { + return tree.nodeData(.root, .root).slice(tree); +} + +const JsTree = @This(); + +const std = @import("std"); +const assert = std.debug.assert; + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; + +const NullTerminatedString = InternPool.NullTerminatedString; +const String = InternPool.String; + +const Comp = @import("Comp.zig"); +const Decl = Comp.Decl; + +const dod = @import("array.zig"); + +const Unused = @import("Ast.zig").Unused; diff --git a/src/core/Parse.zig b/src/core/Parse.zig new file mode 100644 index 0000000000000000000000000000000000000000..423adae31bfa917eb9cc976a3a43296e5b1300ab --- /dev/null +++ b/src/core/Parse.zig @@ -0,0 +1,1146 @@ +//! Parser takes a Token list and creates a Node list. It does not own the source code or +//! token data, however transferring into an `Ast` does move token list ownership. +gpa: Allocator, +source: [:0]const u8, +token_tags: []const Token.Tag, +token_starts: []const ByteOffset, + +pos: TokenIndex = 0, +nodes: Node.List = .{}, +extra: ArrayListUnmanaged(u32) = .{}, +scratch: ArrayListUnmanaged(NodeIndex) = .{}, + +ast_error: ?AstError.WithTrace = null, + +depth: u32 = 0, + +/// Instead of implementing a non-recursive parser, the stack size is limited by this constant. +const depth_limit = 4096; + +pub fn parse(gpa: std.mem.Allocator, source: [:0]const u8, ast_error: *AstError.WithTrace) Parse.Error!Ast { + var err_byte: ByteOffset = 0; + var tokens = Tokenizer.tokenize( + source, + gpa, + &err_byte, + ) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + ast_error.* = .{ + .err = .{ .tokenization = .{ + .err = err, + .byte = err_byte, + } }, + .trace = Report.DebugStoredTrace.from(@errorReturnTrace()), + }; + return error.ParseFail; + }, + }; + errdefer tokens.deinit(gpa); + + var parser = Parse.init(gpa, source, tokens); + errdefer parser.deinit(); + + try parser.nodes.ensureUnusedCapacity( + gpa, + @max(1, @divFloor(tokens.len, 4)), + ); + + parser.parseRoot() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseFail => { + ast_error.* = parser.ast_error.?; + return error.ParseFail; + }, + }; + + return try parser.finalize(tokens); +} + +fn init(gpa: Allocator, source: [:0]const u8, tokens: Token.List.Slice) Parse { + return Parse{ + .gpa = gpa, + .source = source, + .token_tags = tokens.items(.tag), + .token_starts = tokens.items(.offset), + }; +} + +/// This transfers the parse result into an Ast +fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast { + const extra = try p.extra.toOwnedSlice(p.gpa); + defer p.* = undefined; + assert(token_slice.items(.tag).ptr == p.token_tags.ptr); + assert(token_slice.items(.offset).ptr == p.token_starts.ptr); + p.scratch.deinit(p.gpa); + + const ast: Ast = .{ + .utf8_source = p.source, + .tokens = token_slice, + .nodes = p.nodes.toOwnedSlice(), + .extra = extra, + }; + + return ast; +} + +/// This discards all parser work +fn deinit(p: *Parse) void { + defer p.* = undefined; + p.nodes.deinit(p.gpa); + p.extra.deinit(p.gpa); + p.scratch.deinit(p.gpa); +} + +pub const Error = error{ + OutOfMemory, + ParseFail, +}; + +pub const ExpectedContext = enum { + param, + argument, + statement, + expression, + else_clause, + + pub fn description(tag: ExpectedContext) []const u8 { + return switch (tag) { + .param => "a parameter", + .argument => "an argument", + .statement => "a statement", + .expression => "an expression", + .else_clause => "an else clause", + }; + } +}; + +/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos` +/// When printing these, token list and Ast is not actually available, so full tokens +/// are stored in the errors. +pub const AstError = union(enum) { + /// Failure during tokenization + tokenization: struct { + err: Tokenizer.TokenizeError, + byte: ByteOffset, + }, + /// Unexpected data, expected a token. + unexpected_token: struct { + expected: Token.Tag, + got: Token, + }, + /// Unexpected data, expected + unexpected_context: struct { + expected: ExpectedContext, + got: Token, + }, + /// Went past depth maximum + depth_limit: Token, + + pub fn toReport(err: AstError, comp: *Comp, file: *Comp.File, trace: Report.DebugStoredTrace) !Report { + return (Report.Err{ .ast = err }).toReport(comp, file, trace); + } + + pub fn build(err: AstError, ctx: *Report.MsgContext) !void { + switch (err) { + .tokenization => |data| { + // TODO: consider making tokenizer emit AstError more directly, + // so that it can just annotate start and ends + try ctx.msg(switch (data.err) { + error.UnterminatedString => .{ .codepoint_eol = data.byte }, + else => .{ .codepoint = data.byte }, + }, "{s}", .{switch (data.err) { + error.OutOfMemory => unreachable, + error.InvalidSyntax => "unexpected token", + error.TrailingNumericSeparator => "trailing numberic separator", + error.UnterminatedString => "unterminated string literal", + error.InvalidEscapeSequence => "invalid escape sequence", + error.InvalidEscapeCodepoint => "invalid escape codepoint", + error.InvalidUtf8 => "file is not valid utf8", + error.GitConflict => "git merge conflict encountered", + }}); + }, + .unexpected_token => |data| { + try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{ + data.expected.description(), + data.got.tag.description(), + }); + }, + .unexpected_context => |data| { + try ctx.msg(.{ .token_stored = data.got }, "expected {s}, but found {s}", .{ + data.expected.description(), + data.got.tag.description(), + }); + }, + .depth_limit => |data| { + try ctx.msg(.{ .token_stored = data }, "syntax tree is nested too deeply (limit: {d})", .{depth_limit}); + }, + } + } + + pub const WithTrace = struct { + err: AstError, + trace: Report.DebugStoredTrace, + + pub fn toReport(err: WithTrace, comp: *Comp, file: *Comp.File) !Report { + return err.err.toReport(comp, file, err.trace); + } + }; +}; + +pub fn parseRoot(p: *Parse) Error!void { + const trace = t.trace(@src()); + defer trace.end(); + + assert(p.nodes.len == 0); + assert(p.extra.items.len == 0); + assert(p.pos == 0); + assert(p.nodes.capacity > 0); + + p.nodes.appendAssumeCapacity(.{ + .tag = .root, + .main_token = 0, + .data = undefined, + }); + + const list = try p.parseModuleMembers(); + p.nodes.items(.data)[0] = .{ + .list = list, + }; +} + +fn parseModuleMembers(p: *Parse) Error!SubList { + p.depth += 1; + defer p.depth -= 1; + if (p.depth > depth_limit) { + return p.fail(.{ .depth_limit = p.fullToken(p.pos) }); + } + + const trace = t.trace(@src()); + defer trace.end(); + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + // skip module doc comment + while (p.eatToken(.module_doc_comment)) |_| {} + + while (true) switch (p.currentTag()) { + .kw_let, + .kw_var, + .kw_fn, + .kw_pub, + => try p.scratch.append(p.gpa, try p.parseModuleDecl()), + + .kw_struct, + .kw_enum, + .kw_trait, + .kw_impl, + => |tag| std.debug.panic("TODO: parse {s}", .{@tagName(tag)}), + + .eof => break, + + else => { + if (!p.isExpressionStart()) { + return p.failUnexpectedToken(.statement); + } + + try p.scratch.append(p.gpa, try p.parseExpression(true)); + _ = try p.expect(.semi); + }, + }; + + return p.scratchToSubList(scratch_top); +} + +fn parseModuleDecl(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + _ = p.eatToken(.kw_pub); + + return switch (p.currentTag()) { + .kw_var, .kw_let => try p.parseVarDecl(), + + .kw_fn => try p.parseFnDecl(), + + else => unreachable, + }; +} + +fn parseFnDecl(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.kw_fn); + + _ = try p.expect(.id); + + _ = try p.expect(.start_paren); + + const params = try p.parseParamList(); + + const return_type = if (p.eatToken(.minus_lt)) |_| + if (!p.isExpressionStart()) + return p.failUnexpectedToken(.expression) + else + try p.parseExpression(false) + else + null; + + try p.expectWithoutAdvance(.start_brace); + const body = try p.parseBlock(); + + const extra = try p.addExtra(Ast.FnDecl, .{ + .params = params, + .return_type = optionalNodeIndex(return_type), + }); + + return try p.addNode(.{ + .tag = .fn_decl, + .main_token = main_token, + .data = .{ .fn_decl = .{ + .body = body, + .extra = extra, + } }, + }); +} + +fn parseParamList(p: *Parse) Error!SubList { + const trace = t.trace(@src()); + defer trace.end(); + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top}); + defer trace_param.end(); + + if (p.currentTag() == .end_paren) break; + + try p.scratch.append(p.gpa, try p.parseParamDecl()); + + switch (p.currentTag()) { + .comma => { + _ = p.nextToken(); + }, + .end_paren => break, + else => return p.failUnexpectedToken(.param), + } + } + + _ = p.assertToken(.end_paren); + + return p.scratchToSubList(scratch_top); +} + +fn parseParamDecl(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.id); + + _ = try p.expect(.colon); + + const ty = try p.parseExpression(false); + + return try p.addNode(.{ + .tag = .param_decl, + .main_token = main_token, + .data = .{ .param_decl = .{ + .type = ty, + } }, + }); +} + +fn parseBlock(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.start_brace); + + const tag, const list = try p.parseBlockContents(); + assert(p.lastTag() == .end_brace); + + return try p.addNode(.{ + .tag = tag, + .main_token = main_token, + .data = .{ + .list = list, + }, + }); +} + +fn parseLabelExpression(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.id); + _ = try p.expect(.colon); + const expr = try p.parseExpression(false); + return try p.addNode(.{ + .tag = .label, + .main_token = main_token, + .data = .{ .label = .{ + .expr = expr, + } }, + }); +} + +fn parseBlockContents(p: *Parse) Error!struct { Node.Tag, SubList } { + const trace = t.trace(@src()); + defer trace.end(); + + assert(p.lastTag() == .start_brace); + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + const tag: Node.Tag = while (true) switch (p.currentTag()) { + .kw_let, + .kw_var, + => try p.scratch.append(p.gpa, try p.parseVarDecl()), + + .end_brace => break .block_void, + + else => { + if (!p.isExpressionStart()) { + return p.failUnexpectedToken(.statement); + } + + const first_tag = p.currentTag(); + + try p.scratch.append(p.gpa, try p.parseExpression(true)); + + switch (first_tag) { + // some control flow and blocks are allowed to not have a + // trailing semicolons. this is a stylistic choice. + .kw_if, + .start_brace, + => { + // ending a block here should also count as an implicit return, + // even though voiding it with a semicolon is not allowed. + if (p.currentTag() == .end_brace) break .block_implicit_return; + }, + + // TODO: make kw_break and kw_return REQUIRE a semicolon, as + // they are noreturn statements, doing implicit break on + // them makes no sense + + else => { + if (p.eatToken(.semi) == null) { + // this must be last statement in block if it isnt, + // the p.expect(.end_brace) later will fail. + break .block_implicit_return; + } + }, + } + }, + }; + + _ = try p.expect(.end_brace); + + return .{ tag, try p.scratchToSubList(scratch_top) }; +} + +/// VarDecl <- (kw_pub)? (kw_var | kw_let) id (colon expr)? equal expr semi +fn parseVarDecl(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const kind = p.currentTag(); + assert(kind == .kw_var or kind == .kw_let); + + const main_token = p.nextToken(); + + _ = try p.expect(.id); + + const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) |_| + optionalNodeIndex(try p.parseExpression(false)) + else + .none; + + _ = try p.expect(.equal); + + const data_init = try p.parseExpression(true); + + _ = try p.expect(.semi); + + return try p.addNode(.{ + .tag = .var_decl, + .main_token = main_token, + .data = .{ .var_decl = .{ + .type = data_type, + .init = data_init, + } }, + }); +} + +fn isExpressionStart(p: *Parse) bool { + return switch (p.currentTag()) { + .id, + .number_int, + .number_float, + .string, + .builtin_id, + .kw_return, + .kw_break, + .kw_if, + .start_paren, + .start_brace, + .bang, + .tilde, + .minus, + => true, + else => false, + }; +} + +const Operator = enum(u8) { + add = @intFromEnum(Token.Tag.plus), + sub = @intFromEnum(Token.Tag.minus), + mul = @intFromEnum(Token.Tag.star), + div = @intFromEnum(Token.Tag.slash), + rem = @intFromEnum(Token.Tag.percent), + pow = @intFromEnum(Token.Tag.star_star), + @"and" = @intFromEnum(Token.Tag.and_and), + @"or" = @intFromEnum(Token.Tag.pipe_pipe), + bit_and = @intFromEnum(Token.Tag.@"and"), + bit_or = @intFromEnum(Token.Tag.pipe), + bit_xor = @intFromEnum(Token.Tag.caret), + shl = @intFromEnum(Token.Tag.lt_lt), + shr = @intFromEnum(Token.Tag.gt_gt), + eq = @intFromEnum(Token.Tag.equal_equal), + ne = @intFromEnum(Token.Tag.bang_equal), + lt = @intFromEnum(Token.Tag.lt), + le = @intFromEnum(Token.Tag.lt_equal), + gt = @intFromEnum(Token.Tag.gt), + ge = @intFromEnum(Token.Tag.gt_equal), + // Following operators are assignment operators + assign = @intFromEnum(Token.Tag.equal), + add_assign = @intFromEnum(Token.Tag.plus_equal), + sub_assign = @intFromEnum(Token.Tag.minus_equal), + mul_assign = @intFromEnum(Token.Tag.star_equal), + div_assign = @intFromEnum(Token.Tag.slash_equal), + rem_assign = @intFromEnum(Token.Tag.percent_equal), + pow_assign = @intFromEnum(Token.Tag.star_star_equal), + and_assign = @intFromEnum(Token.Tag.and_and_equal), + or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal), + bit_and_assign = @intFromEnum(Token.Tag.and_equal), + bit_or_assign = @intFromEnum(Token.Tag.pipe_equal), + bit_xor_assign = @intFromEnum(Token.Tag.caret_equal), + shl_assign = @intFromEnum(Token.Tag.lt_lt_equal), + shr_assign = @intFromEnum(Token.Tag.gt_gt_equal), + + const Direction = enum { ltr, rtl }; + + /// Only the binary operator levels are actually used, + /// but all are listed for reference. + const Level = enum { + /// e. e[] e() + accessor, + /// new + new, + /// ++e, --e, !e, ~e, +e, -e, typeof, void, delete + prefix, + /// ** + exponentiation, + /// *, /, % + multiplicative, + /// +, - + additive, + /// <<, >>, >>> + bitwise_shift, + /// <, <=, >, >= + relational, + /// ==, != + equality, + /// & + bit_and, + /// ^ + bit_xor, + /// | + bit_or, + /// && + logical_and, + /// ||, ?? + logical_or, + /// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, |= + assign, + }; + + pub fn direction(op: Operator) Direction { + return switch (op) { + .pow => .rtl, + + .assign, + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => .rtl, + + else => .ltr, + }; + } + + pub fn level(op: Operator) Level { + return switch (op) { + .pow => .exponentiation, + .mul, .div, .rem => .multiplicative, + .add, .sub => .additive, + .shl, .shr => .bitwise_shift, + .lt, .le, .gt, .ge => .relational, + .eq, .ne => .equality, + .@"and" => .logical_and, + .@"or" => .logical_or, + .bit_and => .bit_and, + .bit_xor => .bit_xor, + .bit_or => .bit_or, + .assign, + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => .assign, + }; + } + + const lut = brk: { + const token_tags = std.enums.values(Token.Tag); + const op_tags = std.enums.values(Operator); + + var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len; + for (op_tags) |op| { + is_operator_lut[@intFromEnum(op)] = true; + } + + var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len; + for (op_tags) |op| { + ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op)); + } + + break :brk struct { + const is_operator = is_operator_lut; + const ast_tag = ast_tag_lut; + }; + }; + + pub fn fromToken(tag: Token.Tag) ?Operator { + return if (lut.is_operator[@intFromEnum(tag)]) + @enumFromInt(@intFromEnum(tag)) + else + null; + } + + pub fn astTag(op: Operator) Node.Tag { + const tag = lut.ast_tag[@intFromEnum(op)]; + assert(tag != .root); // invalid operator or lut construction + return tag; + } + + pub fn isAssign(op: Operator) bool { + return switch (op) { + .assign, + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => true, + else => false, + }; + } +}; + +fn parseExpression(p: *Parse, allow_assign_operators: bool) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + p.depth += 1; + defer p.depth -= 1; + if (p.depth > depth_limit) { + return p.fail(.{ .depth_limit = p.fullToken(p.pos) }); + } + + assert(p.isExpressionStart()); + + // These unary operators are greedy, so handle them first. + switch (p.currentTag()) { + .kw_return => return p.parseReturn(), + .kw_break => return p.parseBreak(), + else => {}, + } + + var expr = try p.parseExpressionPart(); + + while (Operator.fromToken(p.currentTag())) |op| { + if (!allow_assign_operators and op.isAssign()) break; + expr = try p.parseBinarySuffix(expr, op, allow_assign_operators); + } + + return expr; +} + +fn parseExpressionPart(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + var expr = switch (p.currentTag()) { + // Literals + .id => if (p.nextTag() == .colon) + try p.parseLabelExpression() + else + try p.blankNode(.id), + .number_int => try p.blankNode(.number_int), + .number_float => try p.blankNode(.number_float), + .string => try p.blankNode(.string), + + // Complex expressions + .builtin_id => try p.parseBuiltinCall(), + .kw_return => try p.parseReturn(), + .kw_break => try p.parseBreak(), + .kw_if => try p.parseIf(), + .start_paren => try p.parseGroup(), + .start_brace => try p.parseBlock(), + + // Unary operators + inline .bang, .tilde, .minus => |tag| unary: { + const main_token = p.nextToken(); + const expr = try p.parseExpressionPart(); + + break :unary try p.addNode(.{ + .tag = switch (tag) { + .bang => .bool_not, + .tilde => .bit_not, + .minus => .negate, + else => comptime unreachable, + }, + .main_token = main_token, + .data = .{ .un = .{ + .expr = expr, + } }, + }); + }, + + else => return p.failUnexpectedToken(.expression), + }; + + // parse field, calls, and index expressions + while (true) { + const trace2 = t.traceFmt(@src(), "(field/call)", .{}); + defer trace2.end(); + + switch (p.currentTag()) { + .dot => { + const main_token = p.nextToken(); + const field = try p.expect(.id); + + expr = try p.addNode(.{ + .tag = .field, + .main_token = main_token, + .data = .{ .field = .{ + .base = expr, + .field_token = field, + } }, + }); + }, + .start_paren => { + const main_token = p.nextToken(); + const args = try p.parseCallArgs(); + + const extra = try p.addExtra(Ast.Call, .{ + .args = args, + }); + + expr = try p.addNode(.{ + .tag = .call, + .main_token = main_token, + .data = .{ .call = .{ + .base = expr, + .extra = extra, + } }, + }); + }, + .start_bracket => { + const main_token = p.nextToken(); + const index = try p.parseExpression(false); + + expr = try p.addNode(.{ + .tag = .index, + .main_token = main_token, + .data = .{ .bin = .{ + .lhs = expr, + .rhs = index, + } }, + }); + }, + + else => break, + } + } + + return expr; +} + +fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator, allow_assign_operators: bool) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.nextToken(); + const level = op.level(); + var rhs = try p.parseExpressionPart(); + + while (Operator.fromToken(p.currentTag())) |next_op| { + if (!allow_assign_operators and next_op.isAssign()) break; + + const new_level = next_op.level(); + const use = switch (op.direction()) { + .ltr => @intFromEnum(new_level) < @intFromEnum(level), + .rtl => @intFromEnum(new_level) <= @intFromEnum(level), + }; + if (!use) break; + + rhs = try p.parseBinarySuffix(rhs, next_op, allow_assign_operators); + } + + return try p.addNode(.{ + .tag = op.astTag(), + .main_token = main_token, + .data = .{ .bin = .{ + .lhs = lhs, + .rhs = rhs, + } }, + }); +} + +fn parseReturn(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.kw_return); + + const expr = if (p.isExpressionStart()) + try p.parseExpression(false) + else + null; + + return try p.addNode(.{ + .tag = .@"return", + .main_token = main_token, + .data = .{ .@"return" = .{ + .expr = optionalNodeIndex(expr), + } }, + }); +} + +fn parseBreak(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.kw_break); + + const label = if (p.eatToken(.colon)) |_| + try p.expect(.id) + else + null; + + const expr = if (p.isExpressionStart()) + try p.parseExpression(false) + else + null; + + return try p.addNode(.{ + .tag = .@"break", + .main_token = main_token, + .data = .{ .@"break" = .{ + .label = optionalTokenIndex(label), + .value = optionalNodeIndex(expr), + } }, + }); +} + +fn parseBuiltinCall(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.builtin_id); + + _ = try p.expect(.start_paren); + + const args = try p.parseCallArgs(); + + return try p.addNode(.{ + .tag = .builtin_call, + .main_token = main_token, + .data = .{ .list = args }, + }); +} + +fn parseCallArgs(p: *Parse) Error!SubList { + const trace = t.trace(@src()); + defer trace.end(); + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (p.isExpressionStart()) { + try p.scratch.append(p.gpa, try p.parseExpression(false)); + + switch (p.currentTag()) { + .end_paren => { + break; + }, + .comma => { + _ = p.nextToken(); + if (p.currentTag() == .end_paren) + break; + }, + else => return p.failUnexpectedToken(.argument), + } + } + + _ = try p.expect(.end_paren); + + return p.scratchToSubList(scratch_top); +} + +fn parseIf(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.kw_if); + + const cond = try p.parseExpression(false); + + _ = try p.expectWithoutAdvance(.start_brace); + const yes = try p.parseBlock(); + + const no = if (p.eatToken(.kw_else)) |_| + try p.parseElseValue() + else + null; + + const extra = try p.addExtra(Ast.If, .{ + .yes = yes, + .no = optionalNodeIndex(no), + }); + + return try p.addNode(.{ + .tag = .@"if", + .main_token = main_token, + .data = .{ .@"if" = .{ + .cond = cond, + .extra = extra, + } }, + }); +} + +fn parseElseValue(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + assert(p.lastTag() == .kw_else); + + return switch (p.currentTag()) { + .start_brace => p.parseBlock(), + .kw_if => p.parseIf(), + .id => p.parseLabelExpression(), + + else => return p.failUnexpectedToken(.else_clause), + }; +} + +fn parseGroup(p: *Parse) Error!NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const main_token = p.assertToken(.start_paren); + + const expr = try p.parseExpression(false); + + _ = try p.expect(.end_paren); + + return try p.addNode(.{ + .tag = .group, + .main_token = main_token, + .data = .{ .un = .{ + .expr = expr, + } }, + }); +} + +// helpers + +/// Append a node and return it's index. +fn addNode(p: *Parse, node: Node) !NodeIndex { + const trace = t.trace(@src()); + defer trace.end(); + + try p.nodes.append(p.gpa, node); + // @intCast is always safe because 2^32 tokens alwayss produce less than + // 2^32 nodes, and there is a 2^32 limit on source file bytes. + return @intCast(p.nodes.len - 1); +} + +fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex { + const trace = t.trace(@src()); + defer trace.end(); + + const extra_top = p.extra.items.len; + const bytes = std.mem.asBytes(&extra); + try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes)); + return @intCast(extra_top); +} + +fn fail(p: *Parse, ast_error: AstError) Error { + p.ast_error = .{ + .err = ast_error, + .trace = Report.DebugStoredTrace.capture(@returnAddress()), + }; + return Error.ParseFail; +} + +fn failUnexpectedToken(p: *Parse, kind: ExpectedContext) Error { + return p.fail(.{ .unexpected_context = .{ + .got = p.fullToken(p.pos), + .expected = kind, + } }); +} + +fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex { + const main_token = p.assertToken(tag); + return try p.addNode(.{ + .tag = @field(Node.Tag, @tagName(tag)), + .main_token = main_token, + .data = .{ .none = .{} }, + }); +} + +fn currentTag(p: *Parse) Token.Tag { + return p.token_tags[p.pos]; +} + +fn lastTag(p: *Parse) Token.Tag { + return p.token_tags[p.pos - 1]; +} + +fn nextTag(p: *Parse) Token.Tag { + return p.token_tags[p.pos + 1]; +} + +fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex { + return if (p.token_tags[p.pos] == tag) p.nextToken() else null; +} + +fn expect(p: *Parse, tag: Token.Tag) !TokenIndex { + if (p.currentTag() != tag) { + return p.fail(.{ .unexpected_token = .{ + .got = p.fullToken(p.pos), + .expected = tag, + } }); + } + return p.nextToken(); +} + +fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void { + if (p.currentTag() != tag) { + return p.fail(.{ .unexpected_token = .{ + .got = p.fullToken(p.pos), + .expected = tag, + } }); + } +} + +fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex { + const token = p.nextToken(); + assert(p.token_tags[token] == tag); + return token; +} + +fn nextToken(p: *Parse) TokenIndex { + const result = p.pos; + p.pos += 1; + return result; +} + +fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool { + return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null; +} + +fn fullToken(p: *Parse, token: TokenIndex) Token { + return Token{ + .tag = p.token_tags[token], + .offset = p.token_starts[token], + }; +} + +fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList { + const trace = t.trace(@src()); + defer trace.end(); + + const items = p.scratch.items[scratch_top..]; + const extra_top = p.extra.items.len; + try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]); + return .{ + .start = @intCast(extra_top), + .len = @intCast(items.len), + }; +} + +const Parse = @This(); + +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const Allocator = mem.Allocator; +const ArrayListUnmanaged = std.ArrayListUnmanaged; + +const Tokenizer = @import("Tokenizer.zig"); +const Token = Tokenizer.Token; +const ByteOffset = Tokenizer.ByteOffset; + +const Ast = @import("Ast.zig"); +const Node = Ast.Node; +const Data = Node.Data; +const SubList = Ast.SubList; +const NodeIndex = Ast.NodeIndex; +const TokenIndex = Ast.TokenIndex; +const ExtraIndex = Ast.ExtraIndex; +const OptionalNodeIndex = Ast.OptionalNodeIndex; +const OptionalTokenIndex = Ast.OptionalTokenIndex; +const OptionalExtraIndex = Ast.OptionalExtraIndex; +const optionalTokenIndex = OptionalTokenIndex.init; +const optionalNodeIndex = OptionalNodeIndex.init; +const optionalExtraIndex = OptionalExtraIndex.init; + +const t = @import("tracer"); + +const Report = @import("Report.zig"); +const Comp = @import("Comp.zig"); diff --git a/src/core/Report.zig b/src/core/Report.zig new file mode 100644 index 0000000000000000000000000000000000000000..814a0f3510f5d342912441a6b39b4a66df3cd735 --- /dev/null +++ b/src/core/Report.zig @@ -0,0 +1,864 @@ +//! A Report is a list of errors and warnings for a part of a compilation. +//! +//! `Err/Warn` structs are unions of every possible message, the logic +//! for printing them is all done here. +//! +//! Across the compiler, `Report.Builder`s' `.err` and `.warn` methods +//! can add messages by ID. +errors: []Item = &.{}, +warnings: []Item = &.{}, +string_bytes: []u8 = &.{}, + +pub var debug_show_report_trace: if (builtin.mode == .Debug) bool else void = + if (builtin.mode == .Debug) false; + +pub const Item = struct { + /// What is this diagnostic + data: union { @"error": Err, warn: Warn }, + /// What file was it in + url: String, + + debug_stack_trace: DebugStoredTrace, + + /// Type is never stored, inferred from which list it is in. + pub const Type = enum { @"error", warn }; +}; + +pub const Err = union(enum) { + pub const FieldEnum = std.meta.FieldEnum(Err); + + /// Constructs a `Report` with just a single error. + pub fn toReport(err: Err, comp: *Comp, file: *File, trace: DebugStoredTrace) !Report { + const string_bytes = try comp.gpa.dupe(u8, file.file_path); + errdefer comp.gpa.free(string_bytes); + + const errors = try comp.gpa.dupe(Item, &.{.{ + .data = .{ .@"error" = err }, + .url = .{ .start = 0, .len = @intCast(string_bytes.len) }, + .debug_stack_trace = trace, + }}); + errdefer comp.gpa.free(errors); + + return .{ .errors = errors, .string_bytes = string_bytes }; + } + + ast: @import("Parse.zig").AstError, + + unknown_identifier: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgNode(data.node, "unknown identifier \"{s}\"", .{ + ctx.ast.basicNodeSlice(data.node), + }); + } + }, + invalid_builtin: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken( + ctx.ast.nodeToken(data.node), + "unknown built-in function \"{s}\"", + .{ctx.ast.basicNodeSlice(data.node)}, + ); + } + }, + duplicate_function: struct { + node: NodeIndex, + prev: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ + ctx.ast.fullFnDecl(data.node).id, + }); + try ctx.msgToken(ctx.ast.fnDeclIdToken(data.prev), "also declared here", .{}); + // TODO: be able to store a full list of other declarations + // and add them here as "and here" and so on + } + }, + duplicate_param: struct { + node: NodeIndex, + prev: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.fnDeclIdToken(data.node), "re-declaration of \"{s}\"", .{ + ctx.ast.fullParamDecl(data.node).id, + }); + try ctx.msgToken(ctx.ast.nodeToken(data.prev), "also declared here", .{}); + // TODO: be able to store a full list of other declarations + // and add them here as "and here" and so on + } + }, + invalid_return: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgNode( + data.node, + "return cannot be used outside of a function body", + .{}, + ); + } + }, + unresolved_label: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msg( + .{ .two_tokens = ctx.ast.nodeToken(data.node) + 1 }, + "unknown block label \"{s}\"", + .{ctx.ast.tokenSlice(ctx.ast.nodeToken(data.node) + 2)}, + ); + } + }, + shadow_builtin_variable: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken( + ctx.ast.nodeToken(data.node) + 1, + "declaration shadows reserved built-in \"{s}\"", + .{ctx.ast.fullVarDecl(data.node).id}, + ); + } + }, + comptime_float_to_int: struct { + val: f64, + src: LazyNode, + + fn build(data: @This(), ctx: *MsgContext) !void { + const node = try data.src.get(ctx); + try ctx.msgNode(node, "numeric literal \"{d}\" cannot be coerced to \"int\"", .{data.val}); + } + }, + coersion: struct { + from: Type, + to: Type, + src: LazyNode, + + fn build(data: @This(), ctx: *MsgContext) !void { + const node = try data.src.get(ctx); + try ctx.msgNode(node, "expected \"{s}\", got \"{s}\"", .{ + data.to.displayName(&ctx.comp.intern_pool), + data.from.displayName(&ctx.comp.intern_pool), + }); + } + }, + call_non_function: struct { + call: LazyNode, + ty: Type, + + fn build(data: @This(), ctx: *MsgContext) !void { + const node = try data.call.get(ctx); + try ctx.msgNode(node, "\"{s}\" is not callable", .{ + data.ty.displayName(&ctx.comp.intern_pool), + }); + } + }, + call_mismatch_args: struct { + call: LazyNode, + actual: u32, + expected: u32, + + fn build(data: @This(), ctx: *MsgContext) !void { + const node = try data.call.get(ctx); + try ctx.msgNode(node, "function expected {d} arguments, recieved {d}", .{ + data.expected, + data.actual, + }); + } + }, + assign_builtin: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgNode( + data.node, + "cannot re-assign built-in \"{s}\"", + .{ctx.ast.basicNodeSlice(data.node)}, + ); + } + }, + invalid_assignment_symbol: struct { + node: NodeIndex, + decl: NodeIndex, + kind: UnusedKind, + + fn build(data: @This(), ctx: *MsgContext) !void { + const id = switch (ctx.ast.nodeTag(data.decl)) { + .param_decl => ctx.ast.fullParamDecl(data.decl).id, + .fn_decl => ctx.ast.fullFnDecl(data.decl).id, + .var_decl => ctx.ast.fullVarDecl(data.decl).id, + else => unreachable, // not a valid symbol node + }; + + try ctx.msgToken(ctx.ast.idToken(data.node), "cannot assign to {s} \"{s}\"", .{ + @tagName(data.kind), + id, + }); + try ctx.msgToken(ctx.ast.idToken(data.decl), "declared here", .{}); + } + }, + invalid_assignment_target: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgNode( + data.node, + "invalid assignment target", + .{}, + ); + } + }, +}; + +pub const Warn = union(enum) { + pub const FieldEnum = std.meta.FieldEnum(Warn); + + unused: struct { + node: NodeIndex, + kind: UnusedKind, + + fn build(data: @This(), ctx: *MsgContext) !void { + const id = switch (ctx.ast.nodeTag(data.node)) { + .param_decl => ctx.ast.fullParamDecl(data.node).id, + .fn_decl => ctx.ast.fullFnDecl(data.node).id, + .var_decl => ctx.ast.fullVarDecl(data.node).id, + else => unreachable, // not a valid symbol node + }; + + try ctx.msgToken(ctx.ast.idToken(data.node), "unused {s} \"{s}\"", .{ + @tagName(data.kind), + id, + }); + + try ctx.msgAlone("if this was intentional, discard it with \"_ = {s};\"", .{id}); + } + }, + not_mutated_var: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + const id = switch (ctx.ast.nodeTag(data.node)) { + .var_decl => ctx.ast.fullVarDecl(data.node).id, + else => unreachable, // not a valid variable symbol node + }; + + try ctx.msgToken(ctx.ast.idToken(data.node), "variable \"{s}\" not mutated", .{ + id, + }); + } + }, + invalid_discard: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.nodeToken(data.node), "\"let\" is not required for a discard", .{}); + } + }, + discard_builtin: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.nodeToken(data.node), "discarding a built-in symbol is meaningless", .{}); + } + }, + unnecessary_discard: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.nodeToken(data.node), "useless discard", .{}); + } + }, + dangling_identifier: struct { + node: NodeIndex, + + fn build(data: @This(), ctx: *MsgContext) !void { + try ctx.msgToken(ctx.ast.nodeToken(data.node), "dangling identifier does nothing", .{}); + } + }, + numeric_int_var_ambiguity: struct { + node: LazyNode, + + fn build(data: @This(), ctx: *MsgContext) !void { + const node = try data.node.get(ctx); + try ctx.msgNode(node, "constant integer in a \"var\" has an unknown type, assuming \"float\"", .{}); + try ctx.msgAlone("add \": float\" or \": int\" after the variable name to set the type", .{}); + } + }, +}; + +/// A variant of std.builtin.StackTrace that stores it's data within itself +/// instead of being a pointer. This allows capturing the stack trace and +/// saving it for later printing. +pub const StoredTrace = struct { + data: [32]usize, + index: usize, + + pub const empty = .{ + .data = .{0} ** 32, + .index = 0, + }; + + fn trace(stored: *StoredTrace) std.builtin.StackTrace { + return .{ + .index = stored.index, + .instruction_addresses = &stored.data, + }; + } + + pub fn capture(begin: ?usize) StoredTrace { + var stored: StoredTrace = undefined; + var frame = stored.trace(); + std.debug.captureStackTrace(begin orelse @returnAddress(), &frame); + stored.index = frame.index; + return stored; + } + + pub fn from(stack_trace: ?*std.builtin.StackTrace) StoredTrace { + if (stack_trace) |stack| { + var data: [32]usize = undefined; + @memset(&data, 0); + const items = @min(stack.instruction_addresses.len, 32); + @memcpy(data[0..items], stack.instruction_addresses[0..items]); + return .{ + .data = data, + .index = @min(items, stack.index), + }; + } else { + return empty; + } + } +}; + +pub const DebugStoredTrace = if (builtin.mode == .Debug) + StoredTrace +else + struct { + pub fn capture(_: ?usize) @This() { + return .{}; + } + + pub fn from(_: ?*std.builtin.StackTrace) @This() { + return .{}; + } + }; + +pub const UnusedKind = enum { + param, + constant, + variable, + function, +}; + +pub const String = struct { + start: u32, + len: u32, + + pub fn slice(str: String, report: Report) []const u8 { + return report.string_bytes[str.start..][0..str.len]; + } +}; + +const Prefix = enum { + @"error", + warn, + note, + + fn color(prefix: Prefix) std.io.tty.Color { + return switch (prefix) { + .@"error" => .red, + .warn => .yellow, + .note => .blue, + }; + } +}; + +pub fn deinit(report: Report, gpa: Allocator) void { + gpa.free(report.errors); + gpa.free(report.warnings); + gpa.free(report.string_bytes); +} + +pub fn dumpToWriter(report: Report, comp: *Comp, writer: anytype, tty: std.io.tty.Config) !void { + // try writer.print("{d} errors, {d} warnings\n", .{ report.errors.len, report.warnings.len }); + for (report.errors) |item| { + try report.formatItem(item, .@"error", comp, writer, tty); + } + for (report.warnings) |item| { + try report.formatItem(item, .warn, comp, writer, tty); + } +} + +pub fn formatItem( + report: Report, + item: Item, + item_type: Item.Type, + comp: *Comp, + writer: anytype, + tty: std.io.tty.Config, +) !void { + var arena = std.heap.ArenaAllocator.init(comp.gpa); + defer arena.deinit(); + var sfa = std.heap.stackFallback(16384, arena.allocator()); + const alloc = sfa.get(); + + const file = try comp.getFile(item.url.slice(report)); + + const need_load_ast = !(item_type == .@"error" and item.data.@"error" == .ast); + const ast_data = if (need_load_ast) try file.getAst(comp) else undefined; + + var ctx: MsgContext = .{ + .arena = alloc, + .comp = comp, + .file = file, + .ast = &ast_data, + .utf8_source = (try file.getSource(comp.gpa)).code, + .components = ArrayListUnmanaged(Component).initCapacity(alloc, 16) catch + unreachable, // stack-fallback allocator has enough space. + }; + + switch (item_type) { + inline else => |tag| switch (@field(item.data, @tagName(tag))) { + inline else => |payload| try payload.build(&ctx), + }, + } + + assert(ctx.components.items.len > 0); + + for (ctx.components.items, 1..) |component, i| { + const prefix: Prefix = if (i == 1) + switch (item_type) { + inline else => |tag| @field(Prefix, @tagName(tag)), + } + else + .note; + + const is_last = i == ctx.components.items.len; + + if (i != 1) { + try tty.setColor(writer, .dim); + try writer.writeAll(" │ "); + try tty.setColor(writer, .reset); + } + + try tty.setColor(writer, prefix.color()); + try tty.setColor(writer, .bold); + try writer.writeAll(@tagName(prefix)); + try tty.setColor(writer, .reset); + try tty.setColor(writer, .dim); + try writer.writeAll(": "); + try tty.setColor(writer, .reset); + try writer.writeAll(component.message_utf8); + try writer.writeAll("\n"); + + const utf8_source = ctx.utf8_source; + + const at = component.location orelse { + if (is_last) { + try tty.setColor(writer, .dim); + try writer.print(" {s}", .{"┴"}); + try writer.writeAll("\n"); + try tty.setColor(writer, .reset); + } + continue; + }; + + const line_before = utf8_source[at.loc.line_start..at.byte]; + const trimmed_before = std.mem.trimLeft(u8, line_before, " \t\r"); + const line_after = utf8_source[at.byte + at.slice.len .. at.loc.line_end + 1]; + const width = std.fmt.count("{d}", .{at.loc.line.oneBased()}); + + try tty.setColor(writer, .dim); + try writer.writeByteNTimes(' ', width); + try writer.print("{s}{s}:{d}:{d}\n", .{ + if (i == 1) "┬ " else "├ ", + file.file_path, + at.loc.line.oneBased(), + at.loc.column.oneBased(), + }); + try writer.print("{d}", .{at.loc.line.oneBased()}); + try writer.print("{s} ", .{"│"}); + try tty.setColor(writer, .reset); + try writer.writeAll(trimmed_before); + try tty.setColor(writer, prefix.color()); + try tty.setColor(writer, .bold); + try writer.writeAll(at.slice); + try tty.setColor(writer, .reset); + try writer.writeAll(line_after); + try tty.setColor(writer, .dim); + try writer.writeAll(if (is_last) " ╰" else " ├"); + try writer.writeBytesNTimes( + "─", + width + + (at.byte - at.loc.line_start) - + (line_before.len - trimmed_before.len), + ); + try tty.setColor(writer, .reset); + try tty.setColor(writer, prefix.color()); + try tty.setColor(writer, .bold); + try writer.writeBytesNTimes("━", at.slice.len); + try tty.setColor(writer, .reset); + try writer.writeAll("\n"); + } + + if (builtin.mode == .Debug and debug_show_report_trace) { + var trace = item.debug_stack_trace; + std.debug.dumpStackTrace(trace.trace()); + } +} + +pub fn dumpToStdErr(report: Report, comp: *Comp) !void { + const stderr = std.io.getStdErr(); + const tty = std.io.tty.detectConfig(stderr); + + if (false or + // Windows API uses it's own API calls, so a buffered writer + // cannot be used :/ + (builtin.os.tag == .windows and tty == .windows_api) or + // Debug stack trace printer writes directly to standard err, + // so we need to be unbuffered for that. + (builtin.mode == .Debug and debug_show_report_trace)) + { + try dumpToWriter(report, comp, stderr.writer(), tty); + } else { + var buf = std.io.bufferedWriter(stderr.writer()); + const writer = buf.writer(); + try dumpToWriter(report, comp, writer, tty); + try buf.flush(); + } +} + +pub const Builder = struct { + gpa: Allocator, + current_file: *Comp.File, + errors: ArrayListUnmanaged(Item) = .{}, + warnings: ArrayListUnmanaged(Item) = .{}, + string_bytes: ArrayListUnmanaged(u8) = .{}, + current_file_str: ?String = null, + + pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void { + try b.errors.append(b.gpa, .{ + .data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) }, + .url = try b.currentFileString(), + .debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{}, + }); + } + + pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void { + try b.warnings.append(b.gpa, .{ + .data = .{ .warn = @unionInit(Warn, @tagName(kind), data) }, + .url = try b.currentFileString(), + .debug_stack_trace = if (builtin.mode == .Debug) StoredTrace.capture(@returnAddress()) else .{}, + }); + } + + pub fn addString(b: *Builder, str: []const u8) !String { + const top = b.string_bytes.items.len; + try b.string_bytes.appendSlice(b.gpa, str); + return .{ .start = @intCast(top), .len = @intCast(str.len) }; + } + + pub fn currentFileString(b: *Builder) !String { + return b.current_file_str orelse { + b.current_file_str = try b.addString(b.current_file.file_path); + return b.current_file_str.?; + }; + } + + pub fn deinit(b: *Builder) void { + b.errors.deinit(b.gpa); + b.warnings.deinit(b.gpa); + b.string_bytes.deinit(b.gpa); + } + + pub fn finalize(b: *Builder) !Report { + defer b.* = undefined; + errdefer b.deinit(); + + const errors = try b.errors.toOwnedSlice(b.gpa); + errdefer b.gpa.free(errors); + + const warnings = try b.warnings.toOwnedSlice(b.gpa); + errdefer b.gpa.free(warnings); + + const string_bytes = try b.string_bytes.toOwnedSlice(b.gpa); + errdefer b.gpa.free(string_bytes); + + return .{ + .errors = errors, + .warnings = warnings, + .string_bytes = string_bytes, + }; + } +}; + +pub const SliceLoc = struct { + loc: Location, + byte: ByteOffset, + slice: []const u8, +}; + +const Component = struct { + message_utf8: []const u8, + location: ?SliceLoc, +}; + +pub const Highlight = union(enum) { + // No source view + alone, + // Highlight one token + token: TokenIndex, + // Highlight one token without referring to the Ast + token_stored: Tokenizer.Token, + // Highlight two tokens + two_tokens: TokenIndex, + // Highlight a Node + node: NodeIndex, + // Highlight a codepoint + codepoint: ByteOffset, + // Highlight from the start of a codepoint to '\n' + codepoint_eol: ByteOffset, + + pub fn loc(highlight: Highlight, ast: *const Ast) ?SliceLoc { + return switch (highlight) { + .alone => null, + inline .node, + .token, + .two_tokens, + => |payload, tag| loc: { + const token_index: TokenIndex = switch (tag) { + .node => ast.nodeToken(payload), + .token, .two_tokens => payload, + else => comptime unreachable, + }; + + const location = ast.tokenLocation(0, token_index); + const token = ast.tokens.get(token_index); + + break :loc .{ + .loc = location, + .slice = token.slice(ast.utf8_source), + .byte = token.offset, + }; + }, + + .token_stored => |token| loc: { + const location = Tokenizer.tokenLocation(ast.utf8_source, 0, token.offset); + + break :loc .{ + .loc = location, + .slice = token.slice(ast.utf8_source), + .byte = token.offset, + }; + }, + inline .codepoint, + .codepoint_eol, + => |b, tag| loc: { + const len = switch (tag) { + .codepoint => std.unicode.utf8ByteSequenceLength(ast.utf8_source[b]) catch 1, + .codepoint_eol => if (std.mem.indexOfScalar(u8, ast.utf8_source[b..], '\n')) |n| + n + else + (ast.utf8_source.len - b), + else => comptime unreachable, + }; + break :loc .{ + .byte = b, + .slice = ast.utf8_source[b..][0..len], + .loc = .{ + // TODO: + .column = .start, + .line = .start, + .line_start = 0, + .line_end = @intCast(ast.utf8_source.len), + }, + }; + }, + }; + } +}; + +/// This serves as a small API to make creating messages easier. +pub const MsgContext = struct { + file: *File, + comp: *Comp, + ast: *const Ast, + utf8_source: [:0]const u8, + + /// This arena is is a stack fallback with 16kb, then + /// falling back to an arena allocator. This should + /// make the cost of arbitrarily creating `Component` + /// nodes extremely lightweight. + arena: Allocator, + + /// First component is "error" or "warn" + /// Trailing components are "note" + components: ArrayListUnmanaged(Component), + + pub fn msg(m: *MsgContext, highlight: Highlight, comptime fmt: []const u8, args: anytype) !void { + const text = try std.fmt.allocPrint(m.arena, fmt, args); + try m.components.append(m.arena, .{ + .message_utf8 = text, + .location = highlight.loc(m.ast), + }); + } + + pub fn msgNode(m: *MsgContext, node: NodeIndex, comptime fmt: []const u8, args: anytype) !void { + return m.msg(.{ .node = node }, fmt, args); + } + + pub fn msgToken(m: *MsgContext, token: TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return m.msg(.{ .token = token }, fmt, args); + } + + pub fn msgAlone(m: *MsgContext, comptime fmt: []const u8, args: anytype) !void { + return m.msg(.alone, fmt, args); + } +}; + +pub const LazyNode = struct { + decl: Comp.Decl.Index, + inst: Teg.Inst.Index, + operand: Operand, + + pub fn get(lazy: LazyNode, ctx: *MsgContext) !NodeIndex { + const decl = ctx.comp.intern_pool.declPtrConst(lazy.decl); + const ns = ctx.comp.intern_pool.namespacePtr(decl.src_namespace); + + // TODO: This might change if we have errors like + // - function arguments not matching at + // - function declared + // And they are both in different files. + // To resolve this, `Component` needs to store which file it is in. + assert(ns.file_scope == ctx.file); + + if (ns.parent.unwrap() != null) { + @panic("TODO: LazyNode.get when the decl is in a nested namespace"); + } + + const teg = &(ctx.file.teg orelse + // this will be required once incremental compilation hits + @panic("TODO: teg was unloaded")); + + const decl_inst = decl.teg_decl_index.unwrap() orelse + unreachable; // file was not analyzed, yet teg loaded? + + const teg_decl = teg.instData(.declaration, decl_inst.resolve(&ctx.comp.intern_pool)); + + const node_offset = teg.instNodeOffset(lazy.inst); + + const base_node: NodeIndex = @intCast(teg_decl.src_node + node_offset); + if (lazy.operand == .main_node) + return base_node; + + const data = ctx.ast.nodeData(base_node); + + const node_tag = ctx.ast.nodeTag(base_node); + + switch (teg.instTag(lazy.inst)) { + .as => switch (node_tag) { + .var_decl => { + switch (lazy.operand) { + .rhs => return data.var_decl.init, + else => {}, + } + }, + else => {}, + }, + .call => switch (node_tag) { + .call => switch (lazy.operand) { + .operand => return data.call.base, + else => |op| return ctx.ast.extraData(data.call.extra, Ast.Call) + .args.slice(ctx.ast)[op.getArgId()], + }, + else => {}, + }, + .@"break" => switch (node_tag) { + .@"return" => return data.@"return".expr.unwrap().?, + .@"break" => return data.@"break".value.unwrap().?, + .block_implicit_return => { + const statements = ctx.ast.nodeStatements(base_node); + return statements[statements.len - 1]; + }, + else => return base_node, + }, + .memory => switch (node_tag) { + .var_decl => switch (lazy.operand) { + .operand => return data.var_decl.init, + else => {}, + }, + else => {}, + }, + .store => { + assert(node_tag.isAssignBin()); + switch (lazy.operand) { + .lhs => return data.bin.lhs, + .rhs => return data.bin.rhs, + else => unreachable, + } + }, + else => {}, + } + + if (!std.debug.runtime_safety) unreachable; + + std.debug.panic("TODO: operand lookup: .{s} + .{s} + {}", .{ + @tagName(teg.instTag(lazy.inst)), + @tagName(ctx.ast.nodeTag(base_node)), + lazy.operand, + }); + } +}; + +pub const Operand = enum(u32) { + /// Point to the main node of an instruction + main_node = 0, + /// Base of a call, field, or operand to a unary instruction + operand = 1, + /// Return type of a function declaration or lambda node + return_type = 2, + /// Left hand side of a binary instruction + lhs = 4, + /// Right hand side of a binary instruction + rhs = 5, + + /// One of: + /// - An argument of a .call node + /// - A parameter of a .fn_decl node + pub fn argument(id: u32) Operand { + return @enumFromInt(id + 5); + } + + fn getArgId(op: Operand) u32 { + return @intFromEnum(op) - 5; + } +}; + +const Report = @This(); +const std = @import("std"); +const ArrayListUnmanaged = std.ArrayListUnmanaged; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; + +const builtin = @import("builtin"); + +const Tokenizer = @import("Tokenizer.zig"); +const ByteOffset = Tokenizer.ByteOffset; + +const Ast = @import("Ast.zig"); +const TokenIndex = Ast.TokenIndex; +const NodeIndex = Ast.NodeIndex; +const Location = Ast.Location; + +const Teg = @import("Teg.zig"); + +const Comp = @import("Comp.zig"); +const File = Comp.File; + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; +const Type = InternPool.Type; + +const util = @import("util.zig"); diff --git a/src/core/Sema.zig b/src/core/Sema.zig new file mode 100644 index 0000000000000000000000000000000000000000..cae7f9f6fd9a8663acf6d0f2c55f06954ec7e15f --- /dev/null +++ b/src/core/Sema.zig @@ -0,0 +1,999 @@ +/// absolutely stealing zig on this one haha +/// This struct is created per declaration and chills + +// inputs: +comp: *Comp, +gpa: Allocator, +arena: Allocator, +code: Teg, +ip: *InternPool, +decl: *Decl, +decl_index: Decl.Index, +namespace_index: Namespace.Index, + +// temp data: +inst_map: InstMap = undefined, +/// scratch memory. can be used in any place you might temporarily allocate an arraylist +/// this is most notably used to store block contents before the block is finalized +scratch: std.ArrayListUnmanaged(u32) = .{}, +params: []Type = undefined, + +// outputs +report: Report.Builder = undefined, +air_instructions: std.MultiArrayList(Air.Inst) = .{}, +air_extra: dod.ExtraStore(.air, .@"var") = .{}, + +temp_result_map: std.AutoArrayHashMapUnmanaged(Teg.Inst.Index, Type) = .{}, + +pub const Error = error{ + /// Analysis has failed, see error list + AnalysisFail, + // This analysis is waiting on other dependencies to finish + // Interrupt, + + // TODO: add handling of this + TooManyValues, + HasNulls, + + OutOfMemory, +} || Comp.File.GetTegError; + +/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything. +/// This is fine because all instructions are going to be adacent in a single declaration. +const InstMap = struct { + items: []Air.Inst.Ref.Optional, + start: Teg.Inst.Index, + + pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap { + const decl = code.declData(inst_start); + const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count); + @memset(insts, .none); + return .{ + .items = insts, + .start = inst_start, + }; + } + + pub fn deinit(map: *InstMap, allocator: Allocator) void { + allocator.free(map.items); + map.* = undefined; + } + + pub fn get(map: *const InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref { + const source_i = @intFromEnum(teg_code); + const start_i = @intFromEnum(map.start); + assert(source_i >= start_i and source_i <= (start_i + map.items.len)); + return map.items[source_i - start_i].unwrap(); + } + + pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void { + if (assertWithMessage(map.get(from) == null)) { + std.debug.panic("%{d} is already mapped (to #{d})", .{ + @intFromEnum(from), + @intFromEnum(map.get(from).?.decode().index), + }); + } + const source_i = @intFromEnum(from); + const start_i = @intFromEnum(map.start); + map.items[source_i - start_i] = to.toOptional(); + } +}; + +pub fn init(s: *Sema) !void { + const t = tracer.trace(@src()); + defer t.end(); + + s.report = .{ + .gpa = s.gpa, + .current_file = s.ip.namespacePtr(s.namespace_index).file_scope, + }; + + try s.comp.current_compile_decls.append(s.gpa, s.decl_index); + + s.inst_map = try InstMap.init( + s.arena, + &s.code, + s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip), + ); +} + +pub fn deinit(s: *Sema) void { + const t = tracer.trace(@src()); + defer t.end(); + + const report = s.report.finalize() catch @panic("OOM"); + report.dumpToStdErr(s.comp) catch @panic("Error"); + report.deinit(s.gpa); + + defer s.* = undefined; + s.inst_map.deinit(s.arena); + s.scratch.deinit(s.arena); + s.air_extra.deinit(s.gpa); + s.air_instructions.deinit(s.gpa); + s.temp_result_map.deinit(s.arena); +} + +pub fn finalize(s: *Sema) !void { + const air: Air = s.tempAir(); + + if (s.comp.options.dump_ir) { + std.debug.print("=" ** 20, .{}); + std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)}); + air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {}; + } + + try s.comp.codeGenDecl(s.decl, &air); +} + +// --- analyze functions --- + +/// This is the entrypoint to semantic analysis: It all starts at a declaration. +pub fn analyzeDeclaration(s: *Sema) Error!Value { + const t = tracer.trace(@src()); + defer t.end(); + + // TODO: turn this entire function into a nonrecursive state machine, so + // that analysis can pause and continue at any point. for example, when a + // dependency on another declaration is required, but that declaration has + // not been resolved yet. or, we are waiting on a file read to happen + + const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start); + assert(decl.kind == .function); + + s.params = try s.gpa.alloc(Type, fn_decl.params.len); + defer s.gpa.free(s.params); + + for (fn_decl.params.slice(&s.code), 0..) |param, i| { + s.params[i] = try s.resolveType( + s.inst_map.start.op(Report.Operand.argument(@intCast(i))), + param, + ); + } + + const return_type = try s.resolveType( + s.inst_map.start.op(.return_type), + fn_decl.return_type, + ); + + const root_block = fn_decl.body; + switch (root_block.decode()) { + .value => |value| { + const coerced = try s.coerceValue( + s.inst_map.start.op(.operand), + value, + return_type, + ); + const block = try s.addBlock(); + _ = try s.addInstRef(.@"break", .{ + .block_ref = block.block_index, + .value = coerced.toAir(), + }); + }, + .index => |index| { + try s.temp_result_map.putNoClobber(s.arena, index, return_type); + _ = try s.analyzeBlock(index); + }, + } + + const func_prototype = (try s.ip.get(s.gpa, .{ + .type_function = .{ + .param_types = s.params, + .return_type = return_type, + }, + })).toTypeUnsafe(s.ip); + const value = try s.ip.get(s.gpa, .{ + .func = .{ + .ty = func_prototype, + .owner_decl = s.decl_index, + .teg_body_inst = s.decl.teg_decl_index.unwrap().?, + }, + }); + s.decl.val = value; + return value; +} + +pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { + const t = tracer.trace(@src()); + defer t.end(); + + const scratch_top = s.scratch.items.len; + defer s.scratch.shrinkRetainingCapacity(scratch_top); + + const block = try s.addBlock(); + try s.inst_map.add(in, block.block_index.toRef()); + + var last: Air.Inst.Ref = undefined; + const instructions = s.code.blockInstructions(in); + assert(instructions.len != 0); + for (instructions) |inst| { + last = try s.analyzeCode(inst); + } + + if (scratch_top + 1 == s.scratch.items.len) { + const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1]; + if (single_instruction == .@"break") { + const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1] + .decode(.@"break", single_instruction); + if (data.block_ref == block.block_index) { + // This is a block that resolved to just a single value ref with + // no side effects. This means we can just lower this to the ref + // itself, and give codegen an easier time. + + // Delete the block and the single instruction + // Break does not use any extra data, so this call removes both items + s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index)); + + return data.value; + } + } + } + + return (try block.finish(s, scratch_top)).toRef(); +} + +fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref { + const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) }); + defer t.end(); + + const out = try s.analyzeCodeInner(in); + + // Block refs are added automatically + if (s.code.instTag(in) == .block) { + assert(s.inst_map.get(in) != null); + } else { + try s.inst_map.add(in, out); + } + + return out; +} + +fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { + switch (s.code.instTag(in)) { + .string => { + const string: Teg.SlicedString = s.code.instPayload(.string, in); + return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) }); + }, + .number => { + const number: dod.Float = s.code.instPayload(.number, in); + return s.makeValueAir(.{ .comptime_float = number.decode() }); + }, + .js => { + const operand_ref = s.code.instData(.js, in).operand; + const operand = try s.resolveRefComptimeType(in.op(.operand), operand_ref, .string); + return s.addInstRef(.js_global_ref, operand.asString(s.ip)); + }, + .field => { + const field: Teg.Inst.Bin = s.code.instPayload(.field, in); + const base = try s.resolveRefRuntime(field.lhs); + + const rhs = try s.resolveRefComptimeType(in.op(.rhs), field.rhs, .string); + const string = rhs.asString(s.ip); + + const base_type = s.typeOfRef(base); + switch (base_type) { + .@"opaque" => { + return s.addInstRef(.field, .{ + .base = base, + .field = try string.tryToNullTerminatedString(s.ip, s.gpa), + }); + }, + else => |tag| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}), + } + }, + .load_decl => { + const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in); + const decl_name = try s.ip.getOrPutString( + s.gpa, + request.string.slice(s.code.string_bytes), + .no_embedded_nulls, + ); + const decl_index = s.lookupDecl(decl_name) orelse { + std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)}); + return error.AnalysisFail; + }; + const resolved = try s.resolveDecl(decl_index); + return resolved.toAir(); + }, + .param => { + const param: Teg.Inst.DataNode = s.code.instData(.param, in); + return s.addInstRef(.arg, .{ + .type = s.params[param.data], + .arg_number = param.data, + }); + }, + inline .add, + .sub, + .mul, + .div, + .lt, + .gt, + .eq, + => |tag| { + const bin: Teg.Inst.Bin = s.code.instPayload(tag, in); + + const lhs = try s.resolveRefRuntime(bin.lhs); + const rhs = try s.resolveRefRuntime(bin.rhs); + + if (lhs.kind == .value and rhs.kind == .value) { + return (try s.comptimeBin( + @field(BinaryOp, @tagName(tag)), + lhs.decode().value, + rhs.decode().value, + )); + } + + const left_type = s.typeOfRef(lhs); + _ = left_type; // autofix + const right_type = s.typeOfRef(rhs); + _ = right_type; // autofix + + // TODO: validate type for operation + + return s.addInstRef(tag.airTag(), .{ + .lhs = lhs, + .rhs = rhs, + }); + }, + inline .negate, .bool_not => |tag| { + const un: Teg.Inst.RefNode = s.code.instData(tag, in); + + const operand = try s.resolveRefRuntime(un.operand); + + const operand_type = s.typeOfRef(operand); + _ = operand_type; // autofix + + // TODO: validate type for operation + + return s.addInstRef(tag.airTag(), .{ + .operand = operand, + }); + }, + .@"break" => { + const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in); + const operand = try s.resolveRefRuntime(brk.operand); + const block_ref = s.inst_map.get(brk.block).?.decode().index; // block ref is resolved by teg + + const coerced = if (s.temp_result_map.get(brk.block)) |ty| + try s.coerceRef(in.op(.operand), operand, ty) + else + operand; + + return s.addInstRef(.@"break", .{ + .block_ref = block_ref, + .value = coerced, + }); + }, + .call => { + const call: Teg.Inst.Call = s.code.instPayload(.call, in); + const base = try s.resolveRefRuntime(call.base); + const base_type = s.typeOfRef(base); + + const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len); + defer s.arena.free(scratch); + if (base_type != .@"opaque") { + const proto = base_type.toFunctionPrototype(s.ip) orelse { + return s.fail(.call_non_function, .{ + .call = s.nodeLoc(in, .operand), + .ty = base_type, + }); + }; + + if (proto.param_types.len != call.args.len) { + // TODO: also attempt to coerce some of the params to reveal more errors. + return s.fail(.call_mismatch_args, .{ + .call = s.nodeLoc(in, .operand), + .actual = @intCast(call.args.len), + .expected = @intCast(proto.param_types.len), + }); + } + + // validate function args + for (proto.param_types, call.args.slice(&s.code), 0..) |param, arg, i| { + const ref = try s.resolveRefRuntime(arg); + const coerced = try s.coerceRef( + in.op(Report.Operand.argument(@intCast(i))), + ref, + param, + ); + scratch[i] = coerced; + } + } else { + // just copy the args in without checking + for (call.args.slice(&s.code), 0..) |arg, i| { + const ref = try s.resolveRefRuntime(arg); + scratch[i] = ref; + } + } + + const start = s.air_extra.list.items.len; + try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch)); + + return s.addInstRef(.call, .{ + .base = base, + .call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{ + .args = .{ + .start = @intCast(start), + .len = @intCast(scratch.len), + }, + }), + }); + }, + .ensure_void => { + const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false); + const t = s.typeOfRef(operand); + _ = t; + // if (!t.isEnsureVoid(s.ip)) { + // std.debug.print("oh no, we are discarding {} on accident\n", .{ + // t.fmt(s.ip), + // }); + // } + return InternPool.Value.void.toAir(); + }, + .discard => { + const operand = try s.resolveRefRuntimeExtra(s.code.instData(.discard, in).operand, false); + const t = s.typeOfRef(operand); + if (t == .void) { + std.debug.print("oh no, we have a useless _ = void;\n", .{}); + } + return InternPool.Value.void.toAir(); + }, + .@"if" => { + const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code); + + const cond = try s.resolveRefRuntime(data.cond); + + // try s.validateType(s.typeOfRef(cond), .bool_type); + + // If known at comptime, only analyze the taken path + if (cond.kind == .value) { + @panic("TODO: comptime IF"); + } + + // Emit a runtime branch + const yes_block = try s.analyzeRefToBlock(data.yes); + const no_block = try s.analyzeRefToBlock(data.no); + + return try s.addInstRef(.@"if", .{ + .cond = cond, + .data = try s.air_extra.append(s.gpa, Air.If.Data, .{ + .yes = yes_block, + .no = no_block, + .union_ty = s.unionType(s.typeOfRef(yes_block), s.typeOfRef(no_block)), + }), + }); + }, + .block => { + const block = try s.analyzeBlock(in); + switch (block.decode()) { + .index => |t| if (s.tempAir().instTag(t) == .block) { + try s.scratch.append(s.gpa, @intFromEnum(t)); + }, + else => {}, + } + return block; + }, + .memory => { + const data = s.code.instData(.memory, in); + + var initial_value = try s.resolveRefRuntime(data.operand); + + switch (initial_value.decode()) { + .value => |v| switch (v.toKey(s.ip)) { + .comptime_float => |val| { + try s.report.warn(.numeric_int_var_ambiguity, .{ + .node = s.nodeLoc(in, .operand), + }); + initial_value = (try s.makeValue(.{ .float = val })).toAir(); + }, + else => {}, + }, + else => {}, + } + + return try s.addInstRef(.memory, .{ + .init = initial_value, + .ty = s.typeOfRef(initial_value), + }); + }, + .store => { + const data: Teg.Inst.Store = s.code.instPayload(.store, in); + assert(s.code.instTag(data.memory) == .memory); // corrupt store instruction + + const location = s.mapTeg(data.memory.toRef()).decode().index; + s.markInst(location); + const operand = try s.resolveRefRuntime(data.operand); + const operand_coerced = try s.coerceRef( + in.op(.rhs), + operand, + s.typeOfRef(location.toRef()), + ); + return try s.addInstRef(.store, .{ + .memory = location, + .operand = operand_coerced, + }); + }, + .as => { + const as: Teg.Inst.As = s.code.instPayload(.as, in); + const coerce_to = try s.resolveRefComptimeType(in.op(.lhs), as.type, .type); + const operand = try s.resolveRefRuntime(as.operand); + return try s.coerceRef(in.op(.rhs), operand, coerce_to.toType(s.ip).?); + }, + else => |tag| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}), + } +} + +fn markInst(s: *Sema, inst: Air.Inst.Index) void { + const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)]; + flags.uses = flags.uses.increment(); +} + +fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) Error!Air.Inst.Ref { + const t = tracer.trace(@src()); + defer t.end(); + + const scratch_top = s.scratch.items.len; + defer s.scratch.shrinkRetainingCapacity(scratch_top); + + const visited = try s.resolveRefRuntime(in); + + if (scratch_top == s.scratch.items.len and visited.kind == .value) { + return visited; + } + + if (visited.kind == .index and + s.air_instructions.items(.tag)[@intFromEnum(visited.decode().index)] == .block) + { + return visited; + } + + const block = try s.addBlock(); + _ = try s.addInstRef(.@"break", .{ + .block_ref = block.block_index, + .value = visited, + }); + return (try block.finish(s, scratch_top)).toRef(); +} + +fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) Error!Air.Inst.Ref { + const index = ref.toIndex() orelse { + return ref.toValue().?.toAir(); + }; + + if (s.inst_map.get(index)) |air| { + if (mark) { + switch (air.decode()) { + .index => |i| s.markInst(i), + else => {}, + } + } + return air; + } + + return s.analyzeCode(index); +} + +fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) Error!Air.Inst.Ref { + return resolveRefRuntimeExtra(s, ref, true); +} + +fn resolveRefComptimeAny(s: *Sema, ref: Teg.Inst.Ref) !Value { + const index = switch (ref.decode()) { + .value => |value| return value, + .index => |index| index, + }; + + if (s.inst_map.get(index)) |air| { + switch (air.decode()) { + .value => |value| return value, + else => {}, + } + } + + switch (s.code.instTag(index)) { + .string => { + const string: Teg.SlicedString = s.code.instPayload(.string, index); + return s.makeValue(.{ .string = string.slice(s.code.string_bytes) }); + }, + + else => |tag| std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}), + } +} + +fn resolveRefComptimeType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref, ty: Type) !Value { + const any = try s.resolveRefComptimeAny(ref); + return try s.coerceValue(src, any, ty); +} + +fn resolveType(s: *Sema, src: Teg.InstOp, ref: Teg.Inst.Ref) !Type { + return (try s.resolveRefComptimeType(src, ref, .type)).toTypeUnsafe(s.ip); +} + +/// Coerce `value` runtime ref to `coerce_type`, and emit an error at `teg_ref`'s location +fn coerceRef(s: *Sema, src: InstOp, value: Air.Inst.Ref, to: Type) !Air.Inst.Ref { + const from = s.typeOfRef(value); + if (from == to) { + return value; + } + const index = switch (value.decode()) { + .value => |v| return (try s.coerceValue(src, v, to)).toAir(), + .index => |index| index, + }; + switch (to) { + .@"opaque" => { + // + }, + .bool => { + // + }, + .int32 => switch (from) { + .int32 => unreachable, // same type handled + .numeric_literal => unreachable, // must should comptime known + else => {}, + }, + .float => switch (from) { + .float => unreachable, // same type handled + .int32 => return index.toRef(), + else => {}, + }, + .string => { + // + }, + .noreturn => { + // + }, + .type => { + // + }, + .numeric_literal => { + @panic("TODO: error message for not known at compile time"); + }, + .void => { + // + }, + else => {}, + } + return s.coersionError(src, from, to); +} + +fn coerceValue(s: *Sema, src: InstOp, value: Value, to: Type) !Value { + const from = value.typeOf(s.ip); + if (from == to) { + return value; + } + switch (to) { + .@"opaque" => { + // + }, + .bool => { + // + }, + .int32 => switch (from) { + .int32 => unreachable, // same type handled + .numeric_literal, .float => switch (value.toKey(s.ip)) { + .comptime_float, .float => |val| { + if (!math.isFinite(val) or + val < math.minInt(i32) or + val > math.maxInt(i32) or + (@rem(val, 1) != 0)) + { + return s.fail(.comptime_float_to_int, .{ + .val = val, + .src = s.lazyNode(src), + }); + } + + return try s.makeValue(.{ + .int32 = @intFromFloat(val), + }); + }, + else => unreachable, // not a numeric literal + }, + else => return s.coersionError(src, from, to), + }, + .float => switch (from) { + .float => unreachable, // same type handled + .numeric_literal => if (value.toFloat(s.ip)) |val| + return try s.makeValue(.{ + .float = val, + }), + else => {}, + }, + .string => { + // + }, + .noreturn => { + // + }, + .type => { + if (value == .void) return .void; + }, + .numeric_literal => {}, + .void => { + if (value == .void) return .void; + }, + else => {}, + } + return s.coersionError(src, from, to); +} + +fn resolveDecl(s: *Sema, decl_index: Decl.Index) Error!Value { + const decl = s.ip.declPtr(decl_index); + switch (decl.analysis) { + .unreferenced => { + decl.analysis = .in_progress; + + // TODO: make this short-circuit the current analysis, move this analyze + // job into a "deferred" list, and then make this thread analyze something else + + var arena = std.heap.ArenaAllocator.init(s.gpa); + defer arena.deinit(); + + var todo: Report = undefined; + var sema: Sema = .{ + .comp = s.comp, + .gpa = s.gpa, + .ip = s.ip, + .arena = arena.allocator(), + .code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.comp, &todo), + .decl = decl, + .decl_index = decl_index, + .namespace_index = decl.src_namespace, + }; + + try sema.init(); + defer sema.deinit(); + const value = try sema.analyzeDeclaration(); + try sema.finalize(); + + decl.analysis = .complete; + + return value; + }, + else => { + return decl.val; + }, + } +} + +fn unionType(s: *Sema, a: Type, b: Type) Type { + if (a == b) { + return a; + } + + if (a == .noreturn) + return b; + + if (b == .noreturn) + return a; + + std.debug.panic( + "TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}", + .{ + a.dump(s.ip), + b.dump(s.ip), + }, + ); + // return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } }); +} + +const BinaryOp = enum { + add, + sub, + mul, + div, + lt, + gt, + eq, +}; + +fn comptimeBin(s: *Sema, op: BinaryOp, lhs: Value, rhs: Value) !Air.Inst.Ref { + switch (op) { + .add => { + const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add"); + const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / string add"); + return (try s.ip.get(s.gpa, .{ + .float = l + r, + })).toAir(); + }, + inline .sub, .mul, .div => |tag| { + const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); + const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); + + const result = switch (tag) { + .sub => l - r, + .mul => l * r, + .div => l / r, + else => comptime unreachable, + }; + + return (try s.ip.get(s.gpa, .{ + .float = result, + })).toAir(); + }, + inline .lt, + .gt, + => |tag| { + const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); + const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck"); + const result = switch (tag) { + .lt => l < r, + .gt => l > r, + else => comptime unreachable, + }; + return @as( + InternPool.Value, + if (result) .bool_true else .bool_false, + ).toAir(); + }, + .eq, + => { + const l = lhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq"); + const r = rhs.toFloat(s.ip) orelse @panic("TODO: typecheck / non-numeric eq"); + const result = l == r; + return @as( + InternPool.Value, + if (result) .bool_true else .bool_false, + ).toAir(); + }, + } +} + +/// --- helper --- +const IncompleteBlock = struct { + block_index: Air.Inst.Index, + + pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index { + s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top)); + return block.block_index; + } +}; + +fn addBlock(s: *Sema) !IncompleteBlock { + try s.air_instructions.append(s.gpa, .{ + .tag = .block, + .data = undefined, + .flags = .{}, + }); + return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) }; +} + +fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref { + try s.air_instructions.append(s.gpa, .{ + .tag = tag, + .data = Air.EncodedData.encode(tag, data), + .flags = .{}, + }); + const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1); + try s.scratch.append(s.arena, @intFromEnum(index)); + return index.toRef(); +} + +fn setInstData( + s: *Sema, + comptime tag: Air.Inst.Tag, + index: Air.Inst.Index, + data: tag.Data(), +) void { + const i = @intFromEnum(index); + assert(s.air_instructions.items(.tag)[i] == tag); + s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data); +} + +fn makeValue(s: *Sema, key: InternPool.Key) !Value { + return try s.ip.get(s.gpa, key); +} + +fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref { + return (try s.makeValue(key)).toAir(); +} + +fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref { + if (inst.toValue()) |interned| { + return interned.toAir(); + } + const index = inst.toIndex().?; + return s.inst_map.get(index) orelse { + if (std.debug.runtime_safety) { + std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)}); + } + unreachable; + }; +} + +fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList { + const items = s.scratch.items[scratch_top..]; + const extra_top = s.air_extra.list.items.len; + try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]); + return .{ + .start = @intCast(extra_top), + .len = @intCast(items.len), + }; +} + +fn tempAir(s: *const Sema) Air { + return .{ + .insts = s.air_instructions.slice(), + .extra = s.air_extra.constant(), + }; +} + +fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Type { + return s.tempAir().typeOf(ref, s.ip); +} + +fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex { + var nsi = s.namespace_index; + while (true) { + const ns = s.ip.namespacePtr(nsi); + if (ns.lookupDirect(s.comp, name)) |decl_id| { + return decl_id; + } + nsi = ns.parent.unwrap() orelse + unreachable; // transform ensures all identifiers get resolved + } +} + +fn nodeLoc(s: *const Sema, in: Teg.Inst.Index, op: Report.Operand) Report.LazyNode { + return s.lazyNode(in.op(op)); +} + +fn addWarn(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { + try s.report.warn(kind, data); +} + +fn addError(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { + try s.report.err(kind, data); +} + +fn fail(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) Error { + try s.report.err(kind, data); + return Error.AnalysisFail; +} + +fn coersionError(s: *Sema, src: InstOp, from: Type, to: Type) Error { + return s.fail(.coersion, .{ + .from = from, + .to = to, + .src = s.lazyNode(src), + }); +} + +fn lazyNode(s: *const Sema, src: InstOp) Report.LazyNode { + return src.lazyNode(s.decl_index); +} + +const Sema = @This(); + +const std = @import("std"); +const mem = std.mem; +const math = std.math; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const log = std.log.scoped(.sema); + +inline fn assertWithMessage(cond: bool) bool { + if (std.debug.runtime_safety) { + return !cond; + } else if (!cond) { + unreachable; // assertion failure + } +} + +const Comp = @import("Comp.zig"); +const File = Comp.File; +const Decl = Comp.Decl; +const Namespace = Comp.Namespace; + +const Teg = @import("Teg.zig"); +const InstOp = Teg.InstOp; + +const Air = @import("Air.zig"); +const Report = @import("Report.zig"); +const Ast = @import("Ast.zig"); +const dod = @import("array.zig"); +const tracer = @import("tracer"); + +const InternPool = @import("InternPool.zig"); +const Value = InternPool.Value; +const Type = InternPool.Type; +const NullTerminatedString = InternPool.NullTerminatedString; diff --git a/src/core/Teg.Dump.zig b/src/core/Teg.Dump.zig new file mode 100644 index 0000000000000000000000000000000000000000..e50cd07496835e9fabe3f26ca134788c4f1918ab --- /dev/null +++ b/src/core/Teg.Dump.zig @@ -0,0 +1,314 @@ +//! This implements a debug formatter for Teg +teg: Teg, + +const State = struct { + arena: std.mem.Allocator, + teg: *const Teg, + indent: usize = 0, + tty: std.io.tty.Config, + visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{}, + + fn newline(state: State, writer: anytype) !void { + try writer.writeByte('\n'); + if (state.indent > 0) + try writer.writeByteNTimes(' ', state.indent * 2); + } + + fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void { + const teg = s.teg; + const tty = s.tty; + + try writer.print("%{} = ", .{@intFromEnum(inst)}); + + const tag = s.teg.instTag(inst); + + try tty.setColor(writer, .bright_black); + const node_offset = teg.instNodeOffset(inst); + try writer.print("{c}{d} ", .{ + @as(u8, if (node_offset >= 0) '+' else '-'), + @abs(node_offset), + }); + try tty.setColor(writer, .reset); + + try tty.setColor(writer, .cyan); + try writer.print("{s}", .{@tagName(tag)}); + try tty.setColor(writer, .reset); + + if ((try s.visited.fetchPut(s.arena, inst, {})) != null) { + try tty.setColor(writer, .red); + try writer.writeAll(" [error: recursive tree]"); + try tty.setColor(writer, .reset); + return; + } + + switch (tag) { + .module => { + try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}'); + }, + .declaration => { + const decl_info = teg.declData(inst); + try writer.print(" ({s}) ", .{@tagName(decl_info.kind)}); + + const name = decl_info.name.slice(teg.string_bytes); + if (name.len == 0) { + try tty.setColor(writer, .bold); + try writer.writeAll(""); + } else { + try tty.setColor(writer, .magenta); + try writer.writeAll(name); + } + try tty.setColor(writer, .reset); + + try writer.writeAll(" {"); + s.indent += 1; + try s.newline(writer); + + switch (decl_info.kind) { + .function => { + const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1"; + try writer.writeAll("params: "); + try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']'); + try s.newline(writer); + try writer.writeAll("return_type: "); + try s.dumpRef(writer, fn_data.return_type); + try s.newline(writer); + try writer.writeAll("body: "); + try s.dumpRef(writer, fn_data.body); + }, + } + s.indent -= 1; + try s.newline(writer); + try writer.writeAll("}"); + }, + .param => { + const data = teg.instData(.param, inst); + try writer.writeAll("("); + try tty.setColor(writer, .yellow); + try writer.print("{d}", .{data.data}); + try tty.setColor(writer, .reset); + try writer.writeAll(")"); + }, + .block => { + const block = teg.instData(.block, inst); + const index_list = block.extra_index.get(teg); + try s.dumpInstList(writer, index_list.slice(teg), '[', ']'); + }, + .number => { + const number = teg.instData(.number, inst); + const float = number.extra_index.get(teg).decode(); + try tty.setColor(writer, .yellow); + try writer.print(" {d}", .{float}); + try tty.setColor(writer, .reset); + }, + inline .field, + .add, + .sub, + .mul, + .div, + .rem, + .pow, + .@"and", + .@"or", + .bit_and, + .bit_or, + .bit_xor, + .shl, + .shr, + .eq, + .ne, + .lt, + .le, + .gt, + .ge, + => |comptime_tag| { + const data = teg.instData(comptime_tag, inst).extra_index.get(teg); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.lhs); + try writer.writeAll(", "); + try s.dumpRefMarker(writer, data.rhs); + try writer.writeAll(")"); + }, + .as => { + const data = teg.instData(.as, inst).extra_index.get(teg); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.type); + try writer.writeAll(", "); + try s.dumpRefMarker(writer, data.operand); + try writer.writeAll(")"); + }, + .@"break" => { + const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.block.toRef()); + try writer.writeAll(", "); + try s.dumpRefMarker(writer, data.operand); + try writer.writeAll(")"); + }, + .store => { + const data: Teg.Inst.Store = teg.instData(.store, inst).extra_index.get(teg); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.memory.toRef()); + try writer.writeAll(", "); + try s.dumpRefMarker(writer, data.operand); + try writer.writeAll(")"); + }, + .string => { + const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg); + try tty.setColor(writer, .green); + try writer.writeAll(" \""); + try writer.writeAll(data.slice(teg.string_bytes)); + try writer.writeAll("\""); + try tty.setColor(writer, .reset); + }, + inline .memory, + .discard, + .ensure_void, + .ensure_bool, + .negate, + .bool_not, + .bit_not, + .js, + .import, + .fetch, + .compile_error, + => |comptime_tag| { + const data = teg.instData(comptime_tag, inst); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.operand); + try writer.writeAll(")"); + }, + .call => { + const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg); + try writer.writeAll("("); + try s.dumpRefMarker(writer, call.base); + try writer.writeAll(", "); + try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']'); + try writer.writeAll(")"); + }, + .load_decl => { + const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst); + + try tty.setColor(writer, .red); + try writer.print(" {s}", .{call.string.slice(teg.string_bytes)}); + try tty.setColor(writer, .reset); + }, + .@"if" => { + const data: Teg.Inst.If = teg.instPayload(.@"if", inst); + try writer.writeAll("("); + try s.dumpRefMarker(writer, data.cond); + try writer.writeAll(") {"); + s.indent += 1; + try s.newline(writer); + try writer.writeAll("yes: "); + try s.dumpRef(writer, data.yes); + try s.newline(writer); + try writer.writeAll("no: "); + try s.dumpRef(writer, data.no); + s.indent -= 1; + try s.newline(writer); + try writer.writeAll("}"); + }, + } + } + + fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void { + if (ref.toIndex()) |index| { + try s.dumpInst(writer, index); + } else { + try s.tty.setColor(writer, .blue); + try writer.print("%{s}", .{@tagName(ref)}); + try s.tty.setColor(writer, .reset); + } + } + + fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void { + if (ref.toIndex()) |index| { + try s.tty.setColor(writer, .blue); + try writer.print("%{d}", .{@intFromEnum(index)}); + try s.tty.setColor(writer, .reset); + } else { + try s.tty.setColor(writer, .blue); + try writer.print("%{s}", .{@tagName(ref)}); + try s.tty.setColor(writer, .reset); + } + } + + fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + if (list.len > 0) { + state.indent += 1; + + for (list) |inst| { + try state.newline(writer); + try state.dumpInst(writer, inst); + } + + state.indent -= 1; + try state.newline(writer); + } + try writer.print("{c}", .{r}); + } + + fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + if (list.len > 0) { + state.indent += 1; + + for (list) |ref| { + try state.newline(writer); + try state.dumpRef(writer, ref); + } + + state.indent -= 1; + + try state.newline(writer); + } + try writer.print("{c}", .{r}); + } + + fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void { + try writer.print(" {c}", .{l}); + if (list.len > 0) { + state.indent += 1; + + for (list) |ref| { + try state.newline(writer); + try state.dumpRefMarker(writer, ref); + } + + state.indent -= 1; + + try state.newline(writer); + } + try writer.print("{c}", .{r}); + } +}; + +pub fn format( + dump: @This(), + comptime unused_fmt: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, +) !void { + comptime assert(unused_fmt.len == 0); + + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena.deinit(); + + const teg = dump.teg; + var state = State{ + .teg = &teg, + .tty = detectTTYFromWriter(writer), + .arena = arena.allocator(), + }; + try state.dumpInst(writer, .main_module_inst); + try writer.writeAll("\n"); + assert(state.indent == 0); +} + +const std = @import("std"); +const assert = std.debug.assert; + +const Teg = @import("Teg.zig"); + +const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter; diff --git a/src/core/Teg.zig b/src/core/Teg.zig new file mode 100644 index 0000000000000000000000000000000000000000..5b76ced3658ab57f233eca18e6396439a6a21573 --- /dev/null +++ b/src/core/Teg.zig @@ -0,0 +1,486 @@ +//! TODO: rename this to something else. Teg is not a graph + +insts: Inst.List.Slice, +string_bytes: []const u8, +extra: ExtraStore = .{}, +report: Report, + +pub const ExtraStore = dod.ExtraStore(.teg, .@"const"); +pub const EncodedData = dod.EncodedData(Inst.Tag, u64); +pub const ExtraIndex = ExtraStore.Index; +pub const Dump = @import("Teg.Dump.zig"); + +pub const Inst = struct { + pub const List = std.MultiArrayList(Inst); + + tag: Tag, + data: EncodedData, + + // todo: fully document every thing + pub const Tag = enum { + module, + declaration, + + block, + /// Creates a memory cell in the current scope. + memory, + /// Performs a type coersion, such as in variable declarations. + /// Emits a sema error if the type conversion fails. + /// `let x: i32 = 1` + as, + load_decl, + /// Load a parameter + param, + store, + discard, + ensure_void, + ensure_bool, + /// lowered from `break` or `return` + /// ends a block with a resulting value + @"break", + number, + string, + + @"if", + + add, + sub, + mul, + div, + rem, + pow, + @"and", + @"or", + bit_and, + bit_or, + bit_xor, + shl, + shr, + eq, + ne, + lt, + le, + gt, + ge, + + negate, + bool_not, + bit_not, + field, + call, + + /// Implements the `@js` builtin + js, + /// Implements the `@import` builtin + import, + /// Implements the `@fetch` builtin + fetch, + /// Implements the `@compileError` builtin + compile_error, + + pub fn Data(comptime tag: Tag) type { + return switch (tag) { + .module => PlNode(IndexList), + .declaration => PlNode(Decl), + .block => PlNode(IndexList), + .as => PlNode(As), + .@"break" => PlNode(Break), + .number => PlNode(Float), + .string => PlNode(SlicedString), + .call => PlNode(Call), + .@"if" => PlNode(If), + .load_decl => StrNode, + .param => DataNode, + .store => PlNode(Store), + + // Single instruction operand + .memory, + .discard, + .ensure_void, + .ensure_bool, + .negate, + .bool_not, + .bit_not, + .js, + .import, + .fetch, + .compile_error, + => RefNode, + + // Two instruction operands + .field, + .add, + .sub, + .mul, + .div, + .rem, + .pow, + .@"and", + .@"or", + .bit_and, + .bit_or, + .bit_xor, + .shl, + .shr, + .eq, + .ne, + .lt, + .le, + .gt, + .ge, + => PlNode(Bin), + }; + } + + pub fn airTag(comptime tag: Tag) Air.Inst.Tag { + return @field(Air.Inst.Tag, @tagName(tag)); + } + }; + + /// TODO: make enum? idk + const NodeOffset = i32; + + pub fn PlNode(comptime Payload: type) type { + return struct { + pub const T = Payload; + + src_node: NodeOffset, + extra_index: ExtraIndex(Payload), + }; + } + + pub const RefNode = struct { + src_node: NodeOffset, + operand: Inst.Ref, + }; + + pub const StrNode = struct { + src_node: NodeOffset, + string: NullTerminatedString, + }; + + pub const DataNode = struct { + src_node: NodeOffset, + data: u32, + }; + + /// The position of a Teg instruction within the `Teg` instructions array. + pub const Index = enum(u32) { + /// Teg is structured so that the outermost "main" struct of any file + /// is always at index 0. + main_module_inst = 0, + _, + + pub const static_len = @intFromEnum(Ref.empty_string) + 1; + + pub fn toRef(i: Index) Inst.Ref { + return @enumFromInt(static_len + @intFromEnum(i)); + } + + pub fn toOptional(i: Index) OptionalIndex { + return @enumFromInt(@intFromEnum(i)); + } + + pub fn op(i: Index, operand: Report.Operand) InstOp { + return .{ .index = i, .op = operand }; + } + }; + + pub const OptionalIndex = enum(u32) { + /// Teg is structured so that the outermost "main" module of any file + /// is always at index 0. + main_module_inst = 0, + ref_start_index = Index.static_len, + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: OptionalIndex) ?Index { + return if (oi == .none) null else @enumFromInt(@intFromEnum(oi)); + } + }; + + /// A reference to instruction, or to an InternPool index, or neither. + /// + /// If the integer tag value is < InternPool.static_len, then it + /// corresponds to an InternPool index. Otherwise, this refers to a ZIR + /// instruction. + /// + /// The tag type is specified so that it is safe to bitcast between `[]u32` + /// and `[]Ref`. + pub const Ref = enum(u32) { + opaque_type, + bool_type, + int32_type, + float_type, + string_type, + noreturn_type, + type_type, + numeric_literal_type, + + void, + bool_true, + bool_false, + int32_zero, + float_zero, + nan, + empty_string, + standard_library, + + /// This Ref does not correspond to any instruction or constant + /// value and may instead be used as a sentinel to indicate null. + none = std.math.maxInt(u32), + + _, + + pub const Decoded = union(enum) { + value: InternPool.Value, + index: Teg.Inst.Index, + }; + + pub fn decode(ref: Ref) Decoded { + const ref_int = @intFromEnum(ref); + return if (ref_int >= Index.static_len) + .{ .index = @enumFromInt(ref_int - Index.static_len) } + else + .{ .value = @enumFromInt(ref_int) }; + } + + pub fn toIndex(ref: Ref) ?Index { + assert(ref != .none); + const ref_int = @intFromEnum(ref); + if (ref_int >= Index.static_len) { + return @enumFromInt(ref_int - Index.static_len); + } else { + return null; + } + } + + pub fn toIndexAllowNone(inst: Ref) ?Index { + if (inst == .none) return null; + return toIndex(inst); + } + + pub fn toValue(inst: Ref) ?InternPool.Value { + const ref_int = @intFromEnum(inst); + + // todo: is this > or >= + if (ref_int > Index.static_len) + return null; + + return @enumFromInt(ref_int); + } + }; + + pub const Bin = struct { + lhs: Inst.Ref, + rhs: Inst.Ref, + }; + + pub const Store = struct { + memory: Inst.Index, + operand: Inst.Ref, + }; + + pub const As = struct { + type: Inst.Ref, + operand: Inst.Ref, + }; + + pub const Break = struct { + block: Inst.Index, + operand: Inst.Ref, + }; + + pub const Call = struct { + base: Inst.Ref, + args: RefList, + }; + + pub const If = struct { + cond: Inst.Ref, + yes: Inst.Ref, + no: Inst.Ref, + }; + + /// This will contain trailing data depending on the .kind + pub const Decl = struct { + name: NullTerminatedString, + hash: Cache.BinDigest, + kind: Kind, + instruction_count: u32, + + pub const Kind = enum { + function, + // future: struct, let, var, etc + }; + }; + + /// Trailng data for function declarations + pub const DeclFn = struct { + params: RefList, + return_type: Ref, + body: Ref, + }; +}; + +pub const InstOp = struct { + index: Inst.Index, + op: Report.Operand, + + pub fn lazyNode(inst: InstOp, decl: Comp.Decl.Index) LazyNode { + return .{ + .inst = inst.index, + .operand = inst.op, + .decl = decl, + }; + } +}; + +pub const NullTerminatedString = enum(u32) { + empty = 0, + _, + + pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 { + if (str == .empty) return ""; + const sub = string_bytes[@intFromEnum(str)..]; + const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable; + + return sub[0..i :0]; + } +}; + +pub const SlicedString = struct { + start: u32, + len: u32, + + pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 { + return string_bytes[str.start..][0..str.len]; + } +}; + +pub const RefList = struct { + start: u32, + len: u32, + + pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref { + return @ptrCast(teg.extra.list.items[self.start..][0..self.len]); + } +}; + +pub const IndexList = struct { + start: u32, + len: u32, + + pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index { + return @ptrCast(teg.extra.list.items[self.start..][0..self.len]); + } +}; + +pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void { + gpa.free(teg.string_bytes); + teg.extra.deinit(gpa); + teg.insts.deinit(gpa); + teg.report.deinit(gpa); + teg.* = undefined; +} + +pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag { + return teg.insts.items(.tag)[@intFromEnum(index)]; +} + +pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData { + return teg.insts.items(.data)[@intFromEnum(index)]; +} + +pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() { + return teg.instEncodedData(index).decode(tag, teg.instTag(index)); +} + +pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T { + return teg.instData(tag, index).extra_index.get(teg); +} + +pub fn instNodeOffset(teg: *const Teg, index: Inst.Index) i32 { + switch (teg.instTag(index)) { + inline else => |tag| { + return teg.instData(tag, index).src_node; + }, + } +} + +pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index { + return teg.instData(.block, index).extra_index.get(teg).slice(teg); +} + +pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } { + const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg); + const decl_fn = trail.read(&teg.extra, Inst.DeclFn); + return .{ decl, decl_fn }; +} + +pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl { + return teg.instData(.declaration, index).extra_index.get(teg); +} + +pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index { + const module = teg.instData(.module, module_index); + const module_data: IndexList = module.extra_index.get(teg); + const slice = module_data.slice(teg); + if (std.debug.runtime_safety) { + for (slice) |item| { + if (teg.instTag(item) != .declaration) { + std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))}); + unreachable; + } + } + } + return slice; +} + +pub fn rootFunction(teg: *const Teg) Inst.Index { + const decls = teg.moduleDeclarations(.main_module_inst); + const root_fn = decls[decls.len - 1]; + assert(teg.declName(root_fn).len == 0); + return root_fn; +} + +pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 { + const decl = teg.instData(.declaration, decl_index); + const decl_data: Inst.Decl = decl.extra_index.get(teg); + return decl_data.name.slice(teg.string_bytes); +} + +pub fn dumpToWriter(teg: Teg, writer: anytype) !void { + try writer.print("{}", .{Dump{ .teg = teg }}); +} + +const Teg = @This(); + +const std = @import("std"); +const mem = std.mem; +const assert = std.debug.assert; + +const Cache = @import("Cache.zig"); + +const Ast = @import("Ast.zig"); +const Unused = Ast.Unused; +const NodeIndex = Ast.NodeIndex; +const TokenIndex = Ast.TokenIndex; +const OptionalNodeIndex = Ast.OptionalNodeIndex; +const OptionalTokenIndex = Ast.OptionalTokenIndex; +const OptionalExtraIndex = Ast.OptionalExtraIndex; +const optionalTokenIndex = OptionalTokenIndex.init; +const optionalNodeIndex = OptionalNodeIndex.init; +const optionalExtraIndex = OptionalExtraIndex.init; + +const dod = @import("array.zig"); +const Float = dod.Float; + +const InternPool = @import("InternPool.zig"); + +const Air = @import("Air.zig"); + +const Comp = @import("Comp.zig"); + +const Report = @import("Report.zig"); +const LazyNode = Report.LazyNode; diff --git a/src/core/Tokenizer.zig b/src/core/Tokenizer.zig new file mode 100644 index 0000000000000000000000000000000000000000..9afadd8b6918d36cab142ccf359eb25584502c2a --- /dev/null +++ b/src/core/Tokenizer.zig @@ -0,0 +1,995 @@ +//! The tokenizer is extremely simple structure that, given a view of source +//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit +//! offset to where that token is located. +//! +//! Retriving payloads like string/numeric values, identifier names, etc is +//! done separately, either in Parse or Transform, when the respective data +//! is actually needed. Tokenization does guarante that such data is valid. +code: [:0]const u8, +pos: ByteOffset = 0, + +/// Tokenize source code into a list of tokens. +/// When an error occurs, the position of the error is returned in `error_pos`. +/// Caller owns returned slice. +pub fn tokenize( + source_code: [:0]const u8, + gpa: mem.Allocator, + error_pos: *ByteOffset, +) TokenizeError!Token.List.Slice { + const trace = T.trace(@src()); + defer trace.end(); + + var tokens = Token.List{}; + errdefer tokens.deinit(gpa); + + // pre-allocate some space based on a very rough guess + try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8)); + + var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 }; + errdefer error_pos.* = tokenizer.pos; + + tokenizer.skipWhitespace(); + while (try tokenizer.next()) |token| { + try tokens.append(gpa, token); + tokenizer.skipWhitespace(); + } + + try tokens.append(gpa, .{ + .tag = .eof, + .offset = @intCast(source_code.len), + }); + + return tokens.toOwnedSlice(); +} + +pub const ByteOffset = u32; + +pub const Token = struct { + tag: Tag, + offset: ByteOffset, + + pub const List = std.MultiArrayList(Token); + + pub const Tag = enum { + id, + builtin_id, + number_int, + number_float, + string, + multiline_string, + doc_comment, + module_doc_comment, + + // sort-lines: start + kw_assert, + kw_break, + kw_comptime, + kw_defer, + kw_else, + kw_enum, + kw_fn, + kw_infer, + kw_for, + kw_while, + kw_if, + kw_impl, + kw_json, + kw_let, + kw_match, + kw_module, + kw_pub, + kw_pure, + kw_return, + kw_struct, + kw_trait, + kw_var, + // sort-lines: end + + // sort-lines: start + @"and", + and_and, + and_and_equal, + and_equal, + bang, + bang_equal, + caret, + caret_equal, + colon, + comma, + dot, + end_brace, + end_bracket, + end_paren, + equal, + equal_equal, + equal_lt, + gt, + gt_equal, + gt_gt, + gt_gt_equal, + gt_gt_gt, + gt_gt_gt_equal, + lt, + lt_equal, + lt_lt, + lt_lt_equal, + minus, + minus_equal, + minus_lt, + percent, + percent_equal, + pipe, + pipe_equal, + pipe_pipe, + pipe_pipe_equal, + plus, + plus_equal, + semi, + slash, + slash_equal, + star, + star_equal, + star_star, + star_star_equal, + start_brace, + start_bracket, + start_paren, + tilde, + tilde_equal, + // sort-lines: end + + /// Represents a tokenization error. Instead of reporting an error during + /// tokenization, parsing will begin, and an error will be reported when + /// this token is hit. That way, the error can tell you what it expected. + invalid, + /// End of file + /// Used so that parser does not need to do out of bounds checks + eof, + + /// Most tokens have statically known lexeme, such as "struct" or "+=" + pub fn lexeme(tag: Tag) ?[]const u8 { + return switch (tag) { + .invalid => unreachable, + + // do not have a static lexeme + .eof, + .id, + .builtin_id, + .number_int, + .number_float, + .string, + .multiline_string, + .doc_comment, + .module_doc_comment, + => null, + + .@"and" => "&", + .and_and => "&&", + .bang => "!", + .bang_equal => "!=", + .equal_lt => "=<", + .caret => "^", + .end_brace => "}", + .end_bracket => "]", + .end_paren => ")", + .colon => ":", + .comma => ",", + .dot => ".", + .equal => "=", + .equal_equal => "==", + .gt => ">", + .gt_equal => ">=", + .gt_gt => ">>", + .gt_gt_gt => ">>>", + .lt => "<", + .lt_equal => "<=", + .lt_lt => "<<", + .minus => "-", + .start_brace => "{", + .start_bracket => "[", + .start_paren => "(", + .pipe => "|", + .pipe_pipe => "||", + .percent => "%", + .plus => "+", + .minus_lt => "->", + .semi => ";", + .slash => "/", + .star => "*", + .star_star => "**", + .tilde => "~", + .slash_equal => "/=", + .percent_equal => "%=", + .tilde_equal => "~=", + .caret_equal => "^=", + .and_equal => "&=", + .pipe_equal => "|=", + .lt_lt_equal => "<<=", + .gt_gt_equal => ">>=", + .gt_gt_gt_equal => ">>>=", + .plus_equal => "+=", + .minus_equal => "-=", + .star_equal => "*=", + .star_star_equal => "**=", + .pipe_pipe_equal => "||=", + .and_and_equal => "&&=", + + inline else => |keyword| comptime len: { + if (!mem.startsWith(u8, @tagName(keyword), "kw_")) { + @compileError("Token tag is not a keyword: " ++ @tagName(keyword)); + } + break :len @tagName(keyword)[3..]; + }, + }; + } + + pub fn description(tag: Tag) []const u8 { + return switch (tag) { + .invalid => unreachable, + .eof => "end of file", + .id => "an identifier", + .builtin_id => "a builtin identifier", + .number_int => "a numeric literal", + .number_float => "a numeric literal", + .string => "a string literal", + .multiline_string => "a multiline string literal", + .doc_comment => "a documentation comment", + .module_doc_comment => "a module documentation comment", + inline else => |t| comptime "\"" ++ t.lexeme().? ++ "\"", + }; + } + + /// Most tokens have a static length, such as "struct" or "+=" + pub fn staticLen(tag: Tag) ?u32 { + return if (tag.lexeme()) |str| @intCast(str.len) else null; + } + }; + + /// Instead of storing the length of a token, we can recompute this + /// information on demand, as it is not often needed. + /// + /// If you are computing the length of something that is knowable + /// without the source code, use token.tag.staticLen() instead. + pub fn len(token: Token, utf8_source: [:0]const u8) u32 { + return token.tag.staticLen() orelse { + var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset }; + assert( + (tokenizer.next() catch + unreachable // data corruption + orelse unreachable // data corruption + ).tag == token.tag, + ); + return tokenizer.pos - token.offset; + }; + } + + pub fn slice(token: Token, utf8_source: [:0]const u8) []const u8 { + return token.tag.lexeme() orelse + utf8_source[token.offset..][0..token.len(utf8_source)]; + } + + const keywords = std.ComptimeStringMap(Tag, keywords: { + const fields = std.enums.values(Tag); + const KV = struct { []const u8, Tag }; + var kws: [fields.len]KV = undefined; + var i = 0; + for (fields) |field| { + if (mem.startsWith(u8, @tagName(field), "kw_")) { + kws[i] = .{ @tagName(field)[3..], field }; + i += 1; + } + } + break :keywords kws[0..i]; + }); +}; + +/// When any of these error occurs, the tokenizer's position is set to +/// the start of the offending token. +pub const NextTokenError = error{ + /// Generic token syntax error + InvalidSyntax, + /// A trailing '_' was seen in a numeric literal. + TrailingNumericSeparator, + /// Newline or EOF encountered while parsing a string literal. + UnterminatedString, + /// A string escape sequence was invalid. + InvalidEscapeSequence, + /// The codepoint represented by a unicode escape sequence + /// is not a valid unicode codepoint. + InvalidEscapeCodepoint, + /// There are unresolved git conflicts in the source code. + GitConflict, + /// The source code is not valid UTF-8. + InvalidUtf8, +}; + +pub const TokenizeError = mem.Allocator.Error || NextTokenError; + +/// Next assumes the position is already at the start of a token. +/// You must call `skipWhitespace` before calling this. +pub fn next(t: *Tokenizer) !?Token { + while (true) { + const trace = T.trace(@src()); + defer trace.end(); + + switch (t.byte()) { + ' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace + + 'a'...'z', 'A'...'Z', '_', '$' => { + const start = t.pos; + t.pos += 1; + return try t.tokenizeIdentifier(start); + }, + + '@' => switch (t.nextByte()) { + '"' => { + const start = t.pos; + t.pos += 1; + try t.consumeString(); + return .{ .tag = .id, .offset = start }; + }, + else => { + const start = t.pos; + t.pos += 1; + return try t.tokenizeIdentifier(start); + }, + }, + + '0'...'9' => return try t.tokenizeNumber(), + + '.' => switch (t.nextByte()) { + '0'...'9' => return try t.tokenizeNumber(), + else => return t.simpleToken(.dot), + }, + + '/' => switch (t.nextByte()) { + '=' => return t.simpleToken(.slash_equal), + '/' => { + if ((t.byteAt(2) orelse 0 == '/') and + (t.byteAt(3) orelse 0 != '/')) + { + const start = t.pos; + t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len); + return .{ .tag = .doc_comment, .offset = start }; + } + + if (t.byteAt(2) orelse 0 == '!') { + const start = t.pos; + t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); + return .{ .tag = .module_doc_comment, .offset = start }; + } + + // skip the comment + t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); + t.skipWhitespace(); + continue; + }, + else => return t.simpleToken(.slash), + }, + + '\\' => switch (t.nextByte()) { + '\\' => { + const start = t.pos; + t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len); + return .{ .tag = .multiline_string, .offset = start }; + }, + else => return t.maybeUTF8Error(error.InvalidSyntax), + }, + + '"' => return try t.tokenizeString(), + + '&' => switch (t.nextByte()) { + '&' => return t.simpleToken(.and_and), + '=' => return t.simpleToken(.and_equal), + else => return t.simpleToken(.@"and"), + }, + '|' => switch (t.nextByte()) { + '|' => return t.simpleToken(.pipe_pipe), + '=' => return t.simpleToken(.pipe_equal), + else => return t.simpleToken(.pipe), + }, + '^' => switch (t.nextByte()) { + '^' => return t.simpleToken(.caret), + '=' => return t.simpleToken(.caret_equal), + else => return t.simpleToken(.caret), + }, + '!' => switch (t.nextByte()) { + '=' => return t.simpleToken(.equal_equal), + else => return t.simpleToken(.bang_equal), + }, + '=' => switch (t.nextByte()) { + '=' => return t.simpleToken(.equal_equal), + else => return t.simpleToken(.equal), + }, + '>' => switch (t.nextByte()) { + '=' => return t.simpleToken(.gt_equal), + '>' => switch (t.byteAt(2) orelse 0) { + '>' => return t.simpleToken(.gt_gt_gt), + else => return t.simpleToken(.gt_gt), + }, + else => return t.simpleToken(.gt), + }, + '<' => switch (t.nextByte()) { + '=' => return t.simpleToken(.lt_equal), + '<' => switch (t.byteAt(2) orelse 0) { + '=' => return t.simpleToken(.lt_lt_equal), + else => { + // Git merge conflicts are <<<<<<< + const start = t.pos; + t.pos += 2; + var remain: u3 = 4; + while (t.byte() == '<') { + t.pos += 1; + remain -= 1; + if (remain == 0) { + return error.GitConflict; + } + } + t.pos = start; + return t.simpleToken(.lt_lt); + }, + }, + else => return t.simpleToken(.lt), + }, + '+' => switch (t.nextByte()) { + '=' => return t.simpleToken(.plus_equal), + else => return t.simpleToken(.plus), + }, + '-' => switch (t.nextByte()) { + '=' => return t.simpleToken(.minus_equal), + '>' => return t.simpleToken(.minus_lt), + else => return t.simpleToken(.minus), + }, + '*' => switch (t.nextByte()) { + '*' => return t.simpleToken(.star_star), + '=' => return t.simpleToken(.star_equal), + else => return t.simpleToken(.star), + }, + '%' => switch (t.nextByte()) { + '=' => return t.simpleToken(.percent_equal), + else => return t.simpleToken(.percent), + }, + '~' => switch (t.nextByte()) { + '=' => return t.simpleToken(.tilde_equal), + else => return t.simpleToken(.tilde), + }, + + ';' => return t.simpleToken(.semi), + ':' => return t.simpleToken(.colon), + '(' => return t.simpleToken(.start_paren), + ')' => return t.simpleToken(.end_paren), + '[' => return t.simpleToken(.start_bracket), + ']' => return t.simpleToken(.end_bracket), + '{' => return t.simpleToken(.start_brace), + '}' => return t.simpleToken(.end_brace), + ',' => return t.simpleToken(.comma), + + 0x80...0xFF => { + // need to decode utf-8 + const byte_len, const code_point = try t.codePoint(); + + if (unicode_props.isXidStart(code_point)) { + const start = t.pos; + t.pos += byte_len; + return try t.tokenizeIdentifier(start); + } + + return error.InvalidSyntax; + }, + + else => return error.InvalidSyntax, + + 0 => { + if (t.pos == t.code.len) return null; + return error.InvalidSyntax; + }, + } + } +} + +/// Source code is null terminated, so accessing this is safe +fn byte(t: Tokenizer) u8 { + return t.code[t.pos]; +} + +/// Source code is null terminated. +fn nextByte(t: Tokenizer) u8 { + assert(t.byte() < 0x80); + return t.code[t.pos + 1]; +} + +/// Remember that using this only makes sense if you know for certain that all +/// previous bytes are ASCII, otherwise you may hit UB +fn byteAt(t: Tokenizer, comptime offset: u32) ?u8 { + if (offset == 1) @compileError("use nextByte()"); + + // assert that the previous bytes are ASCII + // current byte is allowed to be a continuation byte + for (t.code[t.pos .. t.pos + offset]) |b| { + assert(b < 0x80); + } + return t.code[t.pos + offset]; +} + +fn byteAtRt(t: Tokenizer, offset: u32) ?u8 { + // assert that the previous bytes are ASCII + // current byte is allowed to be a continuation byte + for (t.code[t.pos .. t.pos + offset]) |b| { + assert(b < 0x80); + } + return t.code[t.pos + offset]; +} + +fn codePoint(t: Tokenizer) !struct { u3, u21 } { + const b = t.byte(); // must check the first byte manually + assert(b >= 0x80); // must be a continuation byte + + const byte_len = unicode.utf8ByteSequenceLength(b) catch + return error.InvalidUtf8; + assert(byte_len > 1); + + const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch + return error.InvalidUtf8; + + return .{ byte_len, code_point }; +} + +fn isAtIdentifierStart(t: *Tokenizer) !bool { + switch (t.byte()) { + 'a'...'z', 'A'...'Z', '_', '$' => return true, + else => return false, + 0x80...0xFF => { + _, const code_point = try t.codePoint(); + return unicode_props.isXidStart(code_point); + }, + } +} + +fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token { + const trace = T.trace(@src()); + defer trace.end(); + + while (true) { + switch (t.byte()) { + 'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => { + t.pos += 1; + }, + 0x80...0xFF => { + const byte_len, const code_point = try t.codePoint(); + + t.pos += byte_len; + + if (!unicode_props.isXidContinue(code_point)) { + break; + } + }, + else => break, + } + } + + return .{ + .tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse + if (t.code[start_byte] == '@') + .builtin_id + else + .id, + .offset = start_byte, + }; +} + +/// Numbers have the following rules: +/// - May start with a `.`, but cannot end with `.` +/// - May contain leading zeros, but transform pass will warn. +/// - May contain a `_` separator, cannot have two in a row. +/// - May contain `e` or `E` for exponents. +/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers. +/// The token is tagged as `number_int` or `number_float` depending +/// on if it contained a `.` This is to help later re-parses. +// +// TODO: is it a zig compiler bug that `?` is required here? +fn tokenizeNumber(t: *Tokenizer) !?Token { + const trace = T.trace(@src()); + defer trace.end(); + + // Caller ensures the following invariant: + switch (t.byte()) { + '.' => switch (t.nextByte()) { + '0'...'9' => {}, + else => unreachable, + }, + '0'...'9' => {}, + else => unreachable, + } + + // Parse Hex, Binary, Octal numbers + if (t.byte() == '0') { + switch (t.nextByte()) { + 'x', 'X' => { + t.pos += 2; + try t.consumeIntegerRadix(.hex); + }, + 'b', 'B' => { + t.pos += 2; + try t.consumeIntegerRadix(.binary); + }, + 'o', 'O' => { + t.pos += 2; + try t.consumeIntegerRadix(.octal); + }, + else => {}, + } + } + + const start = t.pos; + var tag: Token.Tag = .number_int; + + if (t.byte() == '.') { + t.pos += 1; + tag = .number_float; + try t.consumeInteger(); + } else { + try t.consumeInteger(); + + // decimal point notation + switch (t.byte()) { + '.' => switch (t.nextByte()) { + '0'...'9' => { + t.pos += 1; + tag = .number_float; + try t.consumeInteger(); + }, + else => { + // we have not advanced the tokenizer past the `.` + assert(tag == .number_int); + return .{ .tag = tag, .offset = start }; + }, + }, + else => {}, + } + } + + switch (t.byte()) { + 'e', 'E' => { + tag = .number_float; + t.pos += 1; + + switch (t.byte()) { + '+', '-' => { + t.pos += 1; + switch (t.byte()) { + '0'...'9' => {}, + else => return t.maybeUTF8Error(error.InvalidSyntax), + } + }, + '0'...'9' => {}, + else => return t.maybeUTF8Error(error.InvalidSyntax), + } + + try t.consumeInteger(); + + // decimal point notation + switch (t.byte()) { + '.' => switch (t.nextByte()) { + '0'...'9' => { + t.pos += 1; + tag = .number_float; + try t.consumeInteger(); + }, + else => {}, + }, + else => {}, + } + }, + else => {}, + } + + // after numbers must be a valid word break the above returns are right + // before parsing `.` so those can be unchecked. + if (try t.isAtIdentifierStart()) { + return error.InvalidSyntax; + } + + return .{ + .tag = tag, + .offset = start, + }; +} + +fn maybeConsumeInteger(t: *Tokenizer) !?void { + const trace = T.trace(@src()); + defer trace.end(); + + switch (t.byte()) { + '0'...'9' => { + t.pos += 1; + }, + else => return null, + } + t.consumeInteger(); +} + +/// Requires the first byte to be a digit +fn consumeInteger(t: *Tokenizer) !void { + const trace = T.trace(@src()); + defer trace.end(); + + switch (t.byte()) { + '0'...'9' => { + t.pos += 1; + }, + else => unreachable, + } + + while (true) { + switch (t.byte()) { + '0'...'9' => { + t.pos += 1; + }, + '_' => { + t.pos += 1; + switch (t.byte()) { + '0'...'9' => { + t.pos += 1; + }, + else => return t.maybeUTF8Error(error.TrailingNumericSeparator), + } + }, + else => break, + } + } +} + +const Radix = enum { hex, binary, octal }; + +fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void { + const trace = T.trace(@src()); + defer trace.end(); + + const digits = switch (radix) { + .hex => "0123456789abcdefABCDEF", + .binary => "01", + .octal => "01234567", + }; + + first: { + const b = t.byte(); + inline for (digits) |digit| { + if (b == digit) { + break :first; + } + } + return t.maybeUTF8Error(error.InvalidSyntax); + } + + t.pos += 1; + + while (true) { + brk: { + const b = t.byte(); + inline for (digits) |digit| { + if (b == digit) { + break :brk; + } + } + return; + } + t.pos += 1; + } +} + +fn tokenizeString(t: *Tokenizer) !Token { + assert(t.byte() == '"'); + const start = t.pos; + try t.consumeString(); + return .{ .tag = .string, .offset = start }; +} + +fn consumeString(t: *Tokenizer) !void { + const trace = T.trace(@src()); + defer trace.end(); + + const start = t.pos; + assert(t.byte() == '"'); + + t.pos += 1; + + while (true) { + switch (t.byte()) { + '"' => { + t.pos += 1; + break; + }, + '\\' => { + switch (t.nextByte()) { + '\n' => { + t.pos = start; + return error.UnterminatedString; + }, + '\\' => t.pos += 2, + '"' => t.pos += 2, + 'n' => t.pos += 2, + 'r' => t.pos += 2, + 't' => t.pos += 2, + '0' => switch (t.byteAt(2) orelse { + t.pos = start; + return error.UnterminatedString; + }) { + // '\0xFF' hex escape + 'x' => {}, + // '\0' is special cased + else => t.pos += 2, + }, + 'u' => { + const esc_start = t.pos; + t.consumeStringUnicodeEscape() catch |err| switch (err) { + error.InvalidEscapeSequence => { + if (t.byte() == '\n') + return error.UnterminatedString; + t.pos = esc_start; + return err; + }, + error.InvalidEscapeCodepoint => { + t.pos = esc_start; + return err; + }, + // error.UnterminatedString => return err, + }; + }, + else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1), + } + }, + '\n' => { + t.pos = start; + return error.UnterminatedString; + }, + else => { + t.pos += 1; + }, + 0x80...0xFF => { + const byte_len, _ = try t.codePoint(); + t.pos += byte_len; + }, + } + } +} + +fn consumeStringUnicodeEscape(t: *Tokenizer) !void { + const trace = T.trace(@src()); + defer trace.end(); + + assert(t.byte() == '\\'); + assert(t.nextByte() == 'u'); + t.pos += 2; + + if (t.byte() != '{') + return error.InvalidEscapeSequence; + t.pos += 1; + + if (t.byte() == '0') { + if (t.nextByte() == 'x') { + t.pos += 2; + } + } + + const start = t.pos; + t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence; + + const slice = t.code[start..t.pos]; + + if (t.byte() != '}') + return error.InvalidEscapeSequence; + + const code_point = std.fmt.parseInt(u21, slice, 16) catch |err| switch (err) { + error.Overflow => return error.InvalidEscapeCodepoint, + error.InvalidCharacter => unreachable, + }; + _ = std.unicode.utf8CodepointSequenceLength(code_point) catch |err| switch (err) { + error.CodepointTooLarge => return error.InvalidEscapeCodepoint, + }; + + t.pos += 1; +} + +fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token { + const start = t.pos; + t.pos += comptime (tag.staticLen() orelse unreachable); + return .{ .tag = tag, .offset = start }; +} + +/// We are about to return an error, such as error.TrailingNumericSeparator. +/// However, the character we ran into might actually be invalid UTF-8. +/// This utility just checks for that to make sure we dont mask error.InvalidUtf8 +fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{ + .ErrorSet = &[_]std.builtin.Type.Error{ + .{ .name = "InvalidUtf8" }, + .{ .name = @errorName(real_err) }, + }, +}) { + switch (t.byte()) { + 0x80...0xFF => { + _ = try t.codePoint(); + return real_err; + }, + else => return real_err, + } +} + +fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{ + .ErrorSet = &[_]std.builtin.Type.Error{ + .{ .name = "InvalidUtf8" }, + .{ .name = @errorName(real_err) }, + }, +}) { + switch (t.byteAtRt(at) orelse return real_err) { + 0x80...0xFF => { + _ = try t.codePoint(); + return real_err; + }, + else => return real_err, + } +} + +pub fn skipWhitespace(t: *Tokenizer) void { + const trace = T.trace(@src()); + defer trace.end(); + + while (true) { + switch (t.byte()) { + ' ', '\t', '\n', '\r' => { + t.pos += 1; + }, + else => break, + } + } +} + +pub fn tokenLocation( + utf8_source: []const u8, + start_offset: ByteOffset, + token_start: ByteOffset, +) Location { + var loc: Location = .{ + .line = .start, + .column = .start, + .line_start = start_offset, + .line_end = @intCast(utf8_source.len), + }; + + // Scan to by line until we go past the token start + while (std.mem.indexOfScalarPos(u8, utf8_source, loc.line_start, '\n')) |i| { + if (i >= token_start) { + break; // Went past + } + loc.line = loc.line.add(1); + loc.line_start = @intCast(i + 1); + } + + const offset = loc.line_start; + for (utf8_source[offset..], 0..) |c, i| { + if (i + offset == token_start) { + loc.line_end = @intCast(i + offset); + while (loc.line_end < utf8_source.len and utf8_source[loc.line_end] != '\n') { + loc.line_end += 1; + } + return loc; + } + if (c == '\n') { + loc.line = loc.line.add(1); + loc.column = .start; + loc.line_start = @intCast(i + 1); + } else { + loc.column = loc.column.add(1); + } + } + + return loc; +} + +const Tokenizer = @This(); +const std = @import("std"); +const unicode = std.unicode; +const assert = std.debug.assert; +const mem = std.mem; +const unicode_props = @import("unicode_props.zig"); +const T = @import("tracer"); +const Location = @import("Ast.zig").Location; diff --git a/src/core/Transform.zig b/src/core/Transform.zig new file mode 100644 index 0000000000000000000000000000000000000000..9652b3edc67059de5b4f81652cff4a753db1f0fb --- /dev/null +++ b/src/core/Transform.zig @@ -0,0 +1,1479 @@ +//! Transform pass +gpa: Allocator, +ast: Ast, +insts: Inst.List = .{}, +string_bytes: std.ArrayListUnmanaged(u8) = .{}, +extra: ExtraStore = .{}, + +scratch: std.ArrayListUnmanaged(u32) = .{}, + +// TODO: refactor this away +// The new home for this memory should be pre-allocated on scratch, +// but then remove the elements, moving scratch items on top of this +// backwards a few. +call_args: std.ArrayListUnmanaged(Inst.Ref) = .{}, +scopes: std.ArrayListUnmanaged(Scope) = .{}, + +report: Report.Builder, + +pub const Error = error{ + OutOfMemory, + TransformFail, +}; + +pub fn transform( + ast: Ast, + gpa: Allocator, + file: *Comp.File, + fail_report: *Report, +) Error!Teg { + const trace = tracer.trace(@src()); + defer trace.end(); + + var t: Transform = .{ + .ast = ast, + .gpa = gpa, + .report = .{ .gpa = gpa, .current_file = file }, + }; + errdefer t.deinit(); + + try t.string_bytes.append(t.gpa, 0); + + const module = t.transformModule(0) catch |err| switch (err) { + error.OutOfMemory => return err, + }; + + const report = try t.report.finalize(); + if (report.errors.len > 0) { + fail_report.* = report; + return error.TransformFail; + } + + assert(@intFromEnum(module) == 0); + + try t.insts.setCapacity(t.gpa, t.insts.len); + + defer t.deinit(); + return Teg{ + .insts = t.insts.toOwnedSlice(), + .string_bytes = try t.string_bytes.toOwnedSlice(t.gpa), + .extra = try t.extra.toOwnedStore(t.gpa), + .report = report, + }; +} + +pub fn deinit(t: *Transform) void { + t.scratch.deinit(t.gpa); + t.call_args.deinit(t.gpa); + assert(t.scopes.items.len == 0); + t.scopes.deinit(t.gpa); + t.string_bytes.deinit(t.gpa); + t.insts.deinit(t.gpa); + t.extra.deinit(t.gpa); + t.* = undefined; +} + +const Scope = struct { + symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{}, + + const Symbol = struct { + node: NodeIndex, + data: Symbol.Data, + // TODO: optimize this structure + used: bool = false, + mutated: bool = false, + discarded: ?NodeIndex = null, + + const Data = union(enum) { + declaration, + param, + mutable: Inst.Index, + immutable: Inst.Ref, + }; + + pub fn addWarnings(symbol: Symbol, t: *Transform) !void { + if (!symbol.mutated and symbol.data == .mutable) { + try t.report.warn(.not_mutated_var, .{ + .node = symbol.node, + }); + } + if (!symbol.used) { + try t.report.warn(.unused, .{ + .node = symbol.node, + .kind = switch (symbol.data) { + .declaration => .function, + .param => .param, + .mutable => .variable, + .immutable => switch (t.ast.nodeTag(symbol.node)) { + .param_decl => .param, + else => .constant, + }, + }, + }); + } else if (symbol.discarded) |discard| { + try t.report.warn(.unnecessary_discard, .{ + .node = discard, + }); + } + } + }; + + fn addWarnings(s: Scope, t: *Transform) !void { + for (s.symbols.values()) |value| { + try value.addWarnings(t); + } + } + + fn deinit(s: *Scope, gpa: Allocator) void { + s.symbols.deinit(gpa); + s.* = undefined; + } +}; + +const ScratchType = enum { + ref, + index, +}; + +const BlockContext = struct { + node_base: i32, + return_target: ?*?Inst.Index = null, + break_target: ?*BreakTarget = null, + + /// This struct is always stored on the stack; the idea is to pass + /// mutable pointers to it. When `break :brk` is encountered + const BreakTarget = struct { + label: ?[]const u8, + /// This allows `continue` as well as implicit break targetting. + is_loop: bool, + /// If `null`, this label was unused. + block_index: ?Inst.Index = null, + parent: ?*BreakTarget = null, + + fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget { + return .{ + .parent = parent, + .label = label, + .is_loop = is_loop, + }; + } + + fn getInst(target: *BreakTarget, t: *Transform) !Inst.Index { + return target.block_index orelse { + const inst = try t.addRootInst(.{ + .tag = .block, + .data = undefined, + }); + target.block_index = inst; + return inst; + }; + } + }; + + fn init(root_node: NodeIndex) BlockContext { + return .{ + .node_base = @intCast(root_node), + }; + } + + fn nodeOffset(self: BlockContext, node: NodeIndex) i32 { + return @as(i32, @intCast(node)) - self.node_base; + } + + fn declOffset(self: BlockContext, node: NodeIndex) BlockContext { + return .{ + .node_base = self.nodeOffset(node), + }; + } + + fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext { + return .{ + .node_base = self.node_base, + .return_target = return_target, + }; + } + + fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext { + return .{ + .node_base = self.node_base, + .return_target = self.return_target, + .break_target = break_target, + }; + } + + fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index { + var break_target = self.break_target; + while (break_target) |target| : (break_target = target.parent) { + if (target.label) |target_label| { + if (mem.eql(u8, label, target_label)) { + return try target.getInst(t); + } + } + } + return null; + } +}; + +const Liveness = enum { alive, dead }; + +const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{ + .{ "void", .void }, + .{ "true", .bool_true }, + .{ "false", .bool_false }, + .{ "int", .int32_type }, + .{ "float", .float_type }, + .{ "string", .string_type }, + // .{ "bigint", .bigint_type }, + .{ "bool", .bool_type }, + .{ "noreturn", .noreturn_type }, + .{ "type", .type_type }, + .{ "std", .standard_library }, +}); + +fn transformModule(t: *Transform, module_node: NodeIndex) Allocator.Error!Inst.Index { + const trace = tracer.trace(@src()); + defer trace.end(); + + const module = try t.addRootInst(.{ + .tag = .module, + .data = undefined, + }); + + const decl = BlockContext.init(module_node); + + try t.pushScope(); + defer t.popScope(); + + const statements = t.ast.nodeStatements(module_node); + for (statements) |statement| { + try t.previsitModuleStatement(statement); + } + + const s = t.scope(); + + const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1); + defer t.gpa.free(temp); + + { + // prevent concurrent access + s.symbols.lockPointers(); + defer s.symbols.unlockPointers(); + for (s.symbols.values(), 0..) |value, i| { + temp[i] = try t.transformFunction(decl, value.node); + } + } + + const top_level_fn = try t.addRootInst(.{ + .tag = .declaration, + .data = undefined, + }); + + temp[temp.len - 1] = top_level_fn; + + const declarations: IndexList = .{ + .start = @intCast(t.extra.list.items.len), + .len = @intCast(temp.len), + }; + try t.extra.list.appendSlice(t.gpa, @ptrCast(temp)); + + const block = try t.addRootInst(.{ + .tag = .block, + .data = undefined, + }); + + { + const scratch_top = t.scratch.items.len; + defer t.scratch.shrinkRetainingCapacity(scratch_top); + + for (statements) |statement| { + try t.transformModuleStatement(decl, statement); + } + + t.getInstData(block).* = EncodedData.encode(.block, .{ + .src_node = decl.nodeOffset(module_node), + .extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)), + }); + } + + const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{ + .kind = .function, + .name = NullTerminatedString.empty, + .hash = .{0} ** 16, + .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn), + }, dod.extraSizeOf(Inst.DeclFn)); + + trail.writeStruct(Inst.DeclFn, .{ + .params = .{ .start = 0, .len = 0 }, + .return_type = Inst.Ref.void, + .body = block.toRef(), + }); + + t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{ + .src_node = decl.nodeOffset(module_node), + .extra_index = decl_extra, + }); + + t.getInstData(module).* = EncodedData.encode(.module, .{ + .src_node = 0, + .extra_index = try t.extra.append(t.gpa, IndexList, declarations), + }); + + try t.scope().addWarnings(t); + + return module; +} + +fn previsitModuleStatement(t: *Transform, node: NodeIndex) Allocator.Error!void { + const trace = tracer.trace(@src()); + defer trace.end(); + + const tag = t.ast.nodes.items(.tag)[node]; + switch (tag) { + .fn_decl => { + const fn_decl = t.ast.fullFnDecl(node); + const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id); + if (gop.found_existing) { + try t.report.err(.duplicate_function, .{ + .node = node, + .prev = gop.value_ptr.node, + }); + return; + } + gop.value_ptr.* = .{ + .node = node, + .data = .declaration, + }; + }, + else => {}, + } +} + +fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void { + const trace = tracer.trace(@src()); + defer trace.end(); + + const tag = t.ast.nodes.items(.tag)[node]; + switch (tag) { + .root, + .param_decl, + => unreachable, + + .fn_decl => {}, // functions analyzed separately + + else => _ = try t.transformBlockStatement(decl, node, false), + } +} + +fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) Allocator.Error!Inst.Index { + const trace = tracer.trace(@src()); + defer trace.end(); + + const decl = outer_decl.declOffset(node); + const fn_decl = t.ast.fullFnDecl(node); + + const func = try t.addRootInst(.{ + .tag = .declaration, + .data = undefined, + }); + + try t.pushScope(); + defer t.popScope(); + + const param_list = brk: { + const scratch_top = t.scratch.items.len; + defer t.scratch.shrinkRetainingCapacity(scratch_top); + + for (fn_decl.params) |param| { + try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param)); + } + + break :brk try t.scratchToRefList(scratch_top); + }; + + const return_type = if (fn_decl.return_type) |return_type_node| + try t.transformExpressionToBlock(decl, return_type_node, false) + else + Inst.Ref.void; + + const body = try t.transformExpressionToBlock(decl, fn_decl.body, true); + + const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{ + .kind = .function, + .name = try t.addNullTerminatedString(fn_decl.id), + .hash = .{0} ** 16, + .instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func), + }); + + _ = try t.extra.append(t.gpa, Inst.DeclFn, .{ + .params = param_list, + .return_type = return_type, + .body = body, + }); + + t.getInstData(func).* = EncodedData.encode(.declaration, .{ + .src_node = outer_decl.nodeOffset(node), + .extra_index = decl_extra, + }); + + try t.scope().addWarnings(t); + + return func; +} + +const TransformFunctionCtx = struct { + param_list: RefList, +}; + +/// Converts expressions into blocks with a break. This is important +/// for places where a block is not allowed, such as a parameter list. +/// This is also used for the function body, even though it is a block. +/// Simple refs turn into just the ref +fn transformExpressionToBlock( + t: *Transform, + decl: BlockContext, + node: NodeIndex, + comptime is_function: bool, +) Allocator.Error!Inst.Ref { + const trace = tracer.trace(@src()); + defer trace.end(); + + const node_tag = t.ast.nodes.items(.tag)[node]; + switch (node_tag) { + .id => { + // Lower builtin ids without using a block + const id = t.ast.basicNodeSlice(node); + if (BuiltInIds.get(id)) |ref| { + return ref; + } + }, + + else => {}, + } + + var block: ?Inst.Index = null; + + const scratch_top = t.scratch.items.len; + defer t.scratch.shrinkRetainingCapacity(scratch_top); + + if (is_function) { + const s = t.scopes.getLast(); + for (s.symbols.values(), 0..) |*param, i| { + assert(param.data == .param); + const ref = try t.addInstRef(.{ + .tag = .param, + .data = EncodedData.encode(.param, .{ + .src_node = decl.nodeOffset(node), + .data = @intCast(i), + }), + }); + param.data = .{ .immutable = ref }; + } + } + + const expr = try t.transformExpression( + if (is_function) + decl.withReturn(&block) + else + decl, + node, + ); + + // if there are no new instructions, we dont have to create a block context + if (scratch_top == t.scratch.items.len) { + const result = expr orelse unreachable; // null means we added break or return + + // TODO: evaluate if `if` blocks would be fine in sema with instruction refs + // with this extra if check here, we filter out non-blocks. + if (result.toIndex() == null) { + return result; + } + } + + if (expr) |result| { + if (result.toIndex()) |index| { + if (t.insts.items(.tag)[@intFromEnum(index)] == .block) { + return result; + } + } + } + + const ret_block = if (block) |b| + b + else + try t.addRootInst(.{ + .tag = .block, + .data = undefined, + }); + + if (expr) |result| { + _ = try t.addInstRef(.{ + .tag = .@"break", + .data = EncodedData.encode(.@"break", .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ + .block = ret_block, + .operand = result, + }), + }), + }); + } + + const content = try t.scratchToIndexList(scratch_top); + + t.getInstData(ret_block).* = EncodedData.encode(.block, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, IndexList, content), + }); + + return ret_block.toRef(); +} + +fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) Allocator.Error!Inst.Ref { + const trace = tracer.trace(@src()); + defer trace.end(); + + const param_decl = t.ast.fullParamDecl(param); + const value = try t.transformExpressionToBlock(decl, param_decl.type, false); + + // add `sym` into the scope so that the next param's type + // can use this param's value, if known at compile time. + const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id); + if (sym.found_existing) { + try t.report.err(.duplicate_param, .{ + .node = param, + .prev = sym.value_ptr.node, + }); + return .void; + } + + sym.value_ptr.* = .{ + .node = param, + .data = .param, + }; + + return value; +} + +const Builtin = enum { + js, + import, + fetch, + compile_error, + + const map = std.ComptimeStringMap(Builtin, .{ + .{ "js", .js }, + .{ "import", .import }, + .{ "fetch", .fetch }, + .{ "compileError", .compile_error }, + }); + + fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref { + const trace = tracer.trace(@src()); + defer trace.end(); + + if (args.len != 1) { + // try t.report.err(.invalid_builtin_args, .{ .id = "js" }); + return .void; + } + + const source = try t.transformExpression(decl, args[0]) orelse return null; + + return try t.addInstRef(.{ + .tag = .js, + .data = EncodedData.encode(.js, .{ + .src_node = decl.nodeOffset(args[0]), + .operand = source, + }), + }); + } +}; + +/// This is the primary transformation function: converting an AST expression into instructions. +/// `null` represents a `return` or `break`, aka this branch of code is dead. +fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Ref { + const node_tag = t.ast.nodes.items(.tag)[node]; + + const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); + defer trace.end(); + + switch (node_tag) { + .root, + .param_decl, + .var_decl, + .fn_decl, + => unreachable, // not an expression + + .id => { + const id = t.ast.basicNodeSlice(node); + if (BuiltInIds.get(id)) |ref| { + return ref; + } + + if (mem.startsWith(u8, id, "@")) { + @panic("TODO: string identifiers"); + } + + const lookup = t.lookupId(id) orelse { + try t.report.err(.unknown_identifier, .{ .node = node }); + return .void; + }; + + lookup.used = true; + + return switch (lookup.data) { + .declaration => try t.addInstRef(.{ + .tag = .load_decl, + .data = EncodedData.encode(.load_decl, .{ + .src_node = decl.nodeOffset(node), + .string = try t.addNullTerminatedString(id), + }), + }), + .mutable => |idx| idx.toRef(), + .immutable => |ref| ref, + .param => unreachable, //unresolved param + }; + }, + + inline .number_int, .number_float => |tag| { + const text = t.ast.basicNodeSlice(node); + const value = switch (tag) { + .number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"), + .number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"), + else => comptime unreachable, + }; + return try t.addInstRef(.{ + .tag = .number, + .data = EncodedData.encode(.number, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)), + }), + }); + }, + + .string => { + const str = t.ast.basicNodeSlice(node); + const sliced = try t.addSlicedString(str[1 .. str.len - 1]); + return try t.addInstRef(.{ + .tag = .string, + .data = EncodedData.encode(.string, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, SlicedString, sliced), + }), + }); + }, + + .assign => { + const ast_bin = t.ast.nodeData(node).bin; + const lhs = try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse + return .void; + const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse + return null; + const assign_inst = try t.addInstRef(.{ + .tag = .store, + .data = EncodedData.encode(.store, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Store, .{ + .memory = lhs, + .operand = rhs, + }), + }), + }); + return assign_inst; + }, + + inline .add, + .sub, + .mul, + .div, + .rem, + .pow, + .@"and", + .@"or", + .bit_and, + .bit_or, + .bit_xor, + .shl, + .shr, + .eq, + .ne, + .lt, + .le, + .gt, + .ge, + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => |tag| { + // This code path implements binary operations, as well as the + // compound compute + assign operation. + const is_assign = comptime tag.isAssignBin(); + const teg_tag = @field( + Teg.Inst.Tag, + if (is_assign) + @tagName(tag)[0 .. @tagName(tag).len - "_assign".len] + else + @tagName(tag), + ); + const ast_bin = t.ast.nodeData(node).bin; + const lhs = if (is_assign) + try t.transformAssignmentTarget(decl, ast_bin.lhs) orelse + return .void + else + try t.transformExpression(decl, ast_bin.lhs) orelse + return null; + const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null; + + const compute_inst = try t.addInstRef(.{ + .tag = teg_tag, + .data = EncodedData.encode(teg_tag, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{ + .lhs = if (is_assign) lhs.toRef() else lhs, + .rhs = rhs, + }), + }), + }); + + if (is_assign) { + const assign_inst = try t.addInstRef(.{ + .tag = .store, + .data = EncodedData.encode(.store, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Store, .{ + .memory = lhs, + .operand = compute_inst, + }), + }), + }); + return assign_inst; + } + + return compute_inst; + }, + + inline .negate, + .bool_not, + .bit_not, + => |tag| { + const ast_un = t.ast.nodeData(node).un; + const teg_tag = @field(Teg.Inst.Tag, @tagName(tag)); + + const operand = try t.transformExpression(decl, ast_un.expr) orelse + return null; + + return try t.addInstRef(.{ + .tag = teg_tag, + .data = EncodedData.encode(teg_tag, .{ + .src_node = decl.nodeOffset(node), + .operand = operand, + }), + }); + }, + + .label => { + const expr = t.ast.nodeData(node).label.expr; + const label = t.ast.tokenSlice(t.ast.nodeToken(node)); + + var target: BlockContext.BreakTarget = .{ + .parent = decl.break_target, + .label = label, + .is_loop = false, + }; + const inner_ctx = decl.withBreak(&target); + + const scratch_top = t.scratch.items.len; + + const inner_expr = try t.transformExpression(inner_ctx, expr); + + // if there are no new instructions, we dont have to create a block context + if (scratch_top == t.scratch.items.len) + return inner_expr orelse unreachable; // null means we added break or return + + if (target.block_index) |block| { + if (inner_expr) |result| { + _ = try t.addInstRef(.{ + .tag = .@"break", + .data = EncodedData.encode(.@"break", .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ + .block = block, + .operand = result, + }), + }), + }); + } + + const content = try t.scratchToIndexList(scratch_top); + t.scratch.shrinkRetainingCapacity(scratch_top); + + t.getInstData(block).* = EncodedData.encode(.block, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, IndexList, content), + }); + + try t.addScratch(Inst.Index, block); + + return block.toRef(); + } else { + std.debug.print("Warn, unused label {s}", .{label}); + } + + return inner_expr; + }, + + inline .block_void, + .block_implicit_return, + => |tag| { + const statements = t.ast.nodeStatements(node); + + var last: Inst.Ref = .void; + for (statements, 1..) |statement, i| { + const is_last = i == statements.len; + last = try t.transformBlockStatement( + decl, + statement, + tag == .block_implicit_return and is_last, + ) orelse { + return null; + }; + } + + return if (tag == .block_implicit_return) last else .void; + }, + + .builtin_call, + => { + const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse { + try t.report.err(.invalid_builtin, .{ .node = node }); + return .void; + }; + + const base = t.ast.nodeData(node).list; + const args = base.slice(&t.ast); + + return switch (builtin) { + .js => Builtin.builtin_js(t, decl, args), + // .import => Builtin.import(t, decl, args), + // .fetch => Builtin.fetch(t, decl, args), + // .compile_error => Builtin.compile_error(t, decl, args), + else => @panic("TODO"), + }; + }, + + .call => { + const call = t.ast.nodeData(node).call; + const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast); + + const base = try t.transformExpression(decl, call.base) orelse return null; + + const args_list: RefList = brk: { + const args_top = t.call_args.items.len; + defer t.call_args.shrinkRetainingCapacity(args_top); + + for (args) |arg| { + try t.call_args.append( + t.gpa, + try t.transformExpression(decl, arg) orelse + return null, + ); + } + + const items = t.call_args.items[args_top..]; + const extra_top = t.extra.list.items.len; + try t.extra.list.appendSlice(t.gpa, @ptrCast(items)); + + break :brk .{ + .start = @intCast(extra_top), + .len = @intCast(items.len), + }; + }; + + const call_inst = try t.addInstRef(.{ + .tag = .call, + .data = EncodedData.encode(.call, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Call, .{ + .base = base, + .args = args_list, + }), + }), + }); + + return call_inst; + }, + + .field => { + const field = t.ast.nodeData(node).field; + + const base = try t.transformExpression(decl, field.base) orelse return null; + + // TODO: handle identifier escapes + const id = t.ast.tokenSlice(field.field_token); + const prop_name_str = try t.addSlicedString(id); + + const string_ref = try t.addInstRef(.{ + .tag = .string, + .data = EncodedData.encode(.string, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str), + }), + }); + + return try t.addInstRef(.{ + .tag = .field, + .data = EncodedData.encode(.field, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Bin, .{ + .lhs = base, + .rhs = string_ref, + }), + }), + }); + }, + + .@"if" => { + const if_node = t.ast.nodeData(node).@"if"; + const if_data = t.ast.extraData(if_node.extra, Ast.If); + + const cond = try t.transformExpression(decl, if_node.cond) orelse + return null; + + const yes = try t.transformExpressionToBlock( + decl, + if_data.yes, + false, + ); + const no = if (if_data.no.unwrap()) |no| + try t.transformExpressionToBlock( + decl, + no, + false, + ) + else + Inst.Ref.void; + + if (yes == .void and no == .void) { + // when both branches do nothing, we can omit the branch, + // however we still need to type-check the condition operand + _ = try t.addInstRef(.{ + .tag = .ensure_bool, + .data = EncodedData.encode(.ensure_bool, .{ + .src_node = decl.nodeOffset(node), + .operand = cond, + }), + }); + return .void; + } + + return try t.addInstRef(.{ + .tag = .@"if", + .data = EncodedData.encode(.@"if", .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.If, .{ + .cond = cond, + .yes = yes, + .no = no, + }), + }), + }); + }, + + .@"return" => { + const ret_node = t.ast.nodeData(node).@"return"; + + const ret_block_ptr = decl.return_target orelse { + try t.report.err(.invalid_return, .{ .node = node }); + return .void; + }; + + const ret_block = if (ret_block_ptr.*) |block| + block + else append_block: { + const block = try t.addRootInst(.{ + .tag = .block, + .data = undefined, + }); + ret_block_ptr.* = block; + break :append_block block; + }; + + const operand: Inst.Ref = if (ret_node.expr.unwrap()) |expr| + try t.transformExpression(decl.withReturn(null), expr) orelse + return null + else + .void; + + _ = try t.addInstRef(.{ + .tag = .@"break", + .data = EncodedData.encode(.@"break", .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ + .block = ret_block, + .operand = operand, + }), + }), + }); + + return null; // control flow ends + }, + + .@"break" => { + const brk_node = t.ast.nodeData(node).@"break"; + const label = t.ast.tokenSlice( + brk_node.label.unwrap() orelse + @panic("TODO: unlabelled breaks"), + ); + + const block = try decl.findAndMarkBreak(t, label) orelse { + try t.report.err(.unresolved_label, .{ .node = node }); + return null; + }; + + const operand: Inst.Ref = if (brk_node.value.unwrap()) |expr| + try t.transformExpression(decl.withReturn(null), expr) orelse + return null + else + .void; + + _ = try t.addInstRef(.{ + .tag = .@"break", + .data = EncodedData.encode(.@"break", .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.Break, .{ + .block = block, + .operand = operand, + }), + }), + }); + + return null; // control flow ends + }, + + else => |tag| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}), + } +} + +fn transformAssignmentTarget(t: *Transform, decl: BlockContext, node: NodeIndex) Allocator.Error!?Inst.Index { + const node_tag = t.ast.nodes.items(.tag)[node]; + + const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); + defer trace.end(); + + switch (node_tag) { + .root, + .param_decl, + .var_decl, + .fn_decl, + => unreachable, // not an expression + + .id => { + const id = t.ast.basicNodeSlice(node); + assert(!mem.eql(u8, id, "_")); // discards are not assignment targets + + if (BuiltInIds.has(id)) { + try t.report.err(.assign_builtin, .{ .node = node }); + return null; + } + + if (mem.startsWith(u8, id, "@")) { + @panic("TODO: string identifiers"); + } + + const lookup = t.lookupId(id) orelse { + try t.report.err(.unknown_identifier, .{ .node = node }); + return null; + }; + + return switch (lookup.data) { + .mutable => |inst| { + lookup.mutated = true; + return inst; + }, + else => { + lookup.used = true; + + try t.report.err(.invalid_assignment_symbol, .{ + .node = node, + .decl = lookup.node, + .kind = switch (lookup.data) { + .mutable => unreachable, + .declaration => .function, + .param => .param, + .immutable => switch (t.ast.nodeTag(lookup.node)) { + .param_decl => .param, + else => .constant, + }, + }, + }); + return null; + }, + }; + }, + + // TODO: allow field expressions, but these need to emit a different store node + + else => { + _ = try t.transformExpression(decl, node); + try t.report.err(.invalid_assignment_target, .{ + .node = node, + }); + return null; + }, + } +} + +fn transformBlockStatement( + t: *Transform, + decl: BlockContext, + node: NodeIndex, + is_implicit_return_expression: bool, +) !?Inst.Ref { + const node_tag = t.ast.nodes.items(.tag)[node]; + + const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)}); + defer trace.end(); + + switch (node_tag) { + .root => unreachable, // cannot appear in a block + .param_decl => unreachable, // cannot appear in a block + + .var_decl => { + const var_decl = t.ast.fullVarDecl(node); + + if (mem.eql(u8, var_decl.id, "_")) { + try t.report.warn(.invalid_discard, .{ .node = node }); + + const ref = switch (t.ast.nodeTag(var_decl.init)) { + .id => { + const id = t.ast.basicNodeSlice(node); + if (BuiltInIds.has(id)) { + try t.report.warn(.discard_builtin, .{ .node = node }); + return .void; + } + + if (mem.startsWith(u8, id, "@")) { + @panic("TODO: string identifiers"); + } + + const lookup = t.lookupId(id) orelse { + try t.report.err(.unknown_identifier, .{ .node = node }); + return .void; + }; + + lookup.discarded = node; + + return .void; + }, + else => try t.transformExpression(decl, var_decl.init) orelse + return null, + }; + + return try t.addInstRef(.{ + .tag = .discard, + .data = EncodedData.encode(.discard, .{ + .src_node = decl.nodeOffset(node), + .operand = ref, + }), + }); + } + + if (BuiltInIds.has(var_decl.id)) { + try t.report.err(.shadow_builtin_variable, .{ .node = node }); + return .void; + } + + const type_ref = if (var_decl.type) |data_type| + try t.transformExpression(decl, data_type) + else + null; + const data_ref = try t.transformExpression(decl, var_decl.init) orelse + return null; + const data_coerced_ref = if (type_ref) |ref| + try t.addInstRef(.{ + .tag = .as, + .data = EncodedData.encode(.as, .{ + .src_node = decl.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Inst.As, .{ + .type = ref, + .operand = data_ref, + }), + }), + }) + else + data_ref; + + const mem_ref = if (var_decl.is_const) + data_coerced_ref + else + try t.addInstRef(.{ + .tag = .memory, + .data = EncodedData.encode(.memory, .{ + .src_node = decl.nodeOffset(node), + .operand = data_coerced_ref, + }), + }); + + const gop = try t.scope().symbols.getOrPut(t.gpa, var_decl.id); + + // if re-declaring, apply unused warnings now + if (gop.found_existing) { + try gop.value_ptr.addWarnings(t); + } + + gop.value_ptr.* = .{ + .node = node, + .data = switch (var_decl.is_const) { + true => .{ .immutable = mem_ref }, + false => .{ .mutable = mem_ref.toIndex().? }, + }, + }; + + return .void; + }, + + .fn_decl, + => |tag| { + std.debug.panic("TODO: block statement {s}", .{@tagName(tag)}); + }, + + .assign, + => { + const bin = t.ast.nodeData(node).bin; + const lhs = bin.lhs; + + if (t.ast.nodeTag(lhs) == .id and + mem.eql(u8, t.ast.basicNodeSlice(lhs), "_")) + { + // discard, instead of appending an .ensure_void check, + // we will add a .discard instruction instead. + // this will emit a warning if given void. + const result = try t.transformExpression(decl, bin.rhs) orelse + return null; + return try t.addInstRef(.{ + .tag = .discard, + .data = EncodedData.encode(.discard, .{ + .src_node = decl.nodeOffset(node), + .operand = result, + }), + }); + } else { + // process this assignment as usual, do not append .ensure_void + return try t.transformExpression(decl, node); + } + }, + + .add_assign, + .sub_assign, + .mul_assign, + .div_assign, + .rem_assign, + .pow_assign, + .and_assign, + .or_assign, + .bit_and_assign, + .bit_or_assign, + .bit_xor_assign, + .shl_assign, + .shr_assign, + => { + // do not append a .ensure_void check + return try t.transformExpression(decl, node); + }, + + else => { + const result = try t.transformExpression(decl, node) orelse + return null; + + if (!is_implicit_return_expression and result != .void) { + if (t.ast.nodeTag(node) == .id) { + try t.report.warn(.dangling_identifier, .{ + .node = node, + }); + return result; + } + + return try t.addInstRef(.{ + .tag = .ensure_void, + .data = EncodedData.encode(.ensure_void, .{ + .src_node = decl.nodeOffset(node), + .operand = result, + }), + }); + } + + return result; + }, + } +} + +fn addRootInst(t: *Transform, inst: Inst) !Inst.Index { + try t.insts.append(t.gpa, inst); + return @enumFromInt(t.insts.len - 1); +} + +fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref { + try t.scratch.ensureUnusedCapacity(t.gpa, 1); + try t.insts.append(t.gpa, inst); + const index: Inst.Index = @enumFromInt(t.insts.len - 1); + t.scratch.appendAssumeCapacity(@intFromEnum(index)); + return index.toRef(); +} + +fn pushScope(t: *Transform) !void { + try t.scopes.append(t.gpa, .{}); +} + +fn popScope(t: *Transform) void { + t.scope().deinit(t.gpa); + t.scopes.items[t.scopes.items.len - 1] = undefined; + t.scopes.items.len -= 1; +} + +fn scope(t: *Transform) *Scope { + return &t.scopes.items[t.scopes.items.len - 1]; +} + +fn getInstData(t: *Transform, index: Inst.Index) *EncodedData { + return &t.insts.items(.data)[@intFromEnum(index)]; +} + +fn addScratch(t: *Transform, comptime T: type, value: T) !void { + try t.scratch.append(t.gpa, @intFromEnum(value)); +} + +fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList { + const items = t.scratch.items[scratch_top..]; + const extra_top = t.extra.list.items.len; + try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]); + return .{ + .start = @intCast(extra_top), + .len = @intCast(items.len), + }; +} + +fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList { + const items = t.scratch.items[scratch_top..]; + const extra_top = t.extra.list.items.len; + try t.extra.list.appendSlice(t.gpa, items); + return .{ + .start = @intCast(extra_top), + .len = @intCast(items.len), + }; +} + +fn lookupId(t: *Transform, id: []const u8) ?*Scope.Symbol { + var i = t.scopes.items.len; + while (i > 0) { + i -= 1; + if (t.scopes.items[i].symbols.getPtr(id)) |sym| { + return sym; + } + } + return null; +} + +fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString { + if (str.len == 0) { + return .empty; + } + // loop through strings trying to find an existing match, and dedupe memory + var pos: usize = 0; + while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) { + const start_index = i - str.len; + if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) { + const string: NullTerminatedString = @enumFromInt(pos); + assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); + return string; + } + } + + const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len); + try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1); + t.string_bytes.appendSliceAssumeCapacity(str); + t.string_bytes.appendAssumeCapacity(0); + assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); + return string; +} + +fn addSlicedString(t: *Transform, str: []const u8) !SlicedString { + if (str.len == 0) { + return .{ .start = 0, .len = 0 }; + } + + // loop through strings trying to find an existing match, and dedupe memory + var pos: usize = 0; + while (mem.indexOfScalarPos(u8, str, pos, 0)) |i| : (pos = i + 1) { + const start_index = i - str.len; + if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) { + const string: SlicedString = .{ + .start = @intCast(start_index), + .len = @intCast(str.len), + }; + assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); + return string; + } + } + + const string: SlicedString = .{ + .start = @intCast(t.string_bytes.items.len), + .len = @intCast(str.len), + }; + try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len); + t.string_bytes.appendSliceAssumeCapacity(str); + assert(mem.eql(u8, string.slice(t.string_bytes.items), str)); + return string; +} + +const Transform = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = mem.Allocator; +const assert = std.debug.assert; + +const Teg = @import("Teg.zig"); +const Inst = Teg.Inst; +const NullTerminatedString = Teg.NullTerminatedString; +const SlicedString = Teg.SlicedString; +const RefList = Teg.RefList; +const IndexList = Teg.IndexList; +const EncodedData = Teg.EncodedData; +const ExtraIndex = Teg.ExtraIndex; + +const dod = @import("array.zig"); +const ExtraStore = dod.ExtraStore(.teg, .@"var"); +const Float = dod.Float; + +const Tokenizer = @import("Tokenizer.zig"); +const Token = Tokenizer.Token; +const ByteOffset = Tokenizer.ByteOffset; + +const Ast = @import("Ast.zig"); +const Unused = Ast.Unused; +const Node = Ast.Node; +const Data = Node.Data; +const SubList = Ast.SubList; +const NodeIndex = Ast.NodeIndex; +const TokenIndex = Ast.TokenIndex; +const OptionalNodeIndex = Ast.OptionalNodeIndex; +const OptionalTokenIndex = Ast.OptionalTokenIndex; +const OptionalExtraIndex = Ast.OptionalExtraIndex; +const optionalTokenIndex = OptionalTokenIndex.init; +const optionalNodeIndex = OptionalNodeIndex.init; +const optionalExtraIndex = OptionalExtraIndex.init; + +const InternPool = @import("InternPool.zig"); + +const Report = @import("Report.zig"); +const Comp = @import("Comp.zig"); + +const tracer = @import("tracer"); diff --git a/src/core/array.zig b/src/core/array.zig new file mode 100644 index 0000000000000000000000000000000000000000..57ccac606608a1b511c3f0f54871592143edb8a1 --- /dev/null +++ b/src/core/array.zig @@ -0,0 +1,390 @@ +const extra = @This(); + +pub const Mutability = enum { @"const", @"var" }; + +fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void { + const fields = std.meta.fields(@TypeOf(payload)); + var i = index; + inline for (fields) |field| { + const value = @field(payload, field.name); + switch (field.type) { + u32 => items[i] = value, + i32 => items[i] = @bitCast(value), + + [16]u8 => { + const u32s: [4]u32 = @bitCast(value); + items[i] = u32s[0]; + items[i + 1] = u32s[1]; + items[i + 2] = u32s[2]; + items[i + 3] = u32s[3]; + }, + + else => brk: { + const info = @typeInfo(field.type); + if (info == .Enum) { + items[i] = @intFromEnum(value); + break :brk; + } + + const struct_data = info.Struct; + if (struct_data.layout == .@"packed") { + items[i] = @bitCast(value); + break :brk; + } + + @call(.always_inline, setAsU32, .{ items, field.type, i, value }); + }, + } + i += extraSizeOf(field.type); + } +} + +fn getAsU32(items: []const u32, comptime T: type, index: usize) T { + const fields = std.meta.fields(T); + var i = index; + var result: T = undefined; + inline for (fields) |field| { + @field(result, field.name) = switch (field.type) { + u32 => items[i], + i32 => @bitCast(items[i]), + + [16]u8 => @bitCast(items[i..][0..4].*), + + else => brk: { + const info = @typeInfo(field.type); + if (info == .Enum) { + break :brk @enumFromInt(items[i]); + } + + const struct_data = info.Struct; + if (struct_data.layout == .@"packed") { + break :brk @bitCast(items[i]); + } + + break :brk @call( + .always_inline, + getAsU32, + .{ items, field.type, i }, + ); + }, + }; + i += extraSizeOf(field.type); + } + return result; +} + +pub const StoreKind = enum { + intern_pool, + teg, + air, + tfr, + + pub fn resolveStoreConst( + comptime kind: StoreKind, + owner: anytype, + ) *const kind.ResolvedStore(@TypeOf(owner)) { + return &owner.extra; + } + + pub fn resolveStoreMut( + comptime kind: StoreKind, + owner: anytype, + ) *kind.ResolvedStore(@TypeOf(owner)) { + return &owner.extra; + } + + pub fn ResolvedStore( + comptime kind: StoreKind, + comptime Owner: type, + ) type { + const ptr = @typeInfo(Owner).Pointer; + assert(ptr.size == .One); + const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) |field| { + if (std.mem.eql(u8, field.name, "extra")) { + break :brk field.type.mutability; + } + } else unreachable; + return ExtraStore(kind, mutability); + } +}; + +fn ExtraWrap(comptime kind: StoreKind) type { + return struct { + pub fn Index(comptime T: type) type { + return enum(u32) { + _, + + // Reads the data from `store.extra` and returns it. + pub fn get(index: @This(), owner: anytype) T { + return kind.resolveStoreConst(owner).get(T, @intFromEnum(index)); + } + + /// Returns data with a trailing reader to read data stored immediately after + /// the main data structure. What is stored should be documented on `T` + pub fn getTrail(index: @This(), owner: anytype) struct { + T, + kind.ResolvedStore(@TypeOf(owner)).TrailReader, + } { + const advance = comptime extraSizeOf(T); + return .{ + index.get(owner), + .{ .index = @intFromEnum(index) + advance }, + }; + } + }; + } + }; +} + +pub fn EncodedData(comptime Tag: type, backing_type: type) type { + return enum(backing_type) { + _, + + pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() { + assert(proof == tag); // incorrect memory access + const Data = tag.Data(); + const info = @typeInfo(Data); + if (info == .Enum) { + return @enumFromInt(@intFromEnum(data)); + } + if (info == .Struct) { + assert(backing_type == u64); + var fields: [2]u32 = @bitCast(@intFromEnum(data)); + return getAsU32(&fields, Data, 0); + } + return @bitCast(@intFromEnum(data)); + } + + pub fn encode(comptime tag: Tag, data: tag.Data()) @This() { + const Data = tag.Data(); + const info = @typeInfo(Data); + if (info == .Enum) { + return @enumFromInt(@intFromEnum(data)); + } + if (info == .Struct) { + assert(backing_type == u64); + var fields: [2]u32 = undefined; + setAsU32(&fields, Data, 0, data); + return @enumFromInt(@as(backing_type, @bitCast(fields))); + } + return @enumFromInt(@as(backing_type, @bitCast(data))); + } + }; +} + +/// The number of 'u32's this type consumes in `ip.extra`. +pub fn extraSizeOf(comptime T: type) usize { + return switch (T) { + u32, + i32, + => return 1, + + [16]u8 => return 4, + + else => { + const info = @typeInfo(T); + if (info == .Enum) { + comptime assert(@sizeOf(info.Enum.tag_type) <= 32); + return 1; + } + + const struct_data = info.Struct; + if (struct_data.layout == .@"packed") { + comptime assert(struct_data.backing_integer == u32); + return 1; + } + + const fields = std.meta.fields(T); + var size: usize = 0; + inline for (fields) |field| { + size += extraSizeOf(field.type); + } + return size; + }, + }; +} + +pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type { + return struct { + list: List = .{}, + + pub const kind = set_kind; + pub const mutability = set_mutability; + + const Store = @This(); + + const index_holder = ExtraWrap(kind); + pub const Index = index_holder.Index; + + pub const List = switch (mutability) { + .@"const" => struct { + items: []const u32 = &.{}, + + pub fn deinit(slice: *@This(), gpa: Allocator) void { + gpa.free(slice.items); + } + }, + .@"var" => std.ArrayListUnmanaged(u32), + }; + + pub fn deinit(store: *Store, gpa: Allocator) void { + store.list.deinit(gpa); + store.* = undefined; + } + + pub const TrailReader = struct { + index: usize, + + pub fn read(trail: *TrailReader, ip: *const Store, comptime T: type) T { + defer trail.index += comptime extraSizeOf(T); + return ip.get(T, trail.index); + } + + pub fn readValueSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Value.Slice { + defer trail.index += len; + return .{ + .start = @intCast(trail.index), + .len = len, + }; + } + + pub fn readTypeSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Type.Slice { + defer trail.index += len; + return .{ + .start = @intCast(trail.index), + .len = len, + }; + } + }; + + pub const TrailWriter = struct { + store: *Store, + leftover: usize, + + pub fn init(store: *Store, len: usize) TrailWriter { + return .{ .store = store, .leftover = len }; + } + + pub fn end(w: *TrailWriter) void { + assert(w.leftover == 0); + } + + pub fn writeU32(w: *TrailWriter, value: u32) void { + assert(w.leftover > 0); + w.ip.extra.appendAssumeCapacity(value); + w.leftover -= 1; + } + + pub fn writeSlice(w: *TrailWriter, slice: anytype) void { + const Child = std.meta.Child(@TypeOf(slice)); + const child = @typeInfo(Child); + comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32)); + assert(w.leftover >= slice.len); + w.store.list.appendSliceAssumeCapacity(@ptrCast(slice)); + w.leftover -= slice.len; + } + + pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void { + _ = appendAssumeCapacity(w.store, T, data); + } + }; + + pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) { + comptime assert(mutability == .@"var"); + try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload))); + return store.appendAssumeCapacity(T, payload); + } + + pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) { + comptime assert(mutability == .@"var"); + const extra_index: u32 = @intCast(ip.list.items.len); + ip.list.items.len += extraSizeOf(@TypeOf(payload)); + ip.set(T, extra_index, payload); + return @enumFromInt(extra_index); + } + + pub fn appendTrailing( + store: *Store, + gpa: Allocator, + comptime T: type, + payload: T, + trailing_len: usize, + ) Allocator.Error!struct { Index(T), TrailWriter } { + comptime assert(mutability == .@"var"); + try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len); + const extra_index: u32 = @intCast(store.list.items.len); + store.list.items.len += extraSizeOf(@TypeOf(payload)); + store.set(T, extra_index, payload); + return .{ + @enumFromInt(extra_index), + TrailWriter.init(store, trailing_len), + }; + } + + fn set(store: *Store, comptime T: type, index: usize, payload: T) void { + comptime assert(mutability == .@"var"); + setAsU32(store.list.items, T, index, payload); + } + + fn get(store: *const Store, comptime T: type, index: usize) T { + return getAsU32(store.list.items, T, index); + } + + pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") { + comptime assert(mutability == .@"var"); + return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } }; + } + + pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") { + return .{ .list = .{ .items = store.list.items } }; + } + }; +} + +pub const Float = struct { + low: u32, + high: u32, + + pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 }; + + pub fn purifyNaN(f: f64) f64 { + if (std.math.isNan(f)) { + return comptime Float.pure_nan.decode(); + } + return f; + } + + pub fn assertPure(f: f64) f64 { + if (std.math.isNan(f)) { + assert(Float.pure_nan.eql(encode(f))); + } + return f; + } + + pub fn encode(decoded: f64) Float { + if (std.math.isNan(decoded)) { + return pure_nan; + } + const as_bytes: [2]u32 = @bitCast(decoded); + return .{ + .low = as_bytes[0], + .high = as_bytes[1], + }; + } + + pub fn decode(encoded: Float) f64 { + const as_bytes: [2]u32 = .{ + encoded.low, + encoded.high, + }; + return @bitCast(as_bytes); + } + + pub fn eql(a: Float, b: Float) bool { + return a.low == b.low and a.high == b.high; + } +}; + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; diff --git a/src/core/main.zig b/src/core/main.zig new file mode 100644 index 0000000000000000000000000000000000000000..b7f63ed496648a3de79e35cd668ca74eabcf1155 --- /dev/null +++ b/src/core/main.zig @@ -0,0 +1,145 @@ +//! This file is used to test the compiler implementation without +//! needing to build the language server and other tooling. +const std = @import("std"); +const builtin = @import("builtin"); +const core = @import("core"); + +const Tokenizer = core.Tokenizer; +const Parse = core.Parse; +const Transform = core.Transform; +const Comp = core.Comp; +const unicode_props = core.unicode_props; +pub const Report = core.Report; + +const tracer = @import("tracer"); +pub const tracer_impl = @field(tracer, @import("build_options").tracing); + +pub fn main() !void { + try tracer.init(); + defer tracer.deinit(); + try tracer.init_thread(null); + defer tracer.deinit_thread(); + + try unicode_props.init(); + + if (builtin.mode == .Debug) { + // zig build system doesnt print a newline :/ + std.debug.print("\r\x1b[K", .{}); + } + + const use_gpa = builtin.mode == .Debug; + var gpa: if (use_gpa) std.heap.GeneralPurposeAllocator(.{ + .stack_trace_frames = 100, + }) else struct {} = .{}; + defer _ = if (use_gpa) gpa.deinit(); + const alloc = if (use_gpa) + gpa.allocator() + else + std.heap.c_allocator; + + var maybe_file_path: ?[]const u8 = null; + defer if (maybe_file_path) |path| alloc.free(path); + + var cwd_buf: [std.fs.max_path_bytes]u8 = undefined; + const cwd = try std.posix.getcwd(&cwd_buf); + + var exe_name: []const u8 = undefined; + var dump_ir = false; + + { + var args = try std.process.argsWithAllocator(alloc); + defer args.deinit(); + + exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli")); + errdefer alloc.free(exe_name); + + while (args.next()) |arg| { + if (std.mem.startsWith(u8, arg, "-")) { + if (builtin.mode == .Debug and std.mem.eql(u8, arg, "--report-trace")) { + Report.debug_show_report_trace = true; + continue; + } + if (std.mem.eql(u8, arg, "--dump-ir")) { + dump_ir = true; + continue; + } + std.debug.print("error: unknown flag '{s}'\n", .{arg}); + maybe_file_path = null; + break; + } + + if (maybe_file_path == null) { + maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg }); + } else { + std.debug.print("error: multiple files specified\n", .{}); + maybe_file_path = null; + break; + } + } + } + defer alloc.free(exe_name); + + const file_path = maybe_file_path orelse { + std.debug.print("Usage: {s} [flags] \n", .{exe_name}); + { + std.debug.print( + // | 60 chars wrap point| + \\ + \\Flags: + \\ --dump-ir Dump intermediate representations. + \\ + , .{}); + } + if (builtin.mode == .Debug) { + std.debug.print( + \\ + \\Debug Mode Flags: + \\ --report-trace Print stack traces per error. Very noisy + \\ + , .{}); + } + std.process.exit(1); + }; + + var comp: Comp = .{ + .gpa = tracer.tracingAllocator(&alloc), + .options = .{ + .dump_ir = dump_ir, + }, + }; + try comp.init(); + defer comp.deinit(); + const main_file = try comp.getFile(file_path); + + var fail_report: Report = undefined; + const teg = main_file.getTeg(&comp, &fail_report) catch |err| switch (err) { + error.TransformFail => { + std.debug.print("Transform failed:\n", .{}); + try fail_report.dumpToStdErr(&comp); + std.process.exit(1); + }, + error.ParseFail => { + // Already printed. + std.process.exit(1); + }, + else => return err, + }; + + if (comp.options.dump_ir) { + const stderr = std.io.getStdErr().writer(); + try teg.dumpToWriter(stderr); + } + + comp.semaEntryPoint(main_file) catch |err| switch (err) { + error.AnalysisFail => std.process.exit(1), + else => return err, + }; + + const output = std.io.getStdOut().writer(); + var bufw = std.io.bufferedWriter(output); + const writer = bufw.writer(); + try comp.link(writer); + try bufw.flush(); + + std.process.cleanExit(); +} diff --git a/src/core/mod.zig b/src/core/mod.zig new file mode 100644 index 0000000000000000000000000000000000000000..77c4714c40beba307c4ee71b5d30cfa0b69867a9 --- /dev/null +++ b/src/core/mod.zig @@ -0,0 +1,14 @@ +pub const Air = @import("Air.zig"); +pub const array = @import("array.zig"); +pub const Cache = @import("Cache.zig"); +pub const Comp = @import("Comp.zig"); +pub const InternPool = @import("InternPool.zig"); +pub const JsLink = @import("JsLink.zig"); +pub const Parse = @import("Parse.zig"); +pub const Report = @import("Report.zig"); +pub const Sema = @import("Sema.zig"); +pub const Teg = @import("Teg.zig"); +pub const Tokenizer = @import("Tokenizer.zig"); +pub const Transform = @import("Transform.zig"); +pub const unicode_props = @import("unicode_props.zig"); +pub const util = @import("util.zig"); diff --git a/src/core/unicode_props.zig b/src/core/unicode_props.zig new file mode 100644 index 0000000000000000000000000000000000000000..2a71ff3f46182203a77145f6f7df67868db3d271 --- /dev/null +++ b/src/core/unicode_props.zig @@ -0,0 +1,103 @@ +//! Wrapper around the 'zg' PropsData library. +const std = @import("std"); +const PropsData = @import("zg:PropsData"); +const tracer = @import("tracer"); + +var props: ?PropsData = undefined; +var props_memory: []u8 = undefined; + +const raw_allocator = if (@import("builtin").target.isWasm()) + std.heap.wasm_allocator +else + std.heap.page_allocator; + +pub fn init() !void { + if (props != null) return; + + const trace = tracer.trace(@src()); + defer trace.end(); + + // We know exactly how much this uses, so let's preallocate these pages and use a FBA. + const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856; + + if (allocation_size == 0) { + var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator); + props = try PropsData.init(logging_alloc.allocator()); + std.log.info("please write the above numbers into unicode.zig to prealloc this", .{}); + std.process.exit(1); + } + + props_memory = try raw_allocator.alloc(u8, allocation_size); + errdefer std.heap.page_allocator.free(props_memory); + + var fba = std.heap.FixedBufferAllocator.init(props_memory); + props = try PropsData.init(fba.allocator()); + std.debug.assert(fba.end_index == allocation_size); +} + +pub fn deinit() void { + std.debug.assert(props != null); + raw_allocator.free(props_memory); + props = null; + props_memory = undefined; +} + +/// True if `cp` is a mathematical symbol. +pub inline fn isMath(cp: u21) bool { + return props.?.isMath(cp); +} + +/// True if `cp` is an alphabetic character. +pub inline fn isAlphabetic(cp: u21) bool { + return props.?.isAlphabetic(cp); +} + +/// True if `cp` is a valid identifier start character. +pub inline fn isIdStart(cp: u21) bool { + return props.?.isIdStart(cp); +} + +/// True if `cp` is a valid identifier continuation character. +pub inline fn isIdContinue(cp: u21) bool { + return props.?.isIdContinue(cp); +} + +/// True if `cp` is a valid extended identifier start character. +pub inline fn isXidStart(cp: u21) bool { + return props.?.isXidStart(cp); +} + +/// True if `cp` is a valid extended identifier continuation character. +pub inline fn isXidContinue(cp: u21) bool { + return props.?.isXidContinue(cp); +} + +/// True if `cp` is a whitespace character. +pub inline fn isWhitespace(cp: u21) bool { + return props.?.isWhitespace(cp); +} + +/// True if `cp` is a hexadecimal digit. +pub inline fn isHexDigit(cp: u21) bool { + return props.?.isHexDigit(cp); +} + +/// True if `cp` is a diacritic mark. +pub inline fn isDiacritic(cp: u21) bool { + return props.?.isDiacritic(cp); +} + +/// True if `cp` is numeric. +pub inline fn isNumeric(cp: u21) bool { + return props.?.isNumeric(cp); +} + +/// True if `cp` is a digit. +pub inline fn isDigit(cp: u21) bool { + return props.?.isDigit(cp); +} + +/// True if `cp` is decimal. +pub inline fn isDecimal(cp: u21) bool { + return props.?.isDecimal(cp); +} diff --git a/src/core/util.zig b/src/core/util.zig new file mode 100644 index 0000000000000000000000000000000000000000..6e244c52b8da94449206f1cf1ce0fcedd0fc369b --- /dev/null +++ b/src/core/util.zig @@ -0,0 +1,56 @@ +pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config { + if (@hasField(@TypeOf(writer), "unbuffered_writer")) { + return detectTTYFromWriter(writer.unbuffered_writer); + } + if (@TypeOf(writer) == std.fs.File.Writer) { + return std.io.tty.detectConfig(writer.context); + } + if (@TypeOf(writer) == std.io.AnyWriter) { + const fileWriteFn = comptime (std.fs.File{ .handle = undefined }) + .writer().any().writeFn; + if (writer.writeFn == fileWriteFn) { + const file: *const std.fs.File = @alignCast(@ptrCast(writer.context)); + return std.io.tty.detectConfig(file.*); + } + } + return .no_color; +} + +pub fn Ordinal(Int: type) type { + comptime assert(@typeInfo(Int).Int.signedness == .unsigned); + return enum(Int) { + start = 0, + _, + + pub fn fromZeroBased(int: Int) @This() { + assert(int != std.math.maxInt(Int)); + return @enumFromInt(int); + } + pub fn fromOneBased(int: Int) @This() { + assert(int > 0); + return @enumFromInt(int - 1); + } + pub fn zeroBased(ord: @This()) Int { + return @intFromEnum(ord); + } + pub fn oneBased(ord: @This()) Int { + return @intFromEnum(ord) + 1; + } + + pub fn add(ord: @This(), inc: Int) @This() { + return fromZeroBased(ord.zeroBased() + inc); + } + + pub fn format( + _: @This(), + comptime _: []const u8, + _: std.fmt.FormatOptions, + _: anytype, + ) !void { + @compileError("Do not format `Ordinal` directly. Call oneBased or zeroBased to specify."); + } + }; +} + +const std = @import("std"); +const assert = std.debug.assert; diff --git a/src/lsp/lsp.zig b/src/lsp/lsp.zig new file mode 100644 index 0000000000000000000000000000000000000000..340c01d5d3878669148836d67c7515697cf04ccb --- /dev/null +++ b/src/lsp/lsp.zig @@ -0,0 +1,41 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const core = @import("core"); +const tracer = @import("tracer"); + +pub const tracer_impl = @field(tracer, @import("build_options").tracing); + +pub fn main() !void { + if (builtin.mode == .Debug) { + // zig build system doesnt print a newline :/ + std.debug.print("\r\x1b[K", .{}); + } + + std.debug.print("All your codebase are belong to us\n", .{}); + + const use_gpa = builtin.mode == .Debug; + var gpa_state: if (use_gpa) std.heap.GeneralPurposeAllocator(.{ + .stack_trace_frames = 100, + }) else struct {} = .{}; + defer _ = if (use_gpa) gpa_state.deinit(); + const gpa = if (use_gpa) + gpa_state.allocator() + else + std.heap.c_allocator; + + // if `parse` returns `error.ParseFail`, this `err` is set. + // + // at some point later this will change to be a `Report` + // so that multiple parse errors may be reported at once. + var err: core.Parse.AstError.WithTrace = undefined; + var ast = try core.Parse.parse(gpa, + \\let x = 1; + \\ + \\pub fn hello() -> float { + \\ 42.2 + \\} + , &err); + defer ast.deinit(gpa); + + try ast.dumpToWriter(std.io.getStdErr().writer()); +}