From 4373fa6782e2ec9979748230b3679883000467aa Mon Sep 17 00:00:00 2001 From: dave caruso Date: Mon, 29 Jul 2024 01:15:50 -0700 Subject: [PATCH] add array index syntax --- src/core/Air.Dump.zig | 3 ++- src/core/Air.zig | 14 +++++++---- src/core/Comp.zig | 2 +- src/core/Parse.zig | 9 ++++++- src/core/Report.zig | 25 +++++++++++++++---- src/core/Sema.zig | 46 ++++++++++++++++++++++++++++------- src/core/Tokenizer.zig | 43 +++++++++++++++++++++++++++++---- src/core/Transform.zig | 53 ++++++++++++++++++++++++++++++++++------- src/core/array.zig | 3 ++- src/core/js/NodeGen.zig | 39 +++++++++++++++++++++++++++--- test/behavior/array | 12 ++++++++++ 11 files changed, 213 insertions(+), 36 deletions(-) create mode 100644 test/behavior/array diff --git a/src/core/Air.Dump.zig b/src/core/Air.Dump.zig index 1cda7665d78cc67f1373bfc3790272043dca9396..9b988834a5e018f7c867110cb12c168dd424ce51 100644 --- a/src/core/Air.Dump.zig +++ b/src/core/Air.Dump.zig @@ -104,6 +104,7 @@ pub const State = struct { }, inline .negate, .bool_not, + .array_len, => |comptime_tag| { const bin: Air.Un = air.instData(comptime_tag, inst); try w.writeAll("("); @@ -193,7 +194,7 @@ pub const State = struct { try w.writeAll("("); try s.dumpValue(w, array_type.item_type.value()); - try s.dumpRefList(w, array_init.list.slice(air), '[', ']'); + try s.dumpRefList(w, array_init.list.get(air).slice(air), '[', ']'); try w.writeAll(")"); }, } diff --git a/src/core/Air.zig b/src/core/Air.zig index 11b470b4dc06cadcf87c037f9014f319d66eeed4..06e749889cf2a5c9db89d40b9a65cc5d507b63b5 100644 --- a/src/core/Air.zig +++ b/src/core/Air.zig @@ -30,6 +30,8 @@ pub const Inst = struct { field_opaque, field_struct, + array_len, + index, // bin @@ -70,7 +72,10 @@ pub const Inst = struct { .gt, .eq, => Bin, - .negate, .bool_not => Un, + .negate, + .bool_not, + .array_len, + => Un, }; } }; @@ -98,8 +103,8 @@ pub const Inst = struct { return if (ref.kind == .value) return @enumFromInt(ref.data) else null; } - pub fn index(ref: Ref) ?Value { - return if (ref.kind == .value) return @enumFromInt(ref.data) else null; + pub fn index(ref: Ref) ?Index { + return if (ref.kind == .index) return @enumFromInt(ref.data) else null; } pub fn toOptional(ref: Ref) Optional { @@ -279,7 +284,7 @@ pub const StructInit = struct { pub const ArrayInit = struct { ty: Type, - list: RefList, + list: ExtraIndex(RefList), }; pub const FieldStruct = struct { @@ -329,6 +334,7 @@ pub fn typeOfIndex(air: *const Air, index: Inst.Index, ip: *const InternPool) In .lt, .gt, .eq, .bool_not => .bool, .@"break" => .noreturn, .field_opaque => .@"opaque", + .array_len => .int32, // For many instructions, the type is stored on it's data directly. This // is avoided when possible to reduce the size of Air. diff --git a/src/core/Comp.zig b/src/core/Comp.zig index d059c601e4825bc4ce300db2cc0f2fff7b54dda7..6141483f912005627b2f9797e8b646f4946c92e5 100644 --- a/src/core/Comp.zig +++ b/src/core/Comp.zig @@ -185,7 +185,7 @@ pub const File = struct { error.ParseFail => { // TODO: let this merge with other report var report = try err_data.toReport(comp, file); - report.dumpToStdErr(comp) catch {}; + report.dumpToWriter(comp, comp.out.writer, comp.out.tty) catch {}; report.deinit(comp.gpa); return err; }, diff --git a/src/core/Parse.zig b/src/core/Parse.zig index 039fb77d812ea42d4285774c7406179fe225c896..587227e180d999d67074fb03311e7a02012844b0 100644 --- a/src/core/Parse.zig +++ b/src/core/Parse.zig @@ -105,6 +105,7 @@ pub const ExpectedContext = enum { else_clause, struct_field, param_list, + static_field, pub fn description(tag: ExpectedContext) []const u8 { return switch (tag) { @@ -115,6 +116,7 @@ pub const ExpectedContext = enum { .expression => "an expression", .else_clause => "an else clause", .struct_field => "a field definition", + .static_field => "a field name or number", }; } }; @@ -897,7 +899,10 @@ fn parseExpressionPart(p: *Parse, mode: ParseExprMode) Error!NodeIndex { switch (p.currentTag()) { .dot => { const main_token = p.nextToken(); - const field = try p.expect(.id); + const field = switch (p.currentTag()) { + .id, .number_int => p.nextToken(), + else => return p.failUnexpectedToken(.static_field), + }; expr = try p.addNode(.{ .tag = .field, @@ -934,6 +939,8 @@ fn parseExpressionPart(p: *Parse, mode: ParseExprMode) Error!NodeIndex { const index = try p.tryParseExpression(.expression); + _ = try p.expect(.end_bracket); + expr = try p.addNode(.{ .tag = .index, .main_token = main_token, diff --git a/src/core/Report.zig b/src/core/Report.zig index 69689da1a87eb35acc66c70dcfe8b902faf15ab9..069f1432a8bdcf5bd3881e233a55df61148b57d5 100644 --- a/src/core/Report.zig +++ b/src/core/Report.zig @@ -770,18 +770,36 @@ pub const Builder = struct { current_file_str: ?String = null, pub fn err(b: *Builder, comptime kind: Err.FieldEnum, data: std.meta.FieldType(Err, kind)) !void { + try errAt(b, kind, data, @returnAddress()); + } + + pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void { + try warnAt(b, kind, data, @returnAddress()); + } + + pub fn errAt( + b: *Builder, + comptime kind: Err.FieldEnum, + data: std.meta.FieldType(Err, kind), + return_address: usize, + ) !void { try b.errors.append(b.gpa, .{ .data = .{ .@"error" = @unionInit(Err, @tagName(kind), data) }, .url = try b.currentFileString(), - .debug_stack_trace = DebugStoredTrace.capture(@returnAddress()), + .debug_stack_trace = DebugStoredTrace.capture(return_address), }); } - pub fn warn(b: *Builder, comptime kind: Warn.FieldEnum, data: std.meta.FieldType(Warn, kind)) !void { + pub fn warnAt( + b: *Builder, + comptime kind: Warn.FieldEnum, + data: std.meta.FieldType(Warn, kind), + return_address: usize, + ) !void { try b.warnings.append(b.gpa, .{ .data = .{ .warn = @unionInit(Warn, @tagName(kind), data) }, .url = try b.currentFileString(), - .debug_stack_trace = DebugStoredTrace.capture(@returnAddress()), + .debug_stack_trace = DebugStoredTrace.capture(return_address), }); } @@ -1078,7 +1096,6 @@ pub const LazyNode = struct { return items[lazy.operand.getArgId()]; }, .field => { - assert(node_tag == .field); return base_node; }, else => {}, diff --git a/src/core/Sema.zig b/src/core/Sema.zig index 2a6a11b990225cba620a5070c600a96b1d326c08..c41d305831a6e5c175ac09cde0b5a863a81626d4 100644 --- a/src/core/Sema.zig +++ b/src/core/Sema.zig @@ -389,12 +389,42 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { .field => { const field: Teg.Bin = s.code.instPayload(.field, in); const base = try s.resolveRef(field.lhs); + const ty = s.typeOfRef(base); + + const rhs_raw = try s.resolveRef(field.rhs); + + if (ty.toArrayType(s.ip)) |_| { + // Special case `.len` + get_array_len: { + const value = rhs_raw.value() orelse break :get_array_len; + if (value.valueTag(s.ip) != .string) break :get_array_len; + const string = value.asString(s.ip).toSlice(s.ip); + if (!std.mem.eql(u8, string, "len")) break :get_array_len; + + switch (base.decode()) { + .value => |v| { + const items_len = v.valueData(s.ip, .array_value).get(s.ip).items_len; + return s.makeValueAir(.{ .int32 = @intCast(items_len) }); + }, + .index => { + return s.addInstRef(.array_len, .{ + .operand = base, + }); + }, + } + } + + // Arrays use runtime known int32s + const rhs = try s.coerceRef(in.op(.rhs), rhs_raw, .int32); + return s.addInstRef(.index, .{ + .lhs = base, + .rhs = rhs, + }); + } const rhs = try s.resolveCoerceValue(in.op(.rhs), field.rhs, .string); const string = rhs.asString(s.ip); - const ty = s.typeOfRef(base); - if (ty == .@"opaque") { return s.addInstRef(.field_opaque, .{ .base = base, @@ -930,10 +960,10 @@ fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) Error!Air.Inst.Ref { // Create a runtime array instantiation return try s.addInstRef(.array_init, .{ .ty = ty, - .list = .{ + .list = try s.air_extra.append(s.gpa, Air.RefList, .{ .start = try s.air_extra.appendSlice(s.gpa, scratch.slice()), .len = node.list.len, - }, + }), }); } }, @@ -1277,7 +1307,7 @@ fn ensureValueRuntime(s: *Sema, src: InstOp, ref: Air.Inst.Ref) Error!Air.Inst.R .numeric_literal => { const float = v.toFloat(s.ip) orelse unreachable; - if (!core.inInt32Range(float)) { + if (core.inInt32Range(float)) { try s.report.warn(.numeric_int_var_ambiguity, .{ .src = s.lazyNode(src), }); @@ -1687,15 +1717,15 @@ fn nodeLoc(s: *const Sema, in: Teg.Inst.Index, op: Report.Operand) Report.LazyNo } fn addWarn(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { - try s.report.warn(kind, data); + try s.report.warnAt(kind, data, @returnAddress()); } fn addError(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) !void { - try s.report.err(kind, data); + try s.report.errAt(kind, data, @returnAddress()); } fn fail(s: *Sema, comptime kind: Report.Err.FieldEnum, data: std.meta.FieldType(Report.Err, kind)) Error { - try s.report.err(kind, data); + try s.report.errAt(kind, data, @returnAddress()); return Error.AnalysisFail; } diff --git a/src/core/Tokenizer.zig b/src/core/Tokenizer.zig index 910ecb2ae6bdb8eab88975580815f139bdf6e79e..c210fd9b6e21cc92c66a368fe20bdab9c733b025 100644 --- a/src/core/Tokenizer.zig +++ b/src/core/Tokenizer.zig @@ -344,11 +344,6 @@ pub fn next(t: *Tokenizer) !?Token { '0'...'9' => return try t.tokenizeNumber(), - '.' => switch (t.nextByte()) { - '0'...'9' => return try t.tokenizeNumber(), - else => return t.simpleToken(.dot), - }, - '/' => switch (t.nextByte()) { '=' => return t.simpleToken(.slash_equal), '/' => { @@ -490,6 +485,7 @@ pub fn next(t: *Tokenizer) !?Token { '{' => return t.simpleToken(.start_brace), '}' => return t.simpleToken(.end_brace), ',' => return t.simpleToken(.comma), + '.' => return t.simpleToken(.dot), 0x80...0xFF => { // need to decode utf-8 @@ -1250,6 +1246,43 @@ test "basic tokenizer example" { try testing.expectEqual(null, (try tokenizer.next())); } +test "field access" { + const testing = std.testing; + try unicode_props.init(); + + const source = + \\.0 hello.1.2 + ; + const expected_tokens = [_]Token{ + .{ .tag = .kw_var, .offset = 0 }, + .{ .tag = .id, .offset = 4 }, + .{ .tag = .equal, .offset = 10 }, + .{ .tag = .number_float, .offset = 12 }, + .{ .tag = .number_int, .offset = 20 }, + .{ .tag = .semi, .offset = 21 }, + .{ .tag = .kw_pub, .offset = 42 }, + .{ .tag = .kw_fn, .offset = 46 }, + .{ .tag = .id, .offset = 49 }, + .{ .tag = .start_paren, .offset = 54 }, + .{ .tag = .colon, .offset = 55 }, + .{ .tag = .plus, .offset = 56 }, + .{ .tag = .id, .offset = 58 }, + .{ .tag = .end_paren, .offset = 62 }, + .{ .tag = .id, .offset = 64 }, + .{ .tag = .kw_return, .offset = 67 }, + .{ .tag = .id, .offset = 74 }, + .{ .tag = .id, .offset = 77 }, + .{ .tag = .string, .offset = 86 }, + }; + var tokenizer = Tokenizer{ .code = source }; + for (expected_tokens) |tok| { + tokenizer.skipWhitespace(); + try testing.expectEqual(tok, (try tokenizer.next()).?); + } + tokenizer.skipWhitespace(); + try testing.expectEqual(null, (try tokenizer.next())); +} + test "string escapes" { try testStringDecode("hello world", "hello world"); try testStringDecode("hello\n\rw\x00or\x5ald", "hello\\n\\rw\\0or\\x5ald"); diff --git a/src/core/Transform.zig b/src/core/Transform.zig index b17994e6ef4b71c00ac73bfc6644ce7295553e2e..ae049391c30bc8a4a4992c64939e3c5eff3b0fae 100644 --- a/src/core/Transform.zig +++ b/src/core/Transform.zig @@ -1174,20 +1174,57 @@ fn transformExpression( var base_result: ResultType = .none; const base = try t.transformExpression(ctx, &base_result, field.base) orelse return null; - // TODO: handle identifier escapes - const id = t.ast.tokenSlice(field.field_token); - const prop_name_str = try t.addSlicedString(id); + const rhs = switch (t.ast.tokens.items(.tag)[field.field_token]) { + .id => rhs: { + const id = t.ast.tokenSlice(field.field_token); + const id_decoded = try t.decodeIdentifier(id); + const prop_name_str = try t.addSlicedString(id_decoded); - const string_ref = try t.addRef(.string, .{ - .src_node = ctx.nodeOffset(node), - .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str), - }); + break :rhs try t.addRef(.string, .{ + .src_node = ctx.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str), + }); + }, + .number_int => rhs: { + const text = t.ast.tokenSlice(field.field_token); + + break :rhs try t.addRef(.number, .{ + .src_node = ctx.nodeOffset(node), + .extra_index = try t.extra.append( + t.gpa, + Float, + Float.encode(std.fmt.parseFloat(f64, text) catch @panic("TODO")), + ), + }); + }, + else => unreachable, // `Parse` filters out all other token tags + }; return try t.addRef(.field, .{ .src_node = ctx.nodeOffset(node), .extra_index = try t.extra.append(t.gpa, Teg.Bin, .{ .lhs = base, - .rhs = string_ref, + .rhs = rhs, + }), + }); + }, + + .index => { + const bin = t.ast.nodeData(node).bin; + + var lhs_result: ResultType = .none; + const lhs = try t.transformExpression(ctx, &lhs_result, bin.lhs) orelse + return null; + + var rhs_result: ResultType = .none; + const rhs = try t.transformExpression(ctx, &rhs_result, bin.rhs) orelse + return null; + + return try t.addRef(.field, .{ + .src_node = ctx.nodeOffset(node), + .extra_index = try t.extra.append(t.gpa, Teg.Bin, .{ + .lhs = lhs, + .rhs = rhs, }), }); }, diff --git a/src/core/array.zig b/src/core/array.zig index e7d984f5302de89b33184a2cb735e5ac636b62fb..9f7ef1cb8d4d5e4d7bd358a67b5bd6dde142acdb 100644 --- a/src/core/array.zig +++ b/src/core/array.zig @@ -168,7 +168,8 @@ pub fn EncodedData(comptime Tag: type, backing_type: type) type { return @enumFromInt(@intFromEnum(data)); } if (info == .Struct) { - assert(backing_type == u64); + comptime assert(backing_type == u64); + comptime assert(@sizeOf(Data) == 8); var fields: [2]u32 = undefined; setAsU32(&fields, Data, 0, data); return @enumFromInt(@as(backing_type, @bitCast(fields))); diff --git a/src/core/js/NodeGen.zig b/src/core/js/NodeGen.zig index 8165ef0cb034a08a275ba0104dc650994604f259..a8416a50a711db7ae18d86e99f2117d0e9d6a481 100644 --- a/src/core/js/NodeGen.zig +++ b/src/core/js/NodeGen.zig @@ -2,7 +2,7 @@ gpa: Allocator, arena: Allocator, air: *const Air, -ip: *const InternPool, +ip: *InternPool, nodes: Node.List, extra: dod.ExtraStore(.js_tree, .@"var"), @@ -28,7 +28,7 @@ const MapEntry = union { }, }; -pub fn init(gpa: Allocator, arena: Allocator, ip: *const InternPool, air: *const Air) !NodeGen { +pub fn init(gpa: Allocator, arena: Allocator, ip: *InternPool, air: *const Air) !NodeGen { const t = tracer.trace(@src()); defer t.end(); @@ -171,6 +171,7 @@ fn codeGenInst(c: *NodeGen, inst: Air.Inst.Index) Error!void { .sub, .mul, .div, + .index, => |tag| { const data: Air.Bin = c.air.instData(tag, inst); const node = try c.add(@field(Tree.Tag, @tagName(tag)), .{ @@ -273,6 +274,18 @@ fn codeGenInst(c: *NodeGen, inst: Air.Inst.Index) Error!void { try c.markExpression(inst, node); }, + .array_len => { + const data: Air.Un = c.air.instData(.array_len, inst); + const node = try c.add(.field, .{ + .base = try c.fetchInstExpression( + data.operand.index() orelse + unreachable, // TODO: make a second version of Un + ), + .field = try c.ip.getOrPutString(c.gpa, "length", .no_embedded_nulls), + }); + try c.markExpression(inst, node); + }, + inline .call, .js_new => |tag| { const data: Air.Call = c.air.instData(tag, inst); const call_data = data.call_data.get(c.air); @@ -422,7 +435,26 @@ fn codeGenInst(c: *NodeGen, inst: Air.Inst.Index) Error!void { try c.markExpression(inst, node); }, - else => |tag| std.debug.panic("TODO JS codeGenInst for .{s}", .{@tagName(tag)}), + .array_init => { + const struct_init: Air.ArrayInit = c.air.instData(.array_init, inst); + + const items = struct_init.list.get(c.air).slice(c.air); + const store_offset = try c.extra.alloc(c.gpa, items.len); + + for ( + items, + store_offset.., + ) |item, item_i| { + const in = try c.fetchExpression(item); + c.extra.list.items[item_i] = @intFromEnum(in); + } + + const node = try c.add(.array, .{ + .start = store_offset, + .len = @intCast(items.len), + }); + try c.markExpression(inst, node); + }, } } @@ -676,6 +708,7 @@ fn canLowerToExpression(c: *const NodeGen, inst: Air.Inst.Ref) bool { .struct_init, .array_init, .index, + .array_len, => true, // JavaScript variable declarations require statements. diff --git a/test/behavior/array b/test/behavior/array new file mode 100644 index 0000000000000000000000000000000000000000..1c51bbc532191886ed78c15e2c860c5ae5121535 --- /dev/null +++ b/test/behavior/array @@ -0,0 +1,12 @@ +fn i(a: int) -> int { a } + +let array = [i(9), i(3), i(8), i(3), i(-1)]; + +let assertEqual = @js("assertEqual"); + +assertEqual(array.len, i(5)); +assertEqual(array[0], i(9)); +assertEqual(array[1], i(3)); +assertEqual(array.2, i(8)); +assertEqual(array.3, i(3)); +assertEqual(array[i(1) + 2] + array[i(2) + 2], i(2)); -- 2.54.0