From b417db0d1e1f2ce026092e1b310e2c98e7de265f Mon Sep 17 00:00:00 2001 From: Lino Le Van <11367844+lino-levan@users.noreply.github.com> Date: Sat, 3 Feb 2024 05:35:35 +0100 Subject: [PATCH] feat(parser): track the location of source tokens (#8) * feat: add loc * chore: start working on transition parser to include loc * chore: fix everything related to TokenType * feat: add loc to parser --- crates/ast/operators.rs | 46 +- crates/js_ast/convert/convert_valid_block.rs | 1 + crates/parser/parse_block.rs | 10 +- crates/parser/parse_call_arguments.rs | 8 +- crates/parser/parse_declaration.rs | 16 +- crates/parser/parse_doc_comment.rs | 11 +- crates/parser/parse_expr.rs | 5 +- crates/parser/parse_expr_basic.rs | 128 ++-- crates/parser/parse_expr_if.rs | 22 +- crates/parser/parse_fn_declaration.rs | 36 +- crates/parser/parse_module.rs | 6 +- crates/parser/parse_param.rs | 6 +- crates/parser/parse_statement.rs | 20 +- crates/parser/parse_type.rs | 4 +- crates/parser/parser.rs | 23 +- crates/tokenizer/tokenizer.rs | 600 ++++++++++--------- 16 files changed, 517 insertions(+), 425 deletions(-) diff --git a/crates/ast/operators.rs b/crates/ast/operators.rs index 622be4fcc24c93e55588e9e341bccc1176e1cc54..9f6efc404a82baca560d40d7531ee4a499cc93b7 100644 --- a/crates/ast/operators.rs +++ b/crates/ast/operators.rs @@ -1,4 +1,4 @@ -use tokenizer::Token; +use tokenizer::TokenType; use crate::Literal; @@ -94,28 +94,28 @@ impl BinaryOperator { String::from(self.to_str()) } - pub fn from_token(token: &Token) -> Option { - match token { - Token::And => Some(Self::BitAnd), - Token::AndAnd => Some(Self::And), - Token::BangEqual => Some(Self::NotEq), - Token::Caret => Some(Self::BitXor), - Token::EqualEqual => Some(Self::Eq), - Token::Gt => Some(Self::Gt), - Token::GtEqual => Some(Self::Gte), - Token::GtGt => Some(Self::Rshift), - Token::GtGtGt => Some(Self::RshiftUnsigned), - Token::Lt => Some(Self::Lt), - Token::LtEqual => Some(Self::Lte), - Token::LtLt => Some(Self::Lshift), - Token::Minus => Some(Self::Sub), - Token::Or => Some(Self::BitOr), - Token::OrOr => Some(Self::Or), - Token::Percent => Some(Self::Mod), - Token::Plus => Some(Self::Add), - Token::Slash => Some(Self::Div), - Token::Star => Some(Self::Mul), - Token::StarStar => Some(Self::Pow), + pub fn from_token(token: &TokenType) -> Option { + match &token { + TokenType::And => Some(Self::BitAnd), + TokenType::AndAnd => Some(Self::And), + TokenType::BangEqual => Some(Self::NotEq), + TokenType::Caret => Some(Self::BitXor), + TokenType::EqualEqual => Some(Self::Eq), + TokenType::Gt => Some(Self::Gt), + TokenType::GtEqual => Some(Self::Gte), + TokenType::GtGt => Some(Self::Rshift), + TokenType::GtGtGt => Some(Self::RshiftUnsigned), + TokenType::Lt => Some(Self::Lt), + TokenType::LtEqual => Some(Self::Lte), + TokenType::LtLt => Some(Self::Lshift), + TokenType::Minus => Some(Self::Sub), + TokenType::Or => Some(Self::BitOr), + TokenType::OrOr => Some(Self::Or), + TokenType::Percent => Some(Self::Mod), + TokenType::Plus => Some(Self::Add), + TokenType::Slash => Some(Self::Div), + TokenType::Star => Some(Self::Mul), + TokenType::StarStar => Some(Self::Pow), _ => None, } } diff --git a/crates/js_ast/convert/convert_valid_block.rs b/crates/js_ast/convert/convert_valid_block.rs index fb40111073d4e3ea44604c0117ed7419b879935f..dcfb761cb34985d1376b79ebf13ddd843c6ef94e 100644 --- a/crates/js_ast/convert/convert_valid_block.rs +++ b/crates/js_ast/convert/convert_valid_block.rs @@ -26,6 +26,7 @@ pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, loc: Loc } JsExpr::new(JsExprData::Comma { items }, loc.clone(), None) } + _ => todo!(), }; // a block without implicit return is to be type void, so we wrap in a void expression diff --git a/crates/parser/parse_block.rs b/crates/parser/parse_block.rs index fc64e74aacce9ebf988d6b7a6c196f2901fd3a56..1dda24b0ba171c4ba8c7bad2d41003fc2d308b2f 100644 --- a/crates/parser/parse_block.rs +++ b/crates/parser/parse_block.rs @@ -1,12 +1,12 @@ use crate::{parse_statement::parse_statement, Parser}; use ast::Block; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; /// Precondition: The first token exists and is '{' pub fn parse_block(p: &mut Parser) -> Result { match p.next() { - Some(Token::OpenBrace) => (), + Some(TokenType::OpenBrace) => (), _ => unreachable!(), } @@ -30,10 +30,10 @@ pub fn parse_block(p: &mut Parser) -> Result { statements.push(statement.clone()); match p.peek() { - Some(Token::Semicolon) => loop { + Some(TokenType::Semicolon) => loop { p.next(); match p.peek() { - Some(Token::ClosedBrace) => { + Some(TokenType::ClosedBrace) => { p.next(); p.is_top_level = was_top_level; return Ok(Block { @@ -44,7 +44,7 @@ pub fn parse_block(p: &mut Parser) -> Result { _ => continue 'outer, } }, - Some(Token::ClosedBrace) => { + Some(TokenType::ClosedBrace) => { p.next(); p.is_top_level = was_top_level; diff --git a/crates/parser/parse_call_arguments.rs b/crates/parser/parse_call_arguments.rs index 4bdda948b2aef6eca565194538d7c4aeac397753..88b709e7e3ef62f594e2e3429588ab6781cfcd84 100644 --- a/crates/parser/parse_call_arguments.rs +++ b/crates/parser/parse_call_arguments.rs @@ -1,6 +1,6 @@ use ast::Expr; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; use crate::{parse_expr::parse_expr, Parser}; @@ -8,7 +8,7 @@ use crate::{parse_expr::parse_expr, Parser}; pub fn parse_call_arguments(p: &mut Parser) -> Result> { let mut args = Vec::new(); - if let Some(Token::ClosedParen) = p.peek() { + if let Some(TokenType::ClosedParen) = p.peek() { _ = p.next(); return Ok(args); } @@ -18,11 +18,11 @@ pub fn parse_call_arguments(p: &mut Parser) -> Result> { args.push(arg); match p.peek() { - Some(Token::Comma) => { + Some(TokenType::Comma) => { _ = p.next(); continue; } - Some(Token::ClosedParen) => { + Some(TokenType::ClosedParen) => { _ = p.next(); return Ok(args); } diff --git a/crates/parser/parse_declaration.rs b/crates/parser/parse_declaration.rs index 80f140076bb711e3fdf03716bbccd416096a0c35..1199f366ac02e86f2cd7f3039927698279792144 100644 --- a/crates/parser/parse_declaration.rs +++ b/crates/parser/parse_declaration.rs @@ -1,6 +1,6 @@ use ast::{DocComment, Type, VarDecl}; use miette::Result; -use tokenizer::Token; +use tokenizer::{Loc, TokenType}; use crate::parse_expr::parse_expr; use crate::parse_type::parse_type; @@ -12,26 +12,26 @@ pub fn parse_declaration( is_pub: bool, comment: Option, ) -> Result { - let loc = p.loc(); + let start = p.index; let is_constant = match p.next() { - Some(Token::Var) => false, - Some(Token::Let) => true, + Some(TokenType::Var) => false, + Some(TokenType::Let) => true, _ => unreachable!(), }; let name = match p.next() { - Some(Token::Identifier(name)) => name, + Some(TokenType::Identifier(name)) => name, Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()), None => return Err(p.error(format!("Unexpected end of file")).into()), }; let return_type = match p.peek() { - Some(Token::Colon) => { + Some(TokenType::Colon) => { p.next(); parse_type(p)? } _ => Type::Infer, }; let init = match p.next() { - Some(Token::Equal) => parse_expr(p)?, + Some(TokenType::Equal) => parse_expr(p)?, _ => { return Err(p .error("Variable must have an initializer".to_string()) @@ -45,6 +45,6 @@ pub fn parse_declaration( is_constant, is_pub, comment, - loc, + loc: Loc::new(start, p.index - start), }) } diff --git a/crates/parser/parse_doc_comment.rs b/crates/parser/parse_doc_comment.rs index f62e9f32b30c867c7c0542d2902956a337f7a86a..d6ae34c376c12c5f18dcbcf15e489375275a1711 100644 --- a/crates/parser/parse_doc_comment.rs +++ b/crates/parser/parse_doc_comment.rs @@ -1,14 +1,17 @@ use ast::DocComment; -use tokenizer::Token; +use tokenizer::{Loc, TokenType}; use crate::Parser; pub fn maybe_read_doc_comment(p: &mut Parser) -> Option { match p.peek() { - Some(Token::DocComment(data)) => { - let loc = p.loc(); + Some(TokenType::DocComment(data)) => { + let start = p.index; p.next(); - Some(DocComment { data, loc }) + Some(DocComment { + data, + loc: Loc::new(start, p.index - start), + }) } _ => None, } diff --git a/crates/parser/parse_expr.rs b/crates/parser/parse_expr.rs index 693053995ac365681335bfdabb3ebdd0f0a96b3b..dab8ffa3c4951c77fe1d3061921ef1ce79e71cdb 100644 --- a/crates/parser/parse_expr.rs +++ b/crates/parser/parse_expr.rs @@ -1,6 +1,7 @@ use ast::Associativity::*; use ast::{BinaryOperator, Expr, ExprData, OperatorLevel}; use miette::Result; +use tokenizer::Loc; use crate::parse_expr_basic::parse_expr_basic; use crate::Parser; @@ -53,7 +54,7 @@ fn parse_expr_binary( left: Expr, level: OperatorLevel, ) -> Result { - let loc = p.loc(); + let start = p.index; let mut right = parse_expr_basic(p)?; loop { @@ -93,7 +94,7 @@ fn parse_expr_binary( left: Box::new(left), right: Box::new(right), }, - loc, + Loc::new(start, p.index - start), None, )) } diff --git a/crates/parser/parse_expr_basic.rs b/crates/parser/parse_expr_basic.rs index ad484d31b2941b035d86aee60b2dacd64ced832f..0bb680cedf2580f82335a12cb6b02dfa9f1e7ecc 100644 --- a/crates/parser/parse_expr_basic.rs +++ b/crates/parser/parse_expr_basic.rs @@ -1,6 +1,6 @@ use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator}; use miette::Result; -use tokenizer::Token; +use tokenizer::{Loc, TokenType}; use crate::{ parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr, @@ -8,61 +8,87 @@ use crate::{ }; pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { - let loc = p.loc(); + let start = p.index; let comments = p.take_comments(); let mut expr = match p.peek() { - Some(Token::Number(number)) => { + Some(TokenType::Number(number)) => { _ = p.next(); Expr::new( ExprData::Literal(Literal::Number(number.clone())), - loc, + Loc::new(start, p.index - start), comments, ) } - Some(Token::String(string)) => { + Some(TokenType::String(string)) => { _ = p.next(); Expr::new( ExprData::Literal(Literal::String(string.clone())), - loc, + Loc::new(start, p.index - start), comments, ) } - Some(Token::True) => { + Some(TokenType::True) => { _ = p.next(); - Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments) + Expr::new( + ExprData::Literal(Literal::Bool(true)), + Loc::new(start, p.index - start), + comments, + ) } - Some(Token::False) => { + Some(TokenType::False) => { _ = p.next(); - Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments) + Expr::new( + ExprData::Literal(Literal::Bool(false)), + Loc::new(start, p.index - start), + comments, + ) } - Some(Token::Void) => { + Some(TokenType::Void) => { _ = p.next(); - Expr::new(ExprData::Literal(Literal::Void), loc, comments) + Expr::new( + ExprData::Literal(Literal::Void), + Loc::new(start, p.index - start), + comments, + ) } - Some(Token::Identifier(name)) => { - let loc = p.loc(); + Some(TokenType::Identifier(name)) => { + let identifier_start = p.index; _ = p.next(); if name.starts_with("@") { if let Some(id) = BuiltinId::from_str(name.as_str()) { - if let Some(Token::OpenParen) = p.next() { + if let Some(TokenType::OpenParen) = p.next() { let arguments = parse_call_arguments(p)?; - Expr::new(ExprData::Builtin { id, arguments }, loc, comments) + Expr::new( + ExprData::Builtin { id, arguments }, + Loc::new(identifier_start, p.index - identifier_start), + comments, + ) } else { return Err(p - .error_at(format!("Built-in function '{}' must be called", name), loc) + .error_at( + format!("Built-in function '{}' must be called", name), + Loc::new(identifier_start, p.index - identifier_start), + ) .into()); } } else { return Err(p - .error_at(format!("Non-existant built-in: '{}'", name), loc) + .error_at( + format!("Non-existant built-in: '{}'", name), + Loc::new(identifier_start, p.index - identifier_start), + ) .into()); } } else { - Expr::new(ExprData::Identifier(name), loc, comments) + Expr::new( + ExprData::Identifier(name), + Loc::new(identifier_start, p.index - identifier_start), + comments, + ) } } - Some(Token::Return) => { + Some(TokenType::Return) => { if p.is_top_level { return Err(p .error("'return' cannot be used at top level".to_string()) @@ -71,19 +97,31 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { _ = p.next(); let inner = maybe_parse_expr_basic(p)?.map(Box::new); - return Ok(Some(Expr::new(ExprData::Return(inner), loc, comments))); + return Ok(Some(Expr::new( + ExprData::Return(inner), + Loc::new(start, p.index - start), + comments, + ))); } } - Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments), + Some(TokenType::If) => Expr::new( + ExprData::If(parse_expr_if(p)?), + Loc::new(start, p.index - start), + comments, + ), - Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments), + Some(TokenType::OpenBrace) => Expr::new( + ExprData::Block(parse_block(p)?), + Loc::new(start, p.index - start), + comments, + ), - Some(Token::OpenParen) => { + Some(TokenType::OpenParen) => { _ = p.next(); let expr = parse_expr(p)?; match p.next() { - Some(Token::ClosedParen) => (), + Some(TokenType::ClosedParen) => (), Some(token) => { return Err(p.error(format!("Expected ')', but got {:?}", token)).into()) } @@ -92,19 +130,19 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { expr } - Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => { + Some(t @ TokenType::Bang) | Some(t @ TokenType::Minus) | Some(t @ TokenType::Tilde) => { _ = p.next(); Expr::new( ExprData::Unary { op: match t { - Token::Bang => UnaryOperator::Not, - Token::Minus => UnaryOperator::Neg, - Token::Tilde => UnaryOperator::BitNot, + TokenType::Bang => UnaryOperator::Not, + TokenType::Minus => UnaryOperator::Neg, + TokenType::Tilde => UnaryOperator::BitNot, _ => unreachable!(), }, value: Box::new(parse_expr_basic(p)?), }, - loc, + Loc::new(start, p.index - start), comments, ) } @@ -115,14 +153,18 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { // Parse function calls and field accesses loop { match p.peek() { - Some(Token::Dot) => { - let loc = p.loc(); + Some(TokenType::Dot) => { + let field_start = p.index; _ = p.next(); let field = match p.peek() { - Some(Token::Identifier(name)) => { - let loc = p.loc(); + Some(TokenType::Identifier(name)) => { + let field_start = p.index; p.next(); - Expr::new(ExprData::Literal(Literal::String(name)), loc, None) + Expr::new( + ExprData::Literal(Literal::String(name)), + Loc::new(field_start, p.index - field_start), + None, + ) } _ => { return Err(p.error("Expected identifier after dot".to_string()).into()); @@ -133,12 +175,12 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { base: Box::new(expr), field: Box::new(field), }, - loc, + Loc::new(field_start, p.index - field_start), None, ); } - Some(Token::OpenParen) => { - let loc = p.loc(); + Some(TokenType::OpenParen) => { + let expr_start = p.index; _ = p.next(); let arguments = parse_call_arguments(p)?; expr = Expr::new( @@ -146,21 +188,21 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result> { base: Box::new(expr), arguments, }, - loc, + Loc::new(expr_start, p.index - expr_start), None, ); } - Some(Token::OpenBracket) => { - let loc = p.loc(); + Some(TokenType::OpenBracket) => { + let expr_start = p.index; _ = p.next(); let index = parse_expr(p)?; - if let Some(Token::ClosedBracket) = p.next() { + if let Some(TokenType::ClosedBracket) = p.next() { expr = Expr::new( ExprData::Field { base: Box::new(expr), field: Box::new(index), }, - loc, + Loc::new(expr_start, p.index - expr_start), None, ); } else { diff --git a/crates/parser/parse_expr_if.rs b/crates/parser/parse_expr_if.rs index 5af7b44fd7fbf2383a3c3d4ed31d243168859558..12ccc2e5adbdb28bd18af2596342d46ab399d4eb 100644 --- a/crates/parser/parse_expr_if.rs +++ b/crates/parser/parse_expr_if.rs @@ -1,6 +1,6 @@ use ast::{BlockOrIf, BlockWithLoc, If}; use miette::Result; -use tokenizer::Token; +use tokenizer::{Loc, TokenType}; use crate::parse_block::parse_block; use crate::parse_expr::parse_expr; @@ -9,12 +9,12 @@ use crate::Parser; /// Precondition: the first token exists and is 'if' pub fn parse_expr_if(p: &mut Parser) -> Result { match p.next() { - Some(Token::If) => (), + Some(TokenType::If) => (), _ => unreachable!(), } match p.peek() { - Some(Token::OpenBrace) => { + Some(TokenType::OpenBrace) => { return Err(p .error( "To use a block as the condition for an if statement, wrap it in parentheses" @@ -28,11 +28,11 @@ pub fn parse_expr_if(p: &mut Parser) -> Result { let condition = Box::new(parse_expr(p)?); let yes = match p.peek() { - Some(Token::OpenBrace) => { - let loc = p.loc(); + Some(TokenType::OpenBrace) => { + let block_start = p.index; BlockWithLoc { block: parse_block(p)?, - loc, + loc: Loc::new(block_start, p.index - block_start), } } _ => { @@ -43,18 +43,18 @@ pub fn parse_expr_if(p: &mut Parser) -> Result { }; let no = match p.peek() { - Some(Token::Else) => { + Some(TokenType::Else) => { _ = p.next(); match p.peek() { - Some(Token::OpenBrace) => { - let loc = p.loc(); + Some(TokenType::OpenBrace) => { + let block_start = p.index; // let comments = p.take_comments(); Some(BlockOrIf::Block(BlockWithLoc { block: parse_block(p)?, - loc, + loc: Loc::new(block_start, p.index - block_start), })) } - Some(Token::If) => { + Some(TokenType::If) => { // TODO(@paperdave): Save location for the if // let loc = p.loc(); // let comments = p.take_comments(); diff --git a/crates/parser/parse_fn_declaration.rs b/crates/parser/parse_fn_declaration.rs index e52179524e2b97e25d29d1e08f60d7d1ee9e65d9..8a2c54e197e2e21d22169f6a2cd67fef29b242b4 100644 --- a/crates/parser/parse_fn_declaration.rs +++ b/crates/parser/parse_fn_declaration.rs @@ -1,6 +1,6 @@ use ast::{DocComment, Expr, ExprData, FnDecl, Type}; use miette::Result; -use tokenizer::Token; +use tokenizer::{Loc, TokenType}; use crate::parse_block::parse_block; use crate::parse_expr::parse_expr; @@ -14,19 +14,19 @@ pub fn parse_fn_declaration( is_pub: bool, comment: Option, ) -> Result { - let loc = p.loc(); + let start = p.index; match p.next() { - Some(Token::Fn) => (), + Some(TokenType::Fn) => (), _ => unreachable!(), } let name = match p.next() { - Some(Token::Identifier(name)) => name, + Some(TokenType::Identifier(name)) => name, _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()), }; match p.peek() { - Some(Token::OpenParen) => p.next(), + Some(TokenType::OpenParen) => p.next(), _ => { return Err(p .error("Expected '(' after function name".to_string()) @@ -38,22 +38,22 @@ pub fn parse_fn_declaration( loop { match p.peek() { - Some(Token::ClosedParen) => { + Some(TokenType::ClosedParen) => { _ = p.next(); break; } - Some(Token::Identifier(_)) => { + Some(TokenType::Identifier(_)) => { params.push(parse_param(p)?); match p.peek() { - Some(Token::Comma) => { + Some(TokenType::Comma) => { p.next(); - if let Some(Token::ClosedParen) = p.peek() { + if let Some(TokenType::ClosedParen) = p.peek() { p.next(); break; } } - Some(Token::ClosedParen) => { + Some(TokenType::ClosedParen) => { p.next(); break; } @@ -77,7 +77,7 @@ pub fn parse_fn_declaration( } let return_type = match p.peek() { - Some(Token::RightArrow) => { + Some(TokenType::RightArrow) => { _ = p.next(); parse_type(p)? } @@ -88,12 +88,16 @@ pub fn parse_fn_declaration( p.is_in_fn = true; let body = match p.peek() { - Some(Token::OpenBrace) => { - let block_loc = p.loc(); + Some(TokenType::OpenBrace) => { + let block_start = p.index; let block = parse_block(p)?; - Expr::new(ExprData::Block(block), block_loc, None) + Expr::new( + ExprData::Block(block), + Loc::new(block_start, p.index - block_start), + None, + ) } - Some(Token::BigRightArrow) => { + Some(TokenType::BigRightArrow) => { _ = p.next(); parse_expr(p)? } @@ -117,6 +121,6 @@ pub fn parse_fn_declaration( body, is_pub, comment, - loc, + loc: Loc::new(start, p.index - start), }) } diff --git a/crates/parser/parse_module.rs b/crates/parser/parse_module.rs index f3521bd641adcb0063461c9deee8e92593eff303..11c0ba62456af2c17b034958b531200191c98a09 100644 --- a/crates/parser/parse_module.rs +++ b/crates/parser/parse_module.rs @@ -3,7 +3,7 @@ use crate::parse_top_level_statement::parse_top_level_statement; use crate::Parser; use ast::Module; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; pub fn parse_module(p: &mut Parser) -> Result { let mut statements = Vec::new(); @@ -18,8 +18,8 @@ pub fn parse_module(p: &mut Parser) -> Result { let statement = parse_top_level_statement(p)?; match p.peek() { - Some(Token::Semicolon) => { - while let Some(Token::Semicolon) = p.peek() { + Some(TokenType::Semicolon) => { + while let Some(TokenType::Semicolon) = p.peek() { p.next(); } } diff --git a/crates/parser/parse_param.rs b/crates/parser/parse_param.rs index 9e8f34be012b33c1811621ec434e413672df8888..d726905cc438ad3064f113cfcdd0cebb165555b6 100644 --- a/crates/parser/parse_param.rs +++ b/crates/parser/parse_param.rs @@ -1,18 +1,18 @@ use ast::Param; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; use crate::{parse_type::parse_type, Parser}; /// Precondition: A token exists and is an identifier pub fn parse_param(p: &mut Parser) -> Result { let name = match p.next() { - Some(Token::Identifier(name)) => name.clone(), + Some(TokenType::Identifier(name)) => name.clone(), _ => unreachable!(), }; let param_type = match p.peek() { - Some(Token::Colon) => { + Some(TokenType::Colon) => { _ = p.next(); parse_type(p)? } diff --git a/crates/parser/parse_statement.rs b/crates/parser/parse_statement.rs index 0e806628140dfb10ce687edc6c95a67a15e0e944..efe4eacd6c821556f025038de194ef8a72b81870 100644 --- a/crates/parser/parse_statement.rs +++ b/crates/parser/parse_statement.rs @@ -1,6 +1,6 @@ use ast::Statement; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; use crate::parse_declaration::parse_declaration; use crate::parse_doc_comment::maybe_read_doc_comment; @@ -24,8 +24,8 @@ pub fn parse_statement(p: &mut Parser) -> Result { } match p.peek() { - Some(Token::DocComment(_)) => unreachable!(), - Some(Token::Pub) => { + Some(TokenType::DocComment(_)) => unreachable!(), + Some(TokenType::Pub) => { if !p.is_top_level { return Err(p .error("Public declarations are not allowed inside of functions".to_string()) @@ -33,12 +33,10 @@ pub fn parse_statement(p: &mut Parser) -> Result { } _ = p.next(); match p.peek() { - Some(Token::Var) | Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration( - p, - true, - m_doc_comment, - )?)), - Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration( + Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl( + parse_declaration(p, true, m_doc_comment)?, + )), + Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration( p, true, m_doc_comment, @@ -47,12 +45,12 @@ pub fn parse_statement(p: &mut Parser) -> Result { None => Err(p.error("Unexpected end of file".to_string()).into()), } } - Some(Token::Var) | Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration( + Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration( p, false, m_doc_comment, )?)), - Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration( + Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration( p, false, m_doc_comment, diff --git a/crates/parser/parse_type.rs b/crates/parser/parse_type.rs index b3e022b0ec6eede07db734f24e5412e6b57ce4c1..c60afcedf0c7494c203089fe4f8c4a4e29a80841 100644 --- a/crates/parser/parse_type.rs +++ b/crates/parser/parse_type.rs @@ -1,13 +1,13 @@ use ast::Type; use miette::Result; -use tokenizer::Token; +use tokenizer::TokenType; use crate::Parser; pub fn parse_type(p: &mut Parser) -> Result { match p.next() { - Some(Token::Identifier(id)) => match id.as_str() { + Some(TokenType::Identifier(id)) => match id.as_str() { "int" => Ok(Type::Int), "float" => Ok(Type::Float), "bigint" => Ok(Type::BigInt), diff --git a/crates/parser/parser.rs b/crates/parser/parser.rs index 8b7d0aa24f1982aae893c71236d0cd6dce03863f..4c02df4b30a082730a615b0a98c2cbadac9e887b 100644 --- a/crates/parser/parser.rs +++ b/crates/parser/parser.rs @@ -19,6 +19,7 @@ use miette::Result; use parse_module::parse_module; use thiserror::Error; use tokenizer::tokenize; +use tokenizer::TokenType; use tokenizer::{Loc, Token}; pub fn parse(tokens: Vec) -> Result { @@ -52,6 +53,9 @@ pub struct Parser { tokens: Vec, cursor: usize, + /// Current index in the string + pub index: usize, + /// Whether or not the parser is at the top level. /// Currently controls the validity of `pub` keywords. pub is_top_level: bool, @@ -67,6 +71,7 @@ impl Parser { Parser { tokens, cursor: 0, + index: 0, is_top_level: true, is_in_fn: false, comments: vec![], @@ -77,14 +82,16 @@ impl Parser { self.cursor == self.tokens.len() } - pub fn next(&mut self) -> Option { + pub fn next(&mut self) -> Option { if self.cursor == self.tokens.len() { return None; } loop { - match self.tokens[self.cursor].clone() { - Token::Comment(comment) => { + let token = self.tokens[self.cursor].clone(); + self.index = token.loc.start; + match token.variant { + TokenType::Comment(comment) => { self.comments.push(comment); self.cursor += 1; continue; @@ -113,14 +120,14 @@ impl Parser { self.comments.len() > 0 } - pub fn peek(&mut self) -> Option { + pub fn peek(&mut self) -> Option { loop { if self.cursor == self.tokens.len() { return None; } - match self.tokens[self.cursor].clone() { - Token::Comment(comment) => { + match self.tokens[self.cursor].clone().variant { + TokenType::Comment(comment) => { self.comments.push(comment); self.cursor += 1; continue; @@ -137,10 +144,6 @@ impl Parser { ParseError::new(error, None) } - fn loc(&self) -> Loc { - return Loc {}; - } - fn error_at(&self, error: String, loc: Loc) -> ParseError { ParseError::new(error, Some(loc)) } diff --git a/crates/tokenizer/tokenizer.rs b/crates/tokenizer/tokenizer.rs index 06f2495729efddae8d1a4a3945f2805341d42201..0b89bea7eb70503cad6885619945a13a4ed79c70 100644 --- a/crates/tokenizer/tokenizer.rs +++ b/crates/tokenizer/tokenizer.rs @@ -2,14 +2,24 @@ use miette::{Diagnostic, Result}; use thiserror::Error; #[derive(Debug, PartialEq, Clone)] -pub struct Loc {} +pub struct Loc { + pub start: usize, + pub length: usize, +} impl Loc { /// TODO(@paperdave): Remove this. every usage of this is incorrect. - pub const EMPTY: Loc = Loc {}; + pub const EMPTY: Loc = Loc { + start: 0, + length: 0, + }; - pub fn new() -> Self { - Self {} + pub fn new(start: usize, length: usize) -> Self { + Self { start, length } + } + + pub fn len(&self) -> usize { + self.length } } @@ -20,8 +30,8 @@ struct UnknownCharacter { error: String, } -#[derive(Debug, Clone)] -pub enum Token { +#[derive(Debug, Clone, PartialEq)] +pub enum TokenType { // Literals Identifier(String), Number(f64), @@ -78,77 +88,92 @@ pub enum Token { Tilde, // ~ } -impl Token { +impl TokenType { pub fn is_start_of_statement_like_expression(&self) -> bool { match &self { - Token::If | Token::OpenBrace => true, + TokenType::If | TokenType::OpenBrace => true, _ => false, } } } -impl std::fmt::Display for Token { +impl std::fmt::Display for TokenType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let token = match self { - Token::Identifier(_) => "identifier".to_string(), + let token = match &self { + TokenType::Identifier(_) => "identifier".to_string(), - Token::Number(n) => n.to_string(), - Token::String(s) => format!("\"{}\"", s), - Token::Comment(_) => "comment".to_string(), - Token::DocComment(_) => "documentation comment".to_string(), + TokenType::Number(n) => n.to_string(), + TokenType::String(s) => format!("\"{}\"", s), + TokenType::Comment(_) => "comment".to_string(), + TokenType::DocComment(_) => "documentation comment".to_string(), - Token::Else => "else".to_string(), - Token::False => "false".to_string(), - Token::Fn => "fn".to_string(), - Token::If => "if".to_string(), - Token::Let => "let".to_string(), - Token::Pub => "pub".to_string(), - Token::Return => "return".to_string(), - Token::True => "true".to_string(), - Token::Var => "var".to_string(), - Token::Void => "void".to_string(), + TokenType::Else => "else".to_string(), + TokenType::False => "false".to_string(), + TokenType::Fn => "fn".to_string(), + TokenType::If => "if".to_string(), + TokenType::Let => "let".to_string(), + TokenType::Pub => "pub".to_string(), + TokenType::Return => "return".to_string(), + TokenType::True => "true".to_string(), + TokenType::Var => "var".to_string(), + TokenType::Void => "void".to_string(), - Token::And => "&".to_string(), - Token::AndAnd => "&&".to_string(), - Token::Bang => "!".to_string(), - Token::BangEqual => "!=".to_string(), - Token::BigRightArrow => "=>".to_string(), - Token::Caret => "^".to_string(), - Token::ClosedBrace => "}".to_string(), - Token::ClosedBracket => "]".to_string(), - Token::ClosedParen => ")".to_string(), - Token::Colon => ":".to_string(), - Token::Comma => ",".to_string(), - Token::Dot => ".".to_string(), - Token::Equal => "=".to_string(), - Token::EqualEqual => "==".to_string(), - Token::Gt => ">".to_string(), - Token::GtEqual => ">=".to_string(), - Token::GtGt => ">>".to_string(), - Token::GtGtGt => ">>>".to_string(), - Token::Lt => "<".to_string(), - Token::LtEqual => "<=".to_string(), - Token::LtLt => "<<".to_string(), - Token::Minus => "-".to_string(), - Token::OpenBrace => "{".to_string(), - Token::OpenBracket => "[".to_string(), - Token::OpenParen => "(".to_string(), - Token::Or => "|".to_string(), - Token::OrOr => "||".to_string(), - Token::Percent => "%".to_string(), - Token::Plus => "+".to_string(), - Token::RightArrow => "->".to_string(), - Token::Semicolon => ";".to_string(), - Token::Slash => "/".to_string(), - Token::Star => "*".to_string(), - Token::StarStar => "**".to_string(), - Token::Tilde => "~".to_string(), + TokenType::And => "&".to_string(), + TokenType::AndAnd => "&&".to_string(), + TokenType::Bang => "!".to_string(), + TokenType::BangEqual => "!=".to_string(), + TokenType::BigRightArrow => "=>".to_string(), + TokenType::Caret => "^".to_string(), + TokenType::ClosedBrace => "}".to_string(), + TokenType::ClosedBracket => "]".to_string(), + TokenType::ClosedParen => ")".to_string(), + TokenType::Colon => ":".to_string(), + TokenType::Comma => ",".to_string(), + TokenType::Dot => ".".to_string(), + TokenType::Equal => "=".to_string(), + TokenType::EqualEqual => "==".to_string(), + TokenType::Gt => ">".to_string(), + TokenType::GtEqual => ">=".to_string(), + TokenType::GtGt => ">>".to_string(), + TokenType::GtGtGt => ">>>".to_string(), + TokenType::Lt => "<".to_string(), + TokenType::LtEqual => "<=".to_string(), + TokenType::LtLt => "<<".to_string(), + TokenType::Minus => "-".to_string(), + TokenType::OpenBrace => "{".to_string(), + TokenType::OpenBracket => "[".to_string(), + TokenType::OpenParen => "(".to_string(), + TokenType::Or => "|".to_string(), + TokenType::OrOr => "||".to_string(), + TokenType::Percent => "%".to_string(), + TokenType::Plus => "+".to_string(), + TokenType::RightArrow => "->".to_string(), + TokenType::Semicolon => ";".to_string(), + TokenType::Slash => "/".to_string(), + TokenType::Star => "*".to_string(), + TokenType::StarStar => "**".to_string(), + TokenType::Tilde => "~".to_string(), }; write!(f, "{}", token) } } +#[derive(Debug, Clone)] +pub struct Token { + pub variant: TokenType, + pub loc: Loc, +} + +impl Token { + pub fn new(variant: TokenType, start: usize, length: usize) -> Self { + Self { + variant, + loc: Loc::new(start, length), + } + } +} + // TODO(@paperdave): rewrite this to not use a Vec. we probably dont need it. also store Loc pub fn tokenize(content: String) -> Result> { let mut tokens: Vec = Vec::new(); @@ -156,251 +181,266 @@ pub fn tokenize(content: String) -> Result> { loop { match chars.next() { - Some((_, c)) => match c { - 'a'..='z' | 'A'..='Z' | '_' | '@' => { - let mut current_token = String::new(); - current_token.push(c); - loop { - let c = chars.peek(); - if c.is_none() { - break; - } - let (_, c) = c.unwrap(); - match c { - 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => { - current_token.push(*c); - chars.next(); + Some((start, c)) => { + let token = match c { + 'a'..='z' | 'A'..='Z' | '_' | '@' => { + let mut current_token = String::new(); + current_token.push(c); + loop { + let c = chars.peek(); + if c.is_none() { + break; + } + let (_, c) = c.unwrap(); + match c { + 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => { + current_token.push(*c); + chars.next(); + } + _ => break, } - _ => break, } - } - match current_token.as_str() { - "else" => tokens.push(Token::Else), - "false" => tokens.push(Token::False), - "fn" => tokens.push(Token::Fn), - "if" => tokens.push(Token::If), - "let" => tokens.push(Token::Let), - "pub" => tokens.push(Token::Pub), - "return" => tokens.push(Token::Return), - "true" => tokens.push(Token::True), - "var" => tokens.push(Token::Var), - "void" => tokens.push(Token::Void), - _ => tokens.push(Token::Identifier(current_token)), + let length = current_token.len(); + + let variant = match current_token.as_str() { + "else" => TokenType::Else, + "false" => TokenType::False, + "fn" => TokenType::Fn, + "if" => TokenType::If, + "let" => TokenType::Let, + "pub" => TokenType::Pub, + "return" => TokenType::Return, + "true" => TokenType::True, + "var" => TokenType::Var, + "void" => TokenType::Void, + _ => TokenType::Identifier(current_token), + }; + + Token::new(variant, start, length) } - } - '"' => { - let mut current_token = String::new(); - loop { - let c = chars.next(); - if c.is_none() { - break; - } - let (_, c) = c.unwrap(); - match c { - '"' => break, - _ => current_token.push(c), + '"' => { + let mut length = 1; + let mut current_token = String::new(); + loop { + let c = chars.next(); + if c.is_none() { + break; + } + let (_, c) = c.unwrap(); + length += 1; + match c { + '"' => break, + _ => current_token.push(c), + } } + Token::new(TokenType::String(current_token), start, length) } - tokens.push(Token::String(current_token)); - } - '0'..='9' => { - let mut current_token = String::new(); - current_token.push(c); - loop { - let c = chars.peek(); - if c.is_none() { - break; + '0'..='9' => { + let mut current_token = String::new(); + current_token.push(c); + loop { + let c = chars.peek(); + if c.is_none() { + break; + } + let (_, c) = c.unwrap(); + match c { + '0'..='9' => { + current_token.push(*c); + chars.next(); + } + _ => break, + } } - let (_, c) = c.unwrap(); - match c { - '0'..='9' => { - current_token.push(*c); - chars.next(); + match current_token.parse::() { + Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()), + Err(_) => { + return Err(UnknownCharacter { + error: format!("Unknown character: {}", current_token), + } + .into()) } - _ => break, } } - match current_token.parse::() { - Ok(v) => tokens.push(Token::Number(v)), - Err(_) => { - return Err(UnknownCharacter { - error: format!("Unknown character: {}", current_token), + ':' => Token::new(TokenType::Colon, start, 1), + '=' => match chars.peek() { + Some((_, '=')) => { + chars.next(); + Token::new(TokenType::EqualEqual, start, 2) + } + Some((_, '>')) => { + chars.next(); + Token::new(TokenType::BigRightArrow, start, 2) + } + _ => Token::new(TokenType::Equal, start, 1), + }, + ';' => Token::new(TokenType::Semicolon, start, 1), + '>' => match chars.peek() { + Some((_, '=')) => { + chars.next(); + Token::new(TokenType::GtEqual, start, 2) + } + Some((_, '>')) => { + chars.next(); + match chars.peek() { + Some((_, '>')) => { + chars.next(); + Token::new(TokenType::GtGtGt, start, 3) + } + _ => Token::new(TokenType::GtGt, start, 2), } - .into()) } - } - } - ':' => tokens.push(Token::Colon), - '=' => match chars.peek() { - Some((_, '=')) => { - chars.next(); - tokens.push(Token::EqualEqual); - } - Some((_, '>')) => { - chars.next(); - tokens.push(Token::BigRightArrow); - } - _ => tokens.push(Token::Equal), - }, - ';' => tokens.push(Token::Semicolon), - '>' => match chars.peek() { - Some((_, '=')) => { - chars.next(); - tokens.push(Token::GtEqual); - } - Some((_, '>')) => { - chars.next(); + _ => Token::new(TokenType::Gt, start, 1), + }, + '<' => match chars.peek() { + Some((_, '=')) => { + chars.next(); + Token::new(TokenType::LtEqual, start, 2) + } + Some((_, '<')) => { + chars.next(); + Token::new(TokenType::LtLt, start, 2) + } + _ => Token::new(TokenType::Lt, start, 1), + }, + '+' => Token::new(TokenType::Plus, start, 1), + '-' => match chars.peek() { + Some((_, '>')) => { + chars.next(); + Token::new(TokenType::RightArrow, start, 2) + } + _ => Token::new(TokenType::Minus, start, 1), + }, + '*' => match chars.peek() { + Some((_, '*')) => { + chars.next(); + Token::new(TokenType::StarStar, start, 2) + } + _ => Token::new(TokenType::Star, start, 1), + }, + '/' => { match chars.peek() { - Some((_, '>')) => { - chars.next(); - tokens.push(Token::GtGtGt); - } - _ => tokens.push(Token::GtGt), - } - } - _ => tokens.push(Token::Gt), - }, - '<' => match chars.peek() { - Some((_, '=')) => { - chars.next(); - tokens.push(Token::LtEqual); - } - Some((_, '<')) => { - chars.next(); - tokens.push(Token::LtLt); - } - _ => tokens.push(Token::Lt), - }, - '+' => tokens.push(Token::Plus), - '-' => match chars.peek() { - Some((_, '>')) => { - chars.next(); - tokens.push(Token::RightArrow); - } - _ => tokens.push(Token::Minus), - }, - '*' => match chars.peek() { - Some((_, '*')) => { - chars.next(); - tokens.push(Token::StarStar); - } - _ => tokens.push(Token::Star), - }, - '/' => { - match chars.peek() { - Some((_, '/')) => { - chars.next(); // Consume the second slash + Some((_, '/')) => { + chars.next(); // Consume the second slash + let mut length = 2; - let is_doc_comment = match chars.peek() { - Some((_, '/')) => { - chars.next(); // Consume the third slash - true - } - _ => false, - }; - - let mut comment = String::new(); - loop { - // skip leading whitespace - while let Some((_, c)) = chars.peek() { - if !c.is_whitespace() { - break; - } - chars.next(); - } - // read the comment - while let Some((_, c)) = chars.next() { - if c == '\n' || c == '\r' { - break; - } - comment.push(c); - } - // trim trailing whitespace on line - comment = comment.trim_end().to_string(); - // skip to next non-whitespace character - while let Some((_, c)) = chars.peek() { - if !c.is_whitespace() { - break; - } - chars.next(); - } - // check if we've hit another comment - let mut tmp = chars.clone(); - - let is_another_comment = match tmp.peek() { + let is_doc_comment = match chars.peek() { Some((_, '/')) => { - tmp.next(); // Consume the second slash - match tmp.peek() { - Some((_, '/')) => { - tmp.next(); // Consume the third slash - is_doc_comment - } - _ => !is_doc_comment, - } + chars.next(); // Consume the third slash + length += 1; + true } _ => false, }; - if is_another_comment { - // apply our temporary iterator to the main iterator - comment.push('\n'); - chars = tmp; - chars.next(); + let mut comment = String::new(); + loop { + // skip leading whitespace + while let Some((_, c)) = chars.peek() { + if !c.is_whitespace() { + break; + } + chars.next(); + length += 1; + } + // read the comment + while let Some((_, c)) = chars.next() { + length += 1; + if c == '\n' || c == '\r' { + break; + } + comment.push(c); + } + // trim trailing whitespace on line + comment = comment.trim_end().to_string(); + // skip to next non-whitespace character + while let Some((_, c)) = chars.peek() { + if !c.is_whitespace() { + break; + } + chars.next(); + length += 1; + } + // check if we've hit another comment + let mut tmp = chars.clone(); + + let is_another_comment = match tmp.peek() { + Some((_, '/')) => { + tmp.next(); // Consume the second slash + match tmp.peek() { + Some((_, '/')) => { + tmp.next(); // Consume the third slash + is_doc_comment + } + _ => !is_doc_comment, + } + } + _ => false, + }; + + if is_another_comment { + // apply our temporary iterator to the main iterator + comment.push('\n'); + chars = tmp; + chars.next(); + length += 1; + } else { + break; + } + } + + if is_doc_comment { + Token::new(TokenType::DocComment(comment), start, length) } else { - break; + Token::new(TokenType::Comment(comment), start, length) } } - - if is_doc_comment { - tokens.push(Token::DocComment(comment)); - } else { - tokens.push(Token::Comment(comment)); - } + _ => Token::new(TokenType::Slash, start, 1), } - _ => tokens.push(Token::Slash), } - } - ',' => tokens.push(Token::Comma), - '(' => tokens.push(Token::OpenParen), - ')' => tokens.push(Token::ClosedParen), - '{' => tokens.push(Token::OpenBrace), - '}' => tokens.push(Token::ClosedBrace), - '[' => tokens.push(Token::OpenBracket), - ']' => tokens.push(Token::ClosedBracket), - '&' => match chars.peek() { - Some((_, '&')) => { - chars.next(); - tokens.push(Token::AndAnd); - } - _ => tokens.push(Token::And), - }, - '|' => match chars.peek() { - Some((_, '|')) => { - chars.next(); - tokens.push(Token::OrOr); - } - _ => tokens.push(Token::Or), - }, - '!' => match chars.peek() { - Some((_, '=')) => { - chars.next(); - tokens.push(Token::BangEqual); - } - _ => tokens.push(Token::Bang), - }, - '^' => tokens.push(Token::Caret), - '%' => tokens.push(Token::Percent), - '~' => tokens.push(Token::Tilde), - '.' => tokens.push(Token::Dot), - s if s.is_whitespace() => (), - _ => { - return Err(UnknownCharacter { - error: format!("Unexpected '{}' while tokenizing", c), + ',' => Token::new(TokenType::Comma, start, 1), + '(' => Token::new(TokenType::OpenParen, start, 1), + ')' => Token::new(TokenType::ClosedParen, start, 1), + '{' => Token::new(TokenType::OpenBrace, start, 1), + '}' => Token::new(TokenType::ClosedBrace, start, 1), + '[' => Token::new(TokenType::OpenBracket, start, 1), + ']' => Token::new(TokenType::ClosedBracket, start, 1), + '&' => match chars.peek() { + Some((_, '&')) => { + chars.next(); + Token::new(TokenType::AndAnd, start, 2) + } + _ => Token::new(TokenType::And, start, 1), + }, + '|' => match chars.peek() { + Some((_, '|')) => { + chars.next(); + Token::new(TokenType::OrOr, start, 2) + } + _ => Token::new(TokenType::Or, start, 1), + }, + '!' => match chars.peek() { + Some((_, '=')) => { + chars.next(); + Token::new(TokenType::BangEqual, start, 2) + } + _ => Token::new(TokenType::Bang, start, 1), + }, + '^' => Token::new(TokenType::Caret, start, 1), + '%' => Token::new(TokenType::Percent, start, 1), + '~' => Token::new(TokenType::Tilde, start, 1), + '.' => Token::new(TokenType::Dot, start, 1), + s if s.is_whitespace() => continue, + _ => { + return Err(UnknownCharacter { + error: format!("Unexpected '{}' while tokenizing", c), + } + .into()) } - .into()) - } - }, + }; + tokens.push(token); + } None => break, } } -- 2.54.0