From b11fe69cd78904be138106b8b7fc9fdd4a86f01e Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Sat, 20 Jan 2024 19:08:18 -0800 Subject: [PATCH] parser changes --- Cargo.lock | 22 ++-- Cargo.toml | 2 +- crates/ast/ast.rs | 41 ++++++- crates/ast/operators.rs | 34 ++++++ crates/cli/Cargo.toml | 2 +- crates/cli/src/main.rs | 2 +- crates/js_ast/src/lib.rs | 1 + crates/parser/parse_declaration.rs | 12 +- crates/parser/parse_doc_comment.rs | 15 +++ crates/parser/parse_expr.rs | 113 ++---------------- crates/parser/parse_expr_basic.rs | 110 +++++++++++++---- crates/parser/parse_expr_if.rs | 4 +- crates/parser/parse_fn_declaration.rs | 14 ++- crates/parser/parse_module.rs | 25 ++-- crates/parser/parse_statement.rs | 44 ++++++- crates/parser/parser.rs | 72 ++++++++--- crates/{checker => sema}/Cargo.toml | 2 +- .../src/ast_type_coersion.rs | 0 crates/{checker => sema}/src/lib.rs | 0 crates/tokenizer/src/lib.rs | 64 +++++++--- examples/basics/comments.blight | 11 +- 21 files changed, 388 insertions(+), 202 deletions(-) create mode 100644 crates/parser/parse_doc_comment.rs rename crates/{checker => sema}/Cargo.toml (90%) rename crates/{checker => sema}/src/ast_type_coersion.rs (100%) rename crates/{checker => sema}/src/lib.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 0d4dfa5e9ed55431cd6d3cfba52ca941ce7660df..4c8ddfacf5557906451853c5edb4394a656e2af5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,16 +126,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "checker" -version = "0.1.0" -dependencies = [ - "ast", - "miette", - "num", - "thiserror", -] - [[package]] name = "clap" version = "4.4.14" @@ -181,12 +171,12 @@ name = "cli" version = "0.1.0" dependencies = [ "ast", - "checker", "clap", "js_ast", "js_printer", "miette", "parser", + "sema", "thiserror", "tokenizer", ] @@ -466,6 +456,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "sema" +version = "0.1.0" +dependencies = [ + "ast", + "miette", + "num", + "thiserror", +] + [[package]] name = "smawk" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index d786a2120adc1dbefba1c1002a3a56ec7fd9c010..fdcf94c7291631b43fbac96e4a24885312c0d65e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/lino-levan/blight" resolver = "2" members = [ "crates/ast", - "crates/checker", + "crates/sema", "crates/cli", "crates/js_ast", "crates/js_printer", diff --git a/crates/ast/ast.rs b/crates/ast/ast.rs index 972216ddbe6d10e928f7160f8f5d999983829948..bf9cd16e386e9321b7bd123de01cb21d622daa9a 100644 --- a/crates/ast/ast.rs +++ b/crates/ast/ast.rs @@ -12,6 +12,7 @@ pub enum BuiltinId { #[derive(Debug, Clone)] pub struct Module { + pub comment: Option, pub statements: Vec, } @@ -31,7 +32,7 @@ pub struct Param { pub struct Decl { pub name: String, - pub decl_type: Type, + pub data_type: Type, pub init: Expr, pub is_constant: bool, pub is_pub: bool, @@ -43,11 +44,12 @@ pub struct Decl { pub struct Expr { pub data: ExprData, pub loc: Loc, + pub comment: Option, } impl Expr { - pub fn new(data: ExprData, loc: Loc) -> Self { - Self { data, loc } + pub fn new(data: ExprData, loc: Loc, comment: Option) -> Self { + Self { data, loc, comment } } } @@ -57,6 +59,35 @@ pub struct DocComment { pub loc: Loc, } +#[derive(Debug, Clone)] +pub struct Comment { + pub data: String, +} + +impl Comment { + pub fn format(&self) -> String { + let mut lines = self.data.lines(); + let first_line = lines.next().unwrap(); + let mut result = format!("// {}", first_line); + for line in lines { + result.push_str(&format!("\n// {}", line)); + } + result + } +} + +impl DocComment { + pub fn format(&self) -> String { + let mut lines = self.data.lines(); + let first_line = lines.next().unwrap(); + let mut result = format!("/// {}", first_line); + for line in lines { + result.push_str(&format!("\n/// {}", line)); + } + result + } +} + #[derive(Debug, Clone)] pub struct BlockWithLoc { pub block: Block, @@ -110,12 +141,14 @@ pub enum ExprData { pub enum Statement { Expr(Expr), Decl(Decl), + Comment(Comment), } #[derive(Debug, Clone)] pub enum TopLevelStatement { Expr(Expr), Decl(Decl), + Comment(Comment), } impl BuiltinId { @@ -149,6 +182,7 @@ impl Statement { match self { Statement::Expr(expr) => TopLevelStatement::Expr(expr), Statement::Decl(decl) => TopLevelStatement::Decl(decl), + Statement::Comment(comment) => TopLevelStatement::Comment(comment), } } } @@ -169,6 +203,7 @@ impl TopLevelStatement { match self { TopLevelStatement::Expr(expr) => Statement::Expr(expr), TopLevelStatement::Decl(decl) => Statement::Decl(decl), + TopLevelStatement::Comment(comment) => Statement::Comment(comment), } } } diff --git a/crates/ast/operators.rs b/crates/ast/operators.rs index f7c7d4d9ffb2783bb85444d8de71dfed4c6ce2c2..18dd5ae68a5961b785e815adca22dd9c71418907 100644 --- a/crates/ast/operators.rs +++ b/crates/ast/operators.rs @@ -169,3 +169,37 @@ impl UnaryOperator { } } } + +/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table +#[derive(PartialEq, PartialOrd)] +pub enum BinaryOperatorLevel { + Exponentiation, // ** + Multiplicative, // *, /, % + Additive, // +, - + BitwiseShift, // <<, >>, >>> + Relational, // <, <=, >, >= + Equality, // ==, != + BitwiseAnd, // & + BitwiseXor, // ^ + BitwiseOr, // | + LogicalAnd, // && + LogicalOr, // ||, ?? +} + +pub fn get_binary_op_level(token: &Token) -> Option { + use BinaryOperatorLevel::*; + match token { + Token::StarStar => Some(Exponentiation), + Token::Star | Token::Slash | Token::Percent => Some(Multiplicative), + Token::Plus | Token::Minus => Some(Additive), + Token::LtLt | Token::GtGt | Token::GtGtGt => Some(BitwiseShift), + Token::Gt | Token::GtEqual | Token::Lt | Token::LtEqual => Some(Relational), + Token::EqualEqual | Token::BangEqual => Some(Equality), + Token::And => Some(BitwiseAnd), + Token::Caret => Some(BitwiseXor), + Token::Or => Some(BitwiseOr), + Token::AndAnd => Some(LogicalAnd), + Token::OrOr => Some(LogicalOr), + _ => None, + } +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 779b647427e55d42b654c970a4baf081f2551ed1..8c257d214dad91041d7de4f1e29bc1ee6641750c 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -17,4 +17,4 @@ ast = { path = "../ast" } parser = { path = "../parser" } js_ast = { path = "../js_ast" } js_printer = { path = "../js_printer" } -checker = { path = "../checker" } +sema = { path = "../sema" } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 4bd03112c4df413316fd9acfe40662d71d26abaf..00945e42be7c877440cc6f4466dfba7fad622f9c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -56,7 +56,7 @@ pub fn transpile(content: String) -> Result { // 2. Parse the tokens into an AST let blight_ast = parser::parse(tokens)?; // 3. Type check the AST - // checker::check_module(&blight_ast)?; + // sema::check_module(&blight_ast)?; // 4. Optimize the AST // optimize(tree.clone())?; // 5. Translate the AST into javascript diff --git a/crates/js_ast/src/lib.rs b/crates/js_ast/src/lib.rs index d0b9dd88049ab72f5f7b4e50cd2d9a480294a454..57a3e02c1bcea90ebbd0ed2d0cbc470aac67610c 100644 --- a/crates/js_ast/src/lib.rs +++ b/crates/js_ast/src/lib.rs @@ -180,6 +180,7 @@ fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) { }); } Statement::Expr(_) => todo!(), + Statement::Comment(_) => todo!(), // Ast::Function { // name, // arguments, diff --git a/crates/parser/parse_declaration.rs b/crates/parser/parse_declaration.rs index 3fbfc4e8ef2e6d7ee91926c2de05acbc3d46e679..9e5176ed38fa2d414c15e048774d8012b62a0f49 100644 --- a/crates/parser/parse_declaration.rs +++ b/crates/parser/parse_declaration.rs @@ -1,4 +1,4 @@ -use ast::{Decl, Type}; +use ast::{Decl, DocComment, Type}; use miette::Result; use tokenizer::Token; @@ -7,7 +7,11 @@ use crate::parse_type::parse_type; use crate::Parser; /// Precondition: The first token exists and is either 'var' or 'let' -pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result { +pub fn parse_declaration( + p: &mut Parser, + is_pub: bool, + comment: Option, +) -> Result { let loc = p.loc(); let is_constant = match p.next() { Some(Token::Var) => false, @@ -36,11 +40,11 @@ pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result { }; Ok(Decl { name: name.clone(), - decl_type: return_type, + data_type: return_type, init, is_constant, is_pub, - comment: None, + comment, loc, }) } diff --git a/crates/parser/parse_doc_comment.rs b/crates/parser/parse_doc_comment.rs new file mode 100644 index 0000000000000000000000000000000000000000..f62e9f32b30c867c7c0542d2902956a337f7a86a --- /dev/null +++ b/crates/parser/parse_doc_comment.rs @@ -0,0 +1,15 @@ +use ast::DocComment; +use tokenizer::Token; + +use crate::Parser; + +pub fn maybe_read_doc_comment(p: &mut Parser) -> Option { + match p.peek() { + Some(Token::DocComment(data)) => { + let loc = p.loc(); + p.next(); + Some(DocComment { data, loc }) + } + _ => None, + } +} diff --git a/crates/parser/parse_expr.rs b/crates/parser/parse_expr.rs index 98d098d3c847c174c0a9ca6a375cc038fa6d2b55..6ca753167cdabc91ac85e5845d4c1fb356ef2748 100644 --- a/crates/parser/parse_expr.rs +++ b/crates/parser/parse_expr.rs @@ -1,45 +1,8 @@ -use ast::{BinaryOperator, Expr, ExprData, Literal}; +use ast::{get_binary_op_level, BinaryOperator, BinaryOperatorLevel, Expr, ExprData}; use miette::Result; -use tokenizer::Token; -use crate::{ - parse_call_arguments::parse_call_arguments, parse_expr_basic::parse_expr_basic, Parser, -}; - -/// Derived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table - -#[derive(PartialEq, PartialOrd)] -enum Level { - Prefix, // +, -, !, ~ - Exponentiation, // ** - Multiplicative, // *, /, % - Additive, // +, - - BitwiseShift, // <<, >>, >>> - Relational, // <, <=, >, >= - Equality, // ==, != - BitwiseAnd, // & - BitwiseXor, // ^ - BitwiseOr, // | - LogicalAnd, // && - LogicalOr, // ||, ?? -} - -pub fn get_postfix_op_level(token: &Token) -> Option { - match token { - Token::StarStar => Some(Level::Exponentiation), - Token::Star | Token::Slash | Token::Percent => Some(Level::Multiplicative), - Token::Plus | Token::Minus => Some(Level::Additive), - Token::LtLt | Token::GtGt | Token::GtGtGt => Some(Level::BitwiseShift), - Token::Gt | Token::GtEqual | Token::Lt | Token::LtEqual => Some(Level::Relational), - Token::EqualEqual | Token::BangEqual => Some(Level::Equality), - Token::And => Some(Level::BitwiseAnd), - Token::Caret => Some(Level::BitwiseXor), - Token::Or => Some(Level::BitwiseOr), - Token::AndAnd => Some(Level::LogicalAnd), - Token::OrOr => Some(Level::LogicalOr), - _ => None, - } -} +use crate::parse_expr_basic::parse_expr_basic; +use crate::Parser; pub fn parse_expr(p: &mut Parser) -> Result { let mut expr = parse_expr_basic(p)?; @@ -49,68 +12,10 @@ pub fn parse_expr(p: &mut Parser) -> Result { return Ok(expr); } - // Parse function calls and field accesses - loop { - match p.peek() { - Some(Token::Dot) => { - let loc = p.loc(); - _ = p.next(); - let field = match p.peek() { - Some(Token::Identifier(name)) => { - let loc = p.loc(); - p.next(); - Expr::new(ExprData::Literal(Literal::String(name)), loc) - } - _ => { - return Err(p.error("Expected identifier after dot".to_string()).into()); - } - }; - expr = Expr::new( - ExprData::Field { - base: Box::new(expr), - field: Box::new(field), - }, - loc, - ); - } - Some(Token::OpenParen) => { - let loc = p.loc(); - _ = p.next(); - let arguments = parse_call_arguments(p)?; - expr = Expr::new( - ExprData::Call { - base: Box::new(expr), - arguments, - }, - loc, - ); - } - Some(Token::OpenBracket) => { - let loc = p.loc(); - _ = p.next(); - let index = parse_expr(p)?; - if let Some(Token::ClosedBracket) = p.next() { - expr = Expr::new( - ExprData::Field { - base: Box::new(expr), - field: Box::new(index), - }, - loc, - ); - } else { - return Err(p - .error("Expected closing bracket after index".to_string()) - .into()); - } - } - _ => break, - } - } - // Parse operators loop { if let Some(token) = p.peek() { - if let Some(level) = get_postfix_op_level(&token) { + if let Some(level) = get_binary_op_level(&token) { _ = p.next(); expr = parse_expr_binary( p, @@ -128,13 +33,18 @@ pub fn parse_expr(p: &mut Parser) -> Result { } /// Prerequisite: `p.peek()` is an operator -fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Level) -> Result { +fn parse_expr_binary( + p: &mut Parser, + op: BinaryOperator, + left: Expr, + level: BinaryOperatorLevel, +) -> Result { let loc = p.loc(); let mut right = parse_expr_basic(p)?; loop { if let Some(token) = p.peek() { - if let Some(next_level) = get_postfix_op_level(&token) { + if let Some(next_level) = get_binary_op_level(&token) { if next_level <= level { _ = p.next(); right = parse_expr_binary( @@ -157,5 +67,6 @@ fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Leve right: Box::new(right), }, loc, + None, )) } diff --git a/crates/parser/parse_expr_basic.rs b/crates/parser/parse_expr_basic.rs index 8cc9c8409c33d5a4a1ae08550f4cf60ea63a8a76..d1949c82fc65ad390e482f92707eb95b24415346 100644 --- a/crates/parser/parse_expr_basic.rs +++ b/crates/parser/parse_expr_basic.rs @@ -9,32 +9,35 @@ use crate::{ pub fn parse_expr_basic(p: &mut Parser) -> Result { let loc = p.loc(); - match p.peek() { + let comments = p.take_comments(); + let mut expr = match p.peek() { Some(Token::Number(number)) => { _ = p.next(); - Ok(Expr::new( + Expr::new( ExprData::Literal(Literal::Number(number.clone())), loc, - )) + comments, + ) } Some(Token::String(string)) => { _ = p.next(); - Ok(Expr::new( + Expr::new( ExprData::Literal(Literal::String(string.clone())), loc, - )) + comments, + ) } Some(Token::True) => { _ = p.next(); - Ok(Expr::new(ExprData::Literal(Literal::Bool(true)), loc)) + Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments) } Some(Token::False) => { _ = p.next(); - Ok(Expr::new(ExprData::Literal(Literal::Bool(false)), loc)) + Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments) } Some(Token::Void) => { _ = p.next(); - Ok(Expr::new(ExprData::Literal(Literal::Void), loc)) + Expr::new(ExprData::Literal(Literal::Void), loc, comments) } Some(Token::Identifier(name)) => { let loc = p.loc(); @@ -43,7 +46,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result { if let Some(id) = BuiltinId::from_str(name.as_str()) { if let Some(Token::OpenParen) = p.next() { let arguments = parse_call_arguments(p)?; - return Ok(Expr::new(ExprData::Builtin { id, arguments }, loc)); + Expr::new(ExprData::Builtin { id, arguments }, loc, comments) } else { return Err(p .error_at(format!("Built-in function '{}' must be called", name), loc) @@ -55,7 +58,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result { .into()); } } else { - Ok(Expr::new(ExprData::Identifier(name), loc)) + Expr::new(ExprData::Identifier(name), loc, comments) } } @@ -66,18 +69,21 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result { .into()); } else { _ = p.next(); - - return Ok(Expr::new(ExprData::Return(Box::new(parse_expr(p)?)), loc)); + return Ok(Expr::new( + ExprData::Return(Box::new(parse_expr(p)?)), + loc, + comments, + )); } } - Some(Token::If) => Ok(Expr::new(ExprData::If(parse_expr_if(p)?), loc)), + Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments), - Some(Token::OpenBrace) => Ok(Expr::new(ExprData::Block(parse_block(p)?), loc)), + Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments), Some(Token::OpenParen) => { _ = p.next(); - let expr = parse_expr_basic(p)?; + let expr = parse_expr(p)?; match p.next() { Some(Token::ClosedParen) => (), Some(token) => { @@ -85,12 +91,12 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result { } None => return Err(p.error("Unexpected end of file".to_string()).into()), } - Ok(expr) + expr } Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => { _ = p.next(); - Ok(Expr::new( + Expr::new( ExprData::Unary { op: match t { Token::Bang => UnaryOperator::Not, @@ -101,10 +107,74 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result { value: Box::new(parse_expr_basic(p)?), }, loc, - )) + comments, + ) } - Some(t) => Err(p.error(format!("Unexpected '{}'", t)).into()), - None => Err(p.error("Unexpected end of file".to_string()).into()), + Some(t) => return Err(p.error(format!("Unexpected '{}'", t)).into()), + None => return Err(p.error("Unexpected end of file".to_string()).into()), + }; + + // Parse function calls and field accesses + loop { + match p.peek() { + Some(Token::Dot) => { + let loc = p.loc(); + _ = p.next(); + let field = match p.peek() { + Some(Token::Identifier(name)) => { + let loc = p.loc(); + p.next(); + Expr::new(ExprData::Literal(Literal::String(name)), loc, None) + } + _ => { + return Err(p.error("Expected identifier after dot".to_string()).into()); + } + }; + expr = Expr::new( + ExprData::Field { + base: Box::new(expr), + field: Box::new(field), + }, + loc, + None, + ); + } + Some(Token::OpenParen) => { + let loc = p.loc(); + _ = p.next(); + let arguments = parse_call_arguments(p)?; + expr = Expr::new( + ExprData::Call { + base: Box::new(expr), + arguments, + }, + loc, + None, + ); + } + Some(Token::OpenBracket) => { + let loc = p.loc(); + _ = p.next(); + let index = parse_expr(p)?; + if let Some(Token::ClosedBracket) = p.next() { + expr = Expr::new( + ExprData::Field { + base: Box::new(expr), + field: Box::new(index), + }, + loc, + None, + ); + } else { + return Err(p + .error("Expected closing bracket after index".to_string()) + .into()); + } + } + _ => break, + } } + + Ok(expr) } diff --git a/crates/parser/parse_expr_if.rs b/crates/parser/parse_expr_if.rs index a956585da772e01c03c550745f8b33687c79356f..d480a09d7d30075449793e398b15c3103f06e751 100644 --- a/crates/parser/parse_expr_if.rs +++ b/crates/parser/parse_expr_if.rs @@ -2,7 +2,9 @@ use ast::{BlockWithLoc, If}; use miette::Result; use tokenizer::Token; -use crate::{parse_block::parse_block, parse_expr::parse_expr, Parser}; +use crate::parse_block::parse_block; +use crate::parse_expr::parse_expr; +use crate::Parser; /// Precondition: the first token exists and is 'if' pub fn parse_expr_if(p: &mut Parser) -> Result { diff --git a/crates/parser/parse_fn_declaration.rs b/crates/parser/parse_fn_declaration.rs index 4b7405c6bf3a57ab3b0bd50e9a2e26f8b12bd664..3b7363a36204bcd95ac287e5d7fb854cc2f06f78 100644 --- a/crates/parser/parse_fn_declaration.rs +++ b/crates/parser/parse_fn_declaration.rs @@ -1,4 +1,4 @@ -use ast::{Decl, Expr, ExprData, Type}; +use ast::{Decl, DocComment, Expr, ExprData, Type}; use miette::Result; use tokenizer::Token; @@ -8,7 +8,11 @@ use crate::parse_type::parse_type; use crate::Parser; /// Precondition: -pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result { +pub fn parse_fn_declaration( + p: &mut Parser, + is_pub: bool, + comment: Option, +) -> Result { let loc = p.loc(); match p.next() { Some(Token::Fn) => (), @@ -96,11 +100,11 @@ pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result { }; Ok(Decl { name, - decl_type: Type::Any, - init: Expr::new(function_body, block_loc), + data_type: Type::Any, + init: Expr::new(function_body, block_loc, None), is_constant: true, is_pub, - comment: None, + comment, loc: loc, }) } else { diff --git a/crates/parser/parse_module.rs b/crates/parser/parse_module.rs index 9ff66244156f39002142e205e7bd4a4ed6f73cd4..9d3f8fbe8e2e9f2ce1138a72a8b1ece27dda0011 100644 --- a/crates/parser/parse_module.rs +++ b/crates/parser/parse_module.rs @@ -1,23 +1,27 @@ -use crate::{parse_top_level_statement::parse_top_level_statement, Parser}; +use crate::parse_doc_comment::maybe_read_doc_comment; +use crate::parse_top_level_statement::parse_top_level_statement; +use crate::Parser; use ast::Module; use miette::Result; use tokenizer::Token; -pub fn parse_module(tokens: &mut Parser) -> Result { +pub fn parse_module(p: &mut Parser) -> Result { let mut statements = Vec::new(); + let file_doc_comment = maybe_read_doc_comment(p); + loop { - let statement = parse_top_level_statement(tokens)?; + let statement = parse_top_level_statement(p)?; - match tokens.peek() { + match p.peek() { Some(Token::Semicolon) => { - while let Some(Token::Semicolon) = tokens.peek() { - tokens.next(); + while let Some(Token::Semicolon) = p.peek() { + p.next(); } } _ => { if statement.requires_semicolon() { - return Err(tokens + return Err(p .error("Expected semicolon after statement".to_string()) .into()); } @@ -26,8 +30,11 @@ pub fn parse_module(tokens: &mut Parser) -> Result { statements.push(statement); - if tokens.empty() { - return Ok(Module { statements }); + if p.empty() { + return Ok(Module { + statements, + comment: file_doc_comment, + }); } } } diff --git a/crates/parser/parse_statement.rs b/crates/parser/parse_statement.rs index c3da1e4f219af484a5af82b0d45c35f68444c92e..a135c1d4e6de8d3d17c0f0b3d7b03fa3f62c4859 100644 --- a/crates/parser/parse_statement.rs +++ b/crates/parser/parse_statement.rs @@ -3,26 +3,60 @@ use miette::Result; use tokenizer::Token; use crate::parse_declaration::parse_declaration; +use crate::parse_doc_comment::maybe_read_doc_comment; use crate::parse_expr::parse_expr; use crate::parse_fn_declaration::parse_fn_declaration; use crate::Parser; pub fn parse_statement(p: &mut Parser) -> Result { + if let Some(comments) = p.take_comments() { + return Ok(Statement::Comment(comments)); + } + + let m_doc_comment = maybe_read_doc_comment(p); + if let Some(doc_comment) = m_doc_comment.clone() { + if p.has_comments() { + // this implies that comments are immediately after the doc + return Err(p + .error_at(format!("Unattached doc comment"), doc_comment.loc) + .into()); + } + } + match p.peek() { + Some(Token::DocComment(_)) => unreachable!(), Some(Token::Pub) => { _ = p.next(); match p.peek() { Some(Token::Var) | Some(Token::Let) => { - Ok(Statement::Decl(parse_declaration(p, true)?)) + Ok(Statement::Decl(parse_declaration(p, true, m_doc_comment)?)) } - Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, true)?)), + Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration( + p, + true, + m_doc_comment, + )?)), Some(_) => Err(p.error("Expected a declaration".to_string()).into()), None => Err(p.error("Unexpected end of file".to_string()).into()), } } - Some(Token::Var) | Some(Token::Let) => Ok(Statement::Decl(parse_declaration(p, false)?)), - Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, false)?)), - Some(_) => Ok(Statement::Expr(parse_expr(p)?)), + Some(Token::Var) | Some(Token::Let) => { + Ok(Statement::Decl(parse_declaration(p, false, m_doc_comment)?)) + } + Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration( + p, + false, + m_doc_comment, + )?)), + Some(_) => { + if let Some(doc_comment) = m_doc_comment { + Err(p + .error_at(format!("Unattached doc comment"), doc_comment.loc) + .into()) + } else { + Ok(Statement::Expr(parse_expr(p)?)) + } + } None => Err(p.error("Unexpected end of file".to_string()).into()), } } diff --git a/crates/parser/parser.rs b/crates/parser/parser.rs index 0dda281df5e97957e5a57c1ef5261894b56034a0..44f7cdd6f8750dee21ec88a0c746afc2b60f03ae 100644 --- a/crates/parser/parser.rs +++ b/crates/parser/parser.rs @@ -1,6 +1,7 @@ mod parse_block; mod parse_call_arguments; mod parse_declaration; +mod parse_doc_comment; mod parse_expr; mod parse_expr_basic; mod parse_expr_if; @@ -11,15 +12,14 @@ mod parse_statement; mod parse_top_level_statement; mod parse_type; -use tokenizer::{Loc, Token}; - -use miette::Diagnostic; -use thiserror::Error; - +use ast::Comment; use ast::Module; +use miette::Diagnostic; use miette::Result; use parse_module::parse_module; +use thiserror::Error; use tokenizer::tokenize; +use tokenizer::{Loc, Token}; pub fn parse(tokens: Vec) -> Result { let mut p = Parser::new(tokens); @@ -47,11 +47,15 @@ impl ParseError { #[derive(Debug)] pub struct Parser { - pub tokens: Vec, - pub cursor: usize, + /// TODO(@paperdave): switch to streaming tokenizer + one that can + /// report line and column info + tokens: Vec, + cursor: usize, /// Whether or not the parser is at the top level pub is_top_level: bool, + + comments: Vec, } impl Parser { @@ -60,6 +64,7 @@ impl Parser { tokens, cursor: 0, is_top_level: true, + comments: vec![], } } @@ -67,26 +72,61 @@ impl Parser { self.cursor == self.tokens.len() } - pub fn back(&mut self) { - self.cursor -= 1; - } + // TODO(@paperdave): fix parser token stuff. its very messy rn pub fn next(&mut self) -> Option { if self.cursor == self.tokens.len() { return None; } - let token = self.tokens[self.cursor].clone(); - self.cursor += 1; - Some(token) + loop { + match self.tokens[self.cursor].clone() { + Token::Comment(comment) => { + self.comments.push(comment); + self.cursor += 1; + continue; + } + token => { + self.cursor += 1; + return Some(token); + } + } + } } - pub fn peek(&self) -> Option { - if self.cursor == self.tokens.len() { + pub fn take_comments(&mut self) -> Option { + if self.comments.len() == 0 { return None; } + let comments = self.comments.clone(); + self.comments = vec![]; + Some(Comment { + data: comments.join("\n"), + }) + } - Some(self.tokens[self.cursor].clone()) + pub fn has_comments(&mut self) -> bool { + self.peek(); + self.comments.len() > 0 + } + + pub fn peek(&mut self) -> Option { + loop { + if self.cursor == self.tokens.len() { + return None; + } + + match self.tokens[self.cursor].clone() { + Token::Comment(comment) => { + self.comments.push(comment); + self.cursor += 1; + continue; + } + token => { + return Some(token); + } + } + } } pub fn error(&self, error: String) -> ParseError { diff --git a/crates/checker/Cargo.toml b/crates/sema/Cargo.toml similarity index 90% rename from crates/checker/Cargo.toml rename to crates/sema/Cargo.toml index 35bcf3fac1c683c696223e428ba9f59ff2bb8c75..151d91227fdbfa2b5b2120a5e072e56796233326 100644 --- a/crates/checker/Cargo.toml +++ b/crates/sema/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "checker" +name = "sema" version = "0.1.0" edition = "2021" diff --git a/crates/checker/src/ast_type_coersion.rs b/crates/sema/src/ast_type_coersion.rs similarity index 100% rename from crates/checker/src/ast_type_coersion.rs rename to crates/sema/src/ast_type_coersion.rs diff --git a/crates/checker/src/lib.rs b/crates/sema/src/lib.rs similarity index 100% rename from crates/checker/src/lib.rs rename to crates/sema/src/lib.rs diff --git a/crates/tokenizer/src/lib.rs b/crates/tokenizer/src/lib.rs index d536ccd14cf9a7bcd522e139e2f8f60d727786ad..5b2095c6d430acbe3f687f6324b22018a349a6dd 100644 --- a/crates/tokenizer/src/lib.rs +++ b/crates/tokenizer/src/lib.rs @@ -71,12 +71,12 @@ pub enum Token { impl std::fmt::Display for Token { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let token = match self { - Token::Identifier(_) => "an identifier".to_string(), + Token::Identifier(_) => "identifier".to_string(), Token::Number(n) => n.to_string(), Token::String(s) => format!("\"{}\"", s), - Token::Comment(_) => "a comment".to_string(), - Token::DocComment(_) => "a documentation comment".to_string(), + Token::Comment(_) => "comment".to_string(), + Token::DocComment(_) => "documentation comment".to_string(), Token::Else => "else".to_string(), Token::False => "false".to_string(), @@ -129,6 +129,7 @@ impl std::fmt::Display for Token { } } +// TODO(@paperdave): rewrite this to not use a Vec. we probably dont need it. also store Loc pub fn tokenize(content: String) -> Result> { let mut tokens: Vec = Vec::new(); let mut chars = content.chars().enumerate().peekable(); @@ -277,21 +278,56 @@ pub fn tokenize(content: String) -> Result> { let mut comment = String::new(); loop { - let c = chars.next(); - if c.is_none() { + // skip leading whitespace + while let Some((_, c)) = chars.peek() { + if !c.is_whitespace() { + break; + } + chars.next(); + } + // read the comment + while let Some((_, c)) = chars.next() { + if c == '\n' || c == '\r' { + break; + } + comment.push(c); + } + // trim trailing whitespace on line + comment = comment.trim_end().to_string(); + // skip to next non-whitespace character + while let Some((_, c)) = chars.peek() { + if !c.is_whitespace() { + break; + } + chars.next(); + } + // check if we've hit another comment + let mut tmp = chars.clone(); + + let is_another_comment = match tmp.peek() { + Some((_, '/')) => { + tmp.next(); // Consume the second slash + match tmp.peek() { + Some((_, '/')) => { + tmp.next(); // Consume the third slash + is_doc_comment + } + _ => !is_doc_comment, + } + } + _ => false, + }; + + if is_another_comment { + // apply our temporary iterator to the main iterator + comment.push('\n'); + chars = tmp; + chars.next(); + } else { break; } - let (_, c) = c.unwrap(); - match c { - '\n' => break, - _ => { - comment.push(c.clone()); - } - } } - comment = comment.trim().to_string(); - if is_doc_comment { tokens.push(Token::DocComment(comment)); } else { diff --git a/examples/basics/comments.blight b/examples/basics/comments.blight index cef4db9adcbbb9a7735efe8c97e0bea4e4a39b12..773d675b9cc0a27ff6d2419988e72893c7a09f62 100644 --- a/examples/basics/comments.blight +++ b/examples/basics/comments.blight @@ -1,9 +1,2 @@ -// this is a regular comment -let x: int = 0; - -/// this is a doc comment -let y: int = 0; - -/// Doc comments can be multiple lines. -/// This can be done by simply adding another triple-slash to the next line. -let z: int = 0; +2 // yapping + + 2; \ No newline at end of file -- 2.54.0