From fd797b948e101371e252f5053d00c77f529a48ad Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Sat, 20 Jan 2024 16:32:51 -0800 Subject: [PATCH] rewrite the parser in rust --- Cargo.lock | 22 +- Cargo.toml | 12 +- crates/ast/Cargo.toml | 3 +- crates/ast/ast.rs | 174 ++++++++++ crates/ast/lib.rs | 7 + crates/ast/operators.rs | 171 +++++++++ crates/ast/src/error.rs | 21 -- crates/ast/src/lib.rs | 27 -- crates/ast/src/parse_block.rs | 55 --- crates/ast/src/parse_expression.rs | 105 ------ crates/ast/src/parse_function_call.rs | 56 --- crates/ast/src/parse_statement.rs | 245 ------------- crates/ast/src/parse_type.rs | 28 -- crates/ast/src/types.rs | 153 -------- crates/ast/types.rs | 208 +++++++++++ crates/checker/Cargo.toml | 3 +- crates/checker/src/ast_type_coersion.rs | 54 +++ crates/checker/src/lib.rs | 299 ++++++---------- crates/cli/Cargo.toml | 1 + crates/cli/src/main.rs | 22 +- crates/js_ast/Cargo.toml | 1 - crates/js_ast/src/lib.rs | 384 ++++++++++----------- crates/js_ast/src/types.rs | 11 +- crates/js_printer/Cargo.toml | 1 - crates/js_printer/src/print_expression.rs | 21 +- crates/parser/Cargo.toml | 15 + crates/parser/parse_block.rs | 64 ++++ crates/parser/parse_call_arguments.rs | 31 ++ crates/parser/parse_declaration.rs | 46 +++ crates/parser/parse_expr.rs | 161 +++++++++ crates/parser/parse_expr_basic.rs | 110 ++++++ crates/parser/parse_expr_if.rs | 61 ++++ crates/parser/parse_fn_declaration.rs | 111 ++++++ crates/parser/parse_module.rs | 33 ++ crates/parser/parse_param.rs | 30 ++ crates/parser/parse_statement.rs | 28 ++ crates/parser/parse_top_level_statement.rs | 10 + crates/parser/parse_type.rs | 26 ++ crates/parser/parser.rs | 104 ++++++ crates/tokenizer/Cargo.toml | 2 - crates/tokenizer/src/lib.rs | 214 ++++++++---- crates/util/Cargo.toml | 8 - crates/util/src/lib.rs | 30 -- examples/basics/control.blight | 18 +- 44 files changed, 1941 insertions(+), 1245 deletions(-) create mode 100644 crates/ast/ast.rs create mode 100644 crates/ast/lib.rs create mode 100644 crates/ast/operators.rs delete mode 100644 crates/ast/src/error.rs delete mode 100644 crates/ast/src/lib.rs delete mode 100644 crates/ast/src/parse_block.rs delete mode 100644 crates/ast/src/parse_expression.rs delete mode 100644 crates/ast/src/parse_function_call.rs delete mode 100644 crates/ast/src/parse_statement.rs delete mode 100644 crates/ast/src/parse_type.rs delete mode 100644 crates/ast/src/types.rs create mode 100644 crates/ast/types.rs create mode 100644 crates/checker/src/ast_type_coersion.rs create mode 100644 crates/parser/Cargo.toml create mode 100644 crates/parser/parse_block.rs create mode 100644 crates/parser/parse_call_arguments.rs create mode 100644 crates/parser/parse_declaration.rs create mode 100644 crates/parser/parse_expr.rs create mode 100644 crates/parser/parse_expr_basic.rs create mode 100644 crates/parser/parse_expr_if.rs create mode 100644 crates/parser/parse_fn_declaration.rs create mode 100644 crates/parser/parse_module.rs create mode 100644 crates/parser/parse_param.rs create mode 100644 crates/parser/parse_statement.rs create mode 100644 crates/parser/parse_top_level_statement.rs create mode 100644 crates/parser/parse_type.rs create mode 100644 crates/parser/parser.rs delete mode 100644 crates/util/Cargo.toml delete mode 100644 crates/util/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8317b65192cbb35945e6d398cc412aea7bcee045..0d4dfa5e9ed55431cd6d3cfba52ca941ce7660df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,9 +70,9 @@ name = "ast" version = "0.1.0" dependencies = [ "miette", + "num", "thiserror", "tokenizer", - "util", ] [[package]] @@ -134,7 +134,6 @@ dependencies = [ "miette", "num", "thiserror", - "util", ] [[package]] @@ -187,6 +186,7 @@ dependencies = [ "js_ast", "js_printer", "miette", + "parser", "thiserror", "tokenizer", ] @@ -250,7 +250,6 @@ dependencies = [ "miette", "num", "thiserror", - "util", ] [[package]] @@ -261,7 +260,6 @@ dependencies = [ "miette", "num", "thiserror", - "util", ] [[package]] @@ -420,6 +418,17 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "parser" +version = "0.1.0" +dependencies = [ + "ast", + "miette", + "num", + "thiserror", + "tokenizer", +] + [[package]] name = "proc-macro2" version = "1.0.76" @@ -555,7 +564,6 @@ version = "0.1.0" dependencies = [ "miette", "thiserror", - "util", ] [[package]] @@ -582,10 +590,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "util" -version = "0.1.0" - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index e0aaabdca72cfe022aef071763d33f4ee6b7997c..d786a2120adc1dbefba1c1002a3a56ec7fd9c010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,12 +3,20 @@ version = "0.1.0" name = "blight" edition = "2023" license = "MIT" -authors = ["Lino Le Van"] +authors = ["Lino Le Van", "Dave Caruso"] repository = "https://github.com/lino-levan/blight" [workspace] resolver = "2" -members = ["crates/cli", "crates/util", "crates/tokenizer", "crates/ast", "crates/js_ast", "crates/js_printer"] +members = [ + "crates/ast", + "crates/checker", + "crates/cli", + "crates/js_ast", + "crates/js_printer", + "crates/parser", + "crates/tokenizer", +] [workspace.dependencies] clap = { version = "4.4.12", features = ["derive"] } diff --git a/crates/ast/Cargo.toml b/crates/ast/Cargo.toml index 2f57e0d3305382b701e02b9ba80ee44d14c7d822..fc7837e1d94befad96581f8d202330bb79e732d8 100644 --- a/crates/ast/Cargo.toml +++ b/crates/ast/Cargo.toml @@ -4,10 +4,11 @@ version = "0.1.0" edition = "2021" [lib] +path = "./lib.rs" [dependencies] miette.workspace = true thiserror.workspace = true +num.workspace = true tokenizer = { path = "../tokenizer" } -util = { path = "../util" } diff --git a/crates/ast/ast.rs b/crates/ast/ast.rs new file mode 100644 index 0000000000000000000000000000000000000000..972216ddbe6d10e928f7160f8f5d999983829948 --- /dev/null +++ b/crates/ast/ast.rs @@ -0,0 +1,174 @@ +use tokenizer::Loc; + +use crate::{ + types::{Literal, Type}, + BinaryOperator, UnaryOperator, +}; + +#[derive(Debug, Clone)] +pub enum BuiltinId { + Raw, +} + +#[derive(Debug, Clone)] +pub struct Module { + pub statements: Vec, +} + +#[derive(Debug, Clone)] +pub struct Block { + pub statements: Vec, + pub has_implicit_return: bool, +} + +#[derive(Debug, Clone)] +pub struct Param { + pub name: String, + pub param_type: Type, +} + +#[derive(Debug, Clone)] + +pub struct Decl { + pub name: String, + pub decl_type: Type, + pub init: Expr, + pub is_constant: bool, + pub is_pub: bool, + pub loc: Loc, + pub comment: Option, +} + +#[derive(Debug, Clone)] +pub struct Expr { + pub data: ExprData, + pub loc: Loc, +} + +impl Expr { + pub fn new(data: ExprData, loc: Loc) -> Self { + Self { data, loc } + } +} + +#[derive(Debug, Clone)] +pub struct DocComment { + pub data: String, + pub loc: Loc, +} + +#[derive(Debug, Clone)] +pub struct BlockWithLoc { + pub block: Block, + pub loc: Loc, +} + +#[derive(Debug, Clone)] +pub struct If { + pub condition: Box, + pub yes: BlockWithLoc, + pub no: Option, +} + +#[derive(Debug, Clone)] +pub enum ExprData { + Block(Block), + Binary { + op: BinaryOperator, + left: Box, + right: Box, + }, + Unary { + op: UnaryOperator, + value: Box, + }, + Fn { + name: String, + params: Vec, + return_type: Type, + block: Block, + }, + Call { + base: Box, + arguments: Vec, + }, + Field { + base: Box, + field: Box, + }, + Builtin { + id: BuiltinId, + arguments: Vec, + }, + If(If), + Return(Box), + Literal(Literal), + Identifier(String), +} + +#[derive(Debug, Clone)] +pub enum Statement { + Expr(Expr), + Decl(Decl), +} + +#[derive(Debug, Clone)] +pub enum TopLevelStatement { + Expr(Expr), + Decl(Decl), +} + +impl BuiltinId { + pub fn display_name(&self) -> &'static str { + match self { + Self::Raw => "@raw", + } + } + + pub fn from_str(name: &str) -> Option { + match name { + "@raw" => Some(Self::Raw), + _ => None, + } + } +} + +impl Statement { + pub fn requires_semicolon(&self) -> bool { + match &self { + Statement::Expr(expr) => match expr.data { + ExprData::Block(_) => false, + ExprData::If(_) => false, + _ => true, + }, + _ => false, + } + } + + pub fn to_top_level(self) -> TopLevelStatement { + match self { + Statement::Expr(expr) => TopLevelStatement::Expr(expr), + Statement::Decl(decl) => TopLevelStatement::Decl(decl), + } + } +} + +impl TopLevelStatement { + pub fn requires_semicolon(&self) -> bool { + match &self { + TopLevelStatement::Expr(expr) => match expr.data { + ExprData::Block(_) => false, + ExprData::If(_) => false, + _ => true, + }, + _ => false, + } + } + + pub fn to_statement(self) -> Statement { + match self { + TopLevelStatement::Expr(expr) => Statement::Expr(expr), + TopLevelStatement::Decl(decl) => Statement::Decl(decl), + } + } +} diff --git a/crates/ast/lib.rs b/crates/ast/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..c35262a7f11a1125135720a5efee113eb1ec8c29 --- /dev/null +++ b/crates/ast/lib.rs @@ -0,0 +1,7 @@ +mod ast; +mod operators; +mod types; + +pub use ast::*; +pub use operators::*; +pub use types::*; diff --git a/crates/ast/operators.rs b/crates/ast/operators.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7c7d4d9ffb2783bb85444d8de71dfed4c6ce2c2 --- /dev/null +++ b/crates/ast/operators.rs @@ -0,0 +1,171 @@ +use tokenizer::Token; + +use crate::Literal; + +#[derive(Debug, Clone, PartialEq)] +pub enum BinaryOperator { + Add, + Sub, + Mul, + Div, + Pow, + Eq, + Gt, + Lt, + Gte, + Lte, + Lshift, + Rshift, + RshiftUnsigned, + BitAnd, + BitOr, + BitXor, + Mod, + NotEq, + And, + Or, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum UnaryOperator { + Neg, + Not, + BitNot, +} + +impl BinaryOperator { + pub fn to_string(&self) -> String { + String::from(match self { + Self::Add => "+", + Self::Sub => "-", + Self::Div => "/", + Self::Mul => "*", + Self::Pow => "**", + Self::Eq => "==", + Self::Gt => ">", + Self::Lt => "<", + Self::Gte => ">=", + Self::Lte => "<=", + Self::Lshift => "<<", + Self::Rshift => ">>", + Self::RshiftUnsigned => ">>>", + Self::BitAnd => "&", + Self::BitOr => "|", + Self::BitXor => "^", + Self::Mod => "%", + Self::NotEq => "!=", + Self::And => "&&", + Self::Or => "||", + }) + } + + pub fn from_token(token: &Token) -> Option { + match token { + Token::And => Some(Self::BitAnd), + Token::AndAnd => Some(Self::And), + Token::BangEqual => Some(Self::NotEq), + Token::Caret => Some(Self::BitXor), + Token::EqualEqual => Some(Self::Eq), + Token::Gt => Some(Self::Gt), + Token::GtEqual => Some(Self::Gte), + Token::GtGt => Some(Self::Rshift), + Token::GtGtGt => Some(Self::RshiftUnsigned), + Token::Lt => Some(Self::Lt), + Token::LtEqual => Some(Self::Lte), + Token::LtLt => Some(Self::Lshift), + Token::Minus => Some(Self::Sub), + Token::Or => Some(Self::BitOr), + Token::OrOr => Some(Self::Or), + Token::Percent => Some(Self::Mod), + Token::Plus => Some(Self::Add), + Token::Slash => Some(Self::Div), + Token::Star => Some(Self::Mul), + Token::StarStar => Some(Self::Pow), + _ => None, + } + } + + /// Error case of this means 'types do not allow this operation' + pub fn perform(&self, left: Literal, right: Literal) -> Result { + match self { + BinaryOperator::Eq => Ok(Literal::Bool(left == right)), + BinaryOperator::NotEq => Ok(Literal::Bool(left != right)), + + BinaryOperator::Or | BinaryOperator::And => match (left, right) { + (Literal::Bool(a), Literal::Bool(b)) => Ok(Literal::Bool(match self { + BinaryOperator::Or => a || b, + BinaryOperator::And => a && b, + _ => unreachable!(), + })), + _ => Err(()), + }, + + BinaryOperator::Add + | BinaryOperator::Sub + | BinaryOperator::Mul + | BinaryOperator::Div + | BinaryOperator::Pow + | BinaryOperator::Gt + | BinaryOperator::Lt + | BinaryOperator::Gte + | BinaryOperator::Lte + | BinaryOperator::Lshift + | BinaryOperator::Rshift + | BinaryOperator::RshiftUnsigned + | BinaryOperator::BitAnd + | BinaryOperator::BitOr + | BinaryOperator::BitXor + | BinaryOperator::Mod => match (left, right) { + (Literal::Number(a), Literal::Number(b)) => Ok(match self { + BinaryOperator::Add => Literal::Number(a + b), + BinaryOperator::Sub => Literal::Number(a - b), + BinaryOperator::Mul => Literal::Number(a * b), + BinaryOperator::Div => Literal::Number(a / b), + BinaryOperator::Pow => Literal::Number(a.powf(b)), + BinaryOperator::Gt => Literal::Bool(a > b), + BinaryOperator::Lt => Literal::Bool(a < b), + BinaryOperator::Gte => Literal::Bool(a >= b), + BinaryOperator::Lte => Literal::Bool(a <= b), + BinaryOperator::Lshift => Literal::Number(((a as i32) << (b as i32)).into()), + BinaryOperator::Rshift => Literal::Number(((a as i32) >> (b as i32)).into()), + BinaryOperator::RshiftUnsigned => { + Literal::Number(((a as u32) >> (b as u32)).into()) + } + BinaryOperator::BitAnd => Literal::Number(((a as i32) & (b as i32)).into()), + BinaryOperator::BitOr => Literal::Number(((a as i32) | (b as i32)).into()), + BinaryOperator::BitXor => Literal::Number(((a as i32) ^ (b as i32)).into()), + BinaryOperator::Mod => Literal::Number(a % b), + _ => unreachable!(), + }), + _ => Err(()), + }, + } + } +} + +impl UnaryOperator { + pub fn to_string(self) -> String { + String::from(match self { + Self::Neg => "-", + Self::Not => "!", + Self::BitNot => "~", + }) + } + + pub fn perform(self, value: Literal) -> Result { + match self { + UnaryOperator::Neg => match value { + Literal::Number(a) => Ok(Literal::Number(-a)), + _ => Err(()), + }, + UnaryOperator::Not => match value { + Literal::Bool(a) => Ok(Literal::Bool(!a)), + _ => Err(()), + }, + UnaryOperator::BitNot => match value { + Literal::Number(a) => Ok(Literal::Number((!(a as i32)).into())), + _ => Err(()), + }, + } + } +} diff --git a/crates/ast/src/error.rs b/crates/ast/src/error.rs deleted file mode 100644 index c04c6362d317fb2623d187a22a7a78dee72982ff..0000000000000000000000000000000000000000 --- a/crates/ast/src/error.rs +++ /dev/null @@ -1,21 +0,0 @@ -use miette::Diagnostic; -use thiserror::Error; - -#[derive(Error, Diagnostic, Debug)] -#[error("AstError")] -pub struct AstError { - #[help] - error: String, -} - -impl AstError { - pub fn new(error: &'static str) -> AstError { - AstError { - error: String::from(error), - } - } - - pub fn from_string(error: String) -> AstError { - AstError { error } - } -} diff --git a/crates/ast/src/lib.rs b/crates/ast/src/lib.rs deleted file mode 100644 index 3810f82f7e30940a92838648832212aeddf5443c..0000000000000000000000000000000000000000 --- a/crates/ast/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -use miette::Result; - -mod error; -mod parse_block; -mod parse_expression; -mod parse_function_call; -mod parse_statement; -mod parse_type; -mod types; - -use tokenizer::Token; - -use parse_statement::parse_statement; -use types::ParserContext; - -pub use types::{Ast, Block, Module, Type}; - -pub fn parse(tokens: Vec) -> Result { - let mut ctx = ParserContext::new(tokens); - let mut ast: Vec = Vec::new(); - - while !ctx.empty() { - ast.push(parse_statement(&mut ctx)?); - } - - Ok(Module { value: ast }) -} diff --git a/crates/ast/src/parse_block.rs b/crates/ast/src/parse_block.rs deleted file mode 100644 index c074466d00afb74240012c5d37631596e5cbc374..0000000000000000000000000000000000000000 --- a/crates/ast/src/parse_block.rs +++ /dev/null @@ -1,55 +0,0 @@ -use miette::Result; - -use tokenizer::Token; - -use super::error::AstError; -use super::parse_expression::parse_expression; -use super::parse_statement::parse_statement; -use super::types::Block; -use super::types::ParserContext; - -pub fn parse_block(tokens: &mut ParserContext) -> Result { - match tokens.next() { - Some(Token::OpenBracket) => (), - _ => return Err(AstError::new("Expected { when parsing block").into()), - } - - let mut value = Vec::new(); - let mut implicit_return = false; - - loop { - match tokens.peek() { - Some(Token::ClosedBracket) => { - tokens.next(); - break; - } - _ => (), - } - - let current_cursor = tokens.cursor; - let result = parse_statement(tokens); - - match result { - Ok(ast) => value.push(ast), - Err(_) => { - tokens.cursor = current_cursor; - value.push(parse_expression(tokens)?); - implicit_return = true; - match tokens.peek() { - Some(Token::ClosedBracket) => (), - _ => { - return Err(AstError::new( - "Expected closed bracket after expression in function body", - ) - .into()); - } - } - } - } - } - - Ok(Block { - value, - implicit_return, - }) -} diff --git a/crates/ast/src/parse_expression.rs b/crates/ast/src/parse_expression.rs deleted file mode 100644 index f71c1cd26b5070d3577eb7bcd607b2f5e4417005..0000000000000000000000000000000000000000 --- a/crates/ast/src/parse_expression.rs +++ /dev/null @@ -1,105 +0,0 @@ -use miette::Result; - -use tokenizer::Token; - -use super::error::AstError; -use super::parse_block::parse_block; -use super::parse_function_call::parse_function_call; -use super::types::{Ast, Block, ParserContext}; - -pub fn parse_expression(tokens: &mut ParserContext) -> Result { - let left = match tokens.next() { - Some(token) => match token { - Token::Number(number) => Ast::Number(number), - Token::String(string) => Ast::String(string.clone()), - Token::BuiltIn(name) => { - match tokens.peek() { - Some(Token::OpenParanthesis) => { - // FRICK we consumed the open paranthesis - tokens.back(); - parse_function_call(tokens)? - } - _ => Ast::BuiltIn(name.clone()), - } - } - Token::True => Ast::Bool(true), - Token::False => Ast::Bool(false), - Token::OpenBracket => { - // FRICK we consumed the open bracket - // we need to put the cursor back - tokens.back(); - Ast::Block(parse_block(tokens)?) - } - Token::If => { - let condition = parse_expression(tokens)?; - - let value = parse_block(tokens)?; - - let mut else_value: Option = None; - - match tokens.peek() { - Some(Token::Else) => { - tokens.next(); - match tokens.peek() { - Some(Token::If) => { - let else_expression = parse_expression(tokens)?; - else_value = Some(Block { - value: vec![else_expression], - implicit_return: true, - }); - } - Some(Token::OpenBracket) => { - else_value = Some(parse_block(tokens)?); - } - _ => { - return Err(AstError::new( - "Expected if or open bracket after else in if statement", - ) - .into()) - } - } - } - _ => (), - } - - return Ok(Ast::If { - condition: Box::new(condition), - value, - else_value, - }); - } - Token::Identifier(name) => { - match tokens.peek() { - Some(Token::OpenParanthesis) => { - // FRICK we consumed the open paranthesis - tokens.back(); - parse_function_call(tokens)? - } - _ => Ast::Identifier(name.clone()), - } - } - _ => { - println!("token: {:?}", token); - return Err(AstError::new("Expected expression, but did not get it").into()); - } - }, - None => return Err(AstError::new("Expected expression, but got nothing instead").into()), - }; - - match tokens.peek() { - Some(token) => match token { - Token::Operator(operator) => { - tokens.next(); - return Ok(Ast::Operation { - operator: operator.clone(), - left: Box::new(left), - right: Box::new(parse_expression(tokens)?), - }); - } - _ => (), - }, - _ => (), - } - - Ok(left) -} diff --git a/crates/ast/src/parse_function_call.rs b/crates/ast/src/parse_function_call.rs deleted file mode 100644 index 57c177d8c77fb7e7aeb1b1d7cbd502c7651ca133..0000000000000000000000000000000000000000 --- a/crates/ast/src/parse_function_call.rs +++ /dev/null @@ -1,56 +0,0 @@ -use miette::Result; - -use tokenizer::Token; - -use super::error::AstError; -use super::parse_expression::parse_expression; -use super::types::{Ast, ParserContext}; - -/// Parses a function call AS AN EXPRESSION -/// This does not consume the semicolon at the end of the function call -/// if it is a statement. -pub fn parse_function_call(tokens: &mut ParserContext) -> Result { - let identifier = match tokens.next() { - Some(Token::Identifier(identifier)) => Ast::Identifier(identifier), - Some(Token::BuiltIn(identifier)) => Ast::BuiltIn(identifier), - _ => return Err(AstError::new("Expected identifier").into()), - }; - - match tokens.peek() { - Some(Token::OpenParanthesis) => { - tokens.next(); - - let mut arguments = Vec::new(); - - loop { - match tokens.peek() { - Some(Token::ClosedParanthesis) => { - tokens.next(); - break; - } - _ => (), - } - arguments.push(parse_expression(tokens)?); - - match tokens.peek() { - Some(Token::Comma) => { - tokens.next(); - } - Some(Token::ClosedParanthesis) => (), - _ => { - return Err(AstError::new( - "Expected ) or , but found something else instead.", - ) - .into()) - } - } - } - - Ok(Ast::FunctionCall { - name: Box::new(identifier), - arguments, - }) - } - _ => Err(AstError::new("Unexpected identifier").into()), - } -} diff --git a/crates/ast/src/parse_statement.rs b/crates/ast/src/parse_statement.rs deleted file mode 100644 index 1286c41fd08bdcc6267f855fa7b1a40df9056004..0000000000000000000000000000000000000000 --- a/crates/ast/src/parse_statement.rs +++ /dev/null @@ -1,245 +0,0 @@ -use miette::Result; - -use tokenizer::Token; - -use crate::Type; - -use super::error::AstError; -use super::parse_block::parse_block; -use super::parse_expression::parse_expression; -use super::parse_function_call::parse_function_call; -use super::parse_type::parse_type; -use super::types::{Argument, Ast, Block, ParserContext}; - -/// Precondition: There is at least one token in the iterator -pub fn parse_statement(tokens: &mut ParserContext) -> Result { - let token = tokens.next().unwrap(); - match token { - Token::BuiltIn(name) => { - match tokens.peek() { - Some(Token::OpenParanthesis) => { - // Okay, so we messed up. We shouldn't have consumed that token. - tokens.back(); - parse_function_call(tokens) - } - _ => Ok(Ast::BuiltIn(name)), - } - } - Token::OpenBracket => { - // FRICK, we have to backtrack - tokens.back(); - Ok(Ast::Block(parse_block(tokens)?)) - } - Token::Pub => { - match tokens.peek() { - Some(Token::Let) | Some(Token::Var) | Some(Token::Function) => (), - _ => return Err(AstError::new("Expected let, var, or fn after pub").into()), - } - - let inner = parse_statement(tokens)?; - - return Ok(Ast::Public(Box::new(inner))); - } - Token::Var | Token::Let => { - let is_constant = match token { - Token::Var => false, - Token::Let => true, - _ => unreachable!(), - }; - let name = match tokens.next() { - Some(Token::Identifier(name)) => name, - _ => return Err(AstError::new("Expected identifier after var").into()), - }; - let return_type = match tokens.peek() { - Some(Token::Colon) => { - tokens.next(); - parse_type(tokens)? - } - _ => Type::Infer, - }; - let value = match tokens.next() { - Some(Token::Equals) => parse_expression(tokens)?, - _ => return Err(AstError::new("Expected equals after identifier").into()), - }; - match tokens.next() { - Some(Token::Semicolon) => (), - _ => { - return Err( - AstError::new("Expected semicolon after constant declaration").into(), - ) - } - }; - Ok(Ast::Declaration { - name: name.clone(), - decl_type: return_type, - value: Box::new(value), - is_constant, - }) - } - Token::Return => { - let expression = parse_expression(tokens)?; - - match tokens.next() { - Some(Token::Semicolon) => (), - _ => return Err(AstError::new("Expected semicolon after return statement").into()), - }; - - Ok(Ast::Return(Box::new(expression))) - } - Token::Function => { - let name = match tokens.next() { - Some(Token::Identifier(name)) => name.clone(), - _ => return Err(AstError::new("Expected identifier after fn").into()), - }; - - match tokens.next() { - Some(Token::OpenParanthesis) => (), - _ => { - return Err(AstError::new( - "Expected open paranthesis after identifier in function", - ) - .into()) - } - }; - - let mut arguments = Vec::new(); - - loop { - match tokens.peek() { - Some(Token::ClosedParanthesis) => { - tokens.next(); - break; - } - _ => (), - } - - let argument_name = match tokens.next() { - Some(Token::Identifier(name)) => name, - _ => { - return Err(AstError::new( - "Expected identifier for argument name in function", - ) - .into()) - } - }; - - let return_type = match tokens.next() { - Some(Token::Colon) => parse_type(tokens)?, - _ => return Err(AstError::new("Expected colon after identifier").into()), - }; - - match tokens.peek() { - Some(Token::Comma) => { - tokens.next(); - } - _ => (), - } - - arguments.push(Argument { - name: argument_name.clone(), - return_type, - }) - } - - let return_type = match tokens.next() { - Some(Token::RightArrow) => parse_type(tokens)?, - _ => { - return Err(AstError::new( - "Expected right arrow after argument list in function", - ) - .into()) - } - }; - - match tokens.peek() { - Some(Token::OpenBracket) => (), - _ => { - return Err(AstError::new( - "Expected open bracket after type in function declaration", - ) - .into()) - } - }; - - let block = parse_block(tokens)?; - - return Ok(Ast::Function { - name, - arguments, - return_type, - block, - }); - } - Token::If => { - let condition = parse_expression(tokens)?; - - let value = parse_block(tokens)?; - - let mut else_value: Option = None; - - match tokens.peek() { - Some(Token::Else) => { - tokens.next(); - match tokens.peek() { - Some(Token::If) => { - let else_if = parse_statement(tokens)?; - else_value = Some(Block { - value: vec![else_if], - implicit_return: false, - }); - } - Some(Token::OpenBracket) => { - else_value = Some(parse_block(tokens)?); - } - _ => { - return Err(AstError::new( - "Expected if or open bracket after else in if statement", - ) - .into()) - } - } - } - _ => (), - } - - return Ok(Ast::If { - condition: Box::new(condition), - value, - else_value, - }); - } - Token::Identifier(_) => match tokens.peek() { - Some(Token::OpenParanthesis) => { - // We need to give the identifier back to the parser - tokens.back(); - let result = parse_function_call(tokens); - match tokens.next() { - Some(Token::Semicolon) => (), - _ => { - return Err(AstError::new( - "Expected semicolon after function call statement", - ) - .into()) - } - }; - result - } - _ => Err(AstError::new("Unexpected identifier").into()), - }, - Token::Comment(comment) => Ok(Ast::Comment(comment.clone())), - Token::DocComment(comment) => { - let mut concatted_comments = comment.clone(); - loop { - match tokens.peek() { - Some(Token::DocComment(comment)) => { - concatted_comments.push_str(format!("\n{}", comment).as_str()); - tokens.next(); - } - _ => break, - } - } - Ok(Ast::DocComment(concatted_comments)) - } - _ => return Err(AstError::new("Unexpected token").into()), - } -} diff --git a/crates/ast/src/parse_type.rs b/crates/ast/src/parse_type.rs deleted file mode 100644 index e481d0d283d7ad230ed89190f60b859134dfce4e..0000000000000000000000000000000000000000 --- a/crates/ast/src/parse_type.rs +++ /dev/null @@ -1,28 +0,0 @@ -use miette::Result; - -use tokenizer::Token; - -use super::error::AstError; -use super::types::{ParserContext, Type}; - -pub fn parse_type(tokens: &mut ParserContext) -> Result { - match tokens.next() { - Some(Token::Identifier(typ)) => match typ.as_str() { - "int" => Ok(Type::Int), - "float" => Ok(Type::Float), - "bigint" => Ok(Type::BigInt), - "bool" => Ok(Type::Bool), - "string" => Ok(Type::String), - _ => { - return Err( - AstError::from_string(format!( - "Invalid type '{}' (TODO: add custom type support)", - typ - ) - ).into()) - } - }, - Some(unexpected) => return Err(AstError::from_string(format!("Expected type, got {}", unexpected)).into()), - None => return Err(AstError::from_string("Expected type, got EOF".to_string()).into()), - } -} diff --git a/crates/ast/src/types.rs b/crates/ast/src/types.rs deleted file mode 100644 index 61a84e57cd474bd406185292dfb87fa0cfeed8b3..0000000000000000000000000000000000000000 --- a/crates/ast/src/types.rs +++ /dev/null @@ -1,153 +0,0 @@ -// TODO(lino-levan): Split up AST into expressions, statements, etc. - -use tokenizer::Token; -use util::Operator; - -#[derive(Debug)] -pub struct ParserContext { - pub tokens: Vec, - pub cursor: usize, -} - -impl ParserContext { - pub fn new(tokens: Vec) -> ParserContext { - ParserContext { tokens, cursor: 0 } - } - - pub fn empty(&self) -> bool { - self.cursor == self.tokens.len() - } - - pub fn back(&mut self) { - self.cursor -= 1; - } - - pub fn next(&mut self) -> Option { - if self.cursor == self.tokens.len() { - return None; - } - - let token = self.tokens[self.cursor].clone(); - self.cursor += 1; - Some(token) - } - - pub fn peek(&self) -> Option { - if self.cursor == self.tokens.len() { - return None; - } - - Some(self.tokens[self.cursor].clone()) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum Type { - /// Variables without an explicit type are assigned this until - /// the type inferencer figures out what this is. There is not a - /// keyword for this. - Infer, - /// The type assigned to numeric literals. This allows integer literals - /// to coerce to integer or float depending on the context, but prevents - /// implicit casting of runtime values. - NumberLiteral(f64), - /// Return value of `@raw`. All type checks are disabled. - Any, - /// noreturn is the type of 'return', aka a value that never exists - NoReturn, - /// 32 signed integer - Int, - /// 64 signed JS float - Float, - /// Boolean - Bool, - /// JS BigInt - BigInt, - /// JS String - String, - /// A union of multiple possible types. Might be removed - Union(Vec), -} - -impl std::fmt::Display for Type { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Type::Infer => write!(f, "infered type"), - Type::Any => write!(f, "opaque js value"), - Type::NumberLiteral(_) => write!(f, "numeric literal"), - Type::Int => write!(f, "int"), - Type::Float => write!(f, "float"), - Type::BigInt => write!(f, "bigint"), - Type::Bool => write!(f, "bool"), - Type::String => write!(f, "string"), - Type::NoReturn => write!(f, "noreturn"), - Type::Union(of) => { - for (i, t) in of.iter().enumerate() { - if i != 0 { - write!(f, " | ")?; - } - write!(f, "{}", t)?; - } - Ok(()) - } - } - } -} - -#[derive(Debug, Clone)] -pub struct Module { - pub value: Vec, -} - -#[derive(Debug, Clone)] -pub struct Block { - pub value: Vec, - pub implicit_return: bool, -} - -#[derive(Debug, Clone)] -pub struct Argument { - pub name: String, - pub return_type: Type, -} - -#[derive(Debug, Clone)] -pub enum Ast { - Declaration { - name: String, - decl_type: Type, - value: Box, - is_constant: bool, - }, - Operation { - operator: Operator, - left: Box, - right: Box, - }, - Function { - name: String, - arguments: Vec, - return_type: Type, - block: Block, - }, - FunctionCall { - /// Identifier or BuiltIn - name: Box, - arguments: Vec, - }, - If { - condition: Box, - value: Block, - else_value: Option, - }, - Return(Box), - Public(Box), - Block(Block), - Number(f64), - Bool(bool), - String(String), - Identifier(String), - BuiltIn(String), - Comment(String), - DocComment(String), -} diff --git a/crates/ast/types.rs b/crates/ast/types.rs new file mode 100644 index 0000000000000000000000000000000000000000..e65220300f5e78fceec6f68eef1fdf5ef51dc984 --- /dev/null +++ b/crates/ast/types.rs @@ -0,0 +1,208 @@ +use miette::{Diagnostic, Result}; +use thiserror::Error; + +/// A value that is known at compile-time +#[derive(Debug, Clone, PartialEq)] +pub enum Literal { + Number(f64), + Bool(bool), + String(String), + Void, +} + +impl std::fmt::Display for Literal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Literal::Number(value) => write!(f, "{}", value), + Literal::Bool(value) => write!(f, "{}", value), + Literal::String(value) => write!(f, "{}", value), + Literal::Void => write!(f, "void"), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Type { + /// Variables without an explicit type are assigned this until + /// the type inferencer figures out what this is. There is not a + /// keyword for this. + Infer, + /// The type assigned to literals. This allows integer literals + /// to coerce to integer or float depending on the context, but + /// prevents implicit casting of runtime values. + Literal(Literal), + /// Return value of `@raw`. All type checks are disabled. + Any, + /// The type of `void`. This is the return type of functions + /// that do not return a value. + Void, + /// noreturn is the type of 'return', aka a value that never exists. + /// unlike void, this includes functions that do not return at all. + NoReturn, + /// 32 signed integer + Int, + /// 64 signed JS float + Float, + /// Boolean + Bool, + /// JS BigInt + BigInt, + /// JS String + String, + /// A union of multiple possible types. Might be removed + Union(Vec), +} + +pub trait TypeCoerceable { + fn to_type(&self) -> Option; +} + +impl TypeCoerceable for Literal { + fn to_type(&self) -> Option { + Some(match self { + Literal::Number(_) => Type::Float, + Literal::Bool(_) => Type::Bool, + Literal::String(_) => Type::String, + Literal::Void => Type::Void, + }) + } +} + +impl TypeCoerceable for Type { + fn to_type(&self) -> Option { + Some(self.clone()) + } +} + +impl std::fmt::Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Type::Infer => write!(f, "infered type"), + Type::Any => write!(f, "opaque js value"), + Type::Literal(value) => write!(f, "{}", value), + Type::Int => write!(f, "int"), + Type::Float => write!(f, "float"), + Type::BigInt => write!(f, "bigint"), + Type::Bool => write!(f, "bool"), + Type::String => write!(f, "string"), + Type::NoReturn => write!(f, "noreturn"), + Type::Void => write!(f, "void"), + Type::Union(of) => { + for (i, t) in of.iter().enumerate() { + if i != 0 { + write!(f, " | ")?; + } + write!(f, "{}", t)?; + } + Ok(()) + } + } + } +} + +pub fn merge_types(a: Type, b: Type) -> Result { + match a { + Type::Infer => return Ok(b), + Type::Any => return Ok(Type::Any), + _ => {} + } + + match b { + Type::Infer => return Ok(a), + Type::Any => return Ok(Type::Any), + _ => {} + } + + if a == b { + return Ok(a); + } + + Err(TypeError::new(format!("TODO: Implement type merging for {} and {}", a, b)).into()) +} + +// pub fn coerce_type(target: Type, source: Type) -> Result { +// match source { +// Type::Any => return Ok(target), +// _ => {} +// } +// match target { +// Type::Infer => Ok(source), +// Type::Any => Ok(Type::Any), + +// Type::Literal(_) => unreachable!(), + +// Type::Int => match source { +// Type::Int => Ok(Type::Int), +// Type::Float => Ok(Type::Int), +// Type::Literal(value) => { +// if value.fract() != 0.0 { +// Err( +// TypeError::new(format!("Numeric Literal '{}' is not an integer", value)) +// .into(), +// ) +// } else if value > i32::MAX.into() || value < i32::MIN.into() { +// Err(TypeError::new(format!( +// "Numeric Literal '{}' is out of range for an integer ({}..{})", +// value, +// i32::MIN, +// i32::MAX +// )) +// .into()) +// } else { +// Ok(Type::Int) +// } +// } +// t => Err(TypeError::new(format!("Cannot coerce {} to int", t)).into()), +// }, + +// Type::BigInt => match source { +// Type::Int => Ok(Type::Int), +// Type::Float => Ok(Type::Int), +// Type::NumberLiteral(value) => { +// if value.fract() != 0.0 { +// Err( +// TypeError::new(format!("Numeric Literal '{}' is not an integer", value)) +// .into(), +// ) +// } else { +// Ok(Type::BigInt) +// } +// } +// t => Err(TypeError::new(format!("Cannot coerce {} to int", t)).into()), +// }, + +// Type::Float => match source { +// Type::Int => Ok(Type::Float), +// Type::Float => Ok(Type::Float), +// Type::NumberLiteral(value) => { +// if value.fract() == 0.0 && value <= f64::MAX.into() && value >= f64::MIN.into() { +// Ok(Type::Float) +// } else { +// Err(TypeError::new(format!("Numeric Literal")).into()) +// } +// } +// _ => Err(TypeError::new(format!("Cannot coerce int to float")).into()), +// }, + +// other => { +// if other == source { +// Ok(other) +// } else { +// Err(TypeError::new(format!("Cannot coerce {} to {}", source, other)).into()) +// } +// } +// } +// } + +#[derive(Error, Diagnostic, Debug)] +#[error("TypeError")] +pub struct TypeError { + #[help] + error: String, +} + +impl TypeError { + pub fn new(error: String) -> Self { + Self { error } + } +} diff --git a/crates/checker/Cargo.toml b/crates/checker/Cargo.toml index 9911221e439591b90a010b7a36b71770a76c2305..35bcf3fac1c683c696223e428ba9f59ff2bb8c75 100644 --- a/crates/checker/Cargo.toml +++ b/crates/checker/Cargo.toml @@ -10,5 +10,4 @@ miette.workspace = true thiserror.workspace = true num.workspace = true -ast = { path = "../ast" } -util = { path = "../util" } +ast = { path = "../ast" } \ No newline at end of file diff --git a/crates/checker/src/ast_type_coersion.rs b/crates/checker/src/ast_type_coersion.rs new file mode 100644 index 0000000000000000000000000000000000000000..411442e62cbed296913068bdf7011e02e4eb4601 --- /dev/null +++ b/crates/checker/src/ast_type_coersion.rs @@ -0,0 +1,54 @@ +// use ast::{BinaryOperator, ExprData, Type}; + +// impl TypeCoerceable for ExprData { +// fn to_type(&self) -> Option { +// match self { +// ExprData::Block(block) => { +// if (block.has_implicit_return) { +// block.statements.last().unwrap().to_type() +// } else { +// Some(Type::Void) +// } +// } +// ExprData::Binary { op, left, right } => { +// // TODO: this likely has to do further type checking +// let left_type = left.to_type(); +// match op { +// BinaryOperator::Add => left_type, +// BinaryOperator::Sub => left_type, +// BinaryOperator::Mul => left_type, +// BinaryOperator::Div => left_type, +// BinaryOperator::Pow => left_type, +// BinaryOperator::Eq => Some(Type::Bool), +// BinaryOperator::Gt => Some(Type::Bool), +// BinaryOperator::Lt => Some(Type::Bool), +// BinaryOperator::Gte => Some(Type::Bool), +// BinaryOperator::Lte => Some(Type::Bool), +// BinaryOperator::Lshift => Some(Type::Int), +// BinaryOperator::Rshift => Some(Type::Int), +// BinaryOperator::BitAnd => Some(Type::Int), +// BinaryOperator::BitOr => Some(Type::Int), +// BinaryOperator::BitXor => Some(Type::Int), +// BinaryOperator::Mod => Some(Type::Int), +// BinaryOperator::NotEq => Some(Type::Int), + +// BinaryOperator::And => Some(Type::Bool), +// BinaryOperator::Or => Some(Type::Bool), +// } +// } +// ExprData::Unary { op, value } => todo!(), +// ExprData::Function { +// name, +// params, +// return_type, +// block, +// } => Some(return_type), +// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable +// ExprData::Builtin { id, arguments } => todo!(), +// ExprData::If(_) => todo!(), +// ExprData::Return(_) => todo!(), +// ExprData::Literal(_) => todo!(), +// ExprData::Identifier(_) => todo!(), +// } +// } +// } diff --git a/crates/checker/src/lib.rs b/crates/checker/src/lib.rs index 7f68543e0b5f8f27ec36090523bf10227dd0770a..c7a930d635ec3bd0f58f49a10c38bc95b5d81d9e 100644 --- a/crates/checker/src/lib.rs +++ b/crates/checker/src/lib.rs @@ -1,203 +1,96 @@ -use ast::{Ast, Module, Type}; -use miette::Result; - -use miette::Diagnostic; -use thiserror::Error; - -#[derive(Error, Diagnostic, Debug)] -#[error("TypeError")] -pub struct TypeError { - #[help] - error: String, -} - -impl TypeError { - pub fn new(error: String) -> Self { - Self { error } - } -} - -pub fn check_module(block: &Module) -> Result<()> { - for statement in &block.value { - check_expression(statement)?; - } - - Ok(()) -} - -pub fn resolve_type(expr: &Ast) -> Result { - match expr { - Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())), - Ast::Declaration { - name, - decl_type, - value, - is_constant, - } => Ok(Type::NoReturn), - Ast::Operation { - operator, - left, - right, - } => merge_types(resolve_type(left)?, resolve_type(right)?), - Ast::Function { - name, - arguments, - return_type, - block, - } => todo!(), - Ast::FunctionCall { name, arguments } => todo!(), - Ast::If { - condition, - value, - else_value, - } => todo!(), - Ast::Return(_) => Ok(Type::NoReturn), - Ast::Public(_) => todo!(), // TODO: remove Public - Ast::Block(_) => todo!(), - Ast::Bool(_) => Ok(Type::Bool), - Ast::String(_) => Ok(Type::String), - Ast::Identifier(_) => todo!(), - Ast::BuiltIn(_) => todo!(), - - Ast::Comment(_) => unreachable!(), - Ast::DocComment(_) => unreachable!(), - } -} - -pub fn merge_types(a: Type, b: Type) -> Result { - match a { - Type::Infer => return Ok(b), - Type::Any => return Ok(Type::Any), - _ => {} - } - - match b { - Type::Infer => return Ok(a), - Type::Any => return Ok(Type::Any), - _ => {} - } - - if a == b { - return Ok(a); - } - - Err(TypeError::new(format!("Cannot merge {} and {}", a, b)).into()) -} - -pub fn coerce_type(target: Type, source: Type) -> Result { - match source { - Type::Any => return Ok(target), - _ => {} - } - match target { - Type::Infer => Ok(source), - Type::Any => Ok(Type::Any), - - Type::NumberLiteral(_) => unreachable!(), - - Type::Int => match source { - Type::Int => Ok(Type::Int), - Type::Float => Ok(Type::Int), - Type::NumberLiteral(value) => { - if value.fract() != 0.0 { - Err( - TypeError::new(format!("Numeric Literal '{}' is not an integer", value)) - .into(), - ) - } else if value > i32::MAX.into() || value < i32::MIN.into() { - Err(TypeError::new(format!( - "Numeric Literal '{}' is out of range for an integer ({}..{})", - value, - i32::MIN, - i32::MAX - )) - .into()) - } else { - Ok(Type::Int) - } - } - t => Err(TypeError::new(format!("Cannot coerce {} to int", t)).into()), - }, - - Type::BigInt => match source { - Type::Int => Ok(Type::Int), - Type::Float => Ok(Type::Int), - Type::NumberLiteral(value) => { - if value.fract() != 0.0 { - Err( - TypeError::new(format!("Numeric Literal '{}' is not an integer", value)) - .into(), - ) - } else { - Ok(Type::BigInt) - } - } - t => Err(TypeError::new(format!("Cannot coerce {} to int", t)).into()), - }, - - Type::Float => match source { - Type::Int => Ok(Type::Float), - Type::Float => Ok(Type::Float), - Type::NumberLiteral(value) => { - if value.fract() == 0.0 && value <= f64::MAX.into() && value >= f64::MIN.into() { - Ok(Type::Float) - } else { - Err(TypeError::new(format!("Numeric Literal")).into()) - } - } - _ => Err(TypeError::new(format!("Cannot coerce int to float")).into()), - }, - - other => { - if other == source { - Ok(other) - } else { - Err(TypeError::new(format!("Cannot coerce {} to {}", source, other)).into()) - } - } - } -} - -pub fn check_expression(expr: &Ast) -> Result<()> { - match expr { - Ast::Declaration { - name, - decl_type: return_type, - value, - is_constant, - } => { - check_expression(value)?; - coerce_type(return_type.clone(), resolve_type(value)?)?; - } - _ => {} - // Ast::Operation { - // operator, - // left, - // right, - // } => Ok(()), - // Ast::Function { - // name, - // arguments, - // return_type, - // block, - // } => Ok(()), - // Ast::FunctionCall { name, arguments } => Ok(()), - // Ast::If { - // condition, - // value, - // else_value, - // } => Ok(()), - // Ast::Return(_) => Ok(()), - // Ast::Public(_) => Ok(()), - // Ast::Block(_) => Ok(()), - // Ast::Number(_) => Ok(()), - // Ast::Bool(_) => Ok(()), - // Ast::String(_) => Ok(()), - // Ast::Identifier(_) => Ok(()), - // Ast::BuiltIn(_) => Ok(()), - // Ast::Comment(_) => Ok(()), - // Ast::DocComment(_) => Ok(()), - } - - Ok(()) -} +// use ast::{ Module, Type}; +// use miette::Result; + +// mod ast_type_coersion; + +// pub fn check_module(block: &Module) -> Result<()> { +// for statement in &block.statements { +// check_expression(statement)?; +// } + +// Ok(()) +// } + +// pub fn resolve_type(expr: &Ast) -> Result { +// match expr { +// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())), +// Ast::Declaration { +// name, +// decl_type, +// value, +// is_constant, +// } => Ok(Type::NoReturn), +// Ast::Binary { +// operator, +// left, +// right, +// } => merge_types(resolve_type(left)?, resolve_type(right)?), +// Ast::Function { +// name, +// arguments, +// return_type, +// block, +// } => todo!(), +// Ast::FunctionCall { name, arguments } => todo!(), +// Ast::If { +// condition, +// value, +// else_value, +// } => todo!(), +// Ast::Return(_) => Ok(Type::NoReturn), +// Ast::Public(_) => todo!(), // TODO: remove Public +// Ast::Block(_) => todo!(), +// Ast::Bool(_) => Ok(Type::Bool), +// Ast::String(_) => Ok(Type::String), +// Ast::Identifier(_) => todo!(), +// Ast::BuiltIn(_) => todo!(), + +// Ast::Comment(_) => unreachable!(), +// Ast::DocComment(_) => unreachable!(), +// } +// } + +// pub fn check_expression(expr: &Ast) -> Result<()> { +// panic!() +// match expr { +// Ast::Declaration { +// name, +// decl_type: return_type, +// value, +// is_constant, +// } => { +// check_expression(value)?; +// // coerce_type(return_type.clone(), resolve_type(value)?)?; +// } +// _ => {} +// // Ast::Operation { +// // operator, +// // left, +// // right, +// // } => Ok(()), +// // Ast::Function { +// // name, +// // arguments, +// // return_type, +// // block, +// // } => Ok(()), +// // Ast::FunctionCall { name, arguments } => Ok(()), +// // Ast::If { +// // condition, +// // value, +// // else_value, +// // } => Ok(()), +// // Ast::Return(_) => Ok(()), +// // Ast::Public(_) => Ok(()), +// // Ast::Block(_) => Ok(()), +// // Ast::Number(_) => Ok(()), +// // Ast::Bool(_) => Ok(()), +// // Ast::String(_) => Ok(()), +// // Ast::Identifier(_) => Ok(()), +// // Ast::BuiltIn(_) => Ok(()), +// // Ast::Comment(_) => Ok(()), +// // Ast::DocComment(_) => Ok(()), +// } + +// Ok(()) +// } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index a0e6f101de7d10297488257b9f58ddb06354076d..779b647427e55d42b654c970a4baf081f2551ed1 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -14,6 +14,7 @@ thiserror.workspace = true tokenizer = { path = "../tokenizer" } ast = { path = "../ast" } +parser = { path = "../parser" } js_ast = { path = "../js_ast" } js_printer = { path = "../js_printer" } checker = { path = "../checker" } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 254bad1c38f34c5af9d8a13b3c5e1b3482e93cd8..4bd03112c4df413316fd9acfe40662d71d26abaf 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,10 +1,8 @@ use std::{io::Write, path::PathBuf}; -use checker; use clap::{Parser, Subcommand}; use miette::Result; -use ast; use js_printer; use tokenizer; @@ -40,6 +38,9 @@ enum Commands { #[arg(short, long, default_value = "deno")] target: String, }, + Astdump { + file: PathBuf, + }, } fn get_runtime(runtime: &String) -> String { @@ -53,9 +54,9 @@ pub fn transpile(content: String) -> Result { // 1. Tokenize the input let tokens = tokenizer::tokenize(content)?; // 2. Parse the tokens into an AST - let blight_ast = ast::parse(tokens)?; + let blight_ast = parser::parse(tokens)?; // 3. Type check the AST - checker::check_module(&blight_ast)?; + // checker::check_module(&blight_ast)?; // 4. Optimize the AST // optimize(tree.clone())?; // 5. Translate the AST into javascript @@ -66,6 +67,14 @@ pub fn transpile(content: String) -> Result { return Ok(javascript); } +pub fn astdump(content: String) -> Result { + // 1. Tokenize the input + let tokens = tokenizer::tokenize(content)?; + // 2. Parse the tokens into an AST + let blight_ast = parser::parse(tokens)?; + return Ok(format!("{:#?}", blight_ast)); +} + fn main() -> Result<()> { let cli = Cli::parse(); @@ -145,6 +154,11 @@ fn main() -> Result<()> { .wait() .unwrap(); } + Commands::Astdump { file } => { + let contents = std::fs::read_to_string(file.clone()).unwrap(); + let result = astdump(contents.clone())?; + println!("{}", result); + } } Ok(()) diff --git a/crates/js_ast/Cargo.toml b/crates/js_ast/Cargo.toml index ae2bc9036ff37259b6f1bbd79effdf1c8e293511..dbbaf2f2c722dce465082656b9ac56a078b9730e 100644 --- a/crates/js_ast/Cargo.toml +++ b/crates/js_ast/Cargo.toml @@ -11,4 +11,3 @@ thiserror.workspace = true num.workspace = true ast = { path = "../ast" } -util = { path = "../util" } diff --git a/crates/js_ast/src/lib.rs b/crates/js_ast/src/lib.rs index 28685a906d0f4fa1e3083a8b6f70bc9a97e3457f..d0b9dd88049ab72f5f7b4e50cd2d9a480294a454 100644 --- a/crates/js_ast/src/lib.rs +++ b/crates/js_ast/src/lib.rs @@ -1,9 +1,8 @@ use std::collections::HashMap; +use ast::{Expr, ExprData, Literal, Module, Statement}; use miette::Result; -use ast::{Ast, Module}; - mod types; pub use types::{JsExpression, JsStatement}; @@ -99,68 +98,69 @@ impl TranslateContext { // here, we recursively expand out expressions into statements // we then store the result of all the expressions in a variable // and then return the name of that variable -fn get_expression_handle(ast: &Ast, ctx: &mut TranslateContext) -> JsExpression { - match ast { - Ast::Number(value) => JsExpression::Number(*value), - Ast::String(value) => JsExpression::String(value.clone()), - Ast::Identifier(value) => JsExpression::Identifier(ctx.get_identifier(value.clone())), - Ast::Bool(value) => JsExpression::Bool(*value), - Ast::FunctionCall { name, arguments } => { +fn get_expression_handle(ast: &Expr, ctx: &mut TranslateContext) -> JsExpression { + match ast.data.clone() { + ExprData::Literal(Literal::String(value)) => JsExpression::String(value), + ExprData::Literal(Literal::Number(value)) => JsExpression::Number(value), + ExprData::Literal(Literal::Void) => JsExpression::Identifier("null".to_string()), + ExprData::Literal(Literal::Bool(value)) => JsExpression::Identifier(if value { + "true".to_string() + } else { + "false".to_string() + }), + + ExprData::Identifier(value) => JsExpression::Identifier(ctx.get_identifier(value.clone())), + ExprData::Call { base, arguments } => { let mut handles = Vec::new(); for argument in arguments { - handles.push(get_expression_handle(argument, ctx)); - } - match *name.clone() { - Ast::Identifier(name) => ctx.push_internal_variable(JsExpression::FunctionCall { - name: Box::new(JsExpression::Identifier(ctx.get_identifier(name.clone()))), - arguments: handles, - }), - Ast::BuiltIn(name) => match name.as_str() { - "@raw" => { - if handles.len() != 1 { - panic!("@raw expects exactly one argument"); - } - match handles.get(0).unwrap() { - JsExpression::String(value) => { - return JsExpression::Raw(value.clone()); - } - _ => panic!("@raw expects a string argument"), - } - } - _ => panic!("Unknown built-in function {}", name), - }, - _ => panic!("Function name must be an identifier"), + handles.push(get_expression_handle(&argument, ctx)); } + let base = get_expression_handle(&base, ctx); + ctx.push_internal_variable(JsExpression::Call { + base: Box::new(base), + arguments: handles, + }) } - Ast::Operation { - operator, - left, - right, - } => { - let left_handle = get_expression_handle(left, ctx); - let right_handle = get_expression_handle(right, ctx); - JsExpression::Operation { - operator: operator.clone(), + ExprData::Builtin { id, arguments } => match id { + ast::BuiltinId::Raw => { + if arguments.len() != 1 { + panic!("@raw expects exactly one argument"); + } + match arguments.get(0).unwrap() { + Expr { + data: ExprData::Literal(Literal::String(value)), + .. + } => JsExpression::Raw(value.clone()), + _ => panic!("@raw expects a string literal"), + } + } + }, + ExprData::Binary { op, left, right } => { + let left_handle = get_expression_handle(&left, ctx); + let right_handle = get_expression_handle(&right, ctx); + JsExpression::Binary { + operator: op.clone(), left: Box::new(left_handle), right: Box::new(right_handle), } } - Ast::Block(block) => { - if !block.implicit_return { + ExprData::Block(block) => { + if !block.has_implicit_return { panic!("Expression blocks must have an implicit return"); } ctx.push_context(); - for (i, statement) in block.value.iter().enumerate() { - if i == block.value.len() - 1 { + for (i, statement) in block.statements.iter().enumerate() { + if i == block.statements.len() - 1 { break; } else { translate_statement(statement, ctx); } } - let result = get_expression_handle(block.value.last().unwrap(), ctx); - let value = ctx.pop_context(); - value.iter().for_each(|x| ctx.push(x.clone())); - result + todo!() + // let result = get_expression_handle(block.statements.last().unwrap(), ctx); + // let value = ctx.pop_context(); + // value.iter().for_each(|x| ctx.push(x.clone())); + // result } _ => { panic!("unimplemented: {:?}", ast); @@ -168,168 +168,164 @@ fn get_expression_handle(ast: &Ast, ctx: &mut TranslateContext) -> JsExpression } } -fn translate_statement(ast: &Ast, ctx: &mut TranslateContext) { +fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) { match ast { - Ast::Declaration { - name, - decl_type: _, - value, - is_constant, - } => { - let handle = get_expression_handle(value, ctx); - let rename = ctx.declare_variable(name.clone()); + Statement::Decl(decl) => { + let handle = get_expression_handle(&decl.init, ctx); + let rename = ctx.declare_variable(decl.name.clone()); ctx.push(JsStatement::Declaration { name: rename, value: handle, - is_constant: is_constant.clone(), + is_constant: decl.is_constant.clone(), }); } - Ast::Function { - name, - arguments, - return_type: _, - block, - } => { - let rename = ctx.declare_variable(name.clone()); - let arguments = arguments - .iter() - .map(|x| ctx.declare_variable(x.name.clone())) - .collect(); + Statement::Expr(_) => todo!(), + // Ast::Function { + // name, + // arguments, + // return_type: _, + // block, + // } => { + // let rename = ctx.declare_variable(name.clone()); + // let arguments = arguments + // .iter() + // .map(|x| ctx.declare_variable(x.name.clone())) + // .collect(); - ctx.push_context(); - for (i, statement) in block.value.iter().enumerate() { - if block.implicit_return && i == block.value.len() - 1 { - let handle = get_expression_handle(statement, ctx); - ctx.push(JsStatement::Return(handle)); - } else { - translate_statement(statement, ctx); - } - } - let value = ctx.pop_context(); + // ctx.push_context(); + // for (i, statement) in block.value.iter().enumerate() { + // if block.implicit_return && i == block.value.len() - 1 { + // let handle = get_expression_handle(statement, ctx); + // ctx.push(JsStatement::Return(handle)); + // } else { + // translate_statement(statement, ctx); + // } + // } + // let value = ctx.pop_context(); - ctx.push(JsStatement::Function { - name: rename, - arguments, - value, - }); - } - Ast::Public(value) => { - translate_statement(value, ctx); - // I'm sorry, I just don't know how to get the original name properly... my bad - match ctx.peek() { - JsStatement::Function { name, .. } | JsStatement::Declaration { name, .. } => { - let original_name = name.split("$").next().unwrap().to_string(); - ctx.push(JsStatement::Export(Box::new(JsStatement::Declaration { - name: original_name, - value: JsExpression::Identifier(name.clone()), - is_constant: false, - }))); - } - _ => { - panic!("Cannot make non-function public"); - } - } - } - Ast::Return(value) => { - let handle = get_expression_handle(value, ctx); - ctx.push(JsStatement::Return(handle)); - } - Ast::Block(block) => { - ctx.push_context(); - for (i, statement) in block.value.iter().enumerate() { - if block.implicit_return && i == block.value.len() - 1 { - let handle = get_expression_handle(statement, ctx); - ctx.push(JsStatement::Return(handle)); - } else { - translate_statement(statement, ctx); - } - } - let value = ctx.pop_context(); - value.iter().for_each(|x| ctx.push(x.clone())); - } - Ast::If { - condition, - value: block, - else_value, - } => { - let handle = get_expression_handle(condition, ctx); + // ctx.push(JsStatement::Function { + // name: rename, + // arguments, + // value, + // }); + // } + // Ast::Public(value) => { + // translate_statement(value, ctx); + // // I'm sorry, I just don't know how to get the original name properly... my bad + // match ctx.peek() { + // JsStatement::Function { name, .. } | JsStatement::Declaration { name, .. } => { + // let original_name = name.split("$").next().unwrap().to_string(); + // ctx.push(JsStatement::Export(Box::new(JsStatement::Declaration { + // name: original_name, + // value: JsExpression::Identifier(name.clone()), + // is_constant: true, + // }))); + // } + // _ => { + // panic!("Cannot make non-function public"); + // } + // } + // } + // Ast::Return(value) => { + // let handle = get_expression_handle(value, ctx); + // ctx.push(JsStatement::Return(handle)); + // } + // Ast::Block(block) => { + // ctx.push_context(); + // for (i, statement) in block.value.iter().enumerate() { + // if block.implicit_return && i == block.value.len() - 1 { + // let handle = get_expression_handle(statement, ctx); + // ctx.push(JsStatement::Return(handle)); + // } else { + // translate_statement(statement, ctx); + // } + // } + // let value = ctx.pop_context(); + // value.iter().for_each(|x| ctx.push(x.clone())); + // } + // Ast::If { + // condition, + // value: block, + // else_value, + // } => { + // let handle = get_expression_handle(condition, ctx); - ctx.push_context(); - for (i, statement) in block.value.iter().enumerate() { - if block.implicit_return && i == block.value.len() - 1 { - let handle = get_expression_handle(statement, ctx); - ctx.push(JsStatement::Return(handle)); - } else { - translate_statement(statement, ctx); - } - } - let value = ctx.pop_context(); - let else_value = match else_value { - Some(block) => { - ctx.push_context(); - for (i, statement) in block.value.iter().enumerate() { - if block.implicit_return && i == block.value.len() - 1 { - let handle = get_expression_handle(statement, ctx); - ctx.push(JsStatement::Return(handle)); - } else { - translate_statement(statement, ctx); - } - } - Some(ctx.pop_context()) - } - None => None, - }; + // ctx.push_context(); + // for (i, statement) in block.value.iter().enumerate() { + // if block.implicit_return && i == block.value.len() - 1 { + // let handle = get_expression_handle(statement, ctx); + // ctx.push(JsStatement::Return(handle)); + // } else { + // translate_statement(statement, ctx); + // } + // } + // let value = ctx.pop_context(); + // let else_value = match else_value { + // Some(block) => { + // ctx.push_context(); + // for (i, statement) in block.value.iter().enumerate() { + // if block.implicit_return && i == block.value.len() - 1 { + // let handle = get_expression_handle(statement, ctx); + // ctx.push(JsStatement::Return(handle)); + // } else { + // translate_statement(statement, ctx); + // } + // } + // Some(ctx.pop_context()) + // } + // None => None, + // }; - ctx.push(JsStatement::If { - condition: handle, - value, - else_value, - }); - } - Ast::FunctionCall { name, arguments } => { - let mut handles = Vec::new(); - for argument in arguments { - handles.push(get_expression_handle(argument, ctx)); - } - match *name.clone() { - Ast::Identifier(name) => { - ctx.push(JsStatement::Expression(JsExpression::FunctionCall { - name: Box::new(JsExpression::Identifier(ctx.get_identifier(name.clone()))), - arguments: handles, - })); - } - Ast::BuiltIn(name) => match name.as_str() { - "@raw" => { - if handles.len() != 1 { - panic!("@raw expects exactly one argument"); - } - match handles.get(0).unwrap() { - JsExpression::String(value) => { - ctx.push(JsStatement::Expression(JsExpression::FunctionCall { - name: Box::new(JsExpression::Raw(value.clone())), - arguments: handles, - })); - } - _ => panic!("@raw expects a string argument"), - } - } - _ => panic!("Unknown built-in function {}", name), - }, - _ => panic!("Function name must be an identifier"), - } - } - Ast::Comment(_) | Ast::DocComment(_) => {} - _ => { - panic!("unimplemented: {:?}", ast); - } + // ctx.push(JsStatement::If { + // condition: handle, + // value, + // else_value, + // }); + // } + // Ast::FunctionCall { name, arguments } => { + // let mut handles = Vec::new(); + // for argument in arguments { + // handles.push(get_expression_handle(argument, ctx)); + // } + // match *name.clone() { + // Ast::Identifier(name) => { + // ctx.push(JsStatement::Expression(JsExpression::Call { + // name: Box::new(JsExpression::Identifier(ctx.get_identifier(name.clone()))), + // arguments: handles, + // })); + // } + // Ast::BuiltIn(name) => match name.as_str() { + // "@raw" => { + // if handles.len() != 1 { + // panic!("@raw expects exactly one argument"); + // } + // match handles.get(0).unwrap() { + // JsExpression::String(value) => { + // ctx.push(JsStatement::Expression(JsExpression::Call { + // name: Box::new(JsExpression::Raw(value.clone())), + // arguments: handles, + // })); + // } + // _ => panic!("@raw expects a string argument"), + // } + // } + // _ => panic!("Unknown built-in function {}", name), + // }, + // _ => panic!("Function name must be an identifier"), + // } + // } + // Ast::Comment(_) | Ast::DocComment(_) => {} + // _ => { + // panic!("unimplemented: {:?}", ast); + // } } } pub fn translate(module: &Module) -> Result> { let mut ctx = TranslateContext::new(); - for ast in module.value.iter() { - translate_statement(&ast, &mut ctx); + for ast in module.statements.iter() { + translate_statement(&ast.clone().to_statement(), &mut ctx); } Ok(ctx.pop_context()) diff --git a/crates/js_ast/src/types.rs b/crates/js_ast/src/types.rs index 3b5f176e4a51e7ccf14eb6fcf3be46c25577f8b2..f53fb3f665655d739d189bd01a1ae334579b7167 100644 --- a/crates/js_ast/src/types.rs +++ b/crates/js_ast/src/types.rs @@ -1,8 +1,7 @@ +use ast::BinaryOperator; use num::bigint::BigInt; use std::collections::HashMap; -use util::Operator; - #[derive(Debug, PartialEq, Clone)] pub enum JsExpression { Number(f64), @@ -13,8 +12,8 @@ pub enum JsExpression { Raw(String), Array(Vec), Object(HashMap), - Operation { - operator: Operator, + Binary { + operator: BinaryOperator, left: Box, right: Box, }, @@ -22,8 +21,8 @@ pub enum JsExpression { arguments: Vec, value: Vec, }, - FunctionCall { - name: Box, + Call { + base: Box, arguments: Vec, }, } diff --git a/crates/js_printer/Cargo.toml b/crates/js_printer/Cargo.toml index 0071929ae0046ab381623d16ae23e397e7f36393..ec58b9490b3338b6f12f18906971afbdadb9f4ca 100644 --- a/crates/js_printer/Cargo.toml +++ b/crates/js_printer/Cargo.toml @@ -11,4 +11,3 @@ thiserror.workspace = true num.workspace = true js_ast = { path = "../js_ast" } -util = { path = "../util" } diff --git a/crates/js_printer/src/print_expression.rs b/crates/js_printer/src/print_expression.rs index e13e3ebbda35bf9db23ae4dd38c3d8cb513c65f6..15810d077d963fe78e1c45c6c4247403dce9677a 100644 --- a/crates/js_printer/src/print_expression.rs +++ b/crates/js_printer/src/print_expression.rs @@ -41,7 +41,7 @@ pub fn print_expression(statement: &JsExpression) -> String { .collect::>() .join(",") ), - JsExpression::Operation { + JsExpression::Binary { operator, left, right, @@ -60,9 +60,9 @@ pub fn print_expression(statement: &JsExpression) -> String { .collect::>() .join("") ), - JsExpression::FunctionCall { name, arguments } => format!( + JsExpression::Call { base, arguments } => format!( "{}({})", - match *name.clone() { + match *base.clone() { JsExpression::Identifier(value) => value.clone(), JsExpression::Raw(value) => value.clone(), _ => panic!("Function call name must be an identifier"), @@ -83,7 +83,6 @@ mod tests { use js_ast::JsStatement; use num::{bigint::Sign, BigInt}; use std::collections::HashMap; - use util::Operator; #[test] fn number() { @@ -184,16 +183,16 @@ mod tests { #[test] fn operation() { assert_eq!( - print_expression(&JsExpression::Operation { - operator: Operator::Add, + print_expression(&JsExpression::Binary { + operator: AnyOperator::Add, left: Box::new(JsExpression::Number(1.0)), right: Box::new(JsExpression::Number(2.0)), }), "1+2" ); assert_eq!( - print_expression(&JsExpression::Operation { - operator: Operator::Subtract, + print_expression(&JsExpression::Binary { + operator: AnyOperator::Sub, left: Box::new(JsExpression::Number(1.0)), right: Box::new(JsExpression::Number(2.0)), }), @@ -242,21 +241,21 @@ mod tests { #[test] fn function_call() { assert_eq!( - print_expression(&JsExpression::FunctionCall { + print_expression(&JsExpression::Call { name: Box::new(JsExpression::Identifier("hello".to_string())), arguments: vec![], }), "hello()" ); assert_eq!( - print_expression(&JsExpression::FunctionCall { + print_expression(&JsExpression::Call { name: Box::new(JsExpression::Identifier("hello".to_string())), arguments: vec![JsExpression::Number(1.0)], }), "hello(1)" ); assert_eq!( - print_expression(&JsExpression::FunctionCall { + print_expression(&JsExpression::Call { name: Box::new(JsExpression::Identifier("hello".to_string())), arguments: vec![JsExpression::Number(1.0), JsExpression::Number(2.0)], }), diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..220932c2a7189d96bb8a2b807c9d0c97292a2d0a --- /dev/null +++ b/crates/parser/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "parser" +version = "0.1.0" +edition = "2021" + +[lib] +path = "./parser.rs" + +[dependencies] +miette.workspace = true +thiserror.workspace = true +num.workspace = true + +ast = { path = "../ast" } +tokenizer = { path = "../tokenizer" } diff --git a/crates/parser/parse_block.rs b/crates/parser/parse_block.rs new file mode 100644 index 0000000000000000000000000000000000000000..4c6dc77d314740afd0a31a442a85885d18403e89 --- /dev/null +++ b/crates/parser/parse_block.rs @@ -0,0 +1,64 @@ +use crate::{parse_statement::parse_statement, Parser}; +use ast::Block; +use miette::Result; +use tokenizer::Token; + +/// Precondition: The first token exists and is '{' +pub fn parse_block(p: &mut Parser) -> Result { + match p.next() { + Some(Token::OpenBrace) => (), + _ => unreachable!(), + } + + let mut statements = Vec::new(); + + let was_top_level = p.is_top_level; + p.is_top_level = false; + + loop { + let statement = match parse_statement(p) { + Ok(statement) => statement, + Err(err) => { + p.is_top_level = was_top_level; + return Err(err); + } + }; + statements.push(statement.clone()); + + match p.next() { + Some(Token::Semicolon) => loop { + match p.peek() { + Some(Token::ClosedBrace) => { + p.next(); + p.is_top_level = was_top_level; + return Ok(Block { + statements, + has_implicit_return: false, + }); + } + _ => continue, + } + }, + Some(Token::ClosedBrace) => { + if statement.requires_semicolon() { + p.is_top_level = was_top_level; + return Err(p + .error("Expected semicolon after this statement".to_string()) + .into()); + } + + p.is_top_level = was_top_level; + return Ok(Block { + statements, + has_implicit_return: true, + }); + } + _ => { + p.is_top_level = was_top_level; + return Err(p + .error("Expected semicolon or '}' after statement".to_string()) + .into()); + } + }; + } +} diff --git a/crates/parser/parse_call_arguments.rs b/crates/parser/parse_call_arguments.rs new file mode 100644 index 0000000000000000000000000000000000000000..afa16cdcfef28ab3c3dd309f6d2036b5403120bb --- /dev/null +++ b/crates/parser/parse_call_arguments.rs @@ -0,0 +1,31 @@ +use ast::Expr; +use miette::Result; +use tokenizer::Token; + +use crate::{parse_expr::parse_expr, Parser}; + +/// Precondition: The '(' was already consumed +pub fn parse_call_arguments(p: &mut Parser) -> Result> { + let mut args = Vec::new(); + + loop { + let arg = parse_expr(p)?; + args.push(arg); + + match p.peek() { + Some(Token::Comma) => { + _ = p.next(); + continue; + } + Some(Token::ClosedParen) => { + _ = p.next(); + return Ok(args); + } + _ => { + return Err(p + .error("Expected ',' or ')' after argument".to_string()) + .into()) + } + } + } +} diff --git a/crates/parser/parse_declaration.rs b/crates/parser/parse_declaration.rs new file mode 100644 index 0000000000000000000000000000000000000000..3fbfc4e8ef2e6d7ee91926c2de05acbc3d46e679 --- /dev/null +++ b/crates/parser/parse_declaration.rs @@ -0,0 +1,46 @@ +use ast::{Decl, Type}; +use miette::Result; +use tokenizer::Token; + +use crate::parse_expr::parse_expr; +use crate::parse_type::parse_type; +use crate::Parser; + +/// Precondition: The first token exists and is either 'var' or 'let' +pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result { + let loc = p.loc(); + let is_constant = match p.next() { + Some(Token::Var) => false, + Some(Token::Let) => true, + _ => unreachable!(), + }; + let name = match p.next() { + Some(Token::Identifier(name)) => name, + Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()), + None => return Err(p.error(format!("Unexpected end of file")).into()), + }; + let return_type = match p.peek() { + Some(Token::Colon) => { + p.next(); + parse_type(p)? + } + _ => Type::Infer, + }; + let init = match p.next() { + Some(Token::Equal) => parse_expr(p)?, + _ => { + return Err(p + .error("Variable must have an initializer".to_string()) + .into()) + } + }; + Ok(Decl { + name: name.clone(), + decl_type: return_type, + init, + is_constant, + is_pub, + comment: None, + loc, + }) +} diff --git a/crates/parser/parse_expr.rs b/crates/parser/parse_expr.rs new file mode 100644 index 0000000000000000000000000000000000000000..98d098d3c847c174c0a9ca6a375cc038fa6d2b55 --- /dev/null +++ b/crates/parser/parse_expr.rs @@ -0,0 +1,161 @@ +use ast::{BinaryOperator, Expr, ExprData, Literal}; +use miette::Result; +use tokenizer::Token; + +use crate::{ + parse_call_arguments::parse_call_arguments, parse_expr_basic::parse_expr_basic, Parser, +}; + +/// Derived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table + +#[derive(PartialEq, PartialOrd)] +enum Level { + Prefix, // +, -, !, ~ + Exponentiation, // ** + Multiplicative, // *, /, % + Additive, // +, - + BitwiseShift, // <<, >>, >>> + Relational, // <, <=, >, >= + Equality, // ==, != + BitwiseAnd, // & + BitwiseXor, // ^ + BitwiseOr, // | + LogicalAnd, // && + LogicalOr, // ||, ?? +} + +pub fn get_postfix_op_level(token: &Token) -> Option { + match token { + Token::StarStar => Some(Level::Exponentiation), + Token::Star | Token::Slash | Token::Percent => Some(Level::Multiplicative), + Token::Plus | Token::Minus => Some(Level::Additive), + Token::LtLt | Token::GtGt | Token::GtGtGt => Some(Level::BitwiseShift), + Token::Gt | Token::GtEqual | Token::Lt | Token::LtEqual => Some(Level::Relational), + Token::EqualEqual | Token::BangEqual => Some(Level::Equality), + Token::And => Some(Level::BitwiseAnd), + Token::Caret => Some(Level::BitwiseXor), + Token::Or => Some(Level::BitwiseOr), + Token::AndAnd => Some(Level::LogicalAnd), + Token::OrOr => Some(Level::LogicalOr), + _ => None, + } +} + +pub fn parse_expr(p: &mut Parser) -> Result { + let mut expr = parse_expr_basic(p)?; + + // Edge case for return. This has already parsed all the way to the end of the expression + if let ExprData::Return(_) = expr.data { + return Ok(expr); + } + + // Parse function calls and field accesses + loop { + match p.peek() { + Some(Token::Dot) => { + let loc = p.loc(); + _ = p.next(); + let field = match p.peek() { + Some(Token::Identifier(name)) => { + let loc = p.loc(); + p.next(); + Expr::new(ExprData::Literal(Literal::String(name)), loc) + } + _ => { + return Err(p.error("Expected identifier after dot".to_string()).into()); + } + }; + expr = Expr::new( + ExprData::Field { + base: Box::new(expr), + field: Box::new(field), + }, + loc, + ); + } + Some(Token::OpenParen) => { + let loc = p.loc(); + _ = p.next(); + let arguments = parse_call_arguments(p)?; + expr = Expr::new( + ExprData::Call { + base: Box::new(expr), + arguments, + }, + loc, + ); + } + Some(Token::OpenBracket) => { + let loc = p.loc(); + _ = p.next(); + let index = parse_expr(p)?; + if let Some(Token::ClosedBracket) = p.next() { + expr = Expr::new( + ExprData::Field { + base: Box::new(expr), + field: Box::new(index), + }, + loc, + ); + } else { + return Err(p + .error("Expected closing bracket after index".to_string()) + .into()); + } + } + _ => break, + } + } + + // Parse operators + loop { + if let Some(token) = p.peek() { + if let Some(level) = get_postfix_op_level(&token) { + _ = p.next(); + expr = parse_expr_binary( + p, + BinaryOperator::from_token(&token).unwrap(), + expr.clone(), + level, + )?; + continue; + } + } + break; + } + + Ok(expr) +} + +/// Prerequisite: `p.peek()` is an operator +fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Level) -> Result { + let loc = p.loc(); + let mut right = parse_expr_basic(p)?; + + loop { + if let Some(token) = p.peek() { + if let Some(next_level) = get_postfix_op_level(&token) { + if next_level <= level { + _ = p.next(); + right = parse_expr_binary( + p, + BinaryOperator::from_token(&token).unwrap(), + right, + next_level, + )?; + continue; + } + } + } + break; + } + + Ok(Expr::new( + ExprData::Binary { + op, + left: Box::new(left), + right: Box::new(right), + }, + loc, + )) +} diff --git a/crates/parser/parse_expr_basic.rs b/crates/parser/parse_expr_basic.rs new file mode 100644 index 0000000000000000000000000000000000000000..8cc9c8409c33d5a4a1ae08550f4cf60ea63a8a76 --- /dev/null +++ b/crates/parser/parse_expr_basic.rs @@ -0,0 +1,110 @@ +use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator}; +use miette::Result; +use tokenizer::Token; + +use crate::{ + parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr, + parse_expr_if::parse_expr_if, Parser, +}; + +pub fn parse_expr_basic(p: &mut Parser) -> Result { + let loc = p.loc(); + match p.peek() { + Some(Token::Number(number)) => { + _ = p.next(); + Ok(Expr::new( + ExprData::Literal(Literal::Number(number.clone())), + loc, + )) + } + Some(Token::String(string)) => { + _ = p.next(); + Ok(Expr::new( + ExprData::Literal(Literal::String(string.clone())), + loc, + )) + } + Some(Token::True) => { + _ = p.next(); + Ok(Expr::new(ExprData::Literal(Literal::Bool(true)), loc)) + } + Some(Token::False) => { + _ = p.next(); + Ok(Expr::new(ExprData::Literal(Literal::Bool(false)), loc)) + } + Some(Token::Void) => { + _ = p.next(); + Ok(Expr::new(ExprData::Literal(Literal::Void), loc)) + } + Some(Token::Identifier(name)) => { + let loc = p.loc(); + _ = p.next(); + if name.starts_with("@") { + if let Some(id) = BuiltinId::from_str(name.as_str()) { + if let Some(Token::OpenParen) = p.next() { + let arguments = parse_call_arguments(p)?; + return Ok(Expr::new(ExprData::Builtin { id, arguments }, loc)); + } else { + return Err(p + .error_at(format!("Built-in function '{}' must be called", name), loc) + .into()); + } + } else { + return Err(p + .error_at(format!("Non-existant built-in: '{}'", name), loc) + .into()); + } + } else { + Ok(Expr::new(ExprData::Identifier(name), loc)) + } + } + + Some(Token::Return) => { + if p.is_top_level { + return Err(p + .error("'return' cannot be used at top level".to_string()) + .into()); + } else { + _ = p.next(); + + return Ok(Expr::new(ExprData::Return(Box::new(parse_expr(p)?)), loc)); + } + } + + Some(Token::If) => Ok(Expr::new(ExprData::If(parse_expr_if(p)?), loc)), + + Some(Token::OpenBrace) => Ok(Expr::new(ExprData::Block(parse_block(p)?), loc)), + + Some(Token::OpenParen) => { + _ = p.next(); + let expr = parse_expr_basic(p)?; + match p.next() { + Some(Token::ClosedParen) => (), + Some(token) => { + return Err(p.error(format!("Expected ')', but got {:?}", token)).into()) + } + None => return Err(p.error("Unexpected end of file".to_string()).into()), + } + Ok(expr) + } + + Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => { + _ = p.next(); + Ok(Expr::new( + ExprData::Unary { + op: match t { + Token::Bang => UnaryOperator::Not, + Token::Minus => UnaryOperator::Neg, + Token::Tilde => UnaryOperator::BitNot, + _ => unreachable!(), + }, + value: Box::new(parse_expr_basic(p)?), + }, + loc, + )) + } + + Some(t) => Err(p.error(format!("Unexpected '{}'", t)).into()), + None => Err(p.error("Unexpected end of file".to_string()).into()), + } +} diff --git a/crates/parser/parse_expr_if.rs b/crates/parser/parse_expr_if.rs new file mode 100644 index 0000000000000000000000000000000000000000..a956585da772e01c03c550745f8b33687c79356f --- /dev/null +++ b/crates/parser/parse_expr_if.rs @@ -0,0 +1,61 @@ +use ast::{BlockWithLoc, If}; +use miette::Result; +use tokenizer::Token; + +use crate::{parse_block::parse_block, parse_expr::parse_expr, Parser}; + +/// Precondition: the first token exists and is 'if' +pub fn parse_expr_if(p: &mut Parser) -> Result { + match p.next() { + Some(Token::If) => (), + _ => unreachable!(), + } + + match p.peek() { + Some(Token::OpenBrace) => { + return Err(p + .error( + "To use a block as the condition for an if statement, wrap it in parentheses" + .to_string(), + ) + .into()); + } + _ => (), + } + + let condition = Box::new(parse_expr(p)?); + + let yes = match p.peek() { + Some(Token::OpenBrace) => { + let loc = p.loc(); + BlockWithLoc { + block: parse_block(p)?, + loc, + } + } + _ => { + return Err(p + .error("Expected a block after condition".to_string()) + .into()) + } + }; + + let no = match p.peek() { + Some(Token::Else) => { + _ = p.next(); + match p.peek() { + Some(Token::OpenBrace) => { + let loc = p.loc(); + Some(BlockWithLoc { + block: parse_block(p)?, + loc, + }) + } + _ => return Err(p.error("Expected a block after else".to_string()).into()), + } + } + _ => None, + }; + + Ok(If { condition, yes, no }) +} diff --git a/crates/parser/parse_fn_declaration.rs b/crates/parser/parse_fn_declaration.rs new file mode 100644 index 0000000000000000000000000000000000000000..4b7405c6bf3a57ab3b0bd50e9a2e26f8b12bd664 --- /dev/null +++ b/crates/parser/parse_fn_declaration.rs @@ -0,0 +1,111 @@ +use ast::{Decl, Expr, ExprData, Type}; +use miette::Result; +use tokenizer::Token; + +use crate::parse_block::parse_block; +use crate::parse_param::parse_param; +use crate::parse_type::parse_type; +use crate::Parser; + +/// Precondition: +pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result { + let loc = p.loc(); + match p.next() { + Some(Token::Fn) => (), + _ => unreachable!(), + } + + let name = match p.next() { + Some(Token::Identifier(name)) => name, + _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()), + }; + + match p.peek() { + Some(Token::OpenParen) => p.next(), + _ => { + return Err(p + .error("Expected '(' after function name".to_string()) + .into()) + } + }; + + let mut params = Vec::new(); + + loop { + match p.peek() { + Some(Token::ClosedParen) => { + _ = p.next(); + break; + } + Some(Token::Identifier(_)) => { + params.push(parse_param(p)?); + + match p.peek() { + Some(Token::Comma) => { + p.next(); + if let Some(Token::ClosedParen) = p.peek() { + p.next(); + break; + } + } + Some(Token::ClosedParen) => break, + _ => { + return Err(p + .error("Expected ',' or ')' after identifier".to_string()) + .into()) + } + }; + } + _ => { + return Err(p + .error("Expected ')' or identifier after '('".to_string()) + .into()) + } + } + + match p.peek() { + Some(Token::Comma) => p.next(), + Some(Token::ClosedParen) => { + _ = p.next(); + break; + } + _ => { + return Err(p + .error("Expected ',' or ')' after identifier".to_string()) + .into()) + } + }; + } + + let return_type = match p.peek() { + Some(Token::RightArrow) => { + _ = p.next(); + parse_type(p)? + } + _ => Type::Infer, + }; + + if let Some(Token::OpenBrace) = p.peek() { + let block_loc = p.loc(); + let block = parse_block(p)?; + let function_body = ExprData::Fn { + name: name.clone(), + params, + return_type, + block, + }; + Ok(Decl { + name, + decl_type: Type::Any, + init: Expr::new(function_body, block_loc), + is_constant: true, + is_pub, + comment: None, + loc: loc, + }) + } else { + return Err(p + .error("Expected a block to follow function declaration".to_string()) + .into()); + } +} diff --git a/crates/parser/parse_module.rs b/crates/parser/parse_module.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ff66244156f39002142e205e7bd4a4ed6f73cd4 --- /dev/null +++ b/crates/parser/parse_module.rs @@ -0,0 +1,33 @@ +use crate::{parse_top_level_statement::parse_top_level_statement, Parser}; +use ast::Module; +use miette::Result; +use tokenizer::Token; + +pub fn parse_module(tokens: &mut Parser) -> Result { + let mut statements = Vec::new(); + + loop { + let statement = parse_top_level_statement(tokens)?; + + match tokens.peek() { + Some(Token::Semicolon) => { + while let Some(Token::Semicolon) = tokens.peek() { + tokens.next(); + } + } + _ => { + if statement.requires_semicolon() { + return Err(tokens + .error("Expected semicolon after statement".to_string()) + .into()); + } + } + }; + + statements.push(statement); + + if tokens.empty() { + return Ok(Module { statements }); + } + } +} diff --git a/crates/parser/parse_param.rs b/crates/parser/parse_param.rs new file mode 100644 index 0000000000000000000000000000000000000000..9e8f34be012b33c1811621ec434e413672df8888 --- /dev/null +++ b/crates/parser/parse_param.rs @@ -0,0 +1,30 @@ +use ast::Param; +use miette::Result; +use tokenizer::Token; + +use crate::{parse_type::parse_type, Parser}; + +/// Precondition: A token exists and is an identifier +pub fn parse_param(p: &mut Parser) -> Result { + let name = match p.next() { + Some(Token::Identifier(name)) => name.clone(), + _ => unreachable!(), + }; + + let param_type = match p.peek() { + Some(Token::Colon) => { + _ = p.next(); + parse_type(p)? + } + _ => { + return Err(p + .error("Function parameters require type annotations".to_string()) + .into()) + } + }; + + Ok(Param { + name: name.clone(), + param_type, + }) +} diff --git a/crates/parser/parse_statement.rs b/crates/parser/parse_statement.rs new file mode 100644 index 0000000000000000000000000000000000000000..c3da1e4f219af484a5af82b0d45c35f68444c92e --- /dev/null +++ b/crates/parser/parse_statement.rs @@ -0,0 +1,28 @@ +use ast::Statement; +use miette::Result; +use tokenizer::Token; + +use crate::parse_declaration::parse_declaration; +use crate::parse_expr::parse_expr; +use crate::parse_fn_declaration::parse_fn_declaration; +use crate::Parser; + +pub fn parse_statement(p: &mut Parser) -> Result { + match p.peek() { + Some(Token::Pub) => { + _ = p.next(); + match p.peek() { + Some(Token::Var) | Some(Token::Let) => { + Ok(Statement::Decl(parse_declaration(p, true)?)) + } + Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, true)?)), + Some(_) => Err(p.error("Expected a declaration".to_string()).into()), + None => Err(p.error("Unexpected end of file".to_string()).into()), + } + } + Some(Token::Var) | Some(Token::Let) => Ok(Statement::Decl(parse_declaration(p, false)?)), + Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, false)?)), + Some(_) => Ok(Statement::Expr(parse_expr(p)?)), + None => Err(p.error("Unexpected end of file".to_string()).into()), + } +} diff --git a/crates/parser/parse_top_level_statement.rs b/crates/parser/parse_top_level_statement.rs new file mode 100644 index 0000000000000000000000000000000000000000..b3383ecfa6134479f8e473c3e85f9023d58294ee --- /dev/null +++ b/crates/parser/parse_top_level_statement.rs @@ -0,0 +1,10 @@ +use ast::TopLevelStatement; +use miette::Result; + +use crate::{parse_statement::parse_statement, Parser}; + +/// Precondition: the next token exists +pub fn parse_top_level_statement(p: &mut Parser) -> Result { + // TODO: we will eventually have real statements that are exclusive to top-level + Ok(parse_statement(p)?.to_top_level()) +} diff --git a/crates/parser/parse_type.rs b/crates/parser/parse_type.rs new file mode 100644 index 0000000000000000000000000000000000000000..b3e022b0ec6eede07db734f24e5412e6b57ce4c1 --- /dev/null +++ b/crates/parser/parse_type.rs @@ -0,0 +1,26 @@ +use ast::Type; +use miette::Result; + +use tokenizer::Token; + +use crate::Parser; + +pub fn parse_type(p: &mut Parser) -> Result { + match p.next() { + Some(Token::Identifier(id)) => match id.as_str() { + "int" => Ok(Type::Int), + "float" => Ok(Type::Float), + "bigint" => Ok(Type::BigInt), + "bool" => Ok(Type::Bool), + "string" => Ok(Type::String), + invalid_type => Err(p + .error(format!( + "Invalid type '{}' (TODO: add custom type support)", + invalid_type + )) + .into()), + }, + Some(unexpected) => Err(p.error(format!("Expected type, got {}", unexpected)).into()), + None => Err(p.error("Expected type, got EOF".to_string()).into()), + } +} diff --git a/crates/parser/parser.rs b/crates/parser/parser.rs new file mode 100644 index 0000000000000000000000000000000000000000..0dda281df5e97957e5a57c1ef5261894b56034a0 --- /dev/null +++ b/crates/parser/parser.rs @@ -0,0 +1,104 @@ +mod parse_block; +mod parse_call_arguments; +mod parse_declaration; +mod parse_expr; +mod parse_expr_basic; +mod parse_expr_if; +mod parse_fn_declaration; +mod parse_module; +mod parse_param; +mod parse_statement; +mod parse_top_level_statement; +mod parse_type; + +use tokenizer::{Loc, Token}; + +use miette::Diagnostic; +use thiserror::Error; + +use ast::Module; +use miette::Result; +use parse_module::parse_module; +use tokenizer::tokenize; + +pub fn parse(tokens: Vec) -> Result { + let mut p = Parser::new(tokens); + parse_module(&mut p) +} + +pub fn parse_from_string(string: String) -> Result { + let tokens = tokenize(string)?; + parse(tokens) +} + +#[derive(Error, Diagnostic, Debug)] +#[error("ParseError")] +pub struct ParseError { + #[help] + error: String, + loc: Option, +} + +impl ParseError { + pub fn new(error: String, loc: Option) -> ParseError { + ParseError { error, loc } + } +} + +#[derive(Debug)] +pub struct Parser { + pub tokens: Vec, + pub cursor: usize, + + /// Whether or not the parser is at the top level + pub is_top_level: bool, +} + +impl Parser { + pub fn new(tokens: Vec) -> Parser { + Parser { + tokens, + cursor: 0, + is_top_level: true, + } + } + + pub fn empty(&self) -> bool { + self.cursor == self.tokens.len() + } + + pub fn back(&mut self) { + self.cursor -= 1; + } + + pub fn next(&mut self) -> Option { + if self.cursor == self.tokens.len() { + return None; + } + + let token = self.tokens[self.cursor].clone(); + self.cursor += 1; + Some(token) + } + + pub fn peek(&self) -> Option { + if self.cursor == self.tokens.len() { + return None; + } + + Some(self.tokens[self.cursor].clone()) + } + + pub fn error(&self, error: String) -> ParseError { + // TODO(@lino-levan): Add location information + ParseError::new(error, None) + } + + fn loc(&self) -> Loc { + return Loc {}; + } + + fn error_at(&self, error: String, loc: Loc) -> ParseError { + ParseError::new(error, Some(loc)) + } +} diff --git a/crates/tokenizer/Cargo.toml b/crates/tokenizer/Cargo.toml index 21243ee5fc409f648340819f1b792d721c214741..50effc8036b4f1705b14ec46fd16ad36622229e7 100644 --- a/crates/tokenizer/Cargo.toml +++ b/crates/tokenizer/Cargo.toml @@ -8,5 +8,3 @@ edition = "2021" [dependencies] miette.workspace = true thiserror.workspace = true - -util = { path = "../util" } diff --git a/crates/tokenizer/src/lib.rs b/crates/tokenizer/src/lib.rs index a35ad44dd5d8122748a865188c70b94544a9fb3a..d536ccd14cf9a7bcd522e139e2f8f60d727786ad 100644 --- a/crates/tokenizer/src/lib.rs +++ b/crates/tokenizer/src/lib.rs @@ -1,7 +1,8 @@ use miette::{Diagnostic, Result}; use thiserror::Error; -use util::Operator; +#[derive(Debug, Clone)] +pub struct Loc {} #[derive(Error, Diagnostic, Debug)] #[error("UnknownCharacter")] @@ -16,69 +17,118 @@ pub enum Token { Identifier(String), Number(f64), String(String), - BuiltIn(String), // Comments Comment(String), DocComment(String), // Keywords - True, + Else, False, - Let, - Var, + Fn, If, - Else, - Return, + Let, Pub, - Function, - RightArrow, - Comma, - OpenParanthesis, - ClosedParanthesis, - OpenBracket, - ClosedBracket, - Colon, - Equals, - Semicolon, - // Operators - Operator(Operator), + Return, + True, + Var, + Void, + // Symbols + And, // & + AndAnd, // && + Bang, // ! + BangEqual, // != + Caret, // ^ + ClosedBrace, // } + ClosedBracket, // ] + ClosedParen, // ) + Colon, // : + Comma, // , + Equal, // = + EqualEqual, // == + Gt, // > + GtEqual, // >= + GtGt, // >> + GtGtGt, // >>> + Lt, // < + LtEqual, // <= + LtLt, // << + Minus, // - + OpenBrace, // { + OpenBracket, // [ + OpenParen, // ( + Or, // | + OrOr, // || + Percent, // % + Plus, // + + RightArrow, // -> + Semicolon, // ; + Slash, // / + Star, // * + StarStar, // ** + Tilde, // ~ + Dot, // . } impl std::fmt::Display for Token { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let token = match self { Token::Identifier(_) => "an identifier".to_string(), + Token::Number(n) => n.to_string(), Token::String(s) => format!("\"{}\"", s), - Token::BuiltIn(_) => "a builtin identifier".to_string(), Token::Comment(_) => "a comment".to_string(), Token::DocComment(_) => "a documentation comment".to_string(), - Token::True => "true".to_string(), + + Token::Else => "else".to_string(), Token::False => "false".to_string(), - Token::Let => "let".to_string(), - Token::Var => "var".to_string(), + Token::Fn => "fn".to_string(), Token::If => "if".to_string(), - Token::Else => "else".to_string(), - Token::Return => "return".to_string(), + Token::Let => "let".to_string(), Token::Pub => "pub".to_string(), - Token::Function => "fn".to_string(), - Token::RightArrow => "->".to_string(), - Token::Comma => ",".to_string(), - Token::OpenParanthesis => "(".to_string(), - Token::ClosedParanthesis => ")".to_string(), - Token::OpenBracket => "{".to_string(), - Token::ClosedBracket => "}".to_string(), + Token::Return => "return".to_string(), + Token::True => "true".to_string(), + Token::Var => "var".to_string(), + Token::Void => "void".to_string(), + + Token::And => "&".to_string(), + Token::AndAnd => "&&".to_string(), + Token::Bang => "!".to_string(), + Token::BangEqual => "!=".to_string(), + Token::Caret => "^".to_string(), + Token::ClosedBrace => "}".to_string(), + Token::ClosedBracket => "]".to_string(), + Token::ClosedParen => ")".to_string(), Token::Colon => ":".to_string(), - Token::Equals => "=".to_string(), + Token::Comma => ",".to_string(), + Token::Equal => "=".to_string(), + Token::EqualEqual => "==".to_string(), + Token::Gt => ">".to_string(), + Token::GtEqual => ">=".to_string(), + Token::GtGt => ">>".to_string(), + Token::Lt => "<".to_string(), + Token::LtEqual => "<=".to_string(), + Token::LtLt => "<<".to_string(), + Token::Minus => "-".to_string(), + Token::OpenBrace => "{".to_string(), + Token::OpenBracket => "[".to_string(), + Token::OpenParen => "(".to_string(), + Token::Or => "|".to_string(), + Token::OrOr => "||".to_string(), + Token::Percent => "%".to_string(), + Token::Plus => "+".to_string(), + Token::RightArrow => "->".to_string(), Token::Semicolon => ";".to_string(), - Token::Operator(o) => o.clone().to_string(), + Token::Slash => "/".to_string(), + Token::Star => "*".to_string(), + Token::StarStar => "**".to_string(), + Token::Tilde => "~".to_string(), + Token::Dot => ".".to_string(), + Token::GtGtGt => ">>>".to_string(), }; write!(f, "{}", token) } } - - pub fn tokenize(content: String) -> Result> { let mut tokens: Vec = Vec::new(); let mut chars = content.chars().enumerate().peekable(); @@ -105,22 +155,17 @@ pub fn tokenize(content: String) -> Result> { } match current_token.as_str() { - "var" => tokens.push(Token::Var), - "let" => tokens.push(Token::Let), - "if" => tokens.push(Token::If), "else" => tokens.push(Token::Else), + "false" => tokens.push(Token::False), + "fn" => tokens.push(Token::Fn), + "if" => tokens.push(Token::If), + "let" => tokens.push(Token::Let), "pub" => tokens.push(Token::Pub), - "fn" => tokens.push(Token::Function), "return" => tokens.push(Token::Return), "true" => tokens.push(Token::True), - "false" => tokens.push(Token::False), - _ => { - if current_token.starts_with('@') { - tokens.push(Token::BuiltIn(current_token)); - } else { - tokens.push(Token::Identifier(current_token)) - } - } + "var" => tokens.push(Token::Var), + "void" => tokens.push(Token::Void), + _ => tokens.push(Token::Identifier(current_token)), } } '"' => { @@ -169,39 +214,53 @@ pub fn tokenize(content: String) -> Result> { '=' => match chars.peek() { Some((_, '=')) => { chars.next(); - tokens.push(Token::Operator(Operator::Equals)); + tokens.push(Token::EqualEqual); } - _ => tokens.push(Token::Equals), + _ => tokens.push(Token::Equal), }, ';' => tokens.push(Token::Semicolon), '>' => match chars.peek() { Some((_, '=')) => { chars.next(); - tokens.push(Token::Operator(Operator::GreaterThanOrEquals)); + tokens.push(Token::GtEqual); } - _ => tokens.push(Token::Operator(Operator::GreaterThan)), + Some((_, '>')) => { + chars.next(); + match chars.peek() { + Some((_, '>')) => { + chars.next(); + tokens.push(Token::GtGtGt); + } + _ => tokens.push(Token::GtGt), + } + } + _ => tokens.push(Token::Gt), }, '<' => match chars.peek() { Some((_, '=')) => { chars.next(); - tokens.push(Token::Operator(Operator::LessThanOrEquals)); + tokens.push(Token::LtEqual); } - _ => tokens.push(Token::Operator(Operator::LessThan)), + Some((_, '<')) => { + chars.next(); + tokens.push(Token::LtLt); + } + _ => tokens.push(Token::Lt), }, - '+' => tokens.push(Token::Operator(Operator::Add)), + '+' => tokens.push(Token::Plus), '-' => match chars.peek() { Some((_, '>')) => { chars.next(); tokens.push(Token::RightArrow); } - _ => tokens.push(Token::Operator(Operator::Subtract)), + _ => tokens.push(Token::Minus), }, '*' => match chars.peek() { Some((_, '*')) => { chars.next(); - tokens.push(Token::Operator(Operator::Power)); + tokens.push(Token::StarStar); } - _ => tokens.push(Token::Operator(Operator::Multiply)), + _ => tokens.push(Token::Star), }, '/' => { match chars.peek() { @@ -239,18 +298,45 @@ pub fn tokenize(content: String) -> Result> { tokens.push(Token::Comment(comment)); } } - _ => tokens.push(Token::Operator(Operator::Divide)), + _ => tokens.push(Token::Slash), } } ',' => tokens.push(Token::Comma), - '(' => tokens.push(Token::OpenParanthesis), - ')' => tokens.push(Token::ClosedParanthesis), - '{' => tokens.push(Token::OpenBracket), - '}' => tokens.push(Token::ClosedBracket), + '(' => tokens.push(Token::OpenParen), + ')' => tokens.push(Token::ClosedParen), + '{' => tokens.push(Token::OpenBrace), + '}' => tokens.push(Token::ClosedBrace), + '[' => tokens.push(Token::OpenBracket), + ']' => tokens.push(Token::ClosedBracket), + '&' => match chars.peek() { + Some((_, '&')) => { + chars.next(); + tokens.push(Token::AndAnd); + } + _ => tokens.push(Token::And), + }, + '|' => match chars.peek() { + Some((_, '|')) => { + chars.next(); + tokens.push(Token::OrOr); + } + _ => tokens.push(Token::Or), + }, + '!' => match chars.peek() { + Some((_, '=')) => { + chars.next(); + tokens.push(Token::BangEqual); + } + _ => tokens.push(Token::Bang), + }, + '^' => tokens.push(Token::Caret), + '%' => tokens.push(Token::Percent), + '~' => tokens.push(Token::Tilde), + '.' => tokens.push(Token::Dot), s if s.is_whitespace() => (), _ => { return Err(UnknownCharacter { - error: format!("Unsupported character: {}", c), + error: format!("Unexpected '{}' while tokenizing", c), } .into()) } diff --git a/crates/util/Cargo.toml b/crates/util/Cargo.toml deleted file mode 100644 index 8729b70b0ce2b0b1139b6defbbcdee89deea81cc..0000000000000000000000000000000000000000 --- a/crates/util/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "util" -version = "0.1.0" -edition = "2021" - -[lib] - -[dependencies] diff --git a/crates/util/src/lib.rs b/crates/util/src/lib.rs deleted file mode 100644 index c79f4c32474b7cedc5f4b6b6acda2330b51a19c9..0000000000000000000000000000000000000000 --- a/crates/util/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -#[derive(Debug, Clone, PartialEq)] -pub enum Operator { - Add, - Subtract, - Multiply, - Divide, - Power, - Equals, - GreaterThan, - LessThan, - GreaterThanOrEquals, - LessThanOrEquals, -} - -impl Operator { - pub fn to_string(self) -> String { - String::from(match self { - Self::Add => "+", - Self::Subtract => "-", - Self::Divide => "/", - Self::Multiply => "*", - Self::Power => "**", - Self::Equals => "===", - Self::GreaterThan => ">", - Self::LessThan => "<", - Self::GreaterThanOrEquals => ">=", - Self::LessThanOrEquals => "<=", - }) - } -} diff --git a/examples/basics/control.blight b/examples/basics/control.blight index f6e3bdb1db079dc14869607c8fc2821eee7a89f2..579f551f87fa6dc390eb6fd92390ef066f287c36 100644 --- a/examples/basics/control.blight +++ b/examples/basics/control.blight @@ -1,17 +1 @@ -let print = @raw("console.log"); - -let x: bool = true; - -if x { - print("x is true!"); -} - -let y: int = 3; - -if y == 1 { - print("y is 1!"); -} else if y == 2 { - print("y is 2!"); -} else { - print("y is not 1 or 2!"); -} +let x: bool = 1 * 3 + 2 / 2 == 4 + 2; -- 2.54.0