authorgravatar for 11367844+lino-levan@users.noreply.github.comLino Le Van <11367844+lino-levan@users.noreply.github.com> 2024-02-03 05:35:35+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-02-02 20:35:35-08:00
logf93117a51b8aae4a8477e086bfe87c61caf63c1b
tree4c7bb64084c4459c10264cf2a00fa4ca9069ef4a
parentc4d73c341e177bab25226dc3f9676b455e9f2b62

feat(parser): track the location of source tokens (#8)

* feat: add loc * chore: start working on transition parser to include loc * chore: fix everything related to TokenType * feat: add loc to parser

16 files changed, 513 insertions(+), 421 deletions(-)

crates/ast/operators.rs+23-23
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1use tokenizer::Token;1use tokenizer::TokenType;
22
3use crate::Literal;3use crate::Literal;
44
...@@ -94,28 +94,28 @@ impl BinaryOperator {...@@ -94,28 +94,28 @@ impl BinaryOperator {
94 String::from(self.to_str())94 String::from(self.to_str())
95 }95 }
9696
97 pub fn from_token(token: &Token) -> Option<BinaryOperator> {97 pub fn from_token(token: &TokenType) -> Option<BinaryOperator> {
98 match token {98 match &token {
99 Token::And => Some(Self::BitAnd),99 TokenType::And => Some(Self::BitAnd),
100 Token::AndAnd => Some(Self::And),100 TokenType::AndAnd => Some(Self::And),
101 Token::BangEqual => Some(Self::NotEq),101 TokenType::BangEqual => Some(Self::NotEq),
102 Token::Caret => Some(Self::BitXor),102 TokenType::Caret => Some(Self::BitXor),
103 Token::EqualEqual => Some(Self::Eq),103 TokenType::EqualEqual => Some(Self::Eq),
104 Token::Gt => Some(Self::Gt),104 TokenType::Gt => Some(Self::Gt),
105 Token::GtEqual => Some(Self::Gte),105 TokenType::GtEqual => Some(Self::Gte),
106 Token::GtGt => Some(Self::Rshift),106 TokenType::GtGt => Some(Self::Rshift),
107 Token::GtGtGt => Some(Self::RshiftUnsigned),107 TokenType::GtGtGt => Some(Self::RshiftUnsigned),
108 Token::Lt => Some(Self::Lt),108 TokenType::Lt => Some(Self::Lt),
109 Token::LtEqual => Some(Self::Lte),109 TokenType::LtEqual => Some(Self::Lte),
110 Token::LtLt => Some(Self::Lshift),110 TokenType::LtLt => Some(Self::Lshift),
111 Token::Minus => Some(Self::Sub),111 TokenType::Minus => Some(Self::Sub),
112 Token::Or => Some(Self::BitOr),112 TokenType::Or => Some(Self::BitOr),
113 Token::OrOr => Some(Self::Or),113 TokenType::OrOr => Some(Self::Or),
114 Token::Percent => Some(Self::Mod),114 TokenType::Percent => Some(Self::Mod),
115 Token::Plus => Some(Self::Add),115 TokenType::Plus => Some(Self::Add),
116 Token::Slash => Some(Self::Div),116 TokenType::Slash => Some(Self::Div),
117 Token::Star => Some(Self::Mul),117 TokenType::Star => Some(Self::Mul),
118 Token::StarStar => Some(Self::Pow),118 TokenType::StarStar => Some(Self::Pow),
119 _ => None,119 _ => None,
120 }120 }
121 }121 }
crates/js_ast/convert/convert_valid_block.rs+1
...@@ -26,6 +26,7 @@ pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, loc: Loc...@@ -26,6 +26,7 @@ pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, loc: Loc
26 }26 }
27 JsExpr::new(JsExprData::Comma { items }, loc.clone(), None)27 JsExpr::new(JsExprData::Comma { items }, loc.clone(), None)
28 }28 }
29 _ => todo!(),
29 };30 };
3031
31 // a block without implicit return is to be type void, so we wrap in a void expression32 // a block without implicit return is to be type void, so we wrap in a void expression
crates/parser/parse_block.rs+5-5
...@@ -1,12 +1,12 @@...@@ -1,12 +1,12 @@
1use crate::{parse_statement::parse_statement, Parser};1use crate::{parse_statement::parse_statement, Parser};
2use ast::Block;2use ast::Block;
3use miette::Result;3use miette::Result;
4use tokenizer::Token;4use tokenizer::TokenType;
55
6/// Precondition: The first token exists and is '{'6/// Precondition: The first token exists and is '{'
7pub fn parse_block(p: &mut Parser) -> Result<Block> {7pub fn parse_block(p: &mut Parser) -> Result<Block> {
8 match p.next() {8 match p.next() {
9 Some(Token::OpenBrace) => (),9 Some(TokenType::OpenBrace) => (),
10 _ => unreachable!(),10 _ => unreachable!(),
11 }11 }
1212
...@@ -30,10 +30,10 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {...@@ -30,10 +30,10 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
30 statements.push(statement.clone());30 statements.push(statement.clone());
3131
32 match p.peek() {32 match p.peek() {
33 Some(Token::Semicolon) => loop {33 Some(TokenType::Semicolon) => loop {
34 p.next();34 p.next();
35 match p.peek() {35 match p.peek() {
36 Some(Token::ClosedBrace) => {36 Some(TokenType::ClosedBrace) => {
37 p.next();37 p.next();
38 p.is_top_level = was_top_level;38 p.is_top_level = was_top_level;
39 return Ok(Block {39 return Ok(Block {
...@@ -44,7 +44,7 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {...@@ -44,7 +44,7 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
44 _ => continue 'outer,44 _ => continue 'outer,
45 }45 }
46 },46 },
47 Some(Token::ClosedBrace) => {47 Some(TokenType::ClosedBrace) => {
48 p.next();48 p.next();
49 p.is_top_level = was_top_level;49 p.is_top_level = was_top_level;
5050
crates/parser/parse_call_arguments.rs+4-4
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::Expr;1use ast::Expr;
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::TokenType;
44
5use crate::{parse_expr::parse_expr, Parser};5use crate::{parse_expr::parse_expr, Parser};
66
...@@ -8,7 +8,7 @@ use crate::{parse_expr::parse_expr, Parser};...@@ -8,7 +8,7 @@ use crate::{parse_expr::parse_expr, Parser};
8pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {8pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
9 let mut args = Vec::new();9 let mut args = Vec::new();
1010
11 if let Some(Token::ClosedParen) = p.peek() {11 if let Some(TokenType::ClosedParen) = p.peek() {
12 _ = p.next();12 _ = p.next();
13 return Ok(args);13 return Ok(args);
14 }14 }
...@@ -18,11 +18,11 @@ pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {...@@ -18,11 +18,11 @@ pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
18 args.push(arg);18 args.push(arg);
1919
20 match p.peek() {20 match p.peek() {
21 Some(Token::Comma) => {21 Some(TokenType::Comma) => {
22 _ = p.next();22 _ = p.next();
23 continue;23 continue;
24 }24 }
25 Some(Token::ClosedParen) => {25 Some(TokenType::ClosedParen) => {
26 _ = p.next();26 _ = p.next();
27 return Ok(args);27 return Ok(args);
28 }28 }
crates/parser/parse_declaration.rs+8-8
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::{DocComment, Type, VarDecl};1use ast::{DocComment, Type, VarDecl};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::{Loc, TokenType};
44
5use crate::parse_expr::parse_expr;5use crate::parse_expr::parse_expr;
6use crate::parse_type::parse_type;6use crate::parse_type::parse_type;
...@@ -12,26 +12,26 @@ pub fn parse_declaration(...@@ -12,26 +12,26 @@ pub fn parse_declaration(
12 is_pub: bool,12 is_pub: bool,
13 comment: Option<DocComment>,13 comment: Option<DocComment>,
14) -> Result<VarDecl> {14) -> Result<VarDecl> {
15 let loc = p.loc();15 let start = p.index;
16 let is_constant = match p.next() {16 let is_constant = match p.next() {
17 Some(Token::Var) => false,17 Some(TokenType::Var) => false,
18 Some(Token::Let) => true,18 Some(TokenType::Let) => true,
19 _ => unreachable!(),19 _ => unreachable!(),
20 };20 };
21 let name = match p.next() {21 let name = match p.next() {
22 Some(Token::Identifier(name)) => name,22 Some(TokenType::Identifier(name)) => name,
23 Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),23 Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),
24 None => return Err(p.error(format!("Unexpected end of file")).into()),24 None => return Err(p.error(format!("Unexpected end of file")).into()),
25 };25 };
26 let return_type = match p.peek() {26 let return_type = match p.peek() {
27 Some(Token::Colon) => {27 Some(TokenType::Colon) => {
28 p.next();28 p.next();
29 parse_type(p)?29 parse_type(p)?
30 }30 }
31 _ => Type::Infer,31 _ => Type::Infer,
32 };32 };
33 let init = match p.next() {33 let init = match p.next() {
34 Some(Token::Equal) => parse_expr(p)?,34 Some(TokenType::Equal) => parse_expr(p)?,
35 _ => {35 _ => {
36 return Err(p36 return Err(p
37 .error("Variable must have an initializer".to_string())37 .error("Variable must have an initializer".to_string())
...@@ -45,6 +45,6 @@ pub fn parse_declaration(...@@ -45,6 +45,6 @@ pub fn parse_declaration(
45 is_constant,45 is_constant,
46 is_pub,46 is_pub,
47 comment,47 comment,
48 loc,48 loc: Loc::new(start, p.index - start),
49 })49 })
50}50}
crates/parser/parse_doc_comment.rs+7-4
...@@ -1,14 +1,17 @@...@@ -1,14 +1,17 @@
1use ast::DocComment;1use ast::DocComment;
2use tokenizer::Token;2use tokenizer::{Loc, TokenType};
33
4use crate::Parser;4use crate::Parser;
55
6pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {6pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
7 match p.peek() {7 match p.peek() {
8 Some(Token::DocComment(data)) => {8 Some(TokenType::DocComment(data)) => {
9 let loc = p.loc();9 let start = p.index;
10 p.next();10 p.next();
11 Some(DocComment { data, loc })11 Some(DocComment {
12 data,
13 loc: Loc::new(start, p.index - start),
14 })
12 }15 }
13 _ => None,16 _ => None,
14 }17 }
crates/parser/parse_expr.rs+3-2
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1use ast::Associativity::*;1use ast::Associativity::*;
2use ast::{BinaryOperator, Expr, ExprData, OperatorLevel};2use ast::{BinaryOperator, Expr, ExprData, OperatorLevel};
3use miette::Result;3use miette::Result;
4use tokenizer::Loc;
45
5use crate::parse_expr_basic::parse_expr_basic;6use crate::parse_expr_basic::parse_expr_basic;
6use crate::Parser;7use crate::Parser;
...@@ -53,7 +54,7 @@ fn parse_expr_binary(...@@ -53,7 +54,7 @@ fn parse_expr_binary(
53 left: Expr,54 left: Expr,
54 level: OperatorLevel,55 level: OperatorLevel,
55) -> Result<Expr> {56) -> Result<Expr> {
56 let loc = p.loc();57 let start = p.index;
57 let mut right = parse_expr_basic(p)?;58 let mut right = parse_expr_basic(p)?;
5859
59 loop {60 loop {
...@@ -93,7 +94,7 @@ fn parse_expr_binary(...@@ -93,7 +94,7 @@ fn parse_expr_binary(
93 left: Box::new(left),94 left: Box::new(left),
94 right: Box::new(right),95 right: Box::new(right),
95 },96 },
96 loc,97 Loc::new(start, p.index - start),
97 None,98 None,
98 ))99 ))
99}100}
crates/parser/parse_expr_basic.rs+85-43
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};1use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::{Loc, TokenType};
44
5use crate::{5use crate::{
6 parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,6 parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,
...@@ -8,61 +8,87 @@ use crate::{...@@ -8,61 +8,87 @@ use crate::{
8};8};
99
10pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {10pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
11 let loc = p.loc();11 let start = p.index;
12 let comments = p.take_comments();12 let comments = p.take_comments();
13 let mut expr = match p.peek() {13 let mut expr = match p.peek() {
14 Some(Token::Number(number)) => {14 Some(TokenType::Number(number)) => {
15 _ = p.next();15 _ = p.next();
16 Expr::new(16 Expr::new(
17 ExprData::Literal(Literal::Number(number.clone())),17 ExprData::Literal(Literal::Number(number.clone())),
18 loc,18 Loc::new(start, p.index - start),
19 comments,19 comments,
20 )20 )
21 }21 }
22 Some(Token::String(string)) => {22 Some(TokenType::String(string)) => {
23 _ = p.next();23 _ = p.next();
24 Expr::new(24 Expr::new(
25 ExprData::Literal(Literal::String(string.clone())),25 ExprData::Literal(Literal::String(string.clone())),
26 loc,26 Loc::new(start, p.index - start),
27 comments,27 comments,
28 )28 )
29 }29 }
30 Some(Token::True) => {30 Some(TokenType::True) => {
31 _ = p.next();31 _ = p.next();
32 Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments)32 Expr::new(
33 ExprData::Literal(Literal::Bool(true)),
34 Loc::new(start, p.index - start),
35 comments,
36 )
33 }37 }
34 Some(Token::False) => {38 Some(TokenType::False) => {
35 _ = p.next();39 _ = p.next();
36 Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments)40 Expr::new(
41 ExprData::Literal(Literal::Bool(false)),
42 Loc::new(start, p.index - start),
43 comments,
44 )
37 }45 }
38 Some(Token::Void) => {46 Some(TokenType::Void) => {
39 _ = p.next();47 _ = p.next();
40 Expr::new(ExprData::Literal(Literal::Void), loc, comments)48 Expr::new(
49 ExprData::Literal(Literal::Void),
50 Loc::new(start, p.index - start),
51 comments,
52 )
41 }53 }
42 Some(Token::Identifier(name)) => {54 Some(TokenType::Identifier(name)) => {
43 let loc = p.loc();55 let identifier_start = p.index;
44 _ = p.next();56 _ = p.next();
45 if name.starts_with("@") {57 if name.starts_with("@") {
46 if let Some(id) = BuiltinId::from_str(name.as_str()) {58 if let Some(id) = BuiltinId::from_str(name.as_str()) {
47 if let Some(Token::OpenParen) = p.next() {59 if let Some(TokenType::OpenParen) = p.next() {
48 let arguments = parse_call_arguments(p)?;60 let arguments = parse_call_arguments(p)?;
49 Expr::new(ExprData::Builtin { id, arguments }, loc, comments)61 Expr::new(
62 ExprData::Builtin { id, arguments },
63 Loc::new(identifier_start, p.index - identifier_start),
64 comments,
65 )
50 } else {66 } else {
51 return Err(p67 return Err(p
52 .error_at(format!("Built-in function '{}' must be called", name), loc)68 .error_at(
69 format!("Built-in function '{}' must be called", name),
70 Loc::new(identifier_start, p.index - identifier_start),
71 )
53 .into());72 .into());
54 }73 }
55 } else {74 } else {
56 return Err(p75 return Err(p
57 .error_at(format!("Non-existant built-in: '{}'", name), loc)76 .error_at(
77 format!("Non-existant built-in: '{}'", name),
78 Loc::new(identifier_start, p.index - identifier_start),
79 )
58 .into());80 .into());
59 }81 }
60 } else {82 } else {
61 Expr::new(ExprData::Identifier(name), loc, comments)83 Expr::new(
84 ExprData::Identifier(name),
85 Loc::new(identifier_start, p.index - identifier_start),
86 comments,
87 )
62 }88 }
63 }89 }
6490
65 Some(Token::Return) => {91 Some(TokenType::Return) => {
66 if p.is_top_level {92 if p.is_top_level {
67 return Err(p93 return Err(p
68 .error("'return' cannot be used at top level".to_string())94 .error("'return' cannot be used at top level".to_string())
...@@ -71,19 +97,31 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {...@@ -71,19 +97,31 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
71 _ = p.next();97 _ = p.next();
72 let inner = maybe_parse_expr_basic(p)?.map(Box::new);98 let inner = maybe_parse_expr_basic(p)?.map(Box::new);
7399
74 return Ok(Some(Expr::new(ExprData::Return(inner), loc, comments)));100 return Ok(Some(Expr::new(
101 ExprData::Return(inner),
102 Loc::new(start, p.index - start),
103 comments,
104 )));
75 }105 }
76 }106 }
77107
78 Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments),108 Some(TokenType::If) => Expr::new(
109 ExprData::If(parse_expr_if(p)?),
110 Loc::new(start, p.index - start),
111 comments,
112 ),
79113
80 Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments),114 Some(TokenType::OpenBrace) => Expr::new(
115 ExprData::Block(parse_block(p)?),
116 Loc::new(start, p.index - start),
117 comments,
118 ),
81119
82 Some(Token::OpenParen) => {120 Some(TokenType::OpenParen) => {
83 _ = p.next();121 _ = p.next();
84 let expr = parse_expr(p)?;122 let expr = parse_expr(p)?;
85 match p.next() {123 match p.next() {
86 Some(Token::ClosedParen) => (),124 Some(TokenType::ClosedParen) => (),
87 Some(token) => {125 Some(token) => {
88 return Err(p.error(format!("Expected ')', but got {:?}", token)).into())126 return Err(p.error(format!("Expected ')', but got {:?}", token)).into())
89 }127 }
...@@ -92,19 +130,19 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {...@@ -92,19 +130,19 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
92 expr130 expr
93 }131 }
94132
95 Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => {133 Some(t @ TokenType::Bang) | Some(t @ TokenType::Minus) | Some(t @ TokenType::Tilde) => {
96 _ = p.next();134 _ = p.next();
97 Expr::new(135 Expr::new(
98 ExprData::Unary {136 ExprData::Unary {
99 op: match t {137 op: match t {
100 Token::Bang => UnaryOperator::Not,138 TokenType::Bang => UnaryOperator::Not,
101 Token::Minus => UnaryOperator::Neg,139 TokenType::Minus => UnaryOperator::Neg,
102 Token::Tilde => UnaryOperator::BitNot,140 TokenType::Tilde => UnaryOperator::BitNot,
103 _ => unreachable!(),141 _ => unreachable!(),
104 },142 },
105 value: Box::new(parse_expr_basic(p)?),143 value: Box::new(parse_expr_basic(p)?),
106 },144 },
107 loc,145 Loc::new(start, p.index - start),
108 comments,146 comments,
109 )147 )
110 }148 }
...@@ -115,14 +153,18 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {...@@ -115,14 +153,18 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
115 // Parse function calls and field accesses153 // Parse function calls and field accesses
116 loop {154 loop {
117 match p.peek() {155 match p.peek() {
118 Some(Token::Dot) => {156 Some(TokenType::Dot) => {
119 let loc = p.loc();157 let field_start = p.index;
120 _ = p.next();158 _ = p.next();
121 let field = match p.peek() {159 let field = match p.peek() {
122 Some(Token::Identifier(name)) => {160 Some(TokenType::Identifier(name)) => {
123 let loc = p.loc();161 let field_start = p.index;
124 p.next();162 p.next();
125 Expr::new(ExprData::Literal(Literal::String(name)), loc, None)163 Expr::new(
164 ExprData::Literal(Literal::String(name)),
165 Loc::new(field_start, p.index - field_start),
166 None,
167 )
126 }168 }
127 _ => {169 _ => {
128 return Err(p.error("Expected identifier after dot".to_string()).into());170 return Err(p.error("Expected identifier after dot".to_string()).into());
...@@ -133,12 +175,12 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {...@@ -133,12 +175,12 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
133 base: Box::new(expr),175 base: Box::new(expr),
134 field: Box::new(field),176 field: Box::new(field),
135 },177 },
136 loc,178 Loc::new(field_start, p.index - field_start),
137 None,179 None,
138 );180 );
139 }181 }
140 Some(Token::OpenParen) => {182 Some(TokenType::OpenParen) => {
141 let loc = p.loc();183 let expr_start = p.index;
142 _ = p.next();184 _ = p.next();
143 let arguments = parse_call_arguments(p)?;185 let arguments = parse_call_arguments(p)?;
144 expr = Expr::new(186 expr = Expr::new(
...@@ -146,21 +188,21 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {...@@ -146,21 +188,21 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
146 base: Box::new(expr),188 base: Box::new(expr),
147 arguments,189 arguments,
148 },190 },
149 loc,191 Loc::new(expr_start, p.index - expr_start),
150 None,192 None,
151 );193 );
152 }194 }
153 Some(Token::OpenBracket) => {195 Some(TokenType::OpenBracket) => {
154 let loc = p.loc();196 let expr_start = p.index;
155 _ = p.next();197 _ = p.next();
156 let index = parse_expr(p)?;198 let index = parse_expr(p)?;
157 if let Some(Token::ClosedBracket) = p.next() {199 if let Some(TokenType::ClosedBracket) = p.next() {
158 expr = Expr::new(200 expr = Expr::new(
159 ExprData::Field {201 ExprData::Field {
160 base: Box::new(expr),202 base: Box::new(expr),
161 field: Box::new(index),203 field: Box::new(index),
162 },204 },
163 loc,205 Loc::new(expr_start, p.index - expr_start),
164 None,206 None,
165 );207 );
166 } else {208 } else {
crates/parser/parse_expr_if.rs+11-11
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::{BlockOrIf, BlockWithLoc, If};1use ast::{BlockOrIf, BlockWithLoc, If};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::{Loc, TokenType};
44
5use crate::parse_block::parse_block;5use crate::parse_block::parse_block;
6use crate::parse_expr::parse_expr;6use crate::parse_expr::parse_expr;
...@@ -9,12 +9,12 @@ use crate::Parser;...@@ -9,12 +9,12 @@ use crate::Parser;
9/// Precondition: the first token exists and is 'if'9/// Precondition: the first token exists and is 'if'
10pub fn parse_expr_if(p: &mut Parser) -> Result<If> {10pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
11 match p.next() {11 match p.next() {
12 Some(Token::If) => (),12 Some(TokenType::If) => (),
13 _ => unreachable!(),13 _ => unreachable!(),
14 }14 }
1515
16 match p.peek() {16 match p.peek() {
17 Some(Token::OpenBrace) => {17 Some(TokenType::OpenBrace) => {
18 return Err(p18 return Err(p
19 .error(19 .error(
20 "To use a block as the condition for an if statement, wrap it in parentheses"20 "To use a block as the condition for an if statement, wrap it in parentheses"
...@@ -28,11 +28,11 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {...@@ -28,11 +28,11 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
28 let condition = Box::new(parse_expr(p)?);28 let condition = Box::new(parse_expr(p)?);
2929
30 let yes = match p.peek() {30 let yes = match p.peek() {
31 Some(Token::OpenBrace) => {31 Some(TokenType::OpenBrace) => {
32 let loc = p.loc();32 let block_start = p.index;
33 BlockWithLoc {33 BlockWithLoc {
34 block: parse_block(p)?,34 block: parse_block(p)?,
35 loc,35 loc: Loc::new(block_start, p.index - block_start),
36 }36 }
37 }37 }
38 _ => {38 _ => {
...@@ -43,18 +43,18 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {...@@ -43,18 +43,18 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
43 };43 };
4444
45 let no = match p.peek() {45 let no = match p.peek() {
46 Some(Token::Else) => {46 Some(TokenType::Else) => {
47 _ = p.next();47 _ = p.next();
48 match p.peek() {48 match p.peek() {
49 Some(Token::OpenBrace) => {49 Some(TokenType::OpenBrace) => {
50 let loc = p.loc();50 let block_start = p.index;
51 // let comments = p.take_comments();51 // let comments = p.take_comments();
52 Some(BlockOrIf::Block(BlockWithLoc {52 Some(BlockOrIf::Block(BlockWithLoc {
53 block: parse_block(p)?,53 block: parse_block(p)?,
54 loc,54 loc: Loc::new(block_start, p.index - block_start),
55 }))55 }))
56 }56 }
57 Some(Token::If) => {57 Some(TokenType::If) => {
58 // TODO(@paperdave): Save location for the if58 // TODO(@paperdave): Save location for the if
59 // let loc = p.loc();59 // let loc = p.loc();
60 // let comments = p.take_comments();60 // let comments = p.take_comments();
crates/parser/parse_fn_declaration.rs+20-16
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::{DocComment, Expr, ExprData, FnDecl, Type};1use ast::{DocComment, Expr, ExprData, FnDecl, Type};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::{Loc, TokenType};
44
5use crate::parse_block::parse_block;5use crate::parse_block::parse_block;
6use crate::parse_expr::parse_expr;6use crate::parse_expr::parse_expr;
...@@ -14,19 +14,19 @@ pub fn parse_fn_declaration(...@@ -14,19 +14,19 @@ pub fn parse_fn_declaration(
14 is_pub: bool,14 is_pub: bool,
15 comment: Option<DocComment>,15 comment: Option<DocComment>,
16) -> Result<FnDecl> {16) -> Result<FnDecl> {
17 let loc = p.loc();17 let start = p.index;
18 match p.next() {18 match p.next() {
19 Some(Token::Fn) => (),19 Some(TokenType::Fn) => (),
20 _ => unreachable!(),20 _ => unreachable!(),
21 }21 }
2222
23 let name = match p.next() {23 let name = match p.next() {
24 Some(Token::Identifier(name)) => name,24 Some(TokenType::Identifier(name)) => name,
25 _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),25 _ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),
26 };26 };
2727
28 match p.peek() {28 match p.peek() {
29 Some(Token::OpenParen) => p.next(),29 Some(TokenType::OpenParen) => p.next(),
30 _ => {30 _ => {
31 return Err(p31 return Err(p
32 .error("Expected '(' after function name".to_string())32 .error("Expected '(' after function name".to_string())
...@@ -38,22 +38,22 @@ pub fn parse_fn_declaration(...@@ -38,22 +38,22 @@ pub fn parse_fn_declaration(
3838
39 loop {39 loop {
40 match p.peek() {40 match p.peek() {
41 Some(Token::ClosedParen) => {41 Some(TokenType::ClosedParen) => {
42 _ = p.next();42 _ = p.next();
43 break;43 break;
44 }44 }
45 Some(Token::Identifier(_)) => {45 Some(TokenType::Identifier(_)) => {
46 params.push(parse_param(p)?);46 params.push(parse_param(p)?);
4747
48 match p.peek() {48 match p.peek() {
49 Some(Token::Comma) => {49 Some(TokenType::Comma) => {
50 p.next();50 p.next();
51 if let Some(Token::ClosedParen) = p.peek() {51 if let Some(TokenType::ClosedParen) = p.peek() {
52 p.next();52 p.next();
53 break;53 break;
54 }54 }
55 }55 }
56 Some(Token::ClosedParen) => {56 Some(TokenType::ClosedParen) => {
57 p.next();57 p.next();
58 break;58 break;
59 }59 }
...@@ -77,7 +77,7 @@ pub fn parse_fn_declaration(...@@ -77,7 +77,7 @@ pub fn parse_fn_declaration(
77 }77 }
7878
79 let return_type = match p.peek() {79 let return_type = match p.peek() {
80 Some(Token::RightArrow) => {80 Some(TokenType::RightArrow) => {
81 _ = p.next();81 _ = p.next();
82 parse_type(p)?82 parse_type(p)?
83 }83 }
...@@ -88,12 +88,16 @@ pub fn parse_fn_declaration(...@@ -88,12 +88,16 @@ pub fn parse_fn_declaration(
88 p.is_in_fn = true;88 p.is_in_fn = true;
8989
90 let body = match p.peek() {90 let body = match p.peek() {
91 Some(Token::OpenBrace) => {91 Some(TokenType::OpenBrace) => {
92 let block_loc = p.loc();92 let block_start = p.index;
93 let block = parse_block(p)?;93 let block = parse_block(p)?;
94 Expr::new(ExprData::Block(block), block_loc, None)94 Expr::new(
95 ExprData::Block(block),
96 Loc::new(block_start, p.index - block_start),
97 None,
98 )
95 }99 }
96 Some(Token::BigRightArrow) => {100 Some(TokenType::BigRightArrow) => {
97 _ = p.next();101 _ = p.next();
98 parse_expr(p)?102 parse_expr(p)?
99 }103 }
...@@ -117,6 +121,6 @@ pub fn parse_fn_declaration(...@@ -117,6 +121,6 @@ pub fn parse_fn_declaration(
117 body,121 body,
118 is_pub,122 is_pub,
119 comment,123 comment,
120 loc,124 loc: Loc::new(start, p.index - start),
121 })125 })
122}126}
crates/parser/parse_module.rs+3-3
...@@ -3,7 +3,7 @@ use crate::parse_top_level_statement::parse_top_level_statement;...@@ -3,7 +3,7 @@ use crate::parse_top_level_statement::parse_top_level_statement;
3use crate::Parser;3use crate::Parser;
4use ast::Module;4use ast::Module;
5use miette::Result;5use miette::Result;
6use tokenizer::Token;6use tokenizer::TokenType;
77
8pub fn parse_module(p: &mut Parser) -> Result<Module> {8pub fn parse_module(p: &mut Parser) -> Result<Module> {
9 let mut statements = Vec::new();9 let mut statements = Vec::new();
...@@ -18,8 +18,8 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {...@@ -18,8 +18,8 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
18 let statement = parse_top_level_statement(p)?;18 let statement = parse_top_level_statement(p)?;
1919
20 match p.peek() {20 match p.peek() {
21 Some(Token::Semicolon) => {21 Some(TokenType::Semicolon) => {
22 while let Some(Token::Semicolon) = p.peek() {22 while let Some(TokenType::Semicolon) = p.peek() {
23 p.next();23 p.next();
24 }24 }
25 }25 }
crates/parser/parse_param.rs+3-3
...@@ -1,18 +1,18 @@...@@ -1,18 +1,18 @@
1use ast::Param;1use ast::Param;
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::TokenType;
44
5use crate::{parse_type::parse_type, Parser};5use crate::{parse_type::parse_type, Parser};
66
7/// Precondition: A token exists and is an identifier7/// Precondition: A token exists and is an identifier
8pub fn parse_param(p: &mut Parser) -> Result<Param> {8pub fn parse_param(p: &mut Parser) -> Result<Param> {
9 let name = match p.next() {9 let name = match p.next() {
10 Some(Token::Identifier(name)) => name.clone(),10 Some(TokenType::Identifier(name)) => name.clone(),
11 _ => unreachable!(),11 _ => unreachable!(),
12 };12 };
1313
14 let param_type = match p.peek() {14 let param_type = match p.peek() {
15 Some(Token::Colon) => {15 Some(TokenType::Colon) => {
16 _ = p.next();16 _ = p.next();
17 parse_type(p)?17 parse_type(p)?
18 }18 }
crates/parser/parse_statement.rs+9-11
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1use ast::Statement;1use ast::Statement;
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::TokenType;
44
5use crate::parse_declaration::parse_declaration;5use crate::parse_declaration::parse_declaration;
6use crate::parse_doc_comment::maybe_read_doc_comment;6use crate::parse_doc_comment::maybe_read_doc_comment;
...@@ -24,8 +24,8 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {...@@ -24,8 +24,8 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
24 }24 }
2525
26 match p.peek() {26 match p.peek() {
27 Some(Token::DocComment(_)) => unreachable!(),27 Some(TokenType::DocComment(_)) => unreachable!(),
28 Some(Token::Pub) => {28 Some(TokenType::Pub) => {
29 if !p.is_top_level {29 if !p.is_top_level {
30 return Err(p30 return Err(p
31 .error("Public declarations are not allowed inside of functions".to_string())31 .error("Public declarations are not allowed inside of functions".to_string())
...@@ -33,12 +33,10 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {...@@ -33,12 +33,10 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
33 }33 }
34 _ = p.next();34 _ = p.next();
35 match p.peek() {35 match p.peek() {
36 Some(Token::Var) | Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration(36 Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(
37 p,37 parse_declaration(p, true, m_doc_comment)?,
38 true,38 )),
39 m_doc_comment,39 Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
40 )?)),
41 Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
42 p,40 p,
43 true,41 true,
44 m_doc_comment,42 m_doc_comment,
...@@ -47,12 +45,12 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {...@@ -47,12 +45,12 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
47 None => Err(p.error("Unexpected end of file".to_string()).into()),45 None => Err(p.error("Unexpected end of file".to_string()).into()),
48 }46 }
49 }47 }
50 Some(Token::Var) | Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration(48 Some(TokenType::Var) | Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration(
51 p,49 p,
52 false,50 false,
53 m_doc_comment,51 m_doc_comment,
54 )?)),52 )?)),
55 Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(53 Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
56 p,54 p,
57 false,55 false,
58 m_doc_comment,56 m_doc_comment,
crates/parser/parse_type.rs+2-2
...@@ -1,13 +1,13 @@...@@ -1,13 +1,13 @@
1use ast::Type;1use ast::Type;
2use miette::Result;2use miette::Result;
33
4use tokenizer::Token;4use tokenizer::TokenType;
55
6use crate::Parser;6use crate::Parser;
77
8pub fn parse_type(p: &mut Parser) -> Result<Type> {8pub fn parse_type(p: &mut Parser) -> Result<Type> {
9 match p.next() {9 match p.next() {
10 Some(Token::Identifier(id)) => match id.as_str() {10 Some(TokenType::Identifier(id)) => match id.as_str() {
11 "int" => Ok(Type::Int),11 "int" => Ok(Type::Int),
12 "float" => Ok(Type::Float),12 "float" => Ok(Type::Float),
13 "bigint" => Ok(Type::BigInt),13 "bigint" => Ok(Type::BigInt),
crates/parser/parser.rs+13-10
...@@ -19,6 +19,7 @@ use miette::Result;...@@ -19,6 +19,7 @@ use miette::Result;
19use parse_module::parse_module;19use parse_module::parse_module;
20use thiserror::Error;20use thiserror::Error;
21use tokenizer::tokenize;21use tokenizer::tokenize;
22use tokenizer::TokenType;
22use tokenizer::{Loc, Token};23use tokenizer::{Loc, Token};
2324
24pub fn parse(tokens: Vec<Token>) -> Result<Module> {25pub fn parse(tokens: Vec<Token>) -> Result<Module> {
...@@ -52,6 +53,9 @@ pub struct Parser {...@@ -52,6 +53,9 @@ pub struct Parser {
52 tokens: Vec<Token>,53 tokens: Vec<Token>,
53 cursor: usize,54 cursor: usize,
5455
56 /// Current index in the string
57 pub index: usize,
58
55 /// Whether or not the parser is at the top level.59 /// Whether or not the parser is at the top level.
56 /// Currently controls the validity of `pub` keywords.60 /// Currently controls the validity of `pub` keywords.
57 pub is_top_level: bool,61 pub is_top_level: bool,
...@@ -67,6 +71,7 @@ impl Parser {...@@ -67,6 +71,7 @@ impl Parser {
67 Parser {71 Parser {
68 tokens,72 tokens,
69 cursor: 0,73 cursor: 0,
74 index: 0,
70 is_top_level: true,75 is_top_level: true,
71 is_in_fn: false,76 is_in_fn: false,
72 comments: vec![],77 comments: vec![],
...@@ -77,14 +82,16 @@ impl Parser {...@@ -77,14 +82,16 @@ impl Parser {
77 self.cursor == self.tokens.len()82 self.cursor == self.tokens.len()
78 }83 }
7984
80 pub fn next(&mut self) -> Option<Token> {85 pub fn next(&mut self) -> Option<TokenType> {
81 if self.cursor == self.tokens.len() {86 if self.cursor == self.tokens.len() {
82 return None;87 return None;
83 }88 }
8489
85 loop {90 loop {
86 match self.tokens[self.cursor].clone() {91 let token = self.tokens[self.cursor].clone();
87 Token::Comment(comment) => {92 self.index = token.loc.start;
93 match token.variant {
94 TokenType::Comment(comment) => {
88 self.comments.push(comment);95 self.comments.push(comment);
89 self.cursor += 1;96 self.cursor += 1;
90 continue;97 continue;
...@@ -113,14 +120,14 @@ impl Parser {...@@ -113,14 +120,14 @@ impl Parser {
113 self.comments.len() > 0120 self.comments.len() > 0
114 }121 }
115122
116 pub fn peek(&mut self) -> Option<Token> {123 pub fn peek(&mut self) -> Option<TokenType> {
117 loop {124 loop {
118 if self.cursor == self.tokens.len() {125 if self.cursor == self.tokens.len() {
119 return None;126 return None;
120 }127 }
121128
122 match self.tokens[self.cursor].clone() {129 match self.tokens[self.cursor].clone().variant {
123 Token::Comment(comment) => {130 TokenType::Comment(comment) => {
124 self.comments.push(comment);131 self.comments.push(comment);
125 self.cursor += 1;132 self.cursor += 1;
126 continue;133 continue;
...@@ -137,10 +144,6 @@ impl Parser {...@@ -137,10 +144,6 @@ impl Parser {
137 ParseError::new(error, None)144 ParseError::new(error, None)
138 }145 }
139146
140 fn loc(&self) -> Loc {
141 return Loc {};
142 }
143
144 fn error_at(&self, error: String, loc: Loc) -> ParseError {147 fn error_at(&self, error: String, loc: Loc) -> ParseError {
145 ParseError::new(error, Some(loc))148 ParseError::new(error, Some(loc))
146 }149 }
crates/tokenizer/tokenizer.rs+316-276
...@@ -2,14 +2,24 @@ use miette::{Diagnostic, Result};...@@ -2,14 +2,24 @@ use miette::{Diagnostic, Result};
2use thiserror::Error;2use thiserror::Error;
33
4#[derive(Debug, PartialEq, Clone)]4#[derive(Debug, PartialEq, Clone)]
5pub struct Loc {}5pub struct Loc {
6 pub start: usize,
7 pub length: usize,
8}
69
7impl Loc {10impl Loc {
8 /// TODO(@paperdave): Remove this. every usage of this is incorrect.11 /// TODO(@paperdave): Remove this. every usage of this is incorrect.
9 pub const EMPTY: Loc = Loc {};12 pub const EMPTY: Loc = Loc {
13 start: 0,
14 length: 0,
15 };
16
17 pub fn new(start: usize, length: usize) -> Self {
18 Self { start, length }
19 }
1020
11 pub fn new() -> Self {21 pub fn len(&self) -> usize {
12 Self {}22 self.length
13 }23 }
14}24}
1525
...@@ -20,8 +30,8 @@ struct UnknownCharacter {...@@ -20,8 +30,8 @@ struct UnknownCharacter {
20 error: String,30 error: String,
21}31}
2232
23#[derive(Debug, Clone)]33#[derive(Debug, Clone, PartialEq)]
24pub enum Token {34pub enum TokenType {
25 // Literals35 // Literals
26 Identifier(String),36 Identifier(String),
27 Number(f64),37 Number(f64),
...@@ -78,77 +88,92 @@ pub enum Token {...@@ -78,77 +88,92 @@ pub enum Token {
78 Tilde, // ~88 Tilde, // ~
79}89}
8090
81impl Token {91impl TokenType {
82 pub fn is_start_of_statement_like_expression(&self) -> bool {92 pub fn is_start_of_statement_like_expression(&self) -> bool {
83 match &self {93 match &self {
84 Token::If | Token::OpenBrace => true,94 TokenType::If | TokenType::OpenBrace => true,
85 _ => false,95 _ => false,
86 }96 }
87 }97 }
88}98}
8999
90impl std::fmt::Display for Token {100impl std::fmt::Display for TokenType {
91 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {101 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
92 let token = match self {102 let token = match &self {
93 Token::Identifier(_) => "identifier".to_string(),103 TokenType::Identifier(_) => "identifier".to_string(),
94104
95 Token::Number(n) => n.to_string(),105 TokenType::Number(n) => n.to_string(),
96 Token::String(s) => format!("\"{}\"", s),106 TokenType::String(s) => format!("\"{}\"", s),
97 Token::Comment(_) => "comment".to_string(),107 TokenType::Comment(_) => "comment".to_string(),
98 Token::DocComment(_) => "documentation comment".to_string(),108 TokenType::DocComment(_) => "documentation comment".to_string(),
99109
100 Token::Else => "else".to_string(),110 TokenType::Else => "else".to_string(),
101 Token::False => "false".to_string(),111 TokenType::False => "false".to_string(),
102 Token::Fn => "fn".to_string(),112 TokenType::Fn => "fn".to_string(),
103 Token::If => "if".to_string(),113 TokenType::If => "if".to_string(),
104 Token::Let => "let".to_string(),114 TokenType::Let => "let".to_string(),
105 Token::Pub => "pub".to_string(),115 TokenType::Pub => "pub".to_string(),
106 Token::Return => "return".to_string(),116 TokenType::Return => "return".to_string(),
107 Token::True => "true".to_string(),117 TokenType::True => "true".to_string(),
108 Token::Var => "var".to_string(),118 TokenType::Var => "var".to_string(),
109 Token::Void => "void".to_string(),119 TokenType::Void => "void".to_string(),
110120
111 Token::And => "&".to_string(),121 TokenType::And => "&".to_string(),
112 Token::AndAnd => "&&".to_string(),122 TokenType::AndAnd => "&&".to_string(),
113 Token::Bang => "!".to_string(),123 TokenType::Bang => "!".to_string(),
114 Token::BangEqual => "!=".to_string(),124 TokenType::BangEqual => "!=".to_string(),
115 Token::BigRightArrow => "=>".to_string(),125 TokenType::BigRightArrow => "=>".to_string(),
116 Token::Caret => "^".to_string(),126 TokenType::Caret => "^".to_string(),
117 Token::ClosedBrace => "}".to_string(),127 TokenType::ClosedBrace => "}".to_string(),
118 Token::ClosedBracket => "]".to_string(),128 TokenType::ClosedBracket => "]".to_string(),
119 Token::ClosedParen => ")".to_string(),129 TokenType::ClosedParen => ")".to_string(),
120 Token::Colon => ":".to_string(),130 TokenType::Colon => ":".to_string(),
121 Token::Comma => ",".to_string(),131 TokenType::Comma => ",".to_string(),
122 Token::Dot => ".".to_string(),132 TokenType::Dot => ".".to_string(),
123 Token::Equal => "=".to_string(),133 TokenType::Equal => "=".to_string(),
124 Token::EqualEqual => "==".to_string(),134 TokenType::EqualEqual => "==".to_string(),
125 Token::Gt => ">".to_string(),135 TokenType::Gt => ">".to_string(),
126 Token::GtEqual => ">=".to_string(),136 TokenType::GtEqual => ">=".to_string(),
127 Token::GtGt => ">>".to_string(),137 TokenType::GtGt => ">>".to_string(),
128 Token::GtGtGt => ">>>".to_string(),138 TokenType::GtGtGt => ">>>".to_string(),
129 Token::Lt => "<".to_string(),139 TokenType::Lt => "<".to_string(),
130 Token::LtEqual => "<=".to_string(),140 TokenType::LtEqual => "<=".to_string(),
131 Token::LtLt => "<<".to_string(),141 TokenType::LtLt => "<<".to_string(),
132 Token::Minus => "-".to_string(),142 TokenType::Minus => "-".to_string(),
133 Token::OpenBrace => "{".to_string(),143 TokenType::OpenBrace => "{".to_string(),
134 Token::OpenBracket => "[".to_string(),144 TokenType::OpenBracket => "[".to_string(),
135 Token::OpenParen => "(".to_string(),145 TokenType::OpenParen => "(".to_string(),
136 Token::Or => "|".to_string(),146 TokenType::Or => "|".to_string(),
137 Token::OrOr => "||".to_string(),147 TokenType::OrOr => "||".to_string(),
138 Token::Percent => "%".to_string(),148 TokenType::Percent => "%".to_string(),
139 Token::Plus => "+".to_string(),149 TokenType::Plus => "+".to_string(),
140 Token::RightArrow => "->".to_string(),150 TokenType::RightArrow => "->".to_string(),
141 Token::Semicolon => ";".to_string(),151 TokenType::Semicolon => ";".to_string(),
142 Token::Slash => "/".to_string(),152 TokenType::Slash => "/".to_string(),
143 Token::Star => "*".to_string(),153 TokenType::Star => "*".to_string(),
144 Token::StarStar => "**".to_string(),154 TokenType::StarStar => "**".to_string(),
145 Token::Tilde => "~".to_string(),155 TokenType::Tilde => "~".to_string(),
146 };156 };
147157
148 write!(f, "{}", token)158 write!(f, "{}", token)
149 }159 }
150}160}
151161
162#[derive(Debug, Clone)]
163pub struct Token {
164 pub variant: TokenType,
165 pub loc: Loc,
166}
167
168impl Token {
169 pub fn new(variant: TokenType, start: usize, length: usize) -> Self {
170 Self {
171 variant,
172 loc: Loc::new(start, length),
173 }
174 }
175}
176
152// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc177// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
153pub fn tokenize(content: String) -> Result<Vec<Token>> {178pub fn tokenize(content: String) -> Result<Vec<Token>> {
154 let mut tokens: Vec<Token> = Vec::new();179 let mut tokens: Vec<Token> = Vec::new();
...@@ -156,251 +181,266 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {...@@ -156,251 +181,266 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {
156181
157 loop {182 loop {
158 match chars.next() {183 match chars.next() {
159 Some((_, c)) => match c {184 Some((start, c)) => {
160 'a'..='z' | 'A'..='Z' | '_' | '@' => {185 let token = match c {
161 let mut current_token = String::new();186 'a'..='z' | 'A'..='Z' | '_' | '@' => {
162 current_token.push(c);187 let mut current_token = String::new();
163 loop {188 current_token.push(c);
164 let c = chars.peek();189 loop {
165 if c.is_none() {190 let c = chars.peek();
166 break;191 if c.is_none() {
167 }192 break;
168 let (_, c) = c.unwrap();193 }
169 match c {194 let (_, c) = c.unwrap();
170 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => {195 match c {
171 current_token.push(*c);196 'a'..='z' | 'A'..='Z' | '_' | '0'..='9' => {
172 chars.next();197 current_token.push(*c);
198 chars.next();
199 }
200 _ => break,
173 }201 }
174 _ => break,
175 }202 }
176 }
177203
178 match current_token.as_str() {204 let length = current_token.len();
179 "else" => tokens.push(Token::Else),205
180 "false" => tokens.push(Token::False),206 let variant = match current_token.as_str() {
181 "fn" => tokens.push(Token::Fn),207 "else" => TokenType::Else,
182 "if" => tokens.push(Token::If),208 "false" => TokenType::False,
183 "let" => tokens.push(Token::Let),209 "fn" => TokenType::Fn,
184 "pub" => tokens.push(Token::Pub),210 "if" => TokenType::If,
185 "return" => tokens.push(Token::Return),211 "let" => TokenType::Let,
186 "true" => tokens.push(Token::True),212 "pub" => TokenType::Pub,
187 "var" => tokens.push(Token::Var),213 "return" => TokenType::Return,
188 "void" => tokens.push(Token::Void),214 "true" => TokenType::True,
189 _ => tokens.push(Token::Identifier(current_token)),215 "var" => TokenType::Var,
216 "void" => TokenType::Void,
217 _ => TokenType::Identifier(current_token),
218 };
219
220 Token::new(variant, start, length)
190 }221 }
191 }222 '"' => {
192 '"' => {223 let mut length = 1;
193 let mut current_token = String::new();224 let mut current_token = String::new();
194 loop {225 loop {
195 let c = chars.next();226 let c = chars.next();
196 if c.is_none() {227 if c.is_none() {
197 break;228 break;
198 }229 }
199 let (_, c) = c.unwrap();230 let (_, c) = c.unwrap();
200 match c {231 length += 1;
201 '"' => break,232 match c {
202 _ => current_token.push(c),233 '"' => break,
234 _ => current_token.push(c),
235 }
203 }236 }
237 Token::new(TokenType::String(current_token), start, length)
204 }238 }
205 tokens.push(Token::String(current_token));239 '0'..='9' => {
206 }240 let mut current_token = String::new();
207 '0'..='9' => {241 current_token.push(c);
208 let mut current_token = String::new();242 loop {
209 current_token.push(c);243 let c = chars.peek();
210 loop {244 if c.is_none() {
211 let c = chars.peek();245 break;
212 if c.is_none() {246 }
213 break;247 let (_, c) = c.unwrap();
214 }248 match c {
215 let (_, c) = c.unwrap();249 '0'..='9' => {
216 match c {250 current_token.push(*c);
217 '0'..='9' => {251 chars.next();
218 current_token.push(*c);252 }
219 chars.next();253 _ => break,
220 }254 }
221 _ => break,
222 }255 }
223 }256 match current_token.parse::<f64>() {
224 match current_token.parse::<f64>() {257 Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()),
225 Ok(v) => tokens.push(Token::Number(v)),258 Err(_) => {
226 Err(_) => {259 return Err(UnknownCharacter {
227 return Err(UnknownCharacter {260 error: format!("Unknown character: {}", current_token),
228 error: format!("Unknown character: {}", current_token),261 }
262 .into())
229 }263 }
230 .into())
231 }264 }
232 }265 }
233 }266 ':' => Token::new(TokenType::Colon, start, 1),
234 ':' => tokens.push(Token::Colon),267 '=' => match chars.peek() {
235 '=' => match chars.peek() {268 Some((_, '=')) => {
236 Some((_, '=')) => {269 chars.next();
237 chars.next();270 Token::new(TokenType::EqualEqual, start, 2)
238 tokens.push(Token::EqualEqual);271 }
239 }272 Some((_, '>')) => {
240 Some((_, '>')) => {273 chars.next();
241 chars.next();274 Token::new(TokenType::BigRightArrow, start, 2)
242 tokens.push(Token::BigRightArrow);275 }
243 }276 _ => Token::new(TokenType::Equal, start, 1),
244 _ => tokens.push(Token::Equal),277 },
245 },278 ';' => Token::new(TokenType::Semicolon, start, 1),
246 ';' => tokens.push(Token::Semicolon),279 '>' => match chars.peek() {
247 '>' => match chars.peek() {280 Some((_, '=')) => {
248 Some((_, '=')) => {281 chars.next();
249 chars.next();282 Token::new(TokenType::GtEqual, start, 2)
250 tokens.push(Token::GtEqual);283 }
251 }284 Some((_, '>')) => {
252 Some((_, '>')) => {285 chars.next();
253 chars.next();286 match chars.peek() {
254 match chars.peek() {287 Some((_, '>')) => {
255 Some((_, '>')) => {288 chars.next();
256 chars.next();289 Token::new(TokenType::GtGtGt, start, 3)
257 tokens.push(Token::GtGtGt);290 }
291 _ => Token::new(TokenType::GtGt, start, 2),
258 }292 }
259 _ => tokens.push(Token::GtGt),
260 }293 }
261 }294 _ => Token::new(TokenType::Gt, start, 1),
262 _ => tokens.push(Token::Gt),295 },
263 },296 '<' => match chars.peek() {
264 '<' => match chars.peek() {297 Some((_, '=')) => {
265 Some((_, '=')) => {298 chars.next();
266 chars.next();299 Token::new(TokenType::LtEqual, start, 2)
267 tokens.push(Token::LtEqual);300 }
268 }301 Some((_, '<')) => {
269 Some((_, '<')) => {302 chars.next();
270 chars.next();303 Token::new(TokenType::LtLt, start, 2)
271 tokens.push(Token::LtLt);304 }
272 }305 _ => Token::new(TokenType::Lt, start, 1),
273 _ => tokens.push(Token::Lt),306 },
274 },307 '+' => Token::new(TokenType::Plus, start, 1),
275 '+' => tokens.push(Token::Plus),308 '-' => match chars.peek() {
276 '-' => match chars.peek() {309 Some((_, '>')) => {
277 Some((_, '>')) => {310 chars.next();
278 chars.next();311 Token::new(TokenType::RightArrow, start, 2)
279 tokens.push(Token::RightArrow);312 }
280 }313 _ => Token::new(TokenType::Minus, start, 1),
281 _ => tokens.push(Token::Minus),314 },
282 },315 '*' => match chars.peek() {
283 '*' => match chars.peek() {316 Some((_, '*')) => {
284 Some((_, '*')) => {317 chars.next();
285 chars.next();318 Token::new(TokenType::StarStar, start, 2)
286 tokens.push(Token::StarStar);319 }
287 }320 _ => Token::new(TokenType::Star, start, 1),
288 _ => tokens.push(Token::Star),321 },
289 },322 '/' => {
290 '/' => {323 match chars.peek() {
291 match chars.peek() {324 Some((_, '/')) => {
292 Some((_, '/')) => {325 chars.next(); // Consume the second slash
293 chars.next(); // Consume the second slash326 let mut length = 2;
294327
295 let is_doc_comment = match chars.peek() {328 let is_doc_comment = match chars.peek() {
296 Some((_, '/')) => {329 Some((_, '/')) => {
297 chars.next(); // Consume the third slash330 chars.next(); // Consume the third slash
298 true331 length += 1;
299 }332 true
300 _ => false,333 }
301 };334 _ => false,
335 };
302336
303 let mut comment = String::new();337 let mut comment = String::new();
304 loop {338 loop {
305 // skip leading whitespace339 // skip leading whitespace
306 while let Some((_, c)) = chars.peek() {340 while let Some((_, c)) = chars.peek() {
307 if !c.is_whitespace() {341 if !c.is_whitespace() {
308 break;342 break;
343 }
344 chars.next();
345 length += 1;
309 }346 }
310 chars.next();347 // read the comment
311 }348 while let Some((_, c)) = chars.next() {
312 // read the comment349 length += 1;
313 while let Some((_, c)) = chars.next() {350 if c == '\n' || c == '\r' {
314 if c == '\n' || c == '\r' {351 break;
315 break;352 }
353 comment.push(c);
316 }354 }
317 comment.push(c);355 // trim trailing whitespace on line
318 }356 comment = comment.trim_end().to_string();
319 // trim trailing whitespace on line357 // skip to next non-whitespace character
320 comment = comment.trim_end().to_string();358 while let Some((_, c)) = chars.peek() {
321 // skip to next non-whitespace character359 if !c.is_whitespace() {
322 while let Some((_, c)) = chars.peek() {360 break;
323 if !c.is_whitespace() {361 }
324 break;362 chars.next();
363 length += 1;
325 }364 }
326 chars.next();365 // check if we've hit another comment
327 }366 let mut tmp = chars.clone();
328 // check if we've hit another comment
329 let mut tmp = chars.clone();
330367
331 let is_another_comment = match tmp.peek() {368 let is_another_comment = match tmp.peek() {
332 Some((_, '/')) => {369 Some((_, '/')) => {
333 tmp.next(); // Consume the second slash370 tmp.next(); // Consume the second slash
334 match tmp.peek() {371 match tmp.peek() {
335 Some((_, '/')) => {372 Some((_, '/')) => {
336 tmp.next(); // Consume the third slash373 tmp.next(); // Consume the third slash
337 is_doc_comment374 is_doc_comment
375 }
376 _ => !is_doc_comment,
338 }377 }
339 _ => !is_doc_comment,
340 }378 }
379 _ => false,
380 };
381
382 if is_another_comment {
383 // apply our temporary iterator to the main iterator
384 comment.push('\n');
385 chars = tmp;
386 chars.next();
387 length += 1;
388 } else {
389 break;
341 }390 }
342 _ => false,391 }
343 };
344392
345 if is_another_comment {393 if is_doc_comment {
346 // apply our temporary iterator to the main iterator394 Token::new(TokenType::DocComment(comment), start, length)
347 comment.push('\n');
348 chars = tmp;
349 chars.next();
350 } else {395 } else {
351 break;396 Token::new(TokenType::Comment(comment), start, length)
352 }397 }
353 }398 }
354399 _ => Token::new(TokenType::Slash, start, 1),
355 if is_doc_comment {
356 tokens.push(Token::DocComment(comment));
357 } else {
358 tokens.push(Token::Comment(comment));
359 }
360 }400 }
361 _ => tokens.push(Token::Slash),
362 }
363 }
364 ',' => tokens.push(Token::Comma),
365 '(' => tokens.push(Token::OpenParen),
366 ')' => tokens.push(Token::ClosedParen),
367 '{' => tokens.push(Token::OpenBrace),
368 '}' => tokens.push(Token::ClosedBrace),
369 '[' => tokens.push(Token::OpenBracket),
370 ']' => tokens.push(Token::ClosedBracket),
371 '&' => match chars.peek() {
372 Some((_, '&')) => {
373 chars.next();
374 tokens.push(Token::AndAnd);
375 }
376 _ => tokens.push(Token::And),
377 },
378 '|' => match chars.peek() {
379 Some((_, '|')) => {
380 chars.next();
381 tokens.push(Token::OrOr);
382 }401 }
383 _ => tokens.push(Token::Or),402 ',' => Token::new(TokenType::Comma, start, 1),
384 },403 '(' => Token::new(TokenType::OpenParen, start, 1),
385 '!' => match chars.peek() {404 ')' => Token::new(TokenType::ClosedParen, start, 1),
386 Some((_, '=')) => {405 '{' => Token::new(TokenType::OpenBrace, start, 1),
387 chars.next();406 '}' => Token::new(TokenType::ClosedBrace, start, 1),
388 tokens.push(Token::BangEqual);407 '[' => Token::new(TokenType::OpenBracket, start, 1),
389 }408 ']' => Token::new(TokenType::ClosedBracket, start, 1),
390 _ => tokens.push(Token::Bang),409 '&' => match chars.peek() {
391 },410 Some((_, '&')) => {
392 '^' => tokens.push(Token::Caret),411 chars.next();
393 '%' => tokens.push(Token::Percent),412 Token::new(TokenType::AndAnd, start, 2)
394 '~' => tokens.push(Token::Tilde),413 }
395 '.' => tokens.push(Token::Dot),414 _ => Token::new(TokenType::And, start, 1),
396 s if s.is_whitespace() => (),415 },
397 _ => {416 '|' => match chars.peek() {
398 return Err(UnknownCharacter {417 Some((_, '|')) => {
399 error: format!("Unexpected '{}' while tokenizing", c),418 chars.next();
419 Token::new(TokenType::OrOr, start, 2)
420 }
421 _ => Token::new(TokenType::Or, start, 1),
422 },
423 '!' => match chars.peek() {
424 Some((_, '=')) => {
425 chars.next();
426 Token::new(TokenType::BangEqual, start, 2)
427 }
428 _ => Token::new(TokenType::Bang, start, 1),
429 },
430 '^' => Token::new(TokenType::Caret, start, 1),
431 '%' => Token::new(TokenType::Percent, start, 1),
432 '~' => Token::new(TokenType::Tilde, start, 1),
433 '.' => Token::new(TokenType::Dot, start, 1),
434 s if s.is_whitespace() => continue,
435 _ => {
436 return Err(UnknownCharacter {
437 error: format!("Unexpected '{}' while tokenizing", c),
438 }
439 .into())
400 }440 }
401 .into())441 };
402 }442 tokens.push(token);
403 },443 }
404 None => break,444 None => break,
405 }445 }
406 }446 }