Commit b417db0d1e1f2ce026092e1b310e2c98e7de265f | ix

author	Lino Le Van <11367844+lino-levan@users.noreply.github.com> 2024-02-03 05:35:35+01:00
committer	GitHub <noreply@github.com> 2024-02-02 20:35:35-08:00
log	b417db0d1e1f2ce026092e1b310e2c98e7de265f
tree	4c7bb64084c4459c10264cf2a00fa4ca9069ef4a
parent	4d577446b981a403ff6131f0b15451f125316af0
signature	Signed by PGP key B5690EEEBB952194

feat(parser): track the location of source tokens (#8)

* feat: add loc

* chore: start working on transition parser to include loc

* chore: fix everything related to TokenType

* feat: add loc to parser

16 files changed, 513 insertions(+), 421 deletions(-)

crates
- ast
  - operators.rs
- js_ast
  - convert
    convert_valid_block.rs
- parser
- tokenizer
  - tokenizer.rs

crates/ast/operators.rs+23-23

...	...	@@ -1,4 +1,4 @@
1		use tokenizer::Token;
	1	use tokenizer::TokenType;
2	2
3	3	use crate::Literal;
4	4
...	...	@@ -94,28 +94,28 @@ impl BinaryOperator {
94	94	String::from(self.to_str())
95	95	}
96	96
97		pub fn from_token(token: &Token) -> Option<BinaryOperator> {
98		match token {
99		Token::And => Some(Self::BitAnd),
100		Token::AndAnd => Some(Self::And),
101		Token::BangEqual => Some(Self::NotEq),
102		Token::Caret => Some(Self::BitXor),
103		Token::EqualEqual => Some(Self::Eq),
104		Token::Gt => Some(Self::Gt),
105		Token::GtEqual => Some(Self::Gte),
106		Token::GtGt => Some(Self::Rshift),
107		Token::GtGtGt => Some(Self::RshiftUnsigned),
108		Token::Lt => Some(Self::Lt),
109		Token::LtEqual => Some(Self::Lte),
110		Token::LtLt => Some(Self::Lshift),
111		Token::Minus => Some(Self::Sub),
112		Token::Or => Some(Self::BitOr),
113		Token::OrOr => Some(Self::Or),
114		Token::Percent => Some(Self::Mod),
115		Token::Plus => Some(Self::Add),
116		Token::Slash => Some(Self::Div),
117		Token::Star => Some(Self::Mul),
118		Token::StarStar => Some(Self::Pow),
	97	pub fn from_token(token: &TokenType) -> Option<BinaryOperator> {
	98	match &token {
	99	TokenType::And => Some(Self::BitAnd),
	100	TokenType::AndAnd => Some(Self::And),
	101	TokenType::BangEqual => Some(Self::NotEq),
	102	TokenType::Caret => Some(Self::BitXor),
	103	TokenType::EqualEqual => Some(Self::Eq),
	104	TokenType::Gt => Some(Self::Gt),
	105	TokenType::GtEqual => Some(Self::Gte),
	106	TokenType::GtGt => Some(Self::Rshift),
	107	TokenType::GtGtGt => Some(Self::RshiftUnsigned),
	108	TokenType::Lt => Some(Self::Lt),
	109	TokenType::LtEqual => Some(Self::Lte),
	110	TokenType::LtLt => Some(Self::Lshift),
	111	TokenType::Minus => Some(Self::Sub),
	112	TokenType::Or => Some(Self::BitOr),
	113	TokenType::OrOr => Some(Self::Or),
	114	TokenType::Percent => Some(Self::Mod),
	115	TokenType::Plus => Some(Self::Add),
	116	TokenType::Slash => Some(Self::Div),
	117	TokenType::Star => Some(Self::Mul),
	118	TokenType::StarStar => Some(Self::Pow),
119	119	_ => None,
120	120	}
121	121	}

crates/js_ast/convert/convert_valid_block.rs+1

...	...	@@ -26,6 +26,7 @@ pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, loc: Loc
26	26	}
27	27	JsExpr::new(JsExprData::Comma { items }, loc.clone(), None)
28	28	}
	29	_ => todo!(),
29	30	};
30	31
31	32	// a block without implicit return is to be type void, so we wrap in a void expression

crates/parser/parse_block.rs+5-5

...	...	@@ -1,12 +1,12 @@
1	1	use crate::{parse_statement::parse_statement, Parser};
2	2	use ast::Block;
3	3	use miette::Result;
4		use tokenizer::Token;
	4	use tokenizer::TokenType;
5	5
6	6	/// Precondition: The first token exists and is '{'
7	7	pub fn parse_block(p: &mut Parser) -> Result<Block> {
8	8	match p.next() {
9		Some(Token::OpenBrace) => (),
	9	Some(TokenType::OpenBrace) => (),
10	10	_ => unreachable!(),
11	11	}
12	12
...	...	@@ -30,10 +30,10 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
30	30	statements.push(statement.clone());
31	31
32	32	match p.peek() {
33		Some(Token::Semicolon) => loop {
	33	Some(TokenType::Semicolon) => loop {
34	34	p.next();
35	35	match p.peek() {
36		Some(Token::ClosedBrace) => {
	36	Some(TokenType::ClosedBrace) => {
37	37	p.next();
38	38	p.is_top_level = was_top_level;
39	39	return Ok(Block {
...	...	@@ -44,7 +44,7 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
44	44	_ => continue 'outer,
45	45	}
46	46	},
47		Some(Token::ClosedBrace) => {
	47	Some(TokenType::ClosedBrace) => {
48	48	p.next();
49	49	p.is_top_level = was_top_level;
50	50

crates/parser/parse_call_arguments.rs+4-4

...	...	@@ -1,6 +1,6 @@
1	1	use ast::Expr;
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::TokenType;
4	4
5	5	use crate::{parse_expr::parse_expr, Parser};
6	6
...	...	@@ -8,7 +8,7 @@ use crate::{parse_expr::parse_expr, Parser};
8	8	pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
9	9	let mut args = Vec::new();
10	10
11		if let Some(Token::ClosedParen) = p.peek() {
	11	if let Some(TokenType::ClosedParen) = p.peek() {
12	12	_ = p.next();
13	13	return Ok(args);
14	14	}
...	...	@@ -18,11 +18,11 @@ pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
18	18	args.push(arg);
19	19
20	20	match p.peek() {
21		Some(Token::Comma) => {
	21	Some(TokenType::Comma) => {
22	22	_ = p.next();
23	23	continue;
24	24	}
25		Some(Token::ClosedParen) => {
	25	Some(TokenType::ClosedParen) => {
26	26	_ = p.next();
27	27	return Ok(args);
28	28	}

crates/parser/parse_declaration.rs+8-8

...	...	@@ -1,6 +1,6 @@
1	1	use ast::{DocComment, Type, VarDecl};
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::{Loc, TokenType};
4	4
5	5	use crate::parse_expr::parse_expr;
6	6	use crate::parse_type::parse_type;
...	...	@@ -12,26 +12,26 @@ pub fn parse_declaration(
12	12	is_pub: bool,
13	13	comment: Option<DocComment>,
14	14	) -> Result<VarDecl> {
15		let loc = p.loc();
	15	let start = p.index;
16	16	let is_constant = match p.next() {
17		Some(Token::Var) => false,
18		Some(Token::Let) => true,
	17	Some(TokenType::Var) => false,
	18	Some(TokenType::Let) => true,
19	19	_ => unreachable!(),
20	20	};
21	21	let name = match p.next() {
22		Some(Token::Identifier(name)) => name,
	22	Some(TokenType::Identifier(name)) => name,
23	23	Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),
24	24	None => return Err(p.error(format!("Unexpected end of file")).into()),
25	25	};
26	26	let return_type = match p.peek() {
27		Some(Token::Colon) => {
	27	Some(TokenType::Colon) => {
28	28	p.next();
29	29	parse_type(p)?
30	30	}
31	31	_ => Type::Infer,
32	32	};
33	33	let init = match p.next() {
34		Some(Token::Equal) => parse_expr(p)?,
	34	Some(TokenType::Equal) => parse_expr(p)?,
35	35	_ => {
36	36	return Err(p
37	37	.error("Variable must have an initializer".to_string())
...	...	@@ -45,6 +45,6 @@ pub fn parse_declaration(
45	45	is_constant,
46	46	is_pub,
47	47	comment,
48		loc,
	48	loc: Loc::new(start, p.index - start),
49	49	})
50	50	}

crates/parser/parse_doc_comment.rs+7-4

...	...	@@ -1,14 +1,17 @@
1	1	use ast::DocComment;
2		use tokenizer::Token;
	2	use tokenizer::{Loc, TokenType};
3	3
4	4	use crate::Parser;
5	5
6	6	pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
7	7	match p.peek() {
8		Some(Token::DocComment(data)) => {
9		let loc = p.loc();
	8	Some(TokenType::DocComment(data)) => {
	9	let start = p.index;
10	10	p.next();
11		Some(DocComment { data, loc })
	11	Some(DocComment {
	12	data,
	13	loc: Loc::new(start, p.index - start),
	14	})
12	15	}
13	16	_ => None,
14	17	}

crates/parser/parse_expr.rs+3-2

...	...	@@ -1,6 +1,7 @@
1	1	use ast::Associativity::*;
2	2	use ast::{BinaryOperator, Expr, ExprData, OperatorLevel};
3	3	use miette::Result;
	4	use tokenizer::Loc;
4	5
5	6	use crate::parse_expr_basic::parse_expr_basic;
6	7	use crate::Parser;
...	...	@@ -53,7 +54,7 @@ fn parse_expr_binary(
53	54	left: Expr,
54	55	level: OperatorLevel,
55	56	) -> Result<Expr> {
56		let loc = p.loc();
	57	let start = p.index;
57	58	let mut right = parse_expr_basic(p)?;
58	59
59	60	loop {
...	...	@@ -93,7 +94,7 @@ fn parse_expr_binary(
93	94	left: Box::new(left),
94	95	right: Box::new(right),
95	96	},
96		loc,
	97	Loc::new(start, p.index - start),
97	98	None,
98	99	))
99	100	}

crates/parser/parse_expr_basic.rs+85-43

...	...	@@ -1,6 +1,6 @@
1	1	use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::{Loc, TokenType};
4	4
5	5	use crate::{
6	6	parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,
...	...	@@ -8,61 +8,87 @@ use crate::{
8	8	};
9	9
10	10	pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
11		let loc = p.loc();
	11	let start = p.index;
12	12	let comments = p.take_comments();
13	13	let mut expr = match p.peek() {
14		Some(Token::Number(number)) => {
	14	Some(TokenType::Number(number)) => {
15	15	_ = p.next();
16	16	Expr::new(
17	17	ExprData::Literal(Literal::Number(number.clone())),
18		loc,
	18	Loc::new(start, p.index - start),
19	19	comments,
20	20	)
21	21	}
22		Some(Token::String(string)) => {
	22	Some(TokenType::String(string)) => {
23	23	_ = p.next();
24	24	Expr::new(
25	25	ExprData::Literal(Literal::String(string.clone())),
26		loc,
	26	Loc::new(start, p.index - start),
27	27	comments,
28	28	)
29	29	}
30		Some(Token::True) => {
	30	Some(TokenType::True) => {
31	31	_ = p.next();
32		Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments)
	32	Expr::new(
	33	ExprData::Literal(Literal::Bool(true)),
	34	Loc::new(start, p.index - start),
	35	comments,
	36	)
33	37	}
34		Some(Token::False) => {
	38	Some(TokenType::False) => {
35	39	_ = p.next();
36		Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments)
	40	Expr::new(
	41	ExprData::Literal(Literal::Bool(false)),
	42	Loc::new(start, p.index - start),
	43	comments,
	44	)
37	45	}
38		Some(Token::Void) => {
	46	Some(TokenType::Void) => {
39	47	_ = p.next();
40		Expr::new(ExprData::Literal(Literal::Void), loc, comments)
	48	Expr::new(
	49	ExprData::Literal(Literal::Void),
	50	Loc::new(start, p.index - start),
	51	comments,
	52	)
41	53	}
42		Some(Token::Identifier(name)) => {
43		let loc = p.loc();
	54	Some(TokenType::Identifier(name)) => {
	55	let identifier_start = p.index;
44	56	_ = p.next();
45	57	if name.starts_with("@") {
46	58	if let Some(id) = BuiltinId::from_str(name.as_str()) {
47		if let Some(Token::OpenParen) = p.next() {
	59	if let Some(TokenType::OpenParen) = p.next() {
48	60	let arguments = parse_call_arguments(p)?;
49		Expr::new(ExprData::Builtin { id, arguments }, loc, comments)
	61	Expr::new(
	62	ExprData::Builtin { id, arguments },
	63	Loc::new(identifier_start, p.index - identifier_start),
	64	comments,
	65	)
50	66	} else {
51	67	return Err(p
52		.error_at(format!("Built-in function '{}' must be called", name), loc)
	68	.error_at(
	69	format!("Built-in function '{}' must be called", name),
	70	Loc::new(identifier_start, p.index - identifier_start),
	71	)
53	72	.into());
54	73	}
55	74	} else {
56	75	return Err(p
57		.error_at(format!("Non-existant built-in: '{}'", name), loc)
	76	.error_at(
	77	format!("Non-existant built-in: '{}'", name),
	78	Loc::new(identifier_start, p.index - identifier_start),
	79	)
58	80	.into());
59	81	}
60	82	} else {
61		Expr::new(ExprData::Identifier(name), loc, comments)
	83	Expr::new(
	84	ExprData::Identifier(name),
	85	Loc::new(identifier_start, p.index - identifier_start),
	86	comments,
	87	)
62	88	}
63	89	}
64	90
65		Some(Token::Return) => {
	91	Some(TokenType::Return) => {
66	92	if p.is_top_level {
67	93	return Err(p
68	94	.error("'return' cannot be used at top level".to_string())
...	...	@@ -71,19 +97,31 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
71	97	_ = p.next();
72	98	let inner = maybe_parse_expr_basic(p)?.map(Box::new);
73	99
74		return Ok(Some(Expr::new(ExprData::Return(inner), loc, comments)));
	100	return Ok(Some(Expr::new(
	101	ExprData::Return(inner),
	102	Loc::new(start, p.index - start),
	103	comments,
	104	)));
75	105	}
76	106	}
77	107
78		Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments),
	108	Some(TokenType::If) => Expr::new(
	109	ExprData::If(parse_expr_if(p)?),
	110	Loc::new(start, p.index - start),
	111	comments,
	112	),
79	113
80		Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments),
	114	Some(TokenType::OpenBrace) => Expr::new(
	115	ExprData::Block(parse_block(p)?),
	116	Loc::new(start, p.index - start),
	117	comments,
	118	),
81	119
82		Some(Token::OpenParen) => {
	120	Some(TokenType::OpenParen) => {
83	121	_ = p.next();
84	122	let expr = parse_expr(p)?;
85	123	match p.next() {
86		Some(Token::ClosedParen) => (),
	124	Some(TokenType::ClosedParen) => (),
87	125	Some(token) => {
88	126	return Err(p.error(format!("Expected ')', but got {:?}", token)).into())
89	127	}
...	...	@@ -92,19 +130,19 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
92	130	expr
93	131	}
94	132
95		Some(t @ Token::Bang) \| Some(t @ Token::Minus) \| Some(t @ Token::Tilde) => {
	133	Some(t @ TokenType::Bang) \| Some(t @ TokenType::Minus) \| Some(t @ TokenType::Tilde) => {
96	134	_ = p.next();
97	135	Expr::new(
98	136	ExprData::Unary {
99	137	op: match t {
100		Token::Bang => UnaryOperator::Not,
101		Token::Minus => UnaryOperator::Neg,
102		Token::Tilde => UnaryOperator::BitNot,
	138	TokenType::Bang => UnaryOperator::Not,
	139	TokenType::Minus => UnaryOperator::Neg,
	140	TokenType::Tilde => UnaryOperator::BitNot,
103	141	_ => unreachable!(),
104	142	},
105	143	value: Box::new(parse_expr_basic(p)?),
106	144	},
107		loc,
	145	Loc::new(start, p.index - start),
108	146	comments,
109	147	)
110	148	}
...	...	@@ -115,14 +153,18 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
115	153	// Parse function calls and field accesses
116	154	loop {
117	155	match p.peek() {
118		Some(Token::Dot) => {
119		let loc = p.loc();
	156	Some(TokenType::Dot) => {
	157	let field_start = p.index;
120	158	_ = p.next();
121	159	let field = match p.peek() {
122		Some(Token::Identifier(name)) => {
123		let loc = p.loc();
	160	Some(TokenType::Identifier(name)) => {
	161	let field_start = p.index;
124	162	p.next();
125		Expr::new(ExprData::Literal(Literal::String(name)), loc, None)
	163	Expr::new(
	164	ExprData::Literal(Literal::String(name)),
	165	Loc::new(field_start, p.index - field_start),
	166	None,
	167	)
126	168	}
127	169	_ => {
128	170	return Err(p.error("Expected identifier after dot".to_string()).into());
...	...	@@ -133,12 +175,12 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
133	175	base: Box::new(expr),
134	176	field: Box::new(field),
135	177	},
136		loc,
	178	Loc::new(field_start, p.index - field_start),
137	179	None,
138	180	);
139	181	}
140		Some(Token::OpenParen) => {
141		let loc = p.loc();
	182	Some(TokenType::OpenParen) => {
	183	let expr_start = p.index;
142	184	_ = p.next();
143	185	let arguments = parse_call_arguments(p)?;
144	186	expr = Expr::new(
...	...	@@ -146,21 +188,21 @@ pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
146	188	base: Box::new(expr),
147	189	arguments,
148	190	},
149		loc,
	191	Loc::new(expr_start, p.index - expr_start),
150	192	None,
151	193	);
152	194	}
153		Some(Token::OpenBracket) => {
154		let loc = p.loc();
	195	Some(TokenType::OpenBracket) => {
	196	let expr_start = p.index;
155	197	_ = p.next();
156	198	let index = parse_expr(p)?;
157		if let Some(Token::ClosedBracket) = p.next() {
	199	if let Some(TokenType::ClosedBracket) = p.next() {
158	200	expr = Expr::new(
159	201	ExprData::Field {
160	202	base: Box::new(expr),
161	203	field: Box::new(index),
162	204	},
163		loc,
	205	Loc::new(expr_start, p.index - expr_start),
164	206	None,
165	207	);
166	208	} else {

crates/parser/parse_expr_if.rs+11-11

...	...	@@ -1,6 +1,6 @@
1	1	use ast::{BlockOrIf, BlockWithLoc, If};
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::{Loc, TokenType};
4	4
5	5	use crate::parse_block::parse_block;
6	6	use crate::parse_expr::parse_expr;
...	...	@@ -9,12 +9,12 @@ use crate::Parser;
9	9	/// Precondition: the first token exists and is 'if'
10	10	pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
11	11	match p.next() {
12		Some(Token::If) => (),
	12	Some(TokenType::If) => (),
13	13	_ => unreachable!(),
14	14	}
15	15
16	16	match p.peek() {
17		Some(Token::OpenBrace) => {
	17	Some(TokenType::OpenBrace) => {
18	18	return Err(p
19	19	.error(
20	20	"To use a block as the condition for an if statement, wrap it in parentheses"
...	...	@@ -28,11 +28,11 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
28	28	let condition = Box::new(parse_expr(p)?);
29	29
30	30	let yes = match p.peek() {
31		Some(Token::OpenBrace) => {
32		let loc = p.loc();
	31	Some(TokenType::OpenBrace) => {
	32	let block_start = p.index;
33	33	BlockWithLoc {
34	34	block: parse_block(p)?,
35		loc,
	35	loc: Loc::new(block_start, p.index - block_start),
36	36	}
37	37	}
38	38	_ => {
...	...	@@ -43,18 +43,18 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
43	43	};
44	44
45	45	let no = match p.peek() {
46		Some(Token::Else) => {
	46	Some(TokenType::Else) => {
47	47	_ = p.next();
48	48	match p.peek() {
49		Some(Token::OpenBrace) => {
50		let loc = p.loc();
	49	Some(TokenType::OpenBrace) => {
	50	let block_start = p.index;
51	51	// let comments = p.take_comments();
52	52	Some(BlockOrIf::Block(BlockWithLoc {
53	53	block: parse_block(p)?,
54		loc,
	54	loc: Loc::new(block_start, p.index - block_start),
55	55	}))
56	56	}
57		Some(Token::If) => {
	57	Some(TokenType::If) => {
58	58	// TODO(@paperdave): Save location for the if
59	59	// let loc = p.loc();
60	60	// let comments = p.take_comments();

crates/parser/parse_fn_declaration.rs+20-16

...	...	@@ -1,6 +1,6 @@
1	1	use ast::{DocComment, Expr, ExprData, FnDecl, Type};
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::{Loc, TokenType};
4	4
5	5	use crate::parse_block::parse_block;
6	6	use crate::parse_expr::parse_expr;
...	...	@@ -14,19 +14,19 @@ pub fn parse_fn_declaration(
14	14	is_pub: bool,
15	15	comment: Option<DocComment>,
16	16	) -> Result<FnDecl> {
17		let loc = p.loc();
	17	let start = p.index;
18	18	match p.next() {
19		Some(Token::Fn) => (),
	19	Some(TokenType::Fn) => (),
20	20	_ => unreachable!(),
21	21	}
22	22
23	23	let name = match p.next() {
24		Some(Token::Identifier(name)) => name,
	24	Some(TokenType::Identifier(name)) => name,
25	25	_ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),
26	26	};
27	27
28	28	match p.peek() {
29		Some(Token::OpenParen) => p.next(),
	29	Some(TokenType::OpenParen) => p.next(),
30	30	_ => {
31	31	return Err(p
32	32	.error("Expected '(' after function name".to_string())
...	...	@@ -38,22 +38,22 @@ pub fn parse_fn_declaration(
38	38
39	39	loop {
40	40	match p.peek() {
41		Some(Token::ClosedParen) => {
	41	Some(TokenType::ClosedParen) => {
42	42	_ = p.next();
43	43	break;
44	44	}
45		Some(Token::Identifier(_)) => {
	45	Some(TokenType::Identifier(_)) => {
46	46	params.push(parse_param(p)?);
47	47
48	48	match p.peek() {
49		Some(Token::Comma) => {
	49	Some(TokenType::Comma) => {
50	50	p.next();
51		if let Some(Token::ClosedParen) = p.peek() {
	51	if let Some(TokenType::ClosedParen) = p.peek() {
52	52	p.next();
53	53	break;
54	54	}
55	55	}
56		Some(Token::ClosedParen) => {
	56	Some(TokenType::ClosedParen) => {
57	57	p.next();
58	58	break;
59	59	}
...	...	@@ -77,7 +77,7 @@ pub fn parse_fn_declaration(
77	77	}
78	78
79	79	let return_type = match p.peek() {
80		Some(Token::RightArrow) => {
	80	Some(TokenType::RightArrow) => {
81	81	_ = p.next();
82	82	parse_type(p)?
83	83	}
...	...	@@ -88,12 +88,16 @@ pub fn parse_fn_declaration(
88	88	p.is_in_fn = true;
89	89
90	90	let body = match p.peek() {
91		Some(Token::OpenBrace) => {
92		let block_loc = p.loc();
	91	Some(TokenType::OpenBrace) => {
	92	let block_start = p.index;
93	93	let block = parse_block(p)?;
94		Expr::new(ExprData::Block(block), block_loc, None)
	94	Expr::new(
	95	ExprData::Block(block),
	96	Loc::new(block_start, p.index - block_start),
	97	None,
	98	)
95	99	}
96		Some(Token::BigRightArrow) => {
	100	Some(TokenType::BigRightArrow) => {
97	101	_ = p.next();
98	102	parse_expr(p)?
99	103	}
...	...	@@ -117,6 +121,6 @@ pub fn parse_fn_declaration(
117	121	body,
118	122	is_pub,
119	123	comment,
120		loc,
	124	loc: Loc::new(start, p.index - start),
121	125	})
122	126	}

crates/parser/parse_module.rs+3-3

...	...	@@ -3,7 +3,7 @@ use crate::parse_top_level_statement::parse_top_level_statement;
3	3	use crate::Parser;
4	4	use ast::Module;
5	5	use miette::Result;
6		use tokenizer::Token;
	6	use tokenizer::TokenType;
7	7
8	8	pub fn parse_module(p: &mut Parser) -> Result<Module> {
9	9	let mut statements = Vec::new();
...	...	@@ -18,8 +18,8 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
18	18	let statement = parse_top_level_statement(p)?;
19	19
20	20	match p.peek() {
21		Some(Token::Semicolon) => {
22		while let Some(Token::Semicolon) = p.peek() {
	21	Some(TokenType::Semicolon) => {
	22	while let Some(TokenType::Semicolon) = p.peek() {
23	23	p.next();
24	24	}
25	25	}

crates/parser/parse_param.rs+3-3

...	...	@@ -1,18 +1,18 @@
1	1	use ast::Param;
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::TokenType;
4	4
5	5	use crate::{parse_type::parse_type, Parser};
6	6
7	7	/// Precondition: A token exists and is an identifier
8	8	pub fn parse_param(p: &mut Parser) -> Result<Param> {
9	9	let name = match p.next() {
10		Some(Token::Identifier(name)) => name.clone(),
	10	Some(TokenType::Identifier(name)) => name.clone(),
11	11	_ => unreachable!(),
12	12	};
13	13
14	14	let param_type = match p.peek() {
15		Some(Token::Colon) => {
	15	Some(TokenType::Colon) => {
16	16	_ = p.next();
17	17	parse_type(p)?
18	18	}

crates/parser/parse_statement.rs+9-11

...	...	@@ -1,6 +1,6 @@
1	1	use ast::Statement;
2	2	use miette::Result;
3		use tokenizer::Token;
	3	use tokenizer::TokenType;
4	4
5	5	use crate::parse_declaration::parse_declaration;
6	6	use crate::parse_doc_comment::maybe_read_doc_comment;
...	...	@@ -24,8 +24,8 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
24	24	}
25	25
26	26	match p.peek() {
27		Some(Token::DocComment(_)) => unreachable!(),
28		Some(Token::Pub) => {
	27	Some(TokenType::DocComment(_)) => unreachable!(),
	28	Some(TokenType::Pub) => {
29	29	if !p.is_top_level {
30	30	return Err(p
31	31	.error("Public declarations are not allowed inside of functions".to_string())
...	...	@@ -33,12 +33,10 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
33	33	}
34	34	_ = p.next();
35	35	match p.peek() {
36		Some(Token::Var) \| Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration(
37		p,
38		true,
39		m_doc_comment,
40		)?)),
41		Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
	36	Some(TokenType::Var) \| Some(TokenType::Let) => Ok(Statement::VarDecl(
	37	parse_declaration(p, true, m_doc_comment)?,
	38	)),
	39	Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
42	40	p,
43	41	true,
44	42	m_doc_comment,
...	...	@@ -47,12 +45,12 @@ pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
47	45	None => Err(p.error("Unexpected end of file".to_string()).into()),
48	46	}
49	47	}
50		Some(Token::Var) \| Some(Token::Let) => Ok(Statement::VarDecl(parse_declaration(
	48	Some(TokenType::Var) \| Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration(
51	49	p,
52	50	false,
53	51	m_doc_comment,
54	52	)?)),
55		Some(Token::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
	53	Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
56	54	p,
57	55	false,
58	56	m_doc_comment,

crates/parser/parse_type.rs+2-2

...	...	@@ -1,13 +1,13 @@
1	1	use ast::Type;
2	2	use miette::Result;
3	3
4		use tokenizer::Token;
	4	use tokenizer::TokenType;
5	5
6	6	use crate::Parser;
7	7
8	8	pub fn parse_type(p: &mut Parser) -> Result<Type> {
9	9	match p.next() {
10		Some(Token::Identifier(id)) => match id.as_str() {
	10	Some(TokenType::Identifier(id)) => match id.as_str() {
11	11	"int" => Ok(Type::Int),
12	12	"float" => Ok(Type::Float),
13	13	"bigint" => Ok(Type::BigInt),

crates/parser/parser.rs+13-10

...	...	@@ -19,6 +19,7 @@ use miette::Result;
19	19	use parse_module::parse_module;
20	20	use thiserror::Error;
21	21	use tokenizer::tokenize;
	22	use tokenizer::TokenType;
22	23	use tokenizer::{Loc, Token};
23	24
24	25	pub fn parse(tokens: Vec<Token>) -> Result<Module> {
...	...	@@ -52,6 +53,9 @@ pub struct Parser {
52	53	tokens: Vec<Token>,
53	54	cursor: usize,
54	55
	56	/// Current index in the string
	57	pub index: usize,
	58
55	59	/// Whether or not the parser is at the top level.
56	60	/// Currently controls the validity of `pub` keywords.
57	61	pub is_top_level: bool,
...	...	@@ -67,6 +71,7 @@ impl Parser {
67	71	Parser {
68	72	tokens,
69	73	cursor: 0,
	74	index: 0,
70	75	is_top_level: true,
71	76	is_in_fn: false,
72	77	comments: vec![],
...	...	@@ -77,14 +82,16 @@ impl Parser {
77	82	self.cursor == self.tokens.len()
78	83	}
79	84
80		pub fn next(&mut self) -> Option<Token> {
	85	pub fn next(&mut self) -> Option<TokenType> {
81	86	if self.cursor == self.tokens.len() {
82	87	return None;
83	88	}
84	89
85	90	loop {
86		match self.tokens[self.cursor].clone() {
87		Token::Comment(comment) => {
	91	let token = self.tokens[self.cursor].clone();
	92	self.index = token.loc.start;
	93	match token.variant {
	94	TokenType::Comment(comment) => {
88	95	self.comments.push(comment);
89	96	self.cursor += 1;
90	97	continue;
...	...	@@ -113,14 +120,14 @@ impl Parser {
113	120	self.comments.len() > 0
114	121	}
115	122
116		pub fn peek(&mut self) -> Option<Token> {
	123	pub fn peek(&mut self) -> Option<TokenType> {
117	124	loop {
118	125	if self.cursor == self.tokens.len() {
119	126	return None;
120	127	}
121	128
122		match self.tokens[self.cursor].clone() {
123		Token::Comment(comment) => {
	129	match self.tokens[self.cursor].clone().variant {
	130	TokenType::Comment(comment) => {
124	131	self.comments.push(comment);
125	132	self.cursor += 1;
126	133	continue;
...	...	@@ -137,10 +144,6 @@ impl Parser {
137	144	ParseError::new(error, None)
138	145	}
139	146
140		fn loc(&self) -> Loc {
141		return Loc {};
142		}
143
144	147	fn error_at(&self, error: String, loc: Loc) -> ParseError {
145	148	ParseError::new(error, Some(loc))
146	149	}

crates/tokenizer/tokenizer.rs+316-276

...	...	@@ -2,14 +2,24 @@ use miette::{Diagnostic, Result};
2	2	use thiserror::Error;
3	3
4	4	#[derive(Debug, PartialEq, Clone)]
5		pub struct Loc {}
	5	pub struct Loc {
	6	pub start: usize,
	7	pub length: usize,
	8	}
6	9
7	10	impl Loc {
8	11	/// TODO(@paperdave): Remove this. every usage of this is incorrect.
9		pub const EMPTY: Loc = Loc {};
	12	pub const EMPTY: Loc = Loc {
	13	start: 0,
	14	length: 0,
	15	};
	16
	17	pub fn new(start: usize, length: usize) -> Self {
	18	Self { start, length }
	19	}
10	20
11		pub fn new() -> Self {
12		Self {}
	21	pub fn len(&self) -> usize {
	22	self.length
13	23	}
14	24	}
15	25
...	...	@@ -20,8 +30,8 @@ struct UnknownCharacter {
20	30	error: String,
21	31	}
22	32
23		#[derive(Debug, Clone)]
24		pub enum Token {
	33	#[derive(Debug, Clone, PartialEq)]
	34	pub enum TokenType {
25	35	// Literals
26	36	Identifier(String),
27	37	Number(f64),
...	...	@@ -78,77 +88,92 @@ pub enum Token {
78	88	Tilde, // ~
79	89	}
80	90
81		impl Token {
	91	impl TokenType {
82	92	pub fn is_start_of_statement_like_expression(&self) -> bool {
83	93	match &self {
84		Token::If \| Token::OpenBrace => true,
	94	TokenType::If \| TokenType::OpenBrace => true,
85	95	_ => false,
86	96	}
87	97	}
88	98	}
89	99
90		impl std::fmt::Display for Token {
	100	impl std::fmt::Display for TokenType {
91	101	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
92		let token = match self {
93		Token::Identifier(_) => "identifier".to_string(),
	102	let token = match &self {
	103	TokenType::Identifier(_) => "identifier".to_string(),
94	104
95		Token::Number(n) => n.to_string(),
96		Token::String(s) => format!("\"{}\"", s),
97		Token::Comment(_) => "comment".to_string(),
98		Token::DocComment(_) => "documentation comment".to_string(),
	105	TokenType::Number(n) => n.to_string(),
	106	TokenType::String(s) => format!("\"{}\"", s),
	107	TokenType::Comment(_) => "comment".to_string(),
	108	TokenType::DocComment(_) => "documentation comment".to_string(),
99	109
100		Token::Else => "else".to_string(),
101		Token::False => "false".to_string(),
102		Token::Fn => "fn".to_string(),
103		Token::If => "if".to_string(),
104		Token::Let => "let".to_string(),
105		Token::Pub => "pub".to_string(),
106		Token::Return => "return".to_string(),
107		Token::True => "true".to_string(),
108		Token::Var => "var".to_string(),
109		Token::Void => "void".to_string(),
	110	TokenType::Else => "else".to_string(),
	111	TokenType::False => "false".to_string(),
	112	TokenType::Fn => "fn".to_string(),
	113	TokenType::If => "if".to_string(),
	114	TokenType::Let => "let".to_string(),
	115	TokenType::Pub => "pub".to_string(),
	116	TokenType::Return => "return".to_string(),
	117	TokenType::True => "true".to_string(),
	118	TokenType::Var => "var".to_string(),
	119	TokenType::Void => "void".to_string(),
110	120
111		Token::And => "&".to_string(),
112		Token::AndAnd => "&&".to_string(),
113		Token::Bang => "!".to_string(),
114		Token::BangEqual => "!=".to_string(),
115		Token::BigRightArrow => "=>".to_string(),
116		Token::Caret => "^".to_string(),
117		Token::ClosedBrace => "}".to_string(),
118		Token::ClosedBracket => "]".to_string(),
119		Token::ClosedParen => ")".to_string(),
120		Token::Colon => ":".to_string(),
121		Token::Comma => ",".to_string(),
122		Token::Dot => ".".to_string(),
123		Token::Equal => "=".to_string(),
124		Token::EqualEqual => "==".to_string(),
125		Token::Gt => ">".to_string(),
126		Token::GtEqual => ">=".to_string(),
127		Token::GtGt => ">>".to_string(),
128		Token::GtGtGt => ">>>".to_string(),
129		Token::Lt => "<".to_string(),
130		Token::LtEqual => "<=".to_string(),
131		Token::LtLt => "<<".to_string(),
132		Token::Minus => "-".to_string(),
133		Token::OpenBrace => "{".to_string(),
134		Token::OpenBracket => "[".to_string(),
135		Token::OpenParen => "(".to_string(),
136		Token::Or => "\|".to_string(),
137		Token::OrOr => "\|\|".to_string(),
138		Token::Percent => "%".to_string(),
139		Token::Plus => "+".to_string(),
140		Token::RightArrow => "->".to_string(),
141		Token::Semicolon => ";".to_string(),
142		Token::Slash => "/".to_string(),
143		Token::Star => "*".to_string(),
144		Token::StarStar => "**".to_string(),
145		Token::Tilde => "~".to_string(),
	121	TokenType::And => "&".to_string(),
	122	TokenType::AndAnd => "&&".to_string(),
	123	TokenType::Bang => "!".to_string(),
	124	TokenType::BangEqual => "!=".to_string(),
	125	TokenType::BigRightArrow => "=>".to_string(),
	126	TokenType::Caret => "^".to_string(),
	127	TokenType::ClosedBrace => "}".to_string(),
	128	TokenType::ClosedBracket => "]".to_string(),
	129	TokenType::ClosedParen => ")".to_string(),
	130	TokenType::Colon => ":".to_string(),
	131	TokenType::Comma => ",".to_string(),
	132	TokenType::Dot => ".".to_string(),
	133	TokenType::Equal => "=".to_string(),
	134	TokenType::EqualEqual => "==".to_string(),
	135	TokenType::Gt => ">".to_string(),
	136	TokenType::GtEqual => ">=".to_string(),
	137	TokenType::GtGt => ">>".to_string(),
	138	TokenType::GtGtGt => ">>>".to_string(),
	139	TokenType::Lt => "<".to_string(),
	140	TokenType::LtEqual => "<=".to_string(),
	141	TokenType::LtLt => "<<".to_string(),
	142	TokenType::Minus => "-".to_string(),
	143	TokenType::OpenBrace => "{".to_string(),
	144	TokenType::OpenBracket => "[".to_string(),
	145	TokenType::OpenParen => "(".to_string(),
	146	TokenType::Or => "\|".to_string(),
	147	TokenType::OrOr => "\|\|".to_string(),
	148	TokenType::Percent => "%".to_string(),
	149	TokenType::Plus => "+".to_string(),
	150	TokenType::RightArrow => "->".to_string(),
	151	TokenType::Semicolon => ";".to_string(),
	152	TokenType::Slash => "/".to_string(),
	153	TokenType::Star => "*".to_string(),
	154	TokenType::StarStar => "**".to_string(),
	155	TokenType::Tilde => "~".to_string(),
146	156	};
147	157
148	158	write!(f, "{}", token)
149	159	}
150	160	}
151	161
	162	#[derive(Debug, Clone)]
	163	pub struct Token {
	164	pub variant: TokenType,
	165	pub loc: Loc,
	166	}
	167
	168	impl Token {
	169	pub fn new(variant: TokenType, start: usize, length: usize) -> Self {
	170	Self {
	171	variant,
	172	loc: Loc::new(start, length),
	173	}
	174	}
	175	}
	176
152	177	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
153	178	pub fn tokenize(content: String) -> Result<Vec<Token>> {
154	179	let mut tokens: Vec<Token> = Vec::new();
...	...	@@ -156,251 +181,266 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {
156	181
157	182	loop {
158	183	match chars.next() {
159		Some((_, c)) => match c {
160		'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
161		let mut current_token = String::new();
162		current_token.push(c);
163		loop {
164		let c = chars.peek();
165		if c.is_none() {
166		break;
167		}
168		let (_, c) = c.unwrap();
169		match c {
170		'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
171		current_token.push(*c);
172		chars.next();
	184	Some((start, c)) => {
	185	let token = match c {
	186	'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
	187	let mut current_token = String::new();
	188	current_token.push(c);
	189	loop {
	190	let c = chars.peek();
	191	if c.is_none() {
	192	break;
	193	}
	194	let (_, c) = c.unwrap();
	195	match c {
	196	'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
	197	current_token.push(*c);
	198	chars.next();
	199	}
	200	_ => break,
173	201	}
174		_ => break,
175	202	}
176		}
177	203
178		match current_token.as_str() {
179		"else" => tokens.push(Token::Else),
180		"false" => tokens.push(Token::False),
181		"fn" => tokens.push(Token::Fn),
182		"if" => tokens.push(Token::If),
183		"let" => tokens.push(Token::Let),
184		"pub" => tokens.push(Token::Pub),
185		"return" => tokens.push(Token::Return),
186		"true" => tokens.push(Token::True),
187		"var" => tokens.push(Token::Var),
188		"void" => tokens.push(Token::Void),
189		_ => tokens.push(Token::Identifier(current_token)),
	204	let length = current_token.len();
	205
	206	let variant = match current_token.as_str() {
	207	"else" => TokenType::Else,
	208	"false" => TokenType::False,
	209	"fn" => TokenType::Fn,
	210	"if" => TokenType::If,
	211	"let" => TokenType::Let,
	212	"pub" => TokenType::Pub,
	213	"return" => TokenType::Return,
	214	"true" => TokenType::True,
	215	"var" => TokenType::Var,
	216	"void" => TokenType::Void,
	217	_ => TokenType::Identifier(current_token),
	218	};
	219
	220	Token::new(variant, start, length)
190	221	}
191		}
192		'"' => {
193		let mut current_token = String::new();
194		loop {
195		let c = chars.next();
196		if c.is_none() {
197		break;
198		}
199		let (_, c) = c.unwrap();
200		match c {
201		'"' => break,
202		_ => current_token.push(c),
	222	'"' => {
	223	let mut length = 1;
	224	let mut current_token = String::new();
	225	loop {
	226	let c = chars.next();
	227	if c.is_none() {
	228	break;
	229	}
	230	let (_, c) = c.unwrap();
	231	length += 1;
	232	match c {
	233	'"' => break,
	234	_ => current_token.push(c),
	235	}
203	236	}
	237	Token::new(TokenType::String(current_token), start, length)
204	238	}
205		tokens.push(Token::String(current_token));
206		}
207		'0'..='9' => {
208		let mut current_token = String::new();
209		current_token.push(c);
210		loop {
211		let c = chars.peek();
212		if c.is_none() {
213		break;
214		}
215		let (_, c) = c.unwrap();
216		match c {
217		'0'..='9' => {
218		current_token.push(*c);
219		chars.next();
	239	'0'..='9' => {
	240	let mut current_token = String::new();
	241	current_token.push(c);
	242	loop {
	243	let c = chars.peek();
	244	if c.is_none() {
	245	break;
	246	}
	247	let (_, c) = c.unwrap();
	248	match c {
	249	'0'..='9' => {
	250	current_token.push(*c);
	251	chars.next();
	252	}
	253	_ => break,
220	254	}
221		_ => break,
222	255	}
223		}
224		match current_token.parse::<f64>() {
225		Ok(v) => tokens.push(Token::Number(v)),
226		Err(_) => {
227		return Err(UnknownCharacter {
228		error: format!("Unknown character: {}", current_token),
	256	match current_token.parse::<f64>() {
	257	Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()),
	258	Err(_) => {
	259	return Err(UnknownCharacter {
	260	error: format!("Unknown character: {}", current_token),
	261	}
	262	.into())
229	263	}
230		.into())
231	264	}
232	265	}
233		}
234		':' => tokens.push(Token::Colon),
235		'=' => match chars.peek() {
236		Some((_, '=')) => {
237		chars.next();
238		tokens.push(Token::EqualEqual);
239		}
240		Some((_, '>')) => {
241		chars.next();
242		tokens.push(Token::BigRightArrow);
243		}
244		_ => tokens.push(Token::Equal),
245		},
246		';' => tokens.push(Token::Semicolon),
247		'>' => match chars.peek() {
248		Some((_, '=')) => {
249		chars.next();
250		tokens.push(Token::GtEqual);
251		}
252		Some((_, '>')) => {
253		chars.next();
254		match chars.peek() {
255		Some((_, '>')) => {
256		chars.next();
257		tokens.push(Token::GtGtGt);
	266	':' => Token::new(TokenType::Colon, start, 1),
	267	'=' => match chars.peek() {
	268	Some((_, '=')) => {
	269	chars.next();
	270	Token::new(TokenType::EqualEqual, start, 2)
	271	}
	272	Some((_, '>')) => {
	273	chars.next();
	274	Token::new(TokenType::BigRightArrow, start, 2)
	275	}
	276	_ => Token::new(TokenType::Equal, start, 1),
	277	},
	278	';' => Token::new(TokenType::Semicolon, start, 1),
	279	'>' => match chars.peek() {
	280	Some((_, '=')) => {
	281	chars.next();
	282	Token::new(TokenType::GtEqual, start, 2)
	283	}
	284	Some((_, '>')) => {
	285	chars.next();
	286	match chars.peek() {
	287	Some((_, '>')) => {
	288	chars.next();
	289	Token::new(TokenType::GtGtGt, start, 3)
	290	}
	291	_ => Token::new(TokenType::GtGt, start, 2),
258	292	}
259		_ => tokens.push(Token::GtGt),
260	293	}
261		}
262		_ => tokens.push(Token::Gt),
263		},
264		'<' => match chars.peek() {
265		Some((_, '=')) => {
266		chars.next();
267		tokens.push(Token::LtEqual);
268		}
269		Some((_, '<')) => {
270		chars.next();
271		tokens.push(Token::LtLt);
272		}
273		_ => tokens.push(Token::Lt),
274		},
275		'+' => tokens.push(Token::Plus),
276		'-' => match chars.peek() {
277		Some((_, '>')) => {
278		chars.next();
279		tokens.push(Token::RightArrow);
280		}
281		_ => tokens.push(Token::Minus),
282		},
283		'*' => match chars.peek() {
284		Some((_, '*')) => {
285		chars.next();
286		tokens.push(Token::StarStar);
287		}
288		_ => tokens.push(Token::Star),
289		},
290		'/' => {
291		match chars.peek() {
292		Some((_, '/')) => {
293		chars.next(); // Consume the second slash
	294	_ => Token::new(TokenType::Gt, start, 1),
	295	},
	296	'<' => match chars.peek() {
	297	Some((_, '=')) => {
	298	chars.next();
	299	Token::new(TokenType::LtEqual, start, 2)
	300	}
	301	Some((_, '<')) => {
	302	chars.next();
	303	Token::new(TokenType::LtLt, start, 2)
	304	}
	305	_ => Token::new(TokenType::Lt, start, 1),
	306	},
	307	'+' => Token::new(TokenType::Plus, start, 1),
	308	'-' => match chars.peek() {
	309	Some((_, '>')) => {
	310	chars.next();
	311	Token::new(TokenType::RightArrow, start, 2)
	312	}
	313	_ => Token::new(TokenType::Minus, start, 1),
	314	},
	315	'*' => match chars.peek() {
	316	Some((_, '*')) => {
	317	chars.next();
	318	Token::new(TokenType::StarStar, start, 2)
	319	}
	320	_ => Token::new(TokenType::Star, start, 1),
	321	},
	322	'/' => {
	323	match chars.peek() {
	324	Some((_, '/')) => {
	325	chars.next(); // Consume the second slash
	326	let mut length = 2;
294	327
295		let is_doc_comment = match chars.peek() {
296		Some((_, '/')) => {
297		chars.next(); // Consume the third slash
298		true
299		}
300		_ => false,
301		};
	328	let is_doc_comment = match chars.peek() {
	329	Some((_, '/')) => {
	330	chars.next(); // Consume the third slash
	331	length += 1;
	332	true
	333	}
	334	_ => false,
	335	};
302	336
303		let mut comment = String::new();
304		loop {
305		// skip leading whitespace
306		while let Some((_, c)) = chars.peek() {
307		if !c.is_whitespace() {
308		break;
	337	let mut comment = String::new();
	338	loop {
	339	// skip leading whitespace
	340	while let Some((_, c)) = chars.peek() {
	341	if !c.is_whitespace() {
	342	break;
	343	}
	344	chars.next();
	345	length += 1;
309	346	}
310		chars.next();
311		}
312		// read the comment
313		while let Some((_, c)) = chars.next() {
314		if c == '\n' \|\| c == '\r' {
315		break;
	347	// read the comment
	348	while let Some((_, c)) = chars.next() {
	349	length += 1;
	350	if c == '\n' \|\| c == '\r' {
	351	break;
	352	}
	353	comment.push(c);
316	354	}
317		comment.push(c);
318		}
319		// trim trailing whitespace on line
320		comment = comment.trim_end().to_string();
321		// skip to next non-whitespace character
322		while let Some((_, c)) = chars.peek() {
323		if !c.is_whitespace() {
324		break;
	355	// trim trailing whitespace on line
	356	comment = comment.trim_end().to_string();
	357	// skip to next non-whitespace character
	358	while let Some((_, c)) = chars.peek() {
	359	if !c.is_whitespace() {
	360	break;
	361	}
	362	chars.next();
	363	length += 1;
325	364	}
326		chars.next();
327		}
328		// check if we've hit another comment
329		let mut tmp = chars.clone();
	365	// check if we've hit another comment
	366	let mut tmp = chars.clone();
330	367
331		let is_another_comment = match tmp.peek() {
332		Some((_, '/')) => {
333		tmp.next(); // Consume the second slash
334		match tmp.peek() {
335		Some((_, '/')) => {
336		tmp.next(); // Consume the third slash
337		is_doc_comment
	368	let is_another_comment = match tmp.peek() {
	369	Some((_, '/')) => {
	370	tmp.next(); // Consume the second slash
	371	match tmp.peek() {
	372	Some((_, '/')) => {
	373	tmp.next(); // Consume the third slash
	374	is_doc_comment
	375	}
	376	_ => !is_doc_comment,
338	377	}
339		_ => !is_doc_comment,
340	378	}
	379	_ => false,
	380	};
	381
	382	if is_another_comment {
	383	// apply our temporary iterator to the main iterator
	384	comment.push('\n');
	385	chars = tmp;
	386	chars.next();
	387	length += 1;
	388	} else {
	389	break;
341	390	}
342		_ => false,
343		};
	391	}
344	392
345		if is_another_comment {
346		// apply our temporary iterator to the main iterator
347		comment.push('\n');
348		chars = tmp;
349		chars.next();
	393	if is_doc_comment {
	394	Token::new(TokenType::DocComment(comment), start, length)
350	395	} else {
351		break;
	396	Token::new(TokenType::Comment(comment), start, length)
352	397	}
353	398	}
354
355		if is_doc_comment {
356		tokens.push(Token::DocComment(comment));
357		} else {
358		tokens.push(Token::Comment(comment));
359		}
	399	_ => Token::new(TokenType::Slash, start, 1),
360	400	}
361		_ => tokens.push(Token::Slash),
362		}
363		}
364		',' => tokens.push(Token::Comma),
365		'(' => tokens.push(Token::OpenParen),
366		')' => tokens.push(Token::ClosedParen),
367		'{' => tokens.push(Token::OpenBrace),
368		'}' => tokens.push(Token::ClosedBrace),
369		'[' => tokens.push(Token::OpenBracket),
370		']' => tokens.push(Token::ClosedBracket),
371		'&' => match chars.peek() {
372		Some((_, '&')) => {
373		chars.next();
374		tokens.push(Token::AndAnd);
375		}
376		_ => tokens.push(Token::And),
377		},
378		'\|' => match chars.peek() {
379		Some((_, '\|')) => {
380		chars.next();
381		tokens.push(Token::OrOr);
382	401	}
383		_ => tokens.push(Token::Or),
384		},
385		'!' => match chars.peek() {
386		Some((_, '=')) => {
387		chars.next();
388		tokens.push(Token::BangEqual);
389		}
390		_ => tokens.push(Token::Bang),
391		},
392		'^' => tokens.push(Token::Caret),
393		'%' => tokens.push(Token::Percent),
394		'~' => tokens.push(Token::Tilde),
395		'.' => tokens.push(Token::Dot),
396		s if s.is_whitespace() => (),
397		_ => {
398		return Err(UnknownCharacter {
399		error: format!("Unexpected '{}' while tokenizing", c),
	402	',' => Token::new(TokenType::Comma, start, 1),
	403	'(' => Token::new(TokenType::OpenParen, start, 1),
	404	')' => Token::new(TokenType::ClosedParen, start, 1),
	405	'{' => Token::new(TokenType::OpenBrace, start, 1),
	406	'}' => Token::new(TokenType::ClosedBrace, start, 1),
	407	'[' => Token::new(TokenType::OpenBracket, start, 1),
	408	']' => Token::new(TokenType::ClosedBracket, start, 1),
	409	'&' => match chars.peek() {
	410	Some((_, '&')) => {
	411	chars.next();
	412	Token::new(TokenType::AndAnd, start, 2)
	413	}
	414	_ => Token::new(TokenType::And, start, 1),
	415	},
	416	'\|' => match chars.peek() {
	417	Some((_, '\|')) => {
	418	chars.next();
	419	Token::new(TokenType::OrOr, start, 2)
	420	}
	421	_ => Token::new(TokenType::Or, start, 1),
	422	},
	423	'!' => match chars.peek() {
	424	Some((_, '=')) => {
	425	chars.next();
	426	Token::new(TokenType::BangEqual, start, 2)
	427	}
	428	_ => Token::new(TokenType::Bang, start, 1),
	429	},
	430	'^' => Token::new(TokenType::Caret, start, 1),
	431	'%' => Token::new(TokenType::Percent, start, 1),
	432	'~' => Token::new(TokenType::Tilde, start, 1),
	433	'.' => Token::new(TokenType::Dot, start, 1),
	434	s if s.is_whitespace() => continue,
	435	_ => {
	436	return Err(UnknownCharacter {
	437	error: format!("Unexpected '{}' while tokenizing", c),
	438	}
	439	.into())
400	440	}
401		.into())
402		}
403		},
	441	};
	442	tokens.push(token);
	443	}
404	444	None => break,
405	445	}
406	446	}