authorgravatar for me@paperdave.netdave caruso <me@paperdave.net> 2024-01-20 19:08:18-08:00
committergravatar for me@paperdave.netdave caruso <me@paperdave.net> 2024-01-20 19:08:18-08:00
logc89bf26922f4448e6f0cb6015f4627792841deab
treede76df5e71a1116de15edc93e1fe53156978c38b
parent17c6ef0255b85120d2bdcae9f48ecd8c88c3c247
signature Commit is signed but in an unrecognized format.

parser changes

24 files changed, 549 insertions(+), 363 deletions(-)

Cargo.lock+11-11
...@@ -126,16 +126,6 @@ version = "1.0.0"...@@ -126,16 +126,6 @@ version = "1.0.0"
126source = "registry+https://github.com/rust-lang/crates.io-index"126source = "registry+https://github.com/rust-lang/crates.io-index"
127checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"127checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
128128
129[[package]]
130name = "checker"
131version = "0.1.0"
132dependencies = [
133 "ast",
134 "miette",
135 "num",
136 "thiserror",
137]
138
139[[package]]129[[package]]
140name = "clap"130name = "clap"
141version = "4.4.14"131version = "4.4.14"
...@@ -181,12 +171,12 @@ name = "cli"...@@ -181,12 +171,12 @@ name = "cli"
181version = "0.1.0"171version = "0.1.0"
182dependencies = [172dependencies = [
183 "ast",173 "ast",
184 "checker",
185 "clap",174 "clap",
186 "js_ast",175 "js_ast",
187 "js_printer",176 "js_printer",
188 "miette",177 "miette",
189 "parser",178 "parser",
179 "sema",
190 "thiserror",180 "thiserror",
191 "tokenizer",181 "tokenizer",
192]182]
...@@ -466,6 +456,16 @@ dependencies = [...@@ -466,6 +456,16 @@ dependencies = [
466 "windows-sys",456 "windows-sys",
467]457]
468458
459[[package]]
460name = "sema"
461version = "0.1.0"
462dependencies = [
463 "ast",
464 "miette",
465 "num",
466 "thiserror",
467]
468
469[[package]]469[[package]]
470name = "smawk"470name = "smawk"
471version = "0.3.2"471version = "0.3.2"
Cargo.toml+1-1
...@@ -10,7 +10,7 @@ repository = "https://github.com/lino-levan/blight"...@@ -10,7 +10,7 @@ repository = "https://github.com/lino-levan/blight"
10resolver = "2"10resolver = "2"
11members = [11members = [
12 "crates/ast",12 "crates/ast",
13 "crates/checker", 13 "crates/sema",
14 "crates/cli",14 "crates/cli",
15 "crates/js_ast",15 "crates/js_ast",
16 "crates/js_printer",16 "crates/js_printer",
crates/ast/ast.rs+38-3
...@@ -12,6 +12,7 @@ pub enum BuiltinId {...@@ -12,6 +12,7 @@ pub enum BuiltinId {
1212
13#[derive(Debug, Clone)]13#[derive(Debug, Clone)]
14pub struct Module {14pub struct Module {
15 pub comment: Option<DocComment>,
15 pub statements: Vec<TopLevelStatement>,16 pub statements: Vec<TopLevelStatement>,
16}17}
1718
...@@ -31,7 +32,7 @@ pub struct Param {...@@ -31,7 +32,7 @@ pub struct Param {
3132
32pub struct Decl {33pub struct Decl {
33 pub name: String,34 pub name: String,
34 pub decl_type: Type,35 pub data_type: Type,
35 pub init: Expr,36 pub init: Expr,
36 pub is_constant: bool,37 pub is_constant: bool,
37 pub is_pub: bool,38 pub is_pub: bool,
...@@ -43,11 +44,12 @@ pub struct Decl {...@@ -43,11 +44,12 @@ pub struct Decl {
43pub struct Expr {44pub struct Expr {
44 pub data: ExprData,45 pub data: ExprData,
45 pub loc: Loc,46 pub loc: Loc,
47 pub comment: Option<Comment>,
46}48}
4749
48impl Expr {50impl Expr {
49 pub fn new(data: ExprData, loc: Loc) -> Self {51 pub fn new(data: ExprData, loc: Loc, comment: Option<Comment>) -> Self {
50 Self { data, loc }52 Self { data, loc, comment }
51 }53 }
52}54}
5355
...@@ -57,6 +59,35 @@ pub struct DocComment {...@@ -57,6 +59,35 @@ pub struct DocComment {
57 pub loc: Loc,59 pub loc: Loc,
58}60}
5961
62#[derive(Debug, Clone)]
63pub struct Comment {
64 pub data: String,
65}
66
67impl Comment {
68 pub fn format(&self) -> String {
69 let mut lines = self.data.lines();
70 let first_line = lines.next().unwrap();
71 let mut result = format!("// {}", first_line);
72 for line in lines {
73 result.push_str(&format!("\n// {}", line));
74 }
75 result
76 }
77}
78
79impl DocComment {
80 pub fn format(&self) -> String {
81 let mut lines = self.data.lines();
82 let first_line = lines.next().unwrap();
83 let mut result = format!("/// {}", first_line);
84 for line in lines {
85 result.push_str(&format!("\n/// {}", line));
86 }
87 result
88 }
89}
90
60#[derive(Debug, Clone)]91#[derive(Debug, Clone)]
61pub struct BlockWithLoc {92pub struct BlockWithLoc {
62 pub block: Block,93 pub block: Block,
...@@ -110,12 +141,14 @@ pub enum ExprData {...@@ -110,12 +141,14 @@ pub enum ExprData {
110pub enum Statement {141pub enum Statement {
111 Expr(Expr),142 Expr(Expr),
112 Decl(Decl),143 Decl(Decl),
144 Comment(Comment),
113}145}
114146
115#[derive(Debug, Clone)]147#[derive(Debug, Clone)]
116pub enum TopLevelStatement {148pub enum TopLevelStatement {
117 Expr(Expr),149 Expr(Expr),
118 Decl(Decl),150 Decl(Decl),
151 Comment(Comment),
119}152}
120153
121impl BuiltinId {154impl BuiltinId {
...@@ -149,6 +182,7 @@ impl Statement {...@@ -149,6 +182,7 @@ impl Statement {
149 match self {182 match self {
150 Statement::Expr(expr) => TopLevelStatement::Expr(expr),183 Statement::Expr(expr) => TopLevelStatement::Expr(expr),
151 Statement::Decl(decl) => TopLevelStatement::Decl(decl),184 Statement::Decl(decl) => TopLevelStatement::Decl(decl),
185 Statement::Comment(comment) => TopLevelStatement::Comment(comment),
152 }186 }
153 }187 }
154}188}
...@@ -169,6 +203,7 @@ impl TopLevelStatement {...@@ -169,6 +203,7 @@ impl TopLevelStatement {
169 match self {203 match self {
170 TopLevelStatement::Expr(expr) => Statement::Expr(expr),204 TopLevelStatement::Expr(expr) => Statement::Expr(expr),
171 TopLevelStatement::Decl(decl) => Statement::Decl(decl),205 TopLevelStatement::Decl(decl) => Statement::Decl(decl),
206 TopLevelStatement::Comment(comment) => Statement::Comment(comment),
172 }207 }
173 }208 }
174}209}
crates/ast/operators.rs+34
...@@ -169,3 +169,37 @@ impl UnaryOperator {...@@ -169,3 +169,37 @@ impl UnaryOperator {
169 }169 }
170 }170 }
171}171}
172
173/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
174#[derive(PartialEq, PartialOrd)]
175pub enum BinaryOperatorLevel {
176 Exponentiation, // **
177 Multiplicative, // *, /, %
178 Additive, // +, -
179 BitwiseShift, // <<, >>, >>>
180 Relational, // <, <=, >, >=
181 Equality, // ==, !=
182 BitwiseAnd, // &
183 BitwiseXor, // ^
184 BitwiseOr, // |
185 LogicalAnd, // &&
186 LogicalOr, // ||, ??
187}
188
189pub fn get_binary_op_level(token: &Token) -> Option<BinaryOperatorLevel> {
190 use BinaryOperatorLevel::*;
191 match token {
192 Token::StarStar => Some(Exponentiation),
193 Token::Star | Token::Slash | Token::Percent => Some(Multiplicative),
194 Token::Plus | Token::Minus => Some(Additive),
195 Token::LtLt | Token::GtGt | Token::GtGtGt => Some(BitwiseShift),
196 Token::Gt | Token::GtEqual | Token::Lt | Token::LtEqual => Some(Relational),
197 Token::EqualEqual | Token::BangEqual => Some(Equality),
198 Token::And => Some(BitwiseAnd),
199 Token::Caret => Some(BitwiseXor),
200 Token::Or => Some(BitwiseOr),
201 Token::AndAnd => Some(LogicalAnd),
202 Token::OrOr => Some(LogicalOr),
203 _ => None,
204 }
205}
crates/checker/Cargo.toml deleted-13
...@@ -1,13 +0,0 @@
1[package]
2name = "checker"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7
8[dependencies]
9miette.workspace = true
10thiserror.workspace = true
11num.workspace = true
12
13ast = { path = "../ast" }
\ No newline at end of file
crates/checker/src/ast_type_coersion.rs deleted-54
...@@ -1,54 +0,0 @@
1// use ast::{BinaryOperator, ExprData, Type};
2
3// impl TypeCoerceable for ExprData {
4// fn to_type(&self) -> Option<Type> {
5// match self {
6// ExprData::Block(block) => {
7// if (block.has_implicit_return) {
8// block.statements.last().unwrap().to_type()
9// } else {
10// Some(Type::Void)
11// }
12// }
13// ExprData::Binary { op, left, right } => {
14// // TODO: this likely has to do further type checking
15// let left_type = left.to_type();
16// match op {
17// BinaryOperator::Add => left_type,
18// BinaryOperator::Sub => left_type,
19// BinaryOperator::Mul => left_type,
20// BinaryOperator::Div => left_type,
21// BinaryOperator::Pow => left_type,
22// BinaryOperator::Eq => Some(Type::Bool),
23// BinaryOperator::Gt => Some(Type::Bool),
24// BinaryOperator::Lt => Some(Type::Bool),
25// BinaryOperator::Gte => Some(Type::Bool),
26// BinaryOperator::Lte => Some(Type::Bool),
27// BinaryOperator::Lshift => Some(Type::Int),
28// BinaryOperator::Rshift => Some(Type::Int),
29// BinaryOperator::BitAnd => Some(Type::Int),
30// BinaryOperator::BitOr => Some(Type::Int),
31// BinaryOperator::BitXor => Some(Type::Int),
32// BinaryOperator::Mod => Some(Type::Int),
33// BinaryOperator::NotEq => Some(Type::Int),
34
35// BinaryOperator::And => Some(Type::Bool),
36// BinaryOperator::Or => Some(Type::Bool),
37// }
38// }
39// ExprData::Unary { op, value } => todo!(),
40// ExprData::Function {
41// name,
42// params,
43// return_type,
44// block,
45// } => Some(return_type),
46// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47// ExprData::Builtin { id, arguments } => todo!(),
48// ExprData::If(_) => todo!(),
49// ExprData::Return(_) => todo!(),
50// ExprData::Literal(_) => todo!(),
51// ExprData::Identifier(_) => todo!(),
52// }
53// }
54// }
crates/checker/src/lib.rs deleted-96
...@@ -1,96 +0,0 @@
1// use ast::{ Module, Type};
2// use miette::Result;
3
4// mod ast_type_coersion;
5
6// pub fn check_module(block: &Module) -> Result<()> {
7// for statement in &block.statements {
8// check_expression(statement)?;
9// }
10
11// Ok(())
12// }
13
14// pub fn resolve_type(expr: &Ast) -> Result<Type> {
15// match expr {
16// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
17// Ast::Declaration {
18// name,
19// decl_type,
20// value,
21// is_constant,
22// } => Ok(Type::NoReturn),
23// Ast::Binary {
24// operator,
25// left,
26// right,
27// } => merge_types(resolve_type(left)?, resolve_type(right)?),
28// Ast::Function {
29// name,
30// arguments,
31// return_type,
32// block,
33// } => todo!(),
34// Ast::FunctionCall { name, arguments } => todo!(),
35// Ast::If {
36// condition,
37// value,
38// else_value,
39// } => todo!(),
40// Ast::Return(_) => Ok(Type::NoReturn),
41// Ast::Public(_) => todo!(), // TODO: remove Public
42// Ast::Block(_) => todo!(),
43// Ast::Bool(_) => Ok(Type::Bool),
44// Ast::String(_) => Ok(Type::String),
45// Ast::Identifier(_) => todo!(),
46// Ast::BuiltIn(_) => todo!(),
47
48// Ast::Comment(_) => unreachable!(),
49// Ast::DocComment(_) => unreachable!(),
50// }
51// }
52
53// pub fn check_expression(expr: &Ast) -> Result<()> {
54// panic!()
55// match expr {
56// Ast::Declaration {
57// name,
58// decl_type: return_type,
59// value,
60// is_constant,
61// } => {
62// check_expression(value)?;
63// // coerce_type(return_type.clone(), resolve_type(value)?)?;
64// }
65// _ => {}
66// // Ast::Operation {
67// // operator,
68// // left,
69// // right,
70// // } => Ok(()),
71// // Ast::Function {
72// // name,
73// // arguments,
74// // return_type,
75// // block,
76// // } => Ok(()),
77// // Ast::FunctionCall { name, arguments } => Ok(()),
78// // Ast::If {
79// // condition,
80// // value,
81// // else_value,
82// // } => Ok(()),
83// // Ast::Return(_) => Ok(()),
84// // Ast::Public(_) => Ok(()),
85// // Ast::Block(_) => Ok(()),
86// // Ast::Number(_) => Ok(()),
87// // Ast::Bool(_) => Ok(()),
88// // Ast::String(_) => Ok(()),
89// // Ast::Identifier(_) => Ok(()),
90// // Ast::BuiltIn(_) => Ok(()),
91// // Ast::Comment(_) => Ok(()),
92// // Ast::DocComment(_) => Ok(()),
93// }
94
95// Ok(())
96// }
crates/cli/Cargo.toml+1-1
...@@ -17,4 +17,4 @@ ast = { path = "../ast" }...@@ -17,4 +17,4 @@ ast = { path = "../ast" }
17parser = { path = "../parser" }17parser = { path = "../parser" }
18js_ast = { path = "../js_ast" }18js_ast = { path = "../js_ast" }
19js_printer = { path = "../js_printer" }19js_printer = { path = "../js_printer" }
20checker = { path = "../checker" }20sema = { path = "../sema" }
crates/cli/src/main.rs+1-1
...@@ -56,7 +56,7 @@ pub fn transpile(content: String) -> Result<String> {...@@ -56,7 +56,7 @@ pub fn transpile(content: String) -> Result<String> {
56 // 2. Parse the tokens into an AST56 // 2. Parse the tokens into an AST
57 let blight_ast = parser::parse(tokens)?;57 let blight_ast = parser::parse(tokens)?;
58 // 3. Type check the AST58 // 3. Type check the AST
59 // checker::check_module(&blight_ast)?;59 // sema::check_module(&blight_ast)?;
60 // 4. Optimize the AST60 // 4. Optimize the AST
61 // optimize(tree.clone())?;61 // optimize(tree.clone())?;
62 // 5. Translate the AST into javascript62 // 5. Translate the AST into javascript
crates/js_ast/src/lib.rs+1
...@@ -180,6 +180,7 @@ fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) {...@@ -180,6 +180,7 @@ fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) {
180 });180 });
181 }181 }
182 Statement::Expr(_) => todo!(),182 Statement::Expr(_) => todo!(),
183 Statement::Comment(_) => todo!(),
183 // Ast::Function {184 // Ast::Function {
184 // name,185 // name,
185 // arguments,186 // arguments,
crates/parser/parse_declaration.rs+8-4
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1use ast::{Decl, Type};1use ast::{Decl, DocComment, Type};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::Token;
44
...@@ -7,7 +7,11 @@ use crate::parse_type::parse_type;...@@ -7,7 +7,11 @@ use crate::parse_type::parse_type;
7use crate::Parser;7use crate::Parser;
88
9/// Precondition: The first token exists and is either 'var' or 'let'9/// Precondition: The first token exists and is either 'var' or 'let'
10pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {10pub fn parse_declaration(
11 p: &mut Parser,
12 is_pub: bool,
13 comment: Option<DocComment>,
14) -> Result<Decl> {
11 let loc = p.loc();15 let loc = p.loc();
12 let is_constant = match p.next() {16 let is_constant = match p.next() {
13 Some(Token::Var) => false,17 Some(Token::Var) => false,
...@@ -36,11 +40,11 @@ pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {...@@ -36,11 +40,11 @@ pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
36 };40 };
37 Ok(Decl {41 Ok(Decl {
38 name: name.clone(),42 name: name.clone(),
39 decl_type: return_type,43 data_type: return_type,
40 init,44 init,
41 is_constant,45 is_constant,
42 is_pub,46 is_pub,
43 comment: None,47 comment,
44 loc,48 loc,
45 })49 })
46}50}
crates/parser/parse_doc_comment.rs created+15
...@@ -0,0 +1,15 @@
1use ast::DocComment;
2use tokenizer::Token;
3
4use crate::Parser;
5
6pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
7 match p.peek() {
8 Some(Token::DocComment(data)) => {
9 let loc = p.loc();
10 p.next();
11 Some(DocComment { data, loc })
12 }
13 _ => None,
14 }
15}
crates/parser/parse_expr.rs+12-101
...@@ -1,45 +1,8 @@...@@ -1,45 +1,8 @@
1use ast::{BinaryOperator, Expr, ExprData, Literal};1use ast::{get_binary_op_level, BinaryOperator, BinaryOperatorLevel, Expr, ExprData};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;
43
5use crate::{4use crate::parse_expr_basic::parse_expr_basic;
6 parse_call_arguments::parse_call_arguments, parse_expr_basic::parse_expr_basic, Parser,5use crate::Parser;
7};
8
9/// Derived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
10
11#[derive(PartialEq, PartialOrd)]
12enum Level {
13 Prefix, // +, -, !, ~
14 Exponentiation, // **
15 Multiplicative, // *, /, %
16 Additive, // +, -
17 BitwiseShift, // <<, >>, >>>
18 Relational, // <, <=, >, >=
19 Equality, // ==, !=
20 BitwiseAnd, // &
21 BitwiseXor, // ^
22 BitwiseOr, // |
23 LogicalAnd, // &&
24 LogicalOr, // ||, ??
25}
26
27pub fn get_postfix_op_level(token: &Token) -> Option<Level> {
28 match token {
29 Token::StarStar => Some(Level::Exponentiation),
30 Token::Star | Token::Slash | Token::Percent => Some(Level::Multiplicative),
31 Token::Plus | Token::Minus => Some(Level::Additive),
32 Token::LtLt | Token::GtGt | Token::GtGtGt => Some(Level::BitwiseShift),
33 Token::Gt | Token::GtEqual | Token::Lt | Token::LtEqual => Some(Level::Relational),
34 Token::EqualEqual | Token::BangEqual => Some(Level::Equality),
35 Token::And => Some(Level::BitwiseAnd),
36 Token::Caret => Some(Level::BitwiseXor),
37 Token::Or => Some(Level::BitwiseOr),
38 Token::AndAnd => Some(Level::LogicalAnd),
39 Token::OrOr => Some(Level::LogicalOr),
40 _ => None,
41 }
42}
436
44pub fn parse_expr(p: &mut Parser) -> Result<Expr> {7pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
45 let mut expr = parse_expr_basic(p)?;8 let mut expr = parse_expr_basic(p)?;
...@@ -49,68 +12,10 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {...@@ -49,68 +12,10 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
49 return Ok(expr);12 return Ok(expr);
50 }13 }
5114
52 // Parse function calls and field accesses
53 loop {
54 match p.peek() {
55 Some(Token::Dot) => {
56 let loc = p.loc();
57 _ = p.next();
58 let field = match p.peek() {
59 Some(Token::Identifier(name)) => {
60 let loc = p.loc();
61 p.next();
62 Expr::new(ExprData::Literal(Literal::String(name)), loc)
63 }
64 _ => {
65 return Err(p.error("Expected identifier after dot".to_string()).into());
66 }
67 };
68 expr = Expr::new(
69 ExprData::Field {
70 base: Box::new(expr),
71 field: Box::new(field),
72 },
73 loc,
74 );
75 }
76 Some(Token::OpenParen) => {
77 let loc = p.loc();
78 _ = p.next();
79 let arguments = parse_call_arguments(p)?;
80 expr = Expr::new(
81 ExprData::Call {
82 base: Box::new(expr),
83 arguments,
84 },
85 loc,
86 );
87 }
88 Some(Token::OpenBracket) => {
89 let loc = p.loc();
90 _ = p.next();
91 let index = parse_expr(p)?;
92 if let Some(Token::ClosedBracket) = p.next() {
93 expr = Expr::new(
94 ExprData::Field {
95 base: Box::new(expr),
96 field: Box::new(index),
97 },
98 loc,
99 );
100 } else {
101 return Err(p
102 .error("Expected closing bracket after index".to_string())
103 .into());
104 }
105 }
106 _ => break,
107 }
108 }
109
110 // Parse operators15 // Parse operators
111 loop {16 loop {
112 if let Some(token) = p.peek() {17 if let Some(token) = p.peek() {
113 if let Some(level) = get_postfix_op_level(&token) {18 if let Some(level) = get_binary_op_level(&token) {
114 _ = p.next();19 _ = p.next();
115 expr = parse_expr_binary(20 expr = parse_expr_binary(
116 p,21 p,
...@@ -128,13 +33,18 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {...@@ -128,13 +33,18 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
128}33}
12934
130/// Prerequisite: `p.peek()` is an operator35/// Prerequisite: `p.peek()` is an operator
131fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Level) -> Result<Expr> {36fn parse_expr_binary(
37 p: &mut Parser,
38 op: BinaryOperator,
39 left: Expr,
40 level: BinaryOperatorLevel,
41) -> Result<Expr> {
132 let loc = p.loc();42 let loc = p.loc();
133 let mut right = parse_expr_basic(p)?;43 let mut right = parse_expr_basic(p)?;
13444
135 loop {45 loop {
136 if let Some(token) = p.peek() {46 if let Some(token) = p.peek() {
137 if let Some(next_level) = get_postfix_op_level(&token) {47 if let Some(next_level) = get_binary_op_level(&token) {
138 if next_level <= level {48 if next_level <= level {
139 _ = p.next();49 _ = p.next();
140 right = parse_expr_binary(50 right = parse_expr_binary(
...@@ -157,5 +67,6 @@ fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Leve...@@ -157,5 +67,6 @@ fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Leve
157 right: Box::new(right),67 right: Box::new(right),
158 },68 },
159 loc,69 loc,
70 None,
160 ))71 ))
161}72}
crates/parser/parse_expr_basic.rs+90-20
...@@ -9,32 +9,35 @@ use crate::{...@@ -9,32 +9,35 @@ use crate::{
99
10pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {10pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
11 let loc = p.loc();11 let loc = p.loc();
12 match p.peek() {12 let comments = p.take_comments();
13 let mut expr = match p.peek() {
13 Some(Token::Number(number)) => {14 Some(Token::Number(number)) => {
14 _ = p.next();15 _ = p.next();
15 Ok(Expr::new(16 Expr::new(
16 ExprData::Literal(Literal::Number(number.clone())),17 ExprData::Literal(Literal::Number(number.clone())),
17 loc,18 loc,
18 ))19 comments,
20 )
19 }21 }
20 Some(Token::String(string)) => {22 Some(Token::String(string)) => {
21 _ = p.next();23 _ = p.next();
22 Ok(Expr::new(24 Expr::new(
23 ExprData::Literal(Literal::String(string.clone())),25 ExprData::Literal(Literal::String(string.clone())),
24 loc,26 loc,
25 ))27 comments,
28 )
26 }29 }
27 Some(Token::True) => {30 Some(Token::True) => {
28 _ = p.next();31 _ = p.next();
29 Ok(Expr::new(ExprData::Literal(Literal::Bool(true)), loc))32 Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments)
30 }33 }
31 Some(Token::False) => {34 Some(Token::False) => {
32 _ = p.next();35 _ = p.next();
33 Ok(Expr::new(ExprData::Literal(Literal::Bool(false)), loc))36 Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments)
34 }37 }
35 Some(Token::Void) => {38 Some(Token::Void) => {
36 _ = p.next();39 _ = p.next();
37 Ok(Expr::new(ExprData::Literal(Literal::Void), loc))40 Expr::new(ExprData::Literal(Literal::Void), loc, comments)
38 }41 }
39 Some(Token::Identifier(name)) => {42 Some(Token::Identifier(name)) => {
40 let loc = p.loc();43 let loc = p.loc();
...@@ -43,7 +46,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {...@@ -43,7 +46,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
43 if let Some(id) = BuiltinId::from_str(name.as_str()) {46 if let Some(id) = BuiltinId::from_str(name.as_str()) {
44 if let Some(Token::OpenParen) = p.next() {47 if let Some(Token::OpenParen) = p.next() {
45 let arguments = parse_call_arguments(p)?;48 let arguments = parse_call_arguments(p)?;
46 return Ok(Expr::new(ExprData::Builtin { id, arguments }, loc));49 Expr::new(ExprData::Builtin { id, arguments }, loc, comments)
47 } else {50 } else {
48 return Err(p51 return Err(p
49 .error_at(format!("Built-in function '{}' must be called", name), loc)52 .error_at(format!("Built-in function '{}' must be called", name), loc)
...@@ -55,7 +58,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {...@@ -55,7 +58,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
55 .into());58 .into());
56 }59 }
57 } else {60 } else {
58 Ok(Expr::new(ExprData::Identifier(name), loc))61 Expr::new(ExprData::Identifier(name), loc, comments)
59 }62 }
60 }63 }
6164
...@@ -66,18 +69,21 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {...@@ -66,18 +69,21 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
66 .into());69 .into());
67 } else {70 } else {
68 _ = p.next();71 _ = p.next();
6972 return Ok(Expr::new(
70 return Ok(Expr::new(ExprData::Return(Box::new(parse_expr(p)?)), loc));73 ExprData::Return(Box::new(parse_expr(p)?)),
74 loc,
75 comments,
76 ));
71 }77 }
72 }78 }
7379
74 Some(Token::If) => Ok(Expr::new(ExprData::If(parse_expr_if(p)?), loc)),80 Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments),
7581
76 Some(Token::OpenBrace) => Ok(Expr::new(ExprData::Block(parse_block(p)?), loc)),82 Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments),
7783
78 Some(Token::OpenParen) => {84 Some(Token::OpenParen) => {
79 _ = p.next();85 _ = p.next();
80 let expr = parse_expr_basic(p)?;86 let expr = parse_expr(p)?;
81 match p.next() {87 match p.next() {
82 Some(Token::ClosedParen) => (),88 Some(Token::ClosedParen) => (),
83 Some(token) => {89 Some(token) => {
...@@ -85,12 +91,12 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {...@@ -85,12 +91,12 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
85 }91 }
86 None => return Err(p.error("Unexpected end of file".to_string()).into()),92 None => return Err(p.error("Unexpected end of file".to_string()).into()),
87 }93 }
88 Ok(expr)94 expr
89 }95 }
9096
91 Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => {97 Some(t @ Token::Bang) | Some(t @ Token::Minus) | Some(t @ Token::Tilde) => {
92 _ = p.next();98 _ = p.next();
93 Ok(Expr::new(99 Expr::new(
94 ExprData::Unary {100 ExprData::Unary {
95 op: match t {101 op: match t {
96 Token::Bang => UnaryOperator::Not,102 Token::Bang => UnaryOperator::Not,
...@@ -101,10 +107,74 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {...@@ -101,10 +107,74 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
101 value: Box::new(parse_expr_basic(p)?),107 value: Box::new(parse_expr_basic(p)?),
102 },108 },
103 loc,109 loc,
104 ))110 comments,
111 )
105 }112 }
106113
107 Some(t) => Err(p.error(format!("Unexpected '{}'", t)).into()),114 Some(t) => return Err(p.error(format!("Unexpected '{}'", t)).into()),
108 None => Err(p.error("Unexpected end of file".to_string()).into()),115 None => return Err(p.error("Unexpected end of file".to_string()).into()),
116 };
117
118 // Parse function calls and field accesses
119 loop {
120 match p.peek() {
121 Some(Token::Dot) => {
122 let loc = p.loc();
123 _ = p.next();
124 let field = match p.peek() {
125 Some(Token::Identifier(name)) => {
126 let loc = p.loc();
127 p.next();
128 Expr::new(ExprData::Literal(Literal::String(name)), loc, None)
129 }
130 _ => {
131 return Err(p.error("Expected identifier after dot".to_string()).into());
132 }
133 };
134 expr = Expr::new(
135 ExprData::Field {
136 base: Box::new(expr),
137 field: Box::new(field),
138 },
139 loc,
140 None,
141 );
142 }
143 Some(Token::OpenParen) => {
144 let loc = p.loc();
145 _ = p.next();
146 let arguments = parse_call_arguments(p)?;
147 expr = Expr::new(
148 ExprData::Call {
149 base: Box::new(expr),
150 arguments,
151 },
152 loc,
153 None,
154 );
155 }
156 Some(Token::OpenBracket) => {
157 let loc = p.loc();
158 _ = p.next();
159 let index = parse_expr(p)?;
160 if let Some(Token::ClosedBracket) = p.next() {
161 expr = Expr::new(
162 ExprData::Field {
163 base: Box::new(expr),
164 field: Box::new(index),
165 },
166 loc,
167 None,
168 );
169 } else {
170 return Err(p
171 .error("Expected closing bracket after index".to_string())
172 .into());
173 }
174 }
175 _ => break,
176 }
109 }177 }
178
179 Ok(expr)
110}180}
crates/parser/parse_expr_if.rs+3-1
...@@ -2,7 +2,9 @@ use ast::{BlockWithLoc, If};...@@ -2,7 +2,9 @@ use ast::{BlockWithLoc, If};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::Token;
44
5use crate::{parse_block::parse_block, parse_expr::parse_expr, Parser};5use crate::parse_block::parse_block;
6use crate::parse_expr::parse_expr;
7use crate::Parser;
68
7/// Precondition: the first token exists and is 'if'9/// Precondition: the first token exists and is 'if'
8pub fn parse_expr_if(p: &mut Parser) -> Result<If> {10pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
crates/parser/parse_fn_declaration.rs+9-5
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1use ast::{Decl, Expr, ExprData, Type};1use ast::{Decl, DocComment, Expr, ExprData, Type};
2use miette::Result;2use miette::Result;
3use tokenizer::Token;3use tokenizer::Token;
44
...@@ -8,7 +8,11 @@ use crate::parse_type::parse_type;...@@ -8,7 +8,11 @@ use crate::parse_type::parse_type;
8use crate::Parser;8use crate::Parser;
99
10/// Precondition:10/// Precondition:
11pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {11pub fn parse_fn_declaration(
12 p: &mut Parser,
13 is_pub: bool,
14 comment: Option<DocComment>,
15) -> Result<Decl> {
12 let loc = p.loc();16 let loc = p.loc();
13 match p.next() {17 match p.next() {
14 Some(Token::Fn) => (),18 Some(Token::Fn) => (),
...@@ -96,11 +100,11 @@ pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {...@@ -96,11 +100,11 @@ pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
96 };100 };
97 Ok(Decl {101 Ok(Decl {
98 name,102 name,
99 decl_type: Type::Any,103 data_type: Type::Any,
100 init: Expr::new(function_body, block_loc),104 init: Expr::new(function_body, block_loc, None),
101 is_constant: true,105 is_constant: true,
102 is_pub,106 is_pub,
103 comment: None,107 comment,
104 loc: loc,108 loc: loc,
105 })109 })
106 } else {110 } else {
crates/parser/parse_module.rs+16-9
...@@ -1,23 +1,27 @@...@@ -1,23 +1,27 @@
1use crate::{parse_top_level_statement::parse_top_level_statement, Parser};1use crate::parse_doc_comment::maybe_read_doc_comment;
2use crate::parse_top_level_statement::parse_top_level_statement;
3use crate::Parser;
2use ast::Module;4use ast::Module;
3use miette::Result;5use miette::Result;
4use tokenizer::Token;6use tokenizer::Token;
57
6pub fn parse_module(tokens: &mut Parser) -> Result<Module> {8pub fn parse_module(p: &mut Parser) -> Result<Module> {
7 let mut statements = Vec::new();9 let mut statements = Vec::new();
810
11 let file_doc_comment = maybe_read_doc_comment(p);
12
9 loop {13 loop {
10 let statement = parse_top_level_statement(tokens)?;14 let statement = parse_top_level_statement(p)?;
1115
12 match tokens.peek() {16 match p.peek() {
13 Some(Token::Semicolon) => {17 Some(Token::Semicolon) => {
14 while let Some(Token::Semicolon) = tokens.peek() {18 while let Some(Token::Semicolon) = p.peek() {
15 tokens.next();19 p.next();
16 }20 }
17 }21 }
18 _ => {22 _ => {
19 if statement.requires_semicolon() {23 if statement.requires_semicolon() {
20 return Err(tokens24 return Err(p
21 .error("Expected semicolon after statement".to_string())25 .error("Expected semicolon after statement".to_string())
22 .into());26 .into());
23 }27 }
...@@ -26,8 +30,11 @@ pub fn parse_module(tokens: &mut Parser) -> Result<Module> {...@@ -26,8 +30,11 @@ pub fn parse_module(tokens: &mut Parser) -> Result<Module> {
2630
27 statements.push(statement);31 statements.push(statement);
2832
29 if tokens.empty() {33 if p.empty() {
30 return Ok(Module { statements });34 return Ok(Module {
35 statements,
36 comment: file_doc_comment,
37 });
31 }38 }
32 }39 }
33}40}
crates/parser/parse_statement.rs+39-5
...@@ -3,26 +3,60 @@ use miette::Result;...@@ -3,26 +3,60 @@ use miette::Result;
3use tokenizer::Token;3use tokenizer::Token;
44
5use crate::parse_declaration::parse_declaration;5use crate::parse_declaration::parse_declaration;
6use crate::parse_doc_comment::maybe_read_doc_comment;
6use crate::parse_expr::parse_expr;7use crate::parse_expr::parse_expr;
7use crate::parse_fn_declaration::parse_fn_declaration;8use crate::parse_fn_declaration::parse_fn_declaration;
8use crate::Parser;9use crate::Parser;
910
10pub fn parse_statement(p: &mut Parser) -> Result<Statement> {11pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
12 if let Some(comments) = p.take_comments() {
13 return Ok(Statement::Comment(comments));
14 }
15
16 let m_doc_comment = maybe_read_doc_comment(p);
17 if let Some(doc_comment) = m_doc_comment.clone() {
18 if p.has_comments() {
19 // this implies that comments are immediately after the doc
20 return Err(p
21 .error_at(format!("Unattached doc comment"), doc_comment.loc)
22 .into());
23 }
24 }
25
11 match p.peek() {26 match p.peek() {
27 Some(Token::DocComment(_)) => unreachable!(),
12 Some(Token::Pub) => {28 Some(Token::Pub) => {
13 _ = p.next();29 _ = p.next();
14 match p.peek() {30 match p.peek() {
15 Some(Token::Var) | Some(Token::Let) => {31 Some(Token::Var) | Some(Token::Let) => {
16 Ok(Statement::Decl(parse_declaration(p, true)?))32 Ok(Statement::Decl(parse_declaration(p, true, m_doc_comment)?))
17 }33 }
18 Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, true)?)),34 Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(
35 p,
36 true,
37 m_doc_comment,
38 )?)),
19 Some(_) => Err(p.error("Expected a declaration".to_string()).into()),39 Some(_) => Err(p.error("Expected a declaration".to_string()).into()),
20 None => Err(p.error("Unexpected end of file".to_string()).into()),40 None => Err(p.error("Unexpected end of file".to_string()).into()),
21 }41 }
22 }42 }
23 Some(Token::Var) | Some(Token::Let) => Ok(Statement::Decl(parse_declaration(p, false)?)),43 Some(Token::Var) | Some(Token::Let) => {
24 Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, false)?)),44 Ok(Statement::Decl(parse_declaration(p, false, m_doc_comment)?))
25 Some(_) => Ok(Statement::Expr(parse_expr(p)?)),45 }
46 Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(
47 p,
48 false,
49 m_doc_comment,
50 )?)),
51 Some(_) => {
52 if let Some(doc_comment) = m_doc_comment {
53 Err(p
54 .error_at(format!("Unattached doc comment"), doc_comment.loc)
55 .into())
56 } else {
57 Ok(Statement::Expr(parse_expr(p)?))
58 }
59 }
26 None => Err(p.error("Unexpected end of file".to_string()).into()),60 None => Err(p.error("Unexpected end of file".to_string()).into()),
27 }61 }
28}62}
crates/parser/parser.rs+56-16
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1mod parse_block;1mod parse_block;
2mod parse_call_arguments;2mod parse_call_arguments;
3mod parse_declaration;3mod parse_declaration;
4mod parse_doc_comment;
4mod parse_expr;5mod parse_expr;
5mod parse_expr_basic;6mod parse_expr_basic;
6mod parse_expr_if;7mod parse_expr_if;
...@@ -11,15 +12,14 @@ mod parse_statement;...@@ -11,15 +12,14 @@ mod parse_statement;
11mod parse_top_level_statement;12mod parse_top_level_statement;
12mod parse_type;13mod parse_type;
1314
14use tokenizer::{Loc, Token};15use ast::Comment;
15
16use miette::Diagnostic;
17use thiserror::Error;
18
19use ast::Module;16use ast::Module;
17use miette::Diagnostic;
20use miette::Result;18use miette::Result;
21use parse_module::parse_module;19use parse_module::parse_module;
20use thiserror::Error;
22use tokenizer::tokenize;21use tokenizer::tokenize;
22use tokenizer::{Loc, Token};
2323
24pub fn parse(tokens: Vec<Token>) -> Result<Module> {24pub fn parse(tokens: Vec<Token>) -> Result<Module> {
25 let mut p = Parser::new(tokens);25 let mut p = Parser::new(tokens);
...@@ -47,11 +47,15 @@ impl ParseError {...@@ -47,11 +47,15 @@ impl ParseError {
4747
48#[derive(Debug)]48#[derive(Debug)]
49pub struct Parser {49pub struct Parser {
50 pub tokens: Vec<Token>,50 /// TODO(@paperdave): switch to streaming tokenizer + one that can
51 pub cursor: usize,51 /// report line and column info
52 tokens: Vec<Token>,
53 cursor: usize,
5254
53 /// Whether or not the parser is at the top level55 /// Whether or not the parser is at the top level
54 pub is_top_level: bool,56 pub is_top_level: bool,
57
58 comments: Vec<String>,
55}59}
5660
57impl Parser {61impl Parser {
...@@ -60,6 +64,7 @@ impl Parser {...@@ -60,6 +64,7 @@ impl Parser {
60 tokens,64 tokens,
61 cursor: 0,65 cursor: 0,
62 is_top_level: true,66 is_top_level: true,
67 comments: vec![],
63 }68 }
64 }69 }
6570
...@@ -67,26 +72,61 @@ impl Parser {...@@ -67,26 +72,61 @@ impl Parser {
67 self.cursor == self.tokens.len()72 self.cursor == self.tokens.len()
68 }73 }
6974
70 pub fn back(&mut self) {75 // TODO(@paperdave): fix parser token stuff. its very messy rn
71 self.cursor -= 1;
72 }
7376
74 pub fn next(&mut self) -> Option<Token> {77 pub fn next(&mut self) -> Option<Token> {
75 if self.cursor == self.tokens.len() {78 if self.cursor == self.tokens.len() {
76 return None;79 return None;
77 }80 }
7881
79 let token = self.tokens[self.cursor].clone();82 loop {
80 self.cursor += 1;83 match self.tokens[self.cursor].clone() {
81 Some(token)84 Token::Comment(comment) => {
85 self.comments.push(comment);
86 self.cursor += 1;
87 continue;
88 }
89 token => {
90 self.cursor += 1;
91 return Some(token);
92 }
93 }
94 }
82 }95 }
8396
84 pub fn peek(&self) -> Option<Token> {97 pub fn take_comments(&mut self) -> Option<Comment> {
85 if self.cursor == self.tokens.len() {98 if self.comments.len() == 0 {
86 return None;99 return None;
87 }100 }
101 let comments = self.comments.clone();
102 self.comments = vec![];
103 Some(Comment {
104 data: comments.join("\n"),
105 })
106 }
107
108 pub fn has_comments(&mut self) -> bool {
109 self.peek();
110 self.comments.len() > 0
111 }
88112
89 Some(self.tokens[self.cursor].clone())113 pub fn peek(&mut self) -> Option<Token> {
114 loop {
115 if self.cursor == self.tokens.len() {
116 return None;
117 }
118
119 match self.tokens[self.cursor].clone() {
120 Token::Comment(comment) => {
121 self.comments.push(comment);
122 self.cursor += 1;
123 continue;
124 }
125 token => {
126 return Some(token);
127 }
128 }
129 }
90 }130 }
91131
92 pub fn error(&self, error: String) -> ParseError {132 pub fn error(&self, error: String) -> ParseError {
crates/sema/Cargo.toml created+13
...@@ -0,0 +1,13 @@
1[package]
2name = "sema"
3version = "0.1.0"
4edition = "2021"
5
6[lib]
7
8[dependencies]
9miette.workspace = true
10thiserror.workspace = true
11num.workspace = true
12
13ast = { path = "../ast" }
\ No newline at end of file
crates/sema/src/ast_type_coersion.rs created+54
...@@ -0,0 +1,54 @@
1// use ast::{BinaryOperator, ExprData, Type};
2
3// impl TypeCoerceable for ExprData {
4// fn to_type(&self) -> Option<Type> {
5// match self {
6// ExprData::Block(block) => {
7// if (block.has_implicit_return) {
8// block.statements.last().unwrap().to_type()
9// } else {
10// Some(Type::Void)
11// }
12// }
13// ExprData::Binary { op, left, right } => {
14// // TODO: this likely has to do further type checking
15// let left_type = left.to_type();
16// match op {
17// BinaryOperator::Add => left_type,
18// BinaryOperator::Sub => left_type,
19// BinaryOperator::Mul => left_type,
20// BinaryOperator::Div => left_type,
21// BinaryOperator::Pow => left_type,
22// BinaryOperator::Eq => Some(Type::Bool),
23// BinaryOperator::Gt => Some(Type::Bool),
24// BinaryOperator::Lt => Some(Type::Bool),
25// BinaryOperator::Gte => Some(Type::Bool),
26// BinaryOperator::Lte => Some(Type::Bool),
27// BinaryOperator::Lshift => Some(Type::Int),
28// BinaryOperator::Rshift => Some(Type::Int),
29// BinaryOperator::BitAnd => Some(Type::Int),
30// BinaryOperator::BitOr => Some(Type::Int),
31// BinaryOperator::BitXor => Some(Type::Int),
32// BinaryOperator::Mod => Some(Type::Int),
33// BinaryOperator::NotEq => Some(Type::Int),
34
35// BinaryOperator::And => Some(Type::Bool),
36// BinaryOperator::Or => Some(Type::Bool),
37// }
38// }
39// ExprData::Unary { op, value } => todo!(),
40// ExprData::Function {
41// name,
42// params,
43// return_type,
44// block,
45// } => Some(return_type),
46// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47// ExprData::Builtin { id, arguments } => todo!(),
48// ExprData::If(_) => todo!(),
49// ExprData::Return(_) => todo!(),
50// ExprData::Literal(_) => todo!(),
51// ExprData::Identifier(_) => todo!(),
52// }
53// }
54// }
crates/sema/src/lib.rs created+96
...@@ -0,0 +1,96 @@
1// use ast::{ Module, Type};
2// use miette::Result;
3
4// mod ast_type_coersion;
5
6// pub fn check_module(block: &Module) -> Result<()> {
7// for statement in &block.statements {
8// check_expression(statement)?;
9// }
10
11// Ok(())
12// }
13
14// pub fn resolve_type(expr: &Ast) -> Result<Type> {
15// match expr {
16// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
17// Ast::Declaration {
18// name,
19// decl_type,
20// value,
21// is_constant,
22// } => Ok(Type::NoReturn),
23// Ast::Binary {
24// operator,
25// left,
26// right,
27// } => merge_types(resolve_type(left)?, resolve_type(right)?),
28// Ast::Function {
29// name,
30// arguments,
31// return_type,
32// block,
33// } => todo!(),
34// Ast::FunctionCall { name, arguments } => todo!(),
35// Ast::If {
36// condition,
37// value,
38// else_value,
39// } => todo!(),
40// Ast::Return(_) => Ok(Type::NoReturn),
41// Ast::Public(_) => todo!(), // TODO: remove Public
42// Ast::Block(_) => todo!(),
43// Ast::Bool(_) => Ok(Type::Bool),
44// Ast::String(_) => Ok(Type::String),
45// Ast::Identifier(_) => todo!(),
46// Ast::BuiltIn(_) => todo!(),
47
48// Ast::Comment(_) => unreachable!(),
49// Ast::DocComment(_) => unreachable!(),
50// }
51// }
52
53// pub fn check_expression(expr: &Ast) -> Result<()> {
54// panic!()
55// match expr {
56// Ast::Declaration {
57// name,
58// decl_type: return_type,
59// value,
60// is_constant,
61// } => {
62// check_expression(value)?;
63// // coerce_type(return_type.clone(), resolve_type(value)?)?;
64// }
65// _ => {}
66// // Ast::Operation {
67// // operator,
68// // left,
69// // right,
70// // } => Ok(()),
71// // Ast::Function {
72// // name,
73// // arguments,
74// // return_type,
75// // block,
76// // } => Ok(()),
77// // Ast::FunctionCall { name, arguments } => Ok(()),
78// // Ast::If {
79// // condition,
80// // value,
81// // else_value,
82// // } => Ok(()),
83// // Ast::Return(_) => Ok(()),
84// // Ast::Public(_) => Ok(()),
85// // Ast::Block(_) => Ok(()),
86// // Ast::Number(_) => Ok(()),
87// // Ast::Bool(_) => Ok(()),
88// // Ast::String(_) => Ok(()),
89// // Ast::Identifier(_) => Ok(()),
90// // Ast::BuiltIn(_) => Ok(()),
91// // Ast::Comment(_) => Ok(()),
92// // Ast::DocComment(_) => Ok(()),
93// }
94
95// Ok(())
96// }
crates/tokenizer/src/lib.rs+49-13
...@@ -71,12 +71,12 @@ pub enum Token {...@@ -71,12 +71,12 @@ pub enum Token {
71impl std::fmt::Display for Token {71impl std::fmt::Display for Token {
72 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {72 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
73 let token = match self {73 let token = match self {
74 Token::Identifier(_) => "an identifier".to_string(),74 Token::Identifier(_) => "identifier".to_string(),
7575
76 Token::Number(n) => n.to_string(),76 Token::Number(n) => n.to_string(),
77 Token::String(s) => format!("\"{}\"", s),77 Token::String(s) => format!("\"{}\"", s),
78 Token::Comment(_) => "a comment".to_string(),78 Token::Comment(_) => "comment".to_string(),
79 Token::DocComment(_) => "a documentation comment".to_string(),79 Token::DocComment(_) => "documentation comment".to_string(),
8080
81 Token::Else => "else".to_string(),81 Token::Else => "else".to_string(),
82 Token::False => "false".to_string(),82 Token::False => "false".to_string(),
...@@ -129,6 +129,7 @@ impl std::fmt::Display for Token {...@@ -129,6 +129,7 @@ impl std::fmt::Display for Token {
129 }129 }
130}130}
131131
132// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
132pub fn tokenize(content: String) -> Result<Vec<Token>> {133pub fn tokenize(content: String) -> Result<Vec<Token>> {
133 let mut tokens: Vec<Token> = Vec::new();134 let mut tokens: Vec<Token> = Vec::new();
134 let mut chars = content.chars().enumerate().peekable();135 let mut chars = content.chars().enumerate().peekable();
...@@ -277,20 +278,55 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {...@@ -277,20 +278,55 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {
277278
278 let mut comment = String::new();279 let mut comment = String::new();
279 loop {280 loop {
280 let c = chars.next();281 // skip leading whitespace
281 if c.is_none() {282 while let Some((_, c)) = chars.peek() {
282 break;283 if !c.is_whitespace() {
284 break;
285 }
286 chars.next();
283 }287 }
284 let (_, c) = c.unwrap();288 // read the comment
285 match c {289 while let Some((_, c)) = chars.next() {
286 '\n' => break,290 if c == '\n' || c == '\r' {
287 _ => {291 break;
288 comment.push(c.clone());
289 }292 }
293 comment.push(c);
290 }294 }
291 }295 // trim trailing whitespace on line
296 comment = comment.trim_end().to_string();
297 // skip to next non-whitespace character
298 while let Some((_, c)) = chars.peek() {
299 if !c.is_whitespace() {
300 break;
301 }
302 chars.next();
303 }
304 // check if we've hit another comment
305 let mut tmp = chars.clone();
292306
293 comment = comment.trim().to_string();307 let is_another_comment = match tmp.peek() {
308 Some((_, '/')) => {
309 tmp.next(); // Consume the second slash
310 match tmp.peek() {
311 Some((_, '/')) => {
312 tmp.next(); // Consume the third slash
313 is_doc_comment
314 }
315 _ => !is_doc_comment,
316 }
317 }
318 _ => false,
319 };
320
321 if is_another_comment {
322 // apply our temporary iterator to the main iterator
323 comment.push('\n');
324 chars = tmp;
325 chars.next();
326 } else {
327 break;
328 }
329 }
294330
295 if is_doc_comment {331 if is_doc_comment {
296 tokens.push(Token::DocComment(comment));332 tokens.push(Token::DocComment(comment));
examples/basics/comments.blight+2-9
...@@ -1,9 +1,2 @@...@@ -1,9 +1,2 @@
1// this is a regular comment12 // yapping
2let x: int = 0;2 + 2;
3
4/// this is a doc comment
5let y: int = 0;
6
7/// Doc comments can be multiple lines.
8/// This can be done by simply adding another triple-slash to the next line.
9let z: int = 0;
\ No newline at end of file