Commit c89bf26922f4448e6f0cb6015f4627792841deab | ix

Cargo.lock+11-11

...	...	@@ -126,16 +126,6 @@ version = "1.0.0"
126	126	source = "registry+https://github.com/rust-lang/crates.io-index"
127	127	checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
128	128
129		[[package]]
130		name = "checker"
131		version = "0.1.0"
132		dependencies = [
133		"ast",
134		"miette",
135		"num",
136		"thiserror",
137		]
138
139	129	[[package]]
140	130	name = "clap"
141	131	version = "4.4.14"
...	...	@@ -181,12 +171,12 @@ name = "cli"
181	171	version = "0.1.0"
182	172	dependencies = [
183	173	"ast",
184		"checker",
185	174	"clap",
186	175	"js_ast",
187	176	"js_printer",
188	177	"miette",
189	178	"parser",
	179	"sema",
190	180	"thiserror",
191	181	"tokenizer",
192	182	]
...	...	@@ -466,6 +456,16 @@ dependencies = [
466	456	"windows-sys",
467	457	]
468	458
	459	[[package]]
	460	name = "sema"
	461	version = "0.1.0"
	462	dependencies = [
	463	"ast",
	464	"miette",
	465	"num",
	466	"thiserror",
	467	]
	468
469	469	[[package]]
470	470	name = "smawk"
471	471	version = "0.3.2"

Cargo.toml+1-1

...	...	@@ -10,7 +10,7 @@ repository = "https://github.com/lino-levan/blight"
10	10	resolver = "2"
11	11	members = [
12	12	"crates/ast",
13		"crates/checker",
	13	"crates/sema",
14	14	"crates/cli",
15	15	"crates/js_ast",
16	16	"crates/js_printer",

crates/ast/ast.rs+38-3

...	...	@@ -12,6 +12,7 @@ pub enum BuiltinId {
12	12
13	13	#[derive(Debug, Clone)]
14	14	pub struct Module {
	15	pub comment: Option<DocComment>,
15	16	pub statements: Vec<TopLevelStatement>,
16	17	}
17	18
...	...	@@ -31,7 +32,7 @@ pub struct Param {
31	32
32	33	pub struct Decl {
33	34	pub name: String,
34		pub decl_type: Type,
	35	pub data_type: Type,
35	36	pub init: Expr,
36	37	pub is_constant: bool,
37	38	pub is_pub: bool,
...	...	@@ -43,11 +44,12 @@ pub struct Decl {
43	44	pub struct Expr {
44	45	pub data: ExprData,
45	46	pub loc: Loc,
	47	pub comment: Option<Comment>,
46	48	}
47	49
48	50	impl Expr {
49		pub fn new(data: ExprData, loc: Loc) -> Self {
50		Self { data, loc }
	51	pub fn new(data: ExprData, loc: Loc, comment: Option<Comment>) -> Self {
	52	Self { data, loc, comment }
51	53	}
52	54	}
53	55
...	...	@@ -57,6 +59,35 @@ pub struct DocComment {
57	59	pub loc: Loc,
58	60	}
59	61
	62	#[derive(Debug, Clone)]
	63	pub struct Comment {
	64	pub data: String,
	65	}
	66
	67	impl Comment {
	68	pub fn format(&self) -> String {
	69	let mut lines = self.data.lines();
	70	let first_line = lines.next().unwrap();
	71	let mut result = format!("// {}", first_line);
	72	for line in lines {
	73	result.push_str(&format!("\n// {}", line));
	74	}
	75	result
	76	}
	77	}
	78
	79	impl DocComment {
	80	pub fn format(&self) -> String {
	81	let mut lines = self.data.lines();
	82	let first_line = lines.next().unwrap();
	83	let mut result = format!("/// {}", first_line);
	84	for line in lines {
	85	result.push_str(&format!("\n/// {}", line));
	86	}
	87	result
	88	}
	89	}
	90
60	91	#[derive(Debug, Clone)]
61	92	pub struct BlockWithLoc {
62	93	pub block: Block,
...	...	@@ -110,12 +141,14 @@ pub enum ExprData {
110	141	pub enum Statement {
111	142	Expr(Expr),
112	143	Decl(Decl),
	144	Comment(Comment),
113	145	}
114	146
115	147	#[derive(Debug, Clone)]
116	148	pub enum TopLevelStatement {
117	149	Expr(Expr),
118	150	Decl(Decl),
	151	Comment(Comment),
119	152	}
120	153
121	154	impl BuiltinId {
...	...	@@ -149,6 +182,7 @@ impl Statement {
149	182	match self {
150	183	Statement::Expr(expr) => TopLevelStatement::Expr(expr),
151	184	Statement::Decl(decl) => TopLevelStatement::Decl(decl),
	185	Statement::Comment(comment) => TopLevelStatement::Comment(comment),
152	186	}
153	187	}
154	188	}
...	...	@@ -169,6 +203,7 @@ impl TopLevelStatement {
169	203	match self {
170	204	TopLevelStatement::Expr(expr) => Statement::Expr(expr),
171	205	TopLevelStatement::Decl(decl) => Statement::Decl(decl),
	206	TopLevelStatement::Comment(comment) => Statement::Comment(comment),
172	207	}
173	208	}
174	209	}

crates/ast/operators.rs+34

...	...	@@ -169,3 +169,37 @@ impl UnaryOperator {
169	169	}
170	170	}
171	171	}
	172
	173	/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
	174	#[derive(PartialEq, PartialOrd)]
	175	pub enum BinaryOperatorLevel {
	176	Exponentiation, // **
	177	Multiplicative, // *, /, %
	178	Additive, // +, -
	179	BitwiseShift, // <<, >>, >>>
	180	Relational, // <, <=, >, >=
	181	Equality, // ==, !=
	182	BitwiseAnd, // &
	183	BitwiseXor, // ^
	184	BitwiseOr, // \|
	185	LogicalAnd, // &&
	186	LogicalOr, // \|\|, ??
	187	}
	188
	189	pub fn get_binary_op_level(token: &Token) -> Option<BinaryOperatorLevel> {
	190	use BinaryOperatorLevel::*;
	191	match token {
	192	Token::StarStar => Some(Exponentiation),
	193	Token::Star \| Token::Slash \| Token::Percent => Some(Multiplicative),
	194	Token::Plus \| Token::Minus => Some(Additive),
	195	Token::LtLt \| Token::GtGt \| Token::GtGtGt => Some(BitwiseShift),
	196	Token::Gt \| Token::GtEqual \| Token::Lt \| Token::LtEqual => Some(Relational),
	197	Token::EqualEqual \| Token::BangEqual => Some(Equality),
	198	Token::And => Some(BitwiseAnd),
	199	Token::Caret => Some(BitwiseXor),
	200	Token::Or => Some(BitwiseOr),
	201	Token::AndAnd => Some(LogicalAnd),
	202	Token::OrOr => Some(LogicalOr),
	203	_ => None,
	204	}
	205	}

crates/checker/Cargo.toml deleted-13

...	...	@@ -1,13 +0,0 @@
1		[package]
2		name = "checker"
3		version = "0.1.0"
4		edition = "2021"
5
6		[lib]
7
8		[dependencies]
9		miette.workspace = true
10		thiserror.workspace = true
11		num.workspace = true
12
13		ast = { path = "../ast" }
		\ No newline at end of file

crates/checker/src/ast_type_coersion.rs deleted-54

...	...	@@ -1,54 +0,0 @@
1		// use ast::{BinaryOperator, ExprData, Type};
2
3		// impl TypeCoerceable for ExprData {
4		// fn to_type(&self) -> Option<Type> {
5		// match self {
6		// ExprData::Block(block) => {
7		// if (block.has_implicit_return) {
8		// block.statements.last().unwrap().to_type()
9		// } else {
10		// Some(Type::Void)
11		// }
12		// }
13		// ExprData::Binary { op, left, right } => {
14		// // TODO: this likely has to do further type checking
15		// let left_type = left.to_type();
16		// match op {
17		// BinaryOperator::Add => left_type,
18		// BinaryOperator::Sub => left_type,
19		// BinaryOperator::Mul => left_type,
20		// BinaryOperator::Div => left_type,
21		// BinaryOperator::Pow => left_type,
22		// BinaryOperator::Eq => Some(Type::Bool),
23		// BinaryOperator::Gt => Some(Type::Bool),
24		// BinaryOperator::Lt => Some(Type::Bool),
25		// BinaryOperator::Gte => Some(Type::Bool),
26		// BinaryOperator::Lte => Some(Type::Bool),
27		// BinaryOperator::Lshift => Some(Type::Int),
28		// BinaryOperator::Rshift => Some(Type::Int),
29		// BinaryOperator::BitAnd => Some(Type::Int),
30		// BinaryOperator::BitOr => Some(Type::Int),
31		// BinaryOperator::BitXor => Some(Type::Int),
32		// BinaryOperator::Mod => Some(Type::Int),
33		// BinaryOperator::NotEq => Some(Type::Int),
34
35		// BinaryOperator::And => Some(Type::Bool),
36		// BinaryOperator::Or => Some(Type::Bool),
37		// }
38		// }
39		// ExprData::Unary { op, value } => todo!(),
40		// ExprData::Function {
41		// name,
42		// params,
43		// return_type,
44		// block,
45		// } => Some(return_type),
46		// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47		// ExprData::Builtin { id, arguments } => todo!(),
48		// ExprData::If(_) => todo!(),
49		// ExprData::Return(_) => todo!(),
50		// ExprData::Literal(_) => todo!(),
51		// ExprData::Identifier(_) => todo!(),
52		// }
53		// }
54		// }

crates/checker/src/lib.rs deleted-96

...	...	@@ -1,96 +0,0 @@
1		// use ast::{ Module, Type};
2		// use miette::Result;
3
4		// mod ast_type_coersion;
5
6		// pub fn check_module(block: &Module) -> Result<()> {
7		// for statement in &block.statements {
8		// check_expression(statement)?;
9		// }
10
11		// Ok(())
12		// }
13
14		// pub fn resolve_type(expr: &Ast) -> Result<Type> {
15		// match expr {
16		// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
17		// Ast::Declaration {
18		// name,
19		// decl_type,
20		// value,
21		// is_constant,
22		// } => Ok(Type::NoReturn),
23		// Ast::Binary {
24		// operator,
25		// left,
26		// right,
27		// } => merge_types(resolve_type(left)?, resolve_type(right)?),
28		// Ast::Function {
29		// name,
30		// arguments,
31		// return_type,
32		// block,
33		// } => todo!(),
34		// Ast::FunctionCall { name, arguments } => todo!(),
35		// Ast::If {
36		// condition,
37		// value,
38		// else_value,
39		// } => todo!(),
40		// Ast::Return(_) => Ok(Type::NoReturn),
41		// Ast::Public(_) => todo!(), // TODO: remove Public
42		// Ast::Block(_) => todo!(),
43		// Ast::Bool(_) => Ok(Type::Bool),
44		// Ast::String(_) => Ok(Type::String),
45		// Ast::Identifier(_) => todo!(),
46		// Ast::BuiltIn(_) => todo!(),
47
48		// Ast::Comment(_) => unreachable!(),
49		// Ast::DocComment(_) => unreachable!(),
50		// }
51		// }
52
53		// pub fn check_expression(expr: &Ast) -> Result<()> {
54		// panic!()
55		// match expr {
56		// Ast::Declaration {
57		// name,
58		// decl_type: return_type,
59		// value,
60		// is_constant,
61		// } => {
62		// check_expression(value)?;
63		// // coerce_type(return_type.clone(), resolve_type(value)?)?;
64		// }
65		// _ => {}
66		// // Ast::Operation {
67		// // operator,
68		// // left,
69		// // right,
70		// // } => Ok(()),
71		// // Ast::Function {
72		// // name,
73		// // arguments,
74		// // return_type,
75		// // block,
76		// // } => Ok(()),
77		// // Ast::FunctionCall { name, arguments } => Ok(()),
78		// // Ast::If {
79		// // condition,
80		// // value,
81		// // else_value,
82		// // } => Ok(()),
83		// // Ast::Return(_) => Ok(()),
84		// // Ast::Public(_) => Ok(()),
85		// // Ast::Block(_) => Ok(()),
86		// // Ast::Number(_) => Ok(()),
87		// // Ast::Bool(_) => Ok(()),
88		// // Ast::String(_) => Ok(()),
89		// // Ast::Identifier(_) => Ok(()),
90		// // Ast::BuiltIn(_) => Ok(()),
91		// // Ast::Comment(_) => Ok(()),
92		// // Ast::DocComment(_) => Ok(()),
93		// }
94
95		// Ok(())
96		// }

crates/cli/Cargo.toml+1-1

...	...	@@ -17,4 +17,4 @@ ast = { path = "../ast" }
17	17	parser = { path = "../parser" }
18	18	js_ast = { path = "../js_ast" }
19	19	js_printer = { path = "../js_printer" }
20		checker = { path = "../checker" }
	20	sema = { path = "../sema" }

crates/cli/src/main.rs+1-1

...	...	@@ -56,7 +56,7 @@ pub fn transpile(content: String) -> Result<String> {
56	56	// 2. Parse the tokens into an AST
57	57	let blight_ast = parser::parse(tokens)?;
58	58	// 3. Type check the AST
59		// checker::check_module(&blight_ast)?;
	59	// sema::check_module(&blight_ast)?;
60	60	// 4. Optimize the AST
61	61	// optimize(tree.clone())?;
62	62	// 5. Translate the AST into javascript

crates/js_ast/src/lib.rs+1

...	...	@@ -180,6 +180,7 @@ fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) {
180	180	});
181	181	}
182	182	Statement::Expr(_) => todo!(),
	183	Statement::Comment(_) => todo!(),
183	184	// Ast::Function {
184	185	// name,
185	186	// arguments,

crates/parser/parse_declaration.rs+8-4

...	...	@@ -1,4 +1,4 @@
1		use ast::{Decl, Type};
	1	use ast::{Decl, DocComment, Type};
2	2	use miette::Result;
3	3	use tokenizer::Token;
4	4
...	...	@@ -7,7 +7,11 @@ use crate::parse_type::parse_type;
7	7	use crate::Parser;
8	8
9	9	/// Precondition: The first token exists and is either 'var' or 'let'
10		pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
	10	pub fn parse_declaration(
	11	p: &mut Parser,
	12	is_pub: bool,
	13	comment: Option<DocComment>,
	14	) -> Result<Decl> {
11	15	let loc = p.loc();
12	16	let is_constant = match p.next() {
13	17	Some(Token::Var) => false,
...	...	@@ -36,11 +40,11 @@ pub fn parse_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
36	40	};
37	41	Ok(Decl {
38	42	name: name.clone(),
39		decl_type: return_type,
	43	data_type: return_type,
40	44	init,
41	45	is_constant,
42	46	is_pub,
43		comment: None,
	47	comment,
44	48	loc,
45	49	})
46	50	}

crates/parser/parse_doc_comment.rs created+15

...	...	@@ -0,0 +1,15 @@
	1	use ast::DocComment;
	2	use tokenizer::Token;
	3
	4	use crate::Parser;
	5
	6	pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
	7	match p.peek() {
	8	Some(Token::DocComment(data)) => {
	9	let loc = p.loc();
	10	p.next();
	11	Some(DocComment { data, loc })
	12	}
	13	_ => None,
	14	}
	15	}

crates/parser/parse_expr.rs+12-101

...	...	@@ -1,45 +1,8 @@
1		use ast::{BinaryOperator, Expr, ExprData, Literal};
	1	use ast::{get_binary_op_level, BinaryOperator, BinaryOperatorLevel, Expr, ExprData};
2	2	use miette::Result;
3		use tokenizer::Token;
4	3
5		use crate::{
6		parse_call_arguments::parse_call_arguments, parse_expr_basic::parse_expr_basic, Parser,
7		};
8
9		/// Derived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
10
11		#[derive(PartialEq, PartialOrd)]
12		enum Level {
13		Prefix, // +, -, !, ~
14		Exponentiation, // **
15		Multiplicative, // *, /, %
16		Additive, // +, -
17		BitwiseShift, // <<, >>, >>>
18		Relational, // <, <=, >, >=
19		Equality, // ==, !=
20		BitwiseAnd, // &
21		BitwiseXor, // ^
22		BitwiseOr, // \|
23		LogicalAnd, // &&
24		LogicalOr, // \|\|, ??
25		}
26
27		pub fn get_postfix_op_level(token: &Token) -> Option<Level> {
28		match token {
29		Token::StarStar => Some(Level::Exponentiation),
30		Token::Star \| Token::Slash \| Token::Percent => Some(Level::Multiplicative),
31		Token::Plus \| Token::Minus => Some(Level::Additive),
32		Token::LtLt \| Token::GtGt \| Token::GtGtGt => Some(Level::BitwiseShift),
33		Token::Gt \| Token::GtEqual \| Token::Lt \| Token::LtEqual => Some(Level::Relational),
34		Token::EqualEqual \| Token::BangEqual => Some(Level::Equality),
35		Token::And => Some(Level::BitwiseAnd),
36		Token::Caret => Some(Level::BitwiseXor),
37		Token::Or => Some(Level::BitwiseOr),
38		Token::AndAnd => Some(Level::LogicalAnd),
39		Token::OrOr => Some(Level::LogicalOr),
40		_ => None,
41		}
42		}
	4	use crate::parse_expr_basic::parse_expr_basic;
	5	use crate::Parser;
43	6
44	7	pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
45	8	let mut expr = parse_expr_basic(p)?;
...	...	@@ -49,68 +12,10 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
49	12	return Ok(expr);
50	13	}
51	14
52		// Parse function calls and field accesses
53		loop {
54		match p.peek() {
55		Some(Token::Dot) => {
56		let loc = p.loc();
57		_ = p.next();
58		let field = match p.peek() {
59		Some(Token::Identifier(name)) => {
60		let loc = p.loc();
61		p.next();
62		Expr::new(ExprData::Literal(Literal::String(name)), loc)
63		}
64		_ => {
65		return Err(p.error("Expected identifier after dot".to_string()).into());
66		}
67		};
68		expr = Expr::new(
69		ExprData::Field {
70		base: Box::new(expr),
71		field: Box::new(field),
72		},
73		loc,
74		);
75		}
76		Some(Token::OpenParen) => {
77		let loc = p.loc();
78		_ = p.next();
79		let arguments = parse_call_arguments(p)?;
80		expr = Expr::new(
81		ExprData::Call {
82		base: Box::new(expr),
83		arguments,
84		},
85		loc,
86		);
87		}
88		Some(Token::OpenBracket) => {
89		let loc = p.loc();
90		_ = p.next();
91		let index = parse_expr(p)?;
92		if let Some(Token::ClosedBracket) = p.next() {
93		expr = Expr::new(
94		ExprData::Field {
95		base: Box::new(expr),
96		field: Box::new(index),
97		},
98		loc,
99		);
100		} else {
101		return Err(p
102		.error("Expected closing bracket after index".to_string())
103		.into());
104		}
105		}
106		_ => break,
107		}
108		}
109
110	15	// Parse operators
111	16	loop {
112	17	if let Some(token) = p.peek() {
113		if let Some(level) = get_postfix_op_level(&token) {
	18	if let Some(level) = get_binary_op_level(&token) {
114	19	_ = p.next();
115	20	expr = parse_expr_binary(
116	21	p,
...	...	@@ -128,13 +33,18 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
128	33	}
129	34
130	35	/// Prerequisite: `p.peek()` is an operator
131		fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Level) -> Result<Expr> {
	36	fn parse_expr_binary(
	37	p: &mut Parser,
	38	op: BinaryOperator,
	39	left: Expr,
	40	level: BinaryOperatorLevel,
	41	) -> Result<Expr> {
132	42	let loc = p.loc();
133	43	let mut right = parse_expr_basic(p)?;
134	44
135	45	loop {
136	46	if let Some(token) = p.peek() {
137		if let Some(next_level) = get_postfix_op_level(&token) {
	47	if let Some(next_level) = get_binary_op_level(&token) {
138	48	if next_level <= level {
139	49	_ = p.next();
140	50	right = parse_expr_binary(
...	...	@@ -157,5 +67,6 @@ fn parse_expr_binary(p: &mut Parser, op: BinaryOperator, left: Expr, level: Leve
157	67	right: Box::new(right),
158	68	},
159	69	loc,
	70	None,
160	71	))
161	72	}

crates/parser/parse_expr_basic.rs+90-20

...	...	@@ -9,32 +9,35 @@ use crate::{
9	9
10	10	pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
11	11	let loc = p.loc();
12		match p.peek() {
	12	let comments = p.take_comments();
	13	let mut expr = match p.peek() {
13	14	Some(Token::Number(number)) => {
14	15	_ = p.next();
15		Ok(Expr::new(
	16	Expr::new(
16	17	ExprData::Literal(Literal::Number(number.clone())),
17	18	loc,
18		))
	19	comments,
	20	)
19	21	}
20	22	Some(Token::String(string)) => {
21	23	_ = p.next();
22		Ok(Expr::new(
	24	Expr::new(
23	25	ExprData::Literal(Literal::String(string.clone())),
24	26	loc,
25		))
	27	comments,
	28	)
26	29	}
27	30	Some(Token::True) => {
28	31	_ = p.next();
29		Ok(Expr::new(ExprData::Literal(Literal::Bool(true)), loc))
	32	Expr::new(ExprData::Literal(Literal::Bool(true)), loc, comments)
30	33	}
31	34	Some(Token::False) => {
32	35	_ = p.next();
33		Ok(Expr::new(ExprData::Literal(Literal::Bool(false)), loc))
	36	Expr::new(ExprData::Literal(Literal::Bool(false)), loc, comments)
34	37	}
35	38	Some(Token::Void) => {
36	39	_ = p.next();
37		Ok(Expr::new(ExprData::Literal(Literal::Void), loc))
	40	Expr::new(ExprData::Literal(Literal::Void), loc, comments)
38	41	}
39	42	Some(Token::Identifier(name)) => {
40	43	let loc = p.loc();
...	...	@@ -43,7 +46,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
43	46	if let Some(id) = BuiltinId::from_str(name.as_str()) {
44	47	if let Some(Token::OpenParen) = p.next() {
45	48	let arguments = parse_call_arguments(p)?;
46		return Ok(Expr::new(ExprData::Builtin { id, arguments }, loc));
	49	Expr::new(ExprData::Builtin { id, arguments }, loc, comments)
47	50	} else {
48	51	return Err(p
49	52	.error_at(format!("Built-in function '{}' must be called", name), loc)
...	...	@@ -55,7 +58,7 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
55	58	.into());
56	59	}
57	60	} else {
58		Ok(Expr::new(ExprData::Identifier(name), loc))
	61	Expr::new(ExprData::Identifier(name), loc, comments)
59	62	}
60	63	}
61	64
...	...	@@ -66,18 +69,21 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
66	69	.into());
67	70	} else {
68	71	_ = p.next();
69
70		return Ok(Expr::new(ExprData::Return(Box::new(parse_expr(p)?)), loc));
	72	return Ok(Expr::new(
	73	ExprData::Return(Box::new(parse_expr(p)?)),
	74	loc,
	75	comments,
	76	));
71	77	}
72	78	}
73	79
74		Some(Token::If) => Ok(Expr::new(ExprData::If(parse_expr_if(p)?), loc)),
	80	Some(Token::If) => Expr::new(ExprData::If(parse_expr_if(p)?), loc, comments),
75	81
76		Some(Token::OpenBrace) => Ok(Expr::new(ExprData::Block(parse_block(p)?), loc)),
	82	Some(Token::OpenBrace) => Expr::new(ExprData::Block(parse_block(p)?), loc, comments),
77	83
78	84	Some(Token::OpenParen) => {
79	85	_ = p.next();
80		let expr = parse_expr_basic(p)?;
	86	let expr = parse_expr(p)?;
81	87	match p.next() {
82	88	Some(Token::ClosedParen) => (),
83	89	Some(token) => {
...	...	@@ -85,12 +91,12 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
85	91	}
86	92	None => return Err(p.error("Unexpected end of file".to_string()).into()),
87	93	}
88		Ok(expr)
	94	expr
89	95	}
90	96
91	97	Some(t @ Token::Bang) \| Some(t @ Token::Minus) \| Some(t @ Token::Tilde) => {
92	98	_ = p.next();
93		Ok(Expr::new(
	99	Expr::new(
94	100	ExprData::Unary {
95	101	op: match t {
96	102	Token::Bang => UnaryOperator::Not,
...	...	@@ -101,10 +107,74 @@ pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
101	107	value: Box::new(parse_expr_basic(p)?),
102	108	},
103	109	loc,
104		))
	110	comments,
	111	)
105	112	}
106	113
107		Some(t) => Err(p.error(format!("Unexpected '{}'", t)).into()),
108		None => Err(p.error("Unexpected end of file".to_string()).into()),
	114	Some(t) => return Err(p.error(format!("Unexpected '{}'", t)).into()),
	115	None => return Err(p.error("Unexpected end of file".to_string()).into()),
	116	};
	117
	118	// Parse function calls and field accesses
	119	loop {
	120	match p.peek() {
	121	Some(Token::Dot) => {
	122	let loc = p.loc();
	123	_ = p.next();
	124	let field = match p.peek() {
	125	Some(Token::Identifier(name)) => {
	126	let loc = p.loc();
	127	p.next();
	128	Expr::new(ExprData::Literal(Literal::String(name)), loc, None)
	129	}
	130	_ => {
	131	return Err(p.error("Expected identifier after dot".to_string()).into());
	132	}
	133	};
	134	expr = Expr::new(
	135	ExprData::Field {
	136	base: Box::new(expr),
	137	field: Box::new(field),
	138	},
	139	loc,
	140	None,
	141	);
	142	}
	143	Some(Token::OpenParen) => {
	144	let loc = p.loc();
	145	_ = p.next();
	146	let arguments = parse_call_arguments(p)?;
	147	expr = Expr::new(
	148	ExprData::Call {
	149	base: Box::new(expr),
	150	arguments,
	151	},
	152	loc,
	153	None,
	154	);
	155	}
	156	Some(Token::OpenBracket) => {
	157	let loc = p.loc();
	158	_ = p.next();
	159	let index = parse_expr(p)?;
	160	if let Some(Token::ClosedBracket) = p.next() {
	161	expr = Expr::new(
	162	ExprData::Field {
	163	base: Box::new(expr),
	164	field: Box::new(index),
	165	},
	166	loc,
	167	None,
	168	);
	169	} else {
	170	return Err(p
	171	.error("Expected closing bracket after index".to_string())
	172	.into());
	173	}
	174	}
	175	_ => break,
	176	}
109	177	}
	178
	179	Ok(expr)
110	180	}

crates/parser/parse_expr_if.rs+3-1

...	...	@@ -2,7 +2,9 @@ use ast::{BlockWithLoc, If};
2	2	use miette::Result;
3	3	use tokenizer::Token;
4	4
5		use crate::{parse_block::parse_block, parse_expr::parse_expr, Parser};
	5	use crate::parse_block::parse_block;
	6	use crate::parse_expr::parse_expr;
	7	use crate::Parser;
6	8
7	9	/// Precondition: the first token exists and is 'if'
8	10	pub fn parse_expr_if(p: &mut Parser) -> Result<If> {

crates/parser/parse_fn_declaration.rs+9-5

...	...	@@ -1,4 +1,4 @@
1		use ast::{Decl, Expr, ExprData, Type};
	1	use ast::{Decl, DocComment, Expr, ExprData, Type};
2	2	use miette::Result;
3	3	use tokenizer::Token;
4	4
...	...	@@ -8,7 +8,11 @@ use crate::parse_type::parse_type;
8	8	use crate::Parser;
9	9
10	10	/// Precondition:
11		pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
	11	pub fn parse_fn_declaration(
	12	p: &mut Parser,
	13	is_pub: bool,
	14	comment: Option<DocComment>,
	15	) -> Result<Decl> {
12	16	let loc = p.loc();
13	17	match p.next() {
14	18	Some(Token::Fn) => (),
...	...	@@ -96,11 +100,11 @@ pub fn parse_fn_declaration(p: &mut Parser, is_pub: bool) -> Result<Decl> {
96	100	};
97	101	Ok(Decl {
98	102	name,
99		decl_type: Type::Any,
100		init: Expr::new(function_body, block_loc),
	103	data_type: Type::Any,
	104	init: Expr::new(function_body, block_loc, None),
101	105	is_constant: true,
102	106	is_pub,
103		comment: None,
	107	comment,
104	108	loc: loc,
105	109	})
106	110	} else {

crates/parser/parse_module.rs+16-9

...	...	@@ -1,23 +1,27 @@
1		use crate::{parse_top_level_statement::parse_top_level_statement, Parser};
	1	use crate::parse_doc_comment::maybe_read_doc_comment;
	2	use crate::parse_top_level_statement::parse_top_level_statement;
	3	use crate::Parser;
2	4	use ast::Module;
3	5	use miette::Result;
4	6	use tokenizer::Token;
5	7
6		pub fn parse_module(tokens: &mut Parser) -> Result<Module> {
	8	pub fn parse_module(p: &mut Parser) -> Result<Module> {
7	9	let mut statements = Vec::new();
8	10
	11	let file_doc_comment = maybe_read_doc_comment(p);
	12
9	13	loop {
10		let statement = parse_top_level_statement(tokens)?;
	14	let statement = parse_top_level_statement(p)?;
11	15
12		match tokens.peek() {
	16	match p.peek() {
13	17	Some(Token::Semicolon) => {
14		while let Some(Token::Semicolon) = tokens.peek() {
15		tokens.next();
	18	while let Some(Token::Semicolon) = p.peek() {
	19	p.next();
16	20	}
17	21	}
18	22	_ => {
19	23	if statement.requires_semicolon() {
20		return Err(tokens
	24	return Err(p
21	25	.error("Expected semicolon after statement".to_string())
22	26	.into());
23	27	}
...	...	@@ -26,8 +30,11 @@ pub fn parse_module(tokens: &mut Parser) -> Result<Module> {
26	30
27	31	statements.push(statement);
28	32
29		if tokens.empty() {
30		return Ok(Module { statements });
	33	if p.empty() {
	34	return Ok(Module {
	35	statements,
	36	comment: file_doc_comment,
	37	});
31	38	}
32	39	}
33	40	}

crates/parser/parse_statement.rs+39-5

...	...	@@ -3,26 +3,60 @@ use miette::Result;
3	3	use tokenizer::Token;
4	4
5	5	use crate::parse_declaration::parse_declaration;
	6	use crate::parse_doc_comment::maybe_read_doc_comment;
6	7	use crate::parse_expr::parse_expr;
7	8	use crate::parse_fn_declaration::parse_fn_declaration;
8	9	use crate::Parser;
9	10
10	11	pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
	12	if let Some(comments) = p.take_comments() {
	13	return Ok(Statement::Comment(comments));
	14	}
	15
	16	let m_doc_comment = maybe_read_doc_comment(p);
	17	if let Some(doc_comment) = m_doc_comment.clone() {
	18	if p.has_comments() {
	19	// this implies that comments are immediately after the doc
	20	return Err(p
	21	.error_at(format!("Unattached doc comment"), doc_comment.loc)
	22	.into());
	23	}
	24	}
	25
11	26	match p.peek() {
	27	Some(Token::DocComment(_)) => unreachable!(),
12	28	Some(Token::Pub) => {
13	29	_ = p.next();
14	30	match p.peek() {
15	31	Some(Token::Var) \| Some(Token::Let) => {
16		Ok(Statement::Decl(parse_declaration(p, true)?))
	32	Ok(Statement::Decl(parse_declaration(p, true, m_doc_comment)?))
17	33	}
18		Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, true)?)),
	34	Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(
	35	p,
	36	true,
	37	m_doc_comment,
	38	)?)),
19	39	Some(_) => Err(p.error("Expected a declaration".to_string()).into()),
20	40	None => Err(p.error("Unexpected end of file".to_string()).into()),
21	41	}
22	42	}
23		Some(Token::Var) \| Some(Token::Let) => Ok(Statement::Decl(parse_declaration(p, false)?)),
24		Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(p, false)?)),
25		Some(_) => Ok(Statement::Expr(parse_expr(p)?)),
	43	Some(Token::Var) \| Some(Token::Let) => {
	44	Ok(Statement::Decl(parse_declaration(p, false, m_doc_comment)?))
	45	}
	46	Some(Token::Fn) => Ok(Statement::Decl(parse_fn_declaration(
	47	p,
	48	false,
	49	m_doc_comment,
	50	)?)),
	51	Some(_) => {
	52	if let Some(doc_comment) = m_doc_comment {
	53	Err(p
	54	.error_at(format!("Unattached doc comment"), doc_comment.loc)
	55	.into())
	56	} else {
	57	Ok(Statement::Expr(parse_expr(p)?))
	58	}
	59	}
26	60	None => Err(p.error("Unexpected end of file".to_string()).into()),
27	61	}
28	62	}

crates/parser/parser.rs+56-16

...	...	@@ -1,6 +1,7 @@
1	1	mod parse_block;
2	2	mod parse_call_arguments;
3	3	mod parse_declaration;
	4	mod parse_doc_comment;
4	5	mod parse_expr;
5	6	mod parse_expr_basic;
6	7	mod parse_expr_if;
...	...	@@ -11,15 +12,14 @@ mod parse_statement;
11	12	mod parse_top_level_statement;
12	13	mod parse_type;
13	14
14		use tokenizer::{Loc, Token};
15
16		use miette::Diagnostic;
17		use thiserror::Error;
18
	15	use ast::Comment;
19	16	use ast::Module;
	17	use miette::Diagnostic;
20	18	use miette::Result;
21	19	use parse_module::parse_module;
	20	use thiserror::Error;
22	21	use tokenizer::tokenize;
	22	use tokenizer::{Loc, Token};
23	23
24	24	pub fn parse(tokens: Vec<Token>) -> Result<Module> {
25	25	let mut p = Parser::new(tokens);
...	...	@@ -47,11 +47,15 @@ impl ParseError {
47	47
48	48	#[derive(Debug)]
49	49	pub struct Parser {
50		pub tokens: Vec<Token>,
51		pub cursor: usize,
	50	/// TODO(@paperdave): switch to streaming tokenizer + one that can
	51	/// report line and column info
	52	tokens: Vec<Token>,
	53	cursor: usize,
52	54
53	55	/// Whether or not the parser is at the top level
54	56	pub is_top_level: bool,
	57
	58	comments: Vec<String>,
55	59	}
56	60
57	61	impl Parser {
...	...	@@ -60,6 +64,7 @@ impl Parser {
60	64	tokens,
61	65	cursor: 0,
62	66	is_top_level: true,
	67	comments: vec![],
63	68	}
64	69	}
65	70
...	...	@@ -67,26 +72,61 @@ impl Parser {
67	72	self.cursor == self.tokens.len()
68	73	}
69	74
70		pub fn back(&mut self) {
71		self.cursor -= 1;
72		}
	75	// TODO(@paperdave): fix parser token stuff. its very messy rn
73	76
74	77	pub fn next(&mut self) -> Option<Token> {
75	78	if self.cursor == self.tokens.len() {
76	79	return None;
77	80	}
78	81
79		let token = self.tokens[self.cursor].clone();
80		self.cursor += 1;
81		Some(token)
	82	loop {
	83	match self.tokens[self.cursor].clone() {
	84	Token::Comment(comment) => {
	85	self.comments.push(comment);
	86	self.cursor += 1;
	87	continue;
	88	}
	89	token => {
	90	self.cursor += 1;
	91	return Some(token);
	92	}
	93	}
	94	}
82	95	}
83	96
84		pub fn peek(&self) -> Option<Token> {
85		if self.cursor == self.tokens.len() {
	97	pub fn take_comments(&mut self) -> Option<Comment> {
	98	if self.comments.len() == 0 {
86	99	return None;
87	100	}
	101	let comments = self.comments.clone();
	102	self.comments = vec![];
	103	Some(Comment {
	104	data: comments.join("\n"),
	105	})
	106	}
	107
	108	pub fn has_comments(&mut self) -> bool {
	109	self.peek();
	110	self.comments.len() > 0
	111	}
88	112
89		Some(self.tokens[self.cursor].clone())
	113	pub fn peek(&mut self) -> Option<Token> {
	114	loop {
	115	if self.cursor == self.tokens.len() {
	116	return None;
	117	}
	118
	119	match self.tokens[self.cursor].clone() {
	120	Token::Comment(comment) => {
	121	self.comments.push(comment);
	122	self.cursor += 1;
	123	continue;
	124	}
	125	token => {
	126	return Some(token);
	127	}
	128	}
	129	}
90	130	}
91	131
92	132	pub fn error(&self, error: String) -> ParseError {

crates/sema/Cargo.toml created+13

...	...	@@ -0,0 +1,13 @@
	1	[package]
	2	name = "sema"
	3	version = "0.1.0"
	4	edition = "2021"
	5
	6	[lib]
	7
	8	[dependencies]
	9	miette.workspace = true
	10	thiserror.workspace = true
	11	num.workspace = true
	12
	13	ast = { path = "../ast" }
		\ No newline at end of file

crates/sema/src/ast_type_coersion.rs created+54

...	...	@@ -0,0 +1,54 @@
	1	// use ast::{BinaryOperator, ExprData, Type};
	2
	3	// impl TypeCoerceable for ExprData {
	4	// fn to_type(&self) -> Option<Type> {
	5	// match self {
	6	// ExprData::Block(block) => {
	7	// if (block.has_implicit_return) {
	8	// block.statements.last().unwrap().to_type()
	9	// } else {
	10	// Some(Type::Void)
	11	// }
	12	// }
	13	// ExprData::Binary { op, left, right } => {
	14	// // TODO: this likely has to do further type checking
	15	// let left_type = left.to_type();
	16	// match op {
	17	// BinaryOperator::Add => left_type,
	18	// BinaryOperator::Sub => left_type,
	19	// BinaryOperator::Mul => left_type,
	20	// BinaryOperator::Div => left_type,
	21	// BinaryOperator::Pow => left_type,
	22	// BinaryOperator::Eq => Some(Type::Bool),
	23	// BinaryOperator::Gt => Some(Type::Bool),
	24	// BinaryOperator::Lt => Some(Type::Bool),
	25	// BinaryOperator::Gte => Some(Type::Bool),
	26	// BinaryOperator::Lte => Some(Type::Bool),
	27	// BinaryOperator::Lshift => Some(Type::Int),
	28	// BinaryOperator::Rshift => Some(Type::Int),
	29	// BinaryOperator::BitAnd => Some(Type::Int),
	30	// BinaryOperator::BitOr => Some(Type::Int),
	31	// BinaryOperator::BitXor => Some(Type::Int),
	32	// BinaryOperator::Mod => Some(Type::Int),
	33	// BinaryOperator::NotEq => Some(Type::Int),
	34
	35	// BinaryOperator::And => Some(Type::Bool),
	36	// BinaryOperator::Or => Some(Type::Bool),
	37	// }
	38	// }
	39	// ExprData::Unary { op, value } => todo!(),
	40	// ExprData::Function {
	41	// name,
	42	// params,
	43	// return_type,
	44	// block,
	45	// } => Some(return_type),
	46	// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
	47	// ExprData::Builtin { id, arguments } => todo!(),
	48	// ExprData::If(_) => todo!(),
	49	// ExprData::Return(_) => todo!(),
	50	// ExprData::Literal(_) => todo!(),
	51	// ExprData::Identifier(_) => todo!(),
	52	// }
	53	// }
	54	// }

crates/sema/src/lib.rs created+96

...	...	@@ -0,0 +1,96 @@
	1	// use ast::{ Module, Type};
	2	// use miette::Result;
	3
	4	// mod ast_type_coersion;
	5
	6	// pub fn check_module(block: &Module) -> Result<()> {
	7	// for statement in &block.statements {
	8	// check_expression(statement)?;
	9	// }
	10
	11	// Ok(())
	12	// }
	13
	14	// pub fn resolve_type(expr: &Ast) -> Result<Type> {
	15	// match expr {
	16	// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
	17	// Ast::Declaration {
	18	// name,
	19	// decl_type,
	20	// value,
	21	// is_constant,
	22	// } => Ok(Type::NoReturn),
	23	// Ast::Binary {
	24	// operator,
	25	// left,
	26	// right,
	27	// } => merge_types(resolve_type(left)?, resolve_type(right)?),
	28	// Ast::Function {
	29	// name,
	30	// arguments,
	31	// return_type,
	32	// block,
	33	// } => todo!(),
	34	// Ast::FunctionCall { name, arguments } => todo!(),
	35	// Ast::If {
	36	// condition,
	37	// value,
	38	// else_value,
	39	// } => todo!(),
	40	// Ast::Return(_) => Ok(Type::NoReturn),
	41	// Ast::Public(_) => todo!(), // TODO: remove Public
	42	// Ast::Block(_) => todo!(),
	43	// Ast::Bool(_) => Ok(Type::Bool),
	44	// Ast::String(_) => Ok(Type::String),
	45	// Ast::Identifier(_) => todo!(),
	46	// Ast::BuiltIn(_) => todo!(),
	47
	48	// Ast::Comment(_) => unreachable!(),
	49	// Ast::DocComment(_) => unreachable!(),
	50	// }
	51	// }
	52
	53	// pub fn check_expression(expr: &Ast) -> Result<()> {
	54	// panic!()
	55	// match expr {
	56	// Ast::Declaration {
	57	// name,
	58	// decl_type: return_type,
	59	// value,
	60	// is_constant,
	61	// } => {
	62	// check_expression(value)?;
	63	// // coerce_type(return_type.clone(), resolve_type(value)?)?;
	64	// }
	65	// _ => {}
	66	// // Ast::Operation {
	67	// // operator,
	68	// // left,
	69	// // right,
	70	// // } => Ok(()),
	71	// // Ast::Function {
	72	// // name,
	73	// // arguments,
	74	// // return_type,
	75	// // block,
	76	// // } => Ok(()),
	77	// // Ast::FunctionCall { name, arguments } => Ok(()),
	78	// // Ast::If {
	79	// // condition,
	80	// // value,
	81	// // else_value,
	82	// // } => Ok(()),
	83	// // Ast::Return(_) => Ok(()),
	84	// // Ast::Public(_) => Ok(()),
	85	// // Ast::Block(_) => Ok(()),
	86	// // Ast::Number(_) => Ok(()),
	87	// // Ast::Bool(_) => Ok(()),
	88	// // Ast::String(_) => Ok(()),
	89	// // Ast::Identifier(_) => Ok(()),
	90	// // Ast::BuiltIn(_) => Ok(()),
	91	// // Ast::Comment(_) => Ok(()),
	92	// // Ast::DocComment(_) => Ok(()),
	93	// }
	94
	95	// Ok(())
	96	// }

crates/tokenizer/src/lib.rs+49-13

...	...	@@ -71,12 +71,12 @@ pub enum Token {
71	71	impl std::fmt::Display for Token {
72	72	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
73	73	let token = match self {
74		Token::Identifier(_) => "an identifier".to_string(),
	74	Token::Identifier(_) => "identifier".to_string(),
75	75
76	76	Token::Number(n) => n.to_string(),
77	77	Token::String(s) => format!("\"{}\"", s),
78		Token::Comment(_) => "a comment".to_string(),
79		Token::DocComment(_) => "a documentation comment".to_string(),
	78	Token::Comment(_) => "comment".to_string(),
	79	Token::DocComment(_) => "documentation comment".to_string(),
80	80
81	81	Token::Else => "else".to_string(),
82	82	Token::False => "false".to_string(),
...	...	@@ -129,6 +129,7 @@ impl std::fmt::Display for Token {
129	129	}
130	130	}
131	131
	132	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
132	133	pub fn tokenize(content: String) -> Result<Vec<Token>> {
133	134	let mut tokens: Vec<Token> = Vec::new();
134	135	let mut chars = content.chars().enumerate().peekable();
...	...	@@ -277,20 +278,55 @@ pub fn tokenize(content: String) -> Result<Vec<Token>> {
277	278
278	279	let mut comment = String::new();
279	280	loop {
280		let c = chars.next();
281		if c.is_none() {
282		break;
	281	// skip leading whitespace
	282	while let Some((_, c)) = chars.peek() {
	283	if !c.is_whitespace() {
	284	break;
	285	}
	286	chars.next();
283	287	}
284		let (_, c) = c.unwrap();
285		match c {
286		'\n' => break,
287		_ => {
288		comment.push(c.clone());
	288	// read the comment
	289	while let Some((_, c)) = chars.next() {
	290	if c == '\n' \|\| c == '\r' {
	291	break;
289	292	}
	293	comment.push(c);
290	294	}
291		}
	295	// trim trailing whitespace on line
	296	comment = comment.trim_end().to_string();
	297	// skip to next non-whitespace character
	298	while let Some((_, c)) = chars.peek() {
	299	if !c.is_whitespace() {
	300	break;
	301	}
	302	chars.next();
	303	}
	304	// check if we've hit another comment
	305	let mut tmp = chars.clone();
292	306
293		comment = comment.trim().to_string();
	307	let is_another_comment = match tmp.peek() {
	308	Some((_, '/')) => {
	309	tmp.next(); // Consume the second slash
	310	match tmp.peek() {
	311	Some((_, '/')) => {
	312	tmp.next(); // Consume the third slash
	313	is_doc_comment
	314	}
	315	_ => !is_doc_comment,
	316	}
	317	}
	318	_ => false,
	319	};
	320
	321	if is_another_comment {
	322	// apply our temporary iterator to the main iterator
	323	comment.push('\n');
	324	chars = tmp;
	325	chars.next();
	326	} else {
	327	break;
	328	}
	329	}
294	330
295	331	if is_doc_comment {
296	332	tokens.push(Token::DocComment(comment));

examples/basics/comments.blight+2-9

...	...	@@ -1,9 +1,2 @@
1		// this is a regular comment
2		let x: int = 0;
3
4		/// this is a doc comment
5		let y: int = 0;
6
7		/// Doc comments can be multiple lines.
8		/// This can be done by simply adding another triple-slash to the next line.
9		let z: int = 0;
	1	2 // yapping
	2	+ 2;
		\ No newline at end of file

author	dave caruso <me@paperdave.net> 2024-01-20 19:08:18-08:00
committer	dave caruso <me@paperdave.net> 2024-01-20 19:08:18-08:00
log	c89bf26922f4448e6f0cb6015f4627792841deab
tree	de76df5e71a1116de15edc93e1fe53156978c38b
parent	17c6ef0255b85120d2bdcae9f48ecd8c88c3c247
signature	Commit is signed but in an unrecognized format.

parser changes