Commit 12864f7c49c005271c6c5610dfaa2cb71ff56ae7 | ix

author	chloe caruso <git@paperclover.net> 2024-01-22 00:32:12-08:00
committer	chloe caruso <git@paperclover.net> 2024-01-22 00:32:12-08:00
log	12864f7c49c005271c6c5610dfaa2cb71ff56ae7
tree	d4f9793c94bcab05e4855cf894dee62e17476bd3
parent	b9cad6ad285b99192b686be1cea8b1b8cfcdd6c1

finish rewriting the project in rust

35 files changed, 2549 insertions(+), 1958 deletions(-)

Cargo.lock+1

...	...	@@ -239,6 +239,7 @@ dependencies = [
239	239	"ast",
240	240	"miette",
241	241	"num",
	242	"parser",
242	243	"thiserror",
243	244	"tokenizer",
244	245	]

Cargo.toml-1

...	...	@@ -1,6 +1,5 @@
1	1	[workspace.package]
2	2	version = "0.1.0"
3		name = "blight"
4	3	edition = "2023"
5	4	license = "MIT"
6	5	authors = ["Lino Le Van", "Dave Caruso"]

crates/ast/ast.rs+38-66

...	...	@@ -74,6 +74,19 @@ impl Comment {
74	74	}
75	75	result
76	76	}
	77
	78	pub fn to_doc_comment(self, loc: Loc) -> DocComment {
	79	DocComment {
	80	data: self.data,
	81	loc,
	82	}
	83	}
	84	}
	85
	86	impl Into<Comment> for DocComment {
	87	fn into(self) -> Comment {
	88	Comment { data: self.data }
	89	}
77	90	}
78	91
79	92	impl DocComment {
...	...	@@ -98,7 +111,13 @@ pub struct BlockWithLoc {
98	111	pub struct If {
99	112	pub condition: Box<Expr>,
100	113	pub yes: BlockWithLoc,
101		pub no: Option<Box<Expr>>,
	114	pub no: Option<BlockOrIf>,
	115	}
	116
	117	#[derive(Debug, Clone)]
	118	pub enum BlockOrIf {
	119	Block(BlockWithLoc),
	120	If(Box<If>),
102	121	}
103	122
104	123	#[derive(Debug, Clone)]
...	...	@@ -166,67 +185,10 @@ impl BuiltinId {
166	185	}
167	186	}
168	187
169		impl ExprData {
170		pub fn is_valid_js_expression(&self) -> bool {
171		match self {
172		// A block containing just expressions can be an expression
173		ExprData::Block(block) => block.is_valid_js_expression(),
174		ExprData::If(expr) => expr.is_valid_js_expression(),
175		ExprData::Return(_) => false,
176		ExprData::Builtin { id, .. } => match id {
177		BuiltinId::Raw => true,
178		},
179		_ => true,
180		}
181		}
182		}
183
184		impl Block {
185		pub fn is_valid_js_expression(&self) -> bool {
186		self.statements
187		.iter()
188		.all(\|statement\| statement.is_valid_js_expression())
189		}
190		}
191
192		impl Expr {
193		pub fn is_valid_js_expression(&self) -> bool {
194		self.data.is_valid_js_expression()
195		}
196		}
197
198		impl If {
199		pub fn is_valid_js_expression(&self) -> bool {
200		self.condition.is_valid_js_expression()
201		&& self.yes.is_valid_js_expression()
202		&& match &self.no {
203		Some(no) => no.is_valid_js_expression(),
204		None => true,
205		}
206		}
207		}
208
209		impl BlockWithLoc {
210		pub fn is_valid_js_expression(&self) -> bool {
211		self.block.is_valid_js_expression()
212		}
213		}
214
215	188	impl Statement {
216		pub fn requires_semicolon(&self) -> bool {
	189	pub fn is_expression(&self) -> bool {
217	190	match &self {
218		Statement::Expr(expr) => match expr.data {
219		ExprData::Block(_) => false,
220		ExprData::If(_) => false,
221		_ => true,
222		},
223		_ => false,
224		}
225		}
226
227		pub fn is_valid_js_expression(&self) -> bool {
228		match self {
229		Statement::Expr(expr) => expr.data.is_valid_js_expression(),
	191	Statement::Expr(_) => true,
230	192	_ => false,
231	193	}
232	194	}
...	...	@@ -238,16 +200,19 @@ impl Statement {
238	200	Statement::Comment(comment) => TopLevelStatement::Comment(comment),
239	201	}
240	202	}
	203
	204	pub fn is_comment(&self) -> bool {
	205	match self {
	206	Statement::Comment(_) => true,
	207	_ => false,
	208	}
	209	}
241	210	}
242	211
243	212	impl TopLevelStatement {
244		pub fn requires_semicolon(&self) -> bool {
	213	pub fn is_expression(&self) -> bool {
245	214	match &self {
246		TopLevelStatement::Expr(expr) => match expr.data {
247		ExprData::Block(_) => false,
248		ExprData::If(_) => false,
249		_ => true,
250		},
	215	TopLevelStatement::Expr(_) => true,
251	216	_ => false,
252	217	}
253	218	}
...	...	@@ -259,4 +224,11 @@ impl TopLevelStatement {
259	224	TopLevelStatement::Comment(comment) => Statement::Comment(comment),
260	225	}
261	226	}
	227
	228	pub fn is_comment(&self) -> bool {
	229	match self {
	230	TopLevelStatement::Comment(_) => true,
	231	_ => false,
	232	}
	233	}
262	234	}

crates/ast/operators.rs+133-23

...	...	@@ -24,6 +24,22 @@ pub enum BinaryOperator {
24	24	NotEq,
25	25	And,
26	26	Or,
	27
	28	Assign,
	29	AssignAdd,
	30	AssignSub,
	31	AssignMul,
	32	AssignDiv,
	33	AssignPow,
	34	AssignLshift,
	35	AssignRshift,
	36	AssignRshiftUnsigned,
	37	AssignBitAnd,
	38	AssignBitOr,
	39	AssignBitXor,
	40	AssignMod,
	41	AssignAnd,
	42	AssignOr,
27	43	}
28	44
29	45	#[derive(Debug, Clone, PartialEq)]
...	...	@@ -34,8 +50,8 @@ pub enum UnaryOperator {
34	50	}
35	51
36	52	impl BinaryOperator {
37		pub fn to_string(&self) -> String {
38		String::from(match self {
	53	pub fn to_str(&self) -> &'static str {
	54	match self {
39	55	Self::Add => "+",
40	56	Self::Sub => "-",
41	57	Self::Div => "/",
...	...	@@ -56,7 +72,26 @@ impl BinaryOperator {
56	72	Self::NotEq => "!=",
57	73	Self::And => "&&",
58	74	Self::Or => "\|\|",
59		})
	75	Self::AssignAdd => "+=",
	76	Self::AssignSub => "-=",
	77	Self::AssignDiv => "/=",
	78	Self::AssignMul => "*=",
	79	Self::AssignPow => "**=",
	80	Self::Assign => "=",
	81	Self::AssignLshift => "<<=",
	82	Self::AssignRshift => ">>=",
	83	Self::AssignRshiftUnsigned => ">>>=",
	84	Self::AssignBitAnd => "&=",
	85	Self::AssignBitOr => "\|=",
	86	Self::AssignBitXor => "^=",
	87	Self::AssignMod => "%=",
	88	Self::AssignAnd => "&&=",
	89	Self::AssignOr => "\|\|=",
	90	}
	91	}
	92
	93	pub fn to_string(&self) -> String {
	94	String::from(self.to_str())
60	95	}
61	96
62	97	pub fn from_token(token: &Token) -> Option<BinaryOperator> {
...	...	@@ -85,6 +120,10 @@ impl BinaryOperator {
85	120	}
86	121	}
87	122
	123	pub fn level(&self) -> OperatorLevel {
	124	OperatorLevel::from(self)
	125	}
	126
88	127	/// Error case of this means 'types do not allow this operation'
89	128	pub fn perform(&self, left: Literal, right: Literal) -> Result<Literal, ()> {
90	129	match self {
...	...	@@ -139,17 +178,24 @@ impl BinaryOperator {
139	178	}),
140	179	_ => Err(()),
141	180	},
	181
	182	// Impossible condition doing assign-add to a literal
	183	_ => Err(()),
142	184	}
143	185	}
144	186	}
145	187
146	188	impl UnaryOperator {
147		pub fn to_string(self) -> String {
148		String::from(match self {
	189	pub fn to_str(&self) -> &'static str {
	190	match self {
149	191	Self::Neg => "-",
150	192	Self::Not => "!",
151	193	Self::BitNot => "~",
152		})
	194	}
	195	}
	196
	197	pub fn to_string(&self) -> String {
	198	String::from(self.to_str())
153	199	}
154	200
155	201	pub fn perform(self, value: Literal) -> Result<Literal, ()> {
...	...	@@ -171,8 +217,17 @@ impl UnaryOperator {
171	217	}
172	218
173	219	/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
174		#[derive(PartialEq, PartialOrd)]
175		pub enum BinaryOperatorLevel {
	220	///
	221	/// Blight doesn't have all of these operators, but the
	222	/// printer uses it to determine where to put parens.
	223	#[derive(PartialEq, PartialOrd, Debug)]
	224	pub enum OperatorLevel {
	225	Accessor, // e. e[] e()
	226	New, // new
	227	Postfix, // expr++ expr--
	228	Prefix, // ++e, --e, !e, ~e, +e, -e, typeof, void, delete
	229
	230	// these are the binary operators
176	231	Exponentiation, // **
177	232	Multiplicative, // *, /, %
178	233	Additive, // +, -
...	...	@@ -184,22 +239,77 @@ pub enum BinaryOperatorLevel {
184	239	BitwiseOr, // \|
185	240	LogicalAnd, // &&
186	241	LogicalOr, // \|\|, ??
	242
	243	AssignmentAndOther, // =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
	244	Comma, // ,
187	245	}
188	246
189		pub fn get_binary_op_level(token: &Token) -> Option<BinaryOperatorLevel> {
190		use BinaryOperatorLevel::*;
191		match token {
192		Token::StarStar => Some(Exponentiation),
193		Token::Star \| Token::Slash \| Token::Percent => Some(Multiplicative),
194		Token::Plus \| Token::Minus => Some(Additive),
195		Token::LtLt \| Token::GtGt \| Token::GtGtGt => Some(BitwiseShift),
196		Token::Gt \| Token::GtEqual \| Token::Lt \| Token::LtEqual => Some(Relational),
197		Token::EqualEqual \| Token::BangEqual => Some(Equality),
198		Token::And => Some(BitwiseAnd),
199		Token::Caret => Some(BitwiseXor),
200		Token::Or => Some(BitwiseOr),
201		Token::AndAnd => Some(LogicalAnd),
202		Token::OrOr => Some(LogicalOr),
203		_ => None,
	247	pub enum Associativity {
	248	LeftToRight,
	249	RightToLeft,
	250	}
	251
	252	impl OperatorLevel {
	253	pub fn associativity(&self) -> Associativity {
	254	use Associativity::*;
	255	use OperatorLevel::*;
	256
	257	match self {
	258	Accessor => LeftToRight,
	259	Exponentiation => RightToLeft,
	260	Multiplicative => LeftToRight,
	261	Additive => LeftToRight,
	262	BitwiseShift => LeftToRight,
	263	Relational => LeftToRight,
	264	Equality => LeftToRight,
	265	BitwiseAnd => LeftToRight,
	266	BitwiseXor => LeftToRight,
	267	BitwiseOr => LeftToRight,
	268	LogicalAnd => LeftToRight,
	269	LogicalOr => LeftToRight,
	270	AssignmentAndOther => RightToLeft,
	271	Comma => LeftToRight,
	272
	273	// These do not make sense to have an associativity
	274	// You will likely not run into this if you do
	275	// BinaryOperator::level(o).associativity()
	276	New \| Postfix \| Prefix => unreachable!("{:?} does not have an associativity", self),
	277	}
	278	}
	279	}
	280
	281	impl From<&BinaryOperator> for OperatorLevel {
	282	fn from(operator: &BinaryOperator) -> Self {
	283	use BinaryOperator::*;
	284	use OperatorLevel::*;
	285
	286	match operator {
	287	Add \| Sub => Additive,
	288	Mul \| Div \| Mod => Multiplicative,
	289	Pow => Exponentiation,
	290	Gt \| Lt \| Gte \| Lte => Relational,
	291	Lshift \| Rshift \| RshiftUnsigned => BitwiseShift,
	292	BitAnd => BitwiseAnd,
	293	BitOr => BitwiseOr,
	294	BitXor => BitwiseXor,
	295	Eq \| NotEq => Equality,
	296	And => LogicalAnd,
	297	Or => LogicalOr,
	298
	299	Assign \| AssignAdd \| AssignSub \| AssignMul \| AssignDiv \| AssignPow \| AssignLshift
	300	\| AssignRshift \| AssignRshiftUnsigned \| AssignBitAnd \| AssignBitOr \| AssignBitXor
	301	\| AssignMod \| AssignAnd \| AssignOr => AssignmentAndOther,
	302	}
	303	}
	304	}
	305
	306	impl From<UnaryOperator> for OperatorLevel {
	307	fn from(operator: UnaryOperator) -> Self {
	308	use OperatorLevel::*;
	309	use UnaryOperator::*;
	310
	311	match operator {
	312	Neg \| Not \| BitNot => Prefix,
	313	}
204	314	}
205	315	}

crates/cli/Cargo.lock deleted-532

...	...	@@ -1,532 +0,0 @@
1		# This file is automatically @generated by Cargo.
2		# It is not intended for manual editing.
3		version = 3
4
5		[[package]]
6		name = "addr2line"
7		version = "0.21.0"
8		source = "registry+https://github.com/rust-lang/crates.io-index"
9		checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
10		dependencies = [
11		"gimli",
12		]
13
14		[[package]]
15		name = "adler"
16		version = "1.0.2"
17		source = "registry+https://github.com/rust-lang/crates.io-index"
18		checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
19
20		[[package]]
21		name = "anstream"
22		version = "0.6.5"
23		source = "registry+https://github.com/rust-lang/crates.io-index"
24		checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
25		dependencies = [
26		"anstyle",
27		"anstyle-parse",
28		"anstyle-query",
29		"anstyle-wincon",
30		"colorchoice",
31		"utf8parse",
32		]
33
34		[[package]]
35		name = "anstyle"
36		version = "1.0.4"
37		source = "registry+https://github.com/rust-lang/crates.io-index"
38		checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
39
40		[[package]]
41		name = "anstyle-parse"
42		version = "0.2.3"
43		source = "registry+https://github.com/rust-lang/crates.io-index"
44		checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
45		dependencies = [
46		"utf8parse",
47		]
48
49		[[package]]
50		name = "anstyle-query"
51		version = "1.0.2"
52		source = "registry+https://github.com/rust-lang/crates.io-index"
53		checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
54		dependencies = [
55		"windows-sys",
56		]
57
58		[[package]]
59		name = "anstyle-wincon"
60		version = "3.0.2"
61		source = "registry+https://github.com/rust-lang/crates.io-index"
62		checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
63		dependencies = [
64		"anstyle",
65		"windows-sys",
66		]
67
68		[[package]]
69		name = "backtrace"
70		version = "0.3.69"
71		source = "registry+https://github.com/rust-lang/crates.io-index"
72		checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
73		dependencies = [
74		"addr2line",
75		"cc",
76		"cfg-if",
77		"libc",
78		"miniz_oxide",
79		"object",
80		"rustc-demangle",
81		]
82
83		[[package]]
84		name = "backtrace-ext"
85		version = "0.2.1"
86		source = "registry+https://github.com/rust-lang/crates.io-index"
87		checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
88		dependencies = [
89		"backtrace",
90		]
91
92		[[package]]
93		name = "bitflags"
94		version = "2.4.1"
95		source = "registry+https://github.com/rust-lang/crates.io-index"
96		checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
97
98		[[package]]
99		name = "blight"
100		version = "0.1.0"
101		dependencies = [
102		"clap",
103		"miette",
104		"thiserror",
105		]
106
107		[[package]]
108		name = "cc"
109		version = "1.0.83"
110		source = "registry+https://github.com/rust-lang/crates.io-index"
111		checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
112		dependencies = [
113		"libc",
114		]
115
116		[[package]]
117		name = "cfg-if"
118		version = "1.0.0"
119		source = "registry+https://github.com/rust-lang/crates.io-index"
120		checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
121
122		[[package]]
123		name = "clap"
124		version = "4.4.12"
125		source = "registry+https://github.com/rust-lang/crates.io-index"
126		checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d"
127		dependencies = [
128		"clap_builder",
129		"clap_derive",
130		]
131
132		[[package]]
133		name = "clap_builder"
134		version = "4.4.12"
135		source = "registry+https://github.com/rust-lang/crates.io-index"
136		checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
137		dependencies = [
138		"anstream",
139		"anstyle",
140		"clap_lex",
141		"strsim",
142		]
143
144		[[package]]
145		name = "clap_derive"
146		version = "4.4.7"
147		source = "registry+https://github.com/rust-lang/crates.io-index"
148		checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
149		dependencies = [
150		"heck",
151		"proc-macro2",
152		"quote",
153		"syn",
154		]
155
156		[[package]]
157		name = "clap_lex"
158		version = "0.6.0"
159		source = "registry+https://github.com/rust-lang/crates.io-index"
160		checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
161
162		[[package]]
163		name = "colorchoice"
164		version = "1.0.0"
165		source = "registry+https://github.com/rust-lang/crates.io-index"
166		checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
167
168		[[package]]
169		name = "errno"
170		version = "0.3.8"
171		source = "registry+https://github.com/rust-lang/crates.io-index"
172		checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
173		dependencies = [
174		"libc",
175		"windows-sys",
176		]
177
178		[[package]]
179		name = "gimli"
180		version = "0.28.1"
181		source = "registry+https://github.com/rust-lang/crates.io-index"
182		checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
183
184		[[package]]
185		name = "heck"
186		version = "0.4.1"
187		source = "registry+https://github.com/rust-lang/crates.io-index"
188		checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
189
190		[[package]]
191		name = "hermit-abi"
192		version = "0.3.3"
193		source = "registry+https://github.com/rust-lang/crates.io-index"
194		checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
195
196		[[package]]
197		name = "is-terminal"
198		version = "0.4.10"
199		source = "registry+https://github.com/rust-lang/crates.io-index"
200		checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
201		dependencies = [
202		"hermit-abi",
203		"rustix",
204		"windows-sys",
205		]
206
207		[[package]]
208		name = "is_ci"
209		version = "1.1.1"
210		source = "registry+https://github.com/rust-lang/crates.io-index"
211		checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
212
213		[[package]]
214		name = "libc"
215		version = "0.2.151"
216		source = "registry+https://github.com/rust-lang/crates.io-index"
217		checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
218
219		[[package]]
220		name = "linux-raw-sys"
221		version = "0.4.12"
222		source = "registry+https://github.com/rust-lang/crates.io-index"
223		checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
224
225		[[package]]
226		name = "memchr"
227		version = "2.7.1"
228		source = "registry+https://github.com/rust-lang/crates.io-index"
229		checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
230
231		[[package]]
232		name = "miette"
233		version = "5.10.0"
234		source = "registry+https://github.com/rust-lang/crates.io-index"
235		checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
236		dependencies = [
237		"backtrace",
238		"backtrace-ext",
239		"is-terminal",
240		"miette-derive",
241		"once_cell",
242		"owo-colors",
243		"supports-color",
244		"supports-hyperlinks",
245		"supports-unicode",
246		"terminal_size",
247		"textwrap",
248		"thiserror",
249		"unicode-width",
250		]
251
252		[[package]]
253		name = "miette-derive"
254		version = "5.10.0"
255		source = "registry+https://github.com/rust-lang/crates.io-index"
256		checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
257		dependencies = [
258		"proc-macro2",
259		"quote",
260		"syn",
261		]
262
263		[[package]]
264		name = "miniz_oxide"
265		version = "0.7.1"
266		source = "registry+https://github.com/rust-lang/crates.io-index"
267		checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
268		dependencies = [
269		"adler",
270		]
271
272		[[package]]
273		name = "object"
274		version = "0.32.2"
275		source = "registry+https://github.com/rust-lang/crates.io-index"
276		checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
277		dependencies = [
278		"memchr",
279		]
280
281		[[package]]
282		name = "once_cell"
283		version = "1.19.0"
284		source = "registry+https://github.com/rust-lang/crates.io-index"
285		checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
286
287		[[package]]
288		name = "owo-colors"
289		version = "3.5.0"
290		source = "registry+https://github.com/rust-lang/crates.io-index"
291		checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
292
293		[[package]]
294		name = "proc-macro2"
295		version = "1.0.73"
296		source = "registry+https://github.com/rust-lang/crates.io-index"
297		checksum = "2dd5e8a1f1029c43224ad5898e50140c2aebb1705f19e67c918ebf5b9e797fe1"
298		dependencies = [
299		"unicode-ident",
300		]
301
302		[[package]]
303		name = "quote"
304		version = "1.0.34"
305		source = "registry+https://github.com/rust-lang/crates.io-index"
306		checksum = "22a37c9326af5ed140c86a46655b5278de879853be5573c01df185b6f49a580a"
307		dependencies = [
308		"proc-macro2",
309		]
310
311		[[package]]
312		name = "rustc-demangle"
313		version = "0.1.23"
314		source = "registry+https://github.com/rust-lang/crates.io-index"
315		checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
316
317		[[package]]
318		name = "rustix"
319		version = "0.38.28"
320		source = "registry+https://github.com/rust-lang/crates.io-index"
321		checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
322		dependencies = [
323		"bitflags",
324		"errno",
325		"libc",
326		"linux-raw-sys",
327		"windows-sys",
328		]
329
330		[[package]]
331		name = "smawk"
332		version = "0.3.2"
333		source = "registry+https://github.com/rust-lang/crates.io-index"
334		checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
335
336		[[package]]
337		name = "strsim"
338		version = "0.10.0"
339		source = "registry+https://github.com/rust-lang/crates.io-index"
340		checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
341
342		[[package]]
343		name = "supports-color"
344		version = "2.1.0"
345		source = "registry+https://github.com/rust-lang/crates.io-index"
346		checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
347		dependencies = [
348		"is-terminal",
349		"is_ci",
350		]
351
352		[[package]]
353		name = "supports-hyperlinks"
354		version = "2.1.0"
355		source = "registry+https://github.com/rust-lang/crates.io-index"
356		checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d"
357		dependencies = [
358		"is-terminal",
359		]
360
361		[[package]]
362		name = "supports-unicode"
363		version = "2.0.0"
364		source = "registry+https://github.com/rust-lang/crates.io-index"
365		checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7"
366		dependencies = [
367		"is-terminal",
368		]
369
370		[[package]]
371		name = "syn"
372		version = "2.0.44"
373		source = "registry+https://github.com/rust-lang/crates.io-index"
374		checksum = "92d27c2c202598d05175a6dd3af46824b7f747f8d8e9b14c623f19fa5069735d"
375		dependencies = [
376		"proc-macro2",
377		"quote",
378		"unicode-ident",
379		]
380
381		[[package]]
382		name = "terminal_size"
383		version = "0.1.17"
384		source = "registry+https://github.com/rust-lang/crates.io-index"
385		checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
386		dependencies = [
387		"libc",
388		"winapi",
389		]
390
391		[[package]]
392		name = "textwrap"
393		version = "0.15.2"
394		source = "registry+https://github.com/rust-lang/crates.io-index"
395		checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d"
396		dependencies = [
397		"smawk",
398		"unicode-linebreak",
399		"unicode-width",
400		]
401
402		[[package]]
403		name = "thiserror"
404		version = "1.0.53"
405		source = "registry+https://github.com/rust-lang/crates.io-index"
406		checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09"
407		dependencies = [
408		"thiserror-impl",
409		]
410
411		[[package]]
412		name = "thiserror-impl"
413		version = "1.0.53"
414		source = "registry+https://github.com/rust-lang/crates.io-index"
415		checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19"
416		dependencies = [
417		"proc-macro2",
418		"quote",
419		"syn",
420		]
421
422		[[package]]
423		name = "unicode-ident"
424		version = "1.0.12"
425		source = "registry+https://github.com/rust-lang/crates.io-index"
426		checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
427
428		[[package]]
429		name = "unicode-linebreak"
430		version = "0.1.5"
431		source = "registry+https://github.com/rust-lang/crates.io-index"
432		checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
433
434		[[package]]
435		name = "unicode-width"
436		version = "0.1.11"
437		source = "registry+https://github.com/rust-lang/crates.io-index"
438		checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
439
440		[[package]]
441		name = "utf8parse"
442		version = "0.2.1"
443		source = "registry+https://github.com/rust-lang/crates.io-index"
444		checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
445
446		[[package]]
447		name = "winapi"
448		version = "0.3.9"
449		source = "registry+https://github.com/rust-lang/crates.io-index"
450		checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
451		dependencies = [
452		"winapi-i686-pc-windows-gnu",
453		"winapi-x86_64-pc-windows-gnu",
454		]
455
456		[[package]]
457		name = "winapi-i686-pc-windows-gnu"
458		version = "0.4.0"
459		source = "registry+https://github.com/rust-lang/crates.io-index"
460		checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
461
462		[[package]]
463		name = "winapi-x86_64-pc-windows-gnu"
464		version = "0.4.0"
465		source = "registry+https://github.com/rust-lang/crates.io-index"
466		checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
467
468		[[package]]
469		name = "windows-sys"
470		version = "0.52.0"
471		source = "registry+https://github.com/rust-lang/crates.io-index"
472		checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
473		dependencies = [
474		"windows-targets",
475		]
476
477		[[package]]
478		name = "windows-targets"
479		version = "0.52.0"
480		source = "registry+https://github.com/rust-lang/crates.io-index"
481		checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
482		dependencies = [
483		"windows_aarch64_gnullvm",
484		"windows_aarch64_msvc",
485		"windows_i686_gnu",
486		"windows_i686_msvc",
487		"windows_x86_64_gnu",
488		"windows_x86_64_gnullvm",
489		"windows_x86_64_msvc",
490		]
491
492		[[package]]
493		name = "windows_aarch64_gnullvm"
494		version = "0.52.0"
495		source = "registry+https://github.com/rust-lang/crates.io-index"
496		checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
497
498		[[package]]
499		name = "windows_aarch64_msvc"
500		version = "0.52.0"
501		source = "registry+https://github.com/rust-lang/crates.io-index"
502		checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
503
504		[[package]]
505		name = "windows_i686_gnu"
506		version = "0.52.0"
507		source = "registry+https://github.com/rust-lang/crates.io-index"
508		checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
509
510		[[package]]
511		name = "windows_i686_msvc"
512		version = "0.52.0"
513		source = "registry+https://github.com/rust-lang/crates.io-index"
514		checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
515
516		[[package]]
517		name = "windows_x86_64_gnu"
518		version = "0.52.0"
519		source = "registry+https://github.com/rust-lang/crates.io-index"
520		checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
521
522		[[package]]
523		name = "windows_x86_64_gnullvm"
524		version = "0.52.0"
525		source = "registry+https://github.com/rust-lang/crates.io-index"
526		checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
527
528		[[package]]
529		name = "windows_x86_64_msvc"
530		version = "0.52.0"
531		source = "registry+https://github.com/rust-lang/crates.io-index"
532		checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"

crates/cli/Cargo.toml+1-1

...	...	@@ -5,7 +5,7 @@ edition = "2021"
5	5
6	6	[[bin]]
7	7	name = "blight"
8		path = "./src/main.rs"
	8	path = "./main.rs"
9	9
10	10	[dependencies]
11	11	clap.workspace = true

crates/cli/main.rs created+172

...	...	@@ -0,0 +1,172 @@
	1	use std::{io::Write, path::PathBuf};
	2
	3	use clap::{Parser, Subcommand};
	4	use miette::{diagnostic, Result};
	5
	6	use js_printer;
	7	use tokenizer;
	8
	9	#[derive(Parser)]
	10	#[command(author, version, about, long_about = None)]
	11	struct Cli {
	12	/// Turn debugging information on
	13	#[arg(short, long, action = clap::ArgAction::Count)]
	14	debug: u8,
	15
	16	#[command(subcommand)]
	17	command: Commands,
	18	}
	19
	20	#[derive(Subcommand)]
	21	enum Commands {
	22	/// Compiles a file into javascript
	23	Compile {
	24	file: PathBuf,
	25	// Whether to output to stdout instead of a file
	26	#[arg(long, default_value_t = false)]
	27	stdout: bool,
	28	/// What javascript runtime to compile to
	29	#[arg(short, long, default_value = "deno")]
	30	target: String,
	31	/// Format the output javascript?
	32	#[arg(long, default_value_t = true)]
	33	format: bool,
	34	},
	35	Run {
	36	file: PathBuf,
	37	/// What javascript runtime to compile to and use
	38	#[arg(short, long, default_value = "deno")]
	39	target: String,
	40	},
	41	Astdump {
	42	file: PathBuf,
	43	},
	44	}
	45
	46	fn get_runtime(runtime: &String) -> String {
	47	match std::env::var("USER") {
	48	Ok(user) if user == "dave" => String::from("bun"),
	49	_ => runtime.clone(),
	50	}
	51	}
	52
	53	pub fn transpile(content: String, writer: &mut dyn Write) -> Result<()> {
	54	// 1. Tokenize the input
	55	let tokens = tokenizer::tokenize(content)?;
	56	// 2. Parse the tokens into an AST
	57	let mut blight_ast = parser::parse(tokens)?;
	58	// 3. Semantic analysis.
	59	sema::analyze_module(&mut blight_ast)?;
	60	// 4. Optimize the AST
	61	// optimize::optimize_module(&mut blight_ast);
	62	// 5. Translate the AST into javascript
	63	let js_ast = js_ast::blight_to_js(blight_ast);
	64	// 6. Print the JS AST
	65	// TODO(@paperdave): use std::io::Write instead of a string
	66	js_printer::Print::print(&js_ast, writer)
	67	.map_err(\|e\| diagnostic!("Error while printing: {}", e))?;
	68	Ok(())
	69	}
	70
	71	pub fn transpile_to_string(content: String) -> Result<String> {
	72	let mut buffer = Vec::new();
	73	transpile(content, &mut buffer)?;
	74	Ok(String::from_utf8(buffer).unwrap())
	75	}
	76
	77	pub fn astdump(content: String) -> Result<String> {
	78	// 1. Tokenize the input
	79	let tokens = tokenizer::tokenize(content)?;
	80	// 2. Parse the tokens into an AST
	81	let blight_ast = parser::parse(tokens)?;
	82	return Ok(format!("{:#?}", blight_ast));
	83	}
	84
	85	fn main() -> Result<()> {
	86	let cli = Cli::parse();
	87
	88	match cli.command {
	89	Commands::Compile {
	90	file,
	91	stdout,
	92	target: _,
	93	format,
	94	} => {
	95	let contents = match std::fs::read_to_string(file.clone()) {
	96	Ok(contents) => contents,
	97	Err(e) => {
	98	eprintln!("Failed to read {}: {}", file.display(), e);
	99	std::process::exit(1);
	100	}
	101	};
	102
	103	let mut result = transpile_to_string(contents.clone())?;
	104
	105	if format {
	106	let mut formatter = std::process::Command::new("deno")
	107	.stdin(std::process::Stdio::piped())
	108	.stdout(std::process::Stdio::piped())
	109	.stderr(std::process::Stdio::inherit())
	110	.arg("fmt")
	111	.arg("-")
	112	.spawn()
	113	.expect("Failed to execute 'deno fmt' on output");
	114	let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
	115	stdin
	116	.write_all(result.as_bytes())
	117	.expect("Failed to write to stdin");
	118	stdin.flush().expect("Failed to flush stdin");
	119	drop(stdin);
	120	let out = formatter
	121	.wait_with_output()
	122	.expect("Failed to execute 'deno fmt' on output");
	123	result = String::from_utf8(out.stdout).unwrap();
	124	}
	125
	126	if stdout {
	127	println!("{}", result);
	128	} else {
	129	std::fs::write(file.with_extension("js"), result).unwrap();
	130	}
	131	}
	132	Commands::Run { file, target } => {
	133	let contents = std::fs::read_to_string(file.clone()).unwrap();
	134	let result = transpile_to_string(contents.clone())?;
	135	let mut temp_file = std::env::temp_dir();
	136	temp_file.push("temp.js");
	137	std::fs::write(temp_file.clone(), result).unwrap();
	138
	139	let rt = get_runtime(&target);
	140
	141	match rt.as_str() {
	142	"deno" => std::process::Command::new("deno")
	143	.arg("run")
	144	.arg("-A")
	145	.arg(temp_file)
	146	.spawn()
	147	.expect("Failed to execute process"),
	148	"node" => std::process::Command::new("node")
	149	.arg(temp_file)
	150	.spawn()
	151	.expect("Failed to execute process"),
	152	"bun" => std::process::Command::new("bun")
	153	.arg("run")
	154	.arg(temp_file)
	155	.spawn()
	156	.expect("Failed to execute process"),
	157	_ => {
	158	panic!("Unknown runtime: {}", target);
	159	}
	160	}
	161	.wait()
	162	.unwrap();
	163	}
	164	Commands::Astdump { file } => {
	165	let contents = std::fs::read_to_string(file.clone()).unwrap();
	166	let result = astdump(contents.clone())?;
	167	println!("{}", result);
	168	}
	169	}
	170
	171	Ok(())
	172	}

crates/cli/src/main.rs deleted-165

...	...	@@ -1,165 +0,0 @@
1		use std::{io::Write, path::PathBuf};
2
3		use clap::{Parser, Subcommand};
4		use miette::Result;
5
6		use js_printer;
7		use tokenizer;
8
9		#[derive(Parser)]
10		#[command(author, version, about, long_about = None)]
11		struct Cli {
12		/// Turn debugging information on
13		#[arg(short, long, action = clap::ArgAction::Count)]
14		debug: u8,
15
16		#[command(subcommand)]
17		command: Commands,
18		}
19
20		#[derive(Subcommand)]
21		enum Commands {
22		/// Compiles a file into javascript
23		Compile {
24		file: PathBuf,
25		// Whether to output to stdout instead of a file
26		#[arg(long, default_value_t = false)]
27		stdout: bool,
28		/// What javascript runtime to compile to
29		#[arg(short, long, default_value = "deno")]
30		target: String,
31		/// Format the output javascript?
32		#[arg(long, default_value_t = true)]
33		format: bool,
34		},
35		Run {
36		file: PathBuf,
37		/// What javascript runtime to compile to and use
38		#[arg(short, long, default_value = "deno")]
39		target: String,
40		},
41		Astdump {
42		file: PathBuf,
43		},
44		}
45
46		fn get_runtime(runtime: &String) -> String {
47		match std::env::var("USER") {
48		Ok(user) if user == "dave" => String::from("bun"),
49		_ => runtime.clone(),
50		}
51		}
52
53		pub fn transpile(content: String) -> Result<String> {
54		// 1. Tokenize the input
55		let tokens = tokenizer::tokenize(content)?;
56		// 2. Parse the tokens into an AST
57		let blight_ast = parser::parse(tokens)?;
58		// 3. Type check the AST
59		// sema::check_module(&blight_ast)?;
60		// 4. Optimize the AST
61		// optimize(tree.clone())?;
62		// 5. Translate the AST into javascript
63		let js_ast = js_ast::translate(&blight_ast)?;
64		// 6. Print the JS AST
65		let javascript = js_printer::print(js_ast);
66		// 7. Return the javascript
67		return Ok(javascript);
68		}
69
70		pub fn astdump(content: String) -> Result<String> {
71		// 1. Tokenize the input
72		let tokens = tokenizer::tokenize(content)?;
73		// 2. Parse the tokens into an AST
74		let blight_ast = parser::parse(tokens)?;
75		return Ok(format!("{:#?}", blight_ast));
76		}
77
78		fn main() -> Result<()> {
79		let cli = Cli::parse();
80
81		match cli.command {
82		Commands::Compile {
83		file,
84		stdout,
85		target: _,
86		format,
87		} => {
88		let contents = match std::fs::read_to_string(file.clone()) {
89		Ok(contents) => contents,
90		Err(e) => {
91		eprintln!("Failed to read {}: {}", file.display(), e);
92		std::process::exit(1);
93		}
94		};
95
96		let mut result = transpile(contents.clone())?;
97
98		if format {
99		let mut formatter = std::process::Command::new("deno")
100		.stdin(std::process::Stdio::piped())
101		.stdout(std::process::Stdio::piped())
102		.stderr(std::process::Stdio::inherit())
103		.arg("fmt")
104		.arg("-")
105		.spawn()
106		.expect("Failed to execute 'deno fmt' on output");
107		let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
108		stdin
109		.write_all(result.as_bytes())
110		.expect("Failed to write to stdin");
111		stdin.flush().expect("Failed to flush stdin");
112		drop(stdin);
113		let out = formatter
114		.wait_with_output()
115		.expect("Failed to execute 'deno fmt' on output");
116		result = String::from_utf8(out.stdout).unwrap();
117		}
118
119		if stdout {
120		println!("{}", result);
121		} else {
122		std::fs::write(file.with_extension("js"), result).unwrap();
123		}
124		}
125		Commands::Run { file, target } => {
126		let contents = std::fs::read_to_string(file.clone()).unwrap();
127		let result = transpile(contents.clone())?;
128		let mut temp_file = std::env::temp_dir();
129		temp_file.push("temp.js");
130		std::fs::write(temp_file.clone(), result).unwrap();
131
132		let rt = get_runtime(&target);
133
134		match rt.as_str() {
135		"deno" => std::process::Command::new("deno")
136		.arg("run")
137		.arg("-A")
138		.arg(temp_file)
139		.spawn()
140		.expect("Failed to execute process"),
141		"node" => std::process::Command::new("node")
142		.arg(temp_file)
143		.spawn()
144		.expect("Failed to execute process"),
145		"bun" => std::process::Command::new("bun")
146		.arg("run")
147		.arg(temp_file)
148		.spawn()
149		.expect("Failed to execute process"),
150		_ => {
151		panic!("Unknown runtime: {}", target);
152		}
153		}
154		.wait()
155		.unwrap();
156		}
157		Commands::Astdump { file } => {
158		let contents = std::fs::read_to_string(file.clone()).unwrap();
159		let result = astdump(contents.clone())?;
160		println!("{}", result);
161		}
162		}
163
164		Ok(())
165		}

crates/js_ast/Cargo.toml+1

...	...	@@ -13,3 +13,4 @@ num.workspace = true
13	13
14	14	ast = { path = "../ast" }
15	15	tokenizer = { path = "../tokenizer" } # TODO: move Loc out of the tokenizer
	16	parser = { path = "../parser" } # TODO: move Loc out of the tokenizer

crates/js_ast/convert_valid_expr.rs created+204

...	...	@@ -0,0 +1,204 @@
	1	//! "Valid" in this context means that the expression does not contain JS statements
	2	//! Meaning conversion is a direct map from a blight as node into a JS one
	3	//!
	4	//! See `is_valid_js_expression` for more details on what qualifies
	5	//!
	6	//! For other nodes, it must hoist statements to the nearest JS block,
	7	//! is done in the `hoist_*` functions in `host.rs`.
	8
	9	use ast::{BlockOrIf, Comment, ExprData, If};
	10	use ast::{BlockWithLoc, BuiltinId, Expr, Literal, Statement};
	11	use tokenizer::Loc;
	12
	13	use crate::is_valid_js_expression::IsValidJSExpression;
	14	use crate::js_ast::*;
	15	use crate::HoistContext;
	16
	17	/// Translate a blight ast expression that is a valid JS expression, to a JS expression
	18	/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
	19	pub fn convert_valid_expr(ctx: &mut HoistContext, expr: Expr) -> JsExpr {
	20	// Assert may be skipped in release builds as all failures are
	21	// annotated with a release-checked panic
	22	debug_assert!(
	23	expr.is_valid_js_expression(),
	24	"must call convert_valid_expr with valid js expression (use hoist instead)"
	25	);
	26
	27	let loc = expr.loc;
	28	let comment = expr.comment;
	29
	30	match expr.data {
	31	ExprData::Identifier(str) => JsExpr::new(
	32	JsExprData::Identifier(ctx.remap_identifier(&str)),
	33	loc,
	34	comment,
	35	),
	36
	37	ExprData::Literal(Literal::Bool(value)) => {
	38	JsExpr::new(JsExprData::Bool(value), loc, comment)
	39	}
	40	ExprData::Literal(Literal::Number(value)) => {
	41	JsExpr::new(JsExprData::Number(value), loc, comment)
	42	}
	43	ExprData::Literal(Literal::String(value)) => {
	44	JsExpr::new(JsExprData::String(value), loc, comment)
	45	}
	46	ExprData::Literal(Literal::Void) => JsExpr::new(JsExprData::Null, loc, comment),
	47
	48	ExprData::Block(block) => convert_valid_block(ctx, BlockWithLoc { block, loc }),
	49
	50	ExprData::Binary { op, left, right } => {
	51	let left = convert_valid_expr(ctx, *left);
	52	let right = convert_valid_expr(ctx, *right);
	53	JsExpr::new(
	54	JsExprData::Binary {
	55	op,
	56	left: Box::new(left),
	57	right: Box::new(right),
	58	},
	59	loc,
	60	comment,
	61	)
	62	}
	63	ExprData::Unary { op, value } => {
	64	let value = convert_valid_expr(ctx, *value);
	65	JsExpr::new(
	66	JsExprData::Unary {
	67	op,
	68	value: Box::new(value),
	69	},
	70	loc,
	71	comment,
	72	)
	73	}
	74	ExprData::Fn { block, params, .. } => JsExpr::new(
	75	JsExprData::ArrowFunction {
	76	params: params.into_iter().map(\|x\| x.name).collect(),
	77	body: Box::new(ctx.hoist_block_to_arrow_fn_body(BlockWithLoc {
	78	block,
	79	loc: loc.clone(),
	80	})),
	81	},
	82	loc,
	83	comment,
	84	),
	85	ExprData::Builtin { id, arguments } => match id {
	86	BuiltinId::Raw => {
	87	let first_arg = arguments.iter().next().unwrap();
	88	// TODO(@paperdave): this string clone is stupid
	89	let str = match first_arg.data.clone() {
	90	ExprData::Literal(Literal::String(s)) => s,
	91	_ => panic!("raw expects a string"),
	92	};
	93	JsExpr::new(JsExprData::Raw(str), loc, comment)
	94	}
	95	},
	96	ExprData::If(expr) => convert_valid_if(ctx, expr, loc, comment),
	97
	98	ExprData::Call { base, arguments } => {
	99	let base = convert_valid_expr(ctx, *base);
	100	let arguments = arguments
	101	.into_iter()
	102	.map(\|x\| convert_valid_expr(ctx, x))
	103	.collect();
	104	JsExpr::new(
	105	JsExprData::Call {
	106	base: Box::new(base),
	107	arguments,
	108	},
	109	loc,
	110	comment,
	111	)
	112	}
	113
	114	ExprData::Field { base, field } => JsExpr::new(
	115	match field.data {
	116	ExprData::Literal(Literal::String(name)) => JsExprData::Dot {
	117	base: Box::new(convert_valid_expr(ctx, *base)),
	118	name,
	119	},
	120	_ => JsExprData::Index {
	121	base: Box::new(convert_valid_expr(ctx, *base)),
	122	index: Box::new(convert_valid_expr(ctx, *field)),
	123	},
	124	},
	125	loc,
	126	comment,
	127	),
	128
	129	ExprData::Return(_) => unreachable!("convert_valid_expr does not support return"),
	130	}
	131	}
	132
	133	pub fn convert_valid_block(ctx: &mut HoistContext, b: BlockWithLoc) -> JsExpr {
	134	debug_assert!(
	135	b.is_valid_js_expression(),
	136	"must call convert_valid_block with valid js expression"
	137	);
	138
	139	let len = b.block.statements.len();
	140	let loc = b.loc;
	141	let block_expr = match len {
	142	0 => JsExpr::new(JsExprData::Null, loc.clone(), None),
	143	// a block expression with a single expression is just that expression
	144	1 => convert_valid_statement_to_expr(ctx, b.block.statements.into_iter().next().unwrap()),
	145	// Since blocks are not valid expressions in JS, we create a
	146	// comma expression from a block with more than one statement.
	147	2.. => {
	148	let mut items = Vec::new();
	149	for statement in b.block.statements.into_iter() {
	150	items.push(convert_valid_statement_to_expr(ctx, statement));
	151	}
	152	JsExpr::new(JsExprData::Comma { items }, loc.clone(), None)
	153	}
	154	};
	155
	156	// a block without implicit return is to be type void, so we wrap in a void expression
	157	if !b.block.has_implicit_return && len > 0 {
	158	JsExpr::new(JsExprData::Void(Box::new(block_expr)), loc, None)
	159	} else {
	160	block_expr
	161	}
	162	}
	163
	164	pub fn convert_valid_statement_to_expr(ctx: &mut HoistContext, statement: Statement) -> JsExpr {
	165	debug_assert!(
	166	statement.is_valid_js_expression(),
	167	"convert_valid_statement_to_expr on an invalid expression (use hoist instead)"
	168	);
	169
	170	match statement {
	171	ast::Statement::Expr(expr) => convert_valid_expr(ctx, expr),
	172	_ => unreachable!("attempt to convert statement to js without hoisting"),
	173	}
	174	}
	175
	176	pub fn convert_valid_statement(ctx: &mut HoistContext, statement: Statement) -> JsStatement {
	177	let expr = convert_valid_statement_to_expr(ctx, statement);
	178	let loc = expr.loc.clone();
	179	JsStatement::new(JsStatementData::Expression(expr), loc, None)
	180	}
	181
	182	pub fn convert_valid_if(
	183	ctx: &mut HoistContext,
	184	expr: If,
	185	loc: Loc,
	186	comment: Option<Comment>,
	187	) -> JsExpr {
	188	let condition = convert_valid_expr(ctx, *expr.condition);
	189	let yes = convert_valid_block(ctx, expr.yes);
	190	let no = match expr.no {
	191	Some(BlockOrIf::Block(block)) => Some(convert_valid_block(ctx, block)),
	192	Some(BlockOrIf::If(else_if)) => Some(convert_valid_if(ctx, *else_if, Loc::EMPTY, None)),
	193	None => None,
	194	};
	195	JsExpr::new(
	196	JsExprData::Ternary {
	197	condition: Box::new(condition),
	198	yes: Box::new(yes),
	199	no: no.map(Box::new),
	200	},
	201	loc,
	202	comment,
	203	)
	204	}

crates/js_ast/has_side_effects.rs created+90

...	...	@@ -0,0 +1,90 @@
	1	use ast::*;
	2
	3	pub trait HasSideEffects {
	4	/// Can this item be removed from the AST without changing the program's behavior?
	5	fn has_side_effects(&self) -> bool;
	6	}
	7
	8	impl HasSideEffects for ExprData {
	9	fn has_side_effects(&self) -> bool {
	10	match self {
	11	ExprData::Literal(_) => false,
	12	ExprData::Identifier(_) => false,
	13
	14	ExprData::Block(block) => block.has_side_effects(),
	15	ExprData::Binary { left, right, .. } => {
	16	left.has_side_effects() \|\| right.has_side_effects()
	17	}
	18	ExprData::Unary { value, .. } => value.has_side_effects(),
	19	ExprData::If(ifexpr) => ifexpr.has_side_effects(),
	20
	21	// note: a function declaration has no side effects, but the call can
	22	ExprData::Fn { .. } => false,
	23
	24	// TODO(@paperdave): implement detection of pure functions
	25	ExprData::Call { base, arguments } => {
	26	base.has_side_effects() \|\| arguments.iter().any(\|x\| x.has_side_effects())
	27	}
	28
	29	// note: we assume field access has side effects, ie: a getter doesn't throw
	30	ExprData::Field { base, field } => base.has_side_effects() \|\| field.has_side_effects(),
	31
	32	// note: we assume the code in @raw always has side effects
	33	ExprData::Builtin { id, .. } => match id {
	34	BuiltinId::Raw => true,
	35	},
	36
	37	ExprData::Return(_) => true,
	38	}
	39	}
	40	}
	41
	42	impl HasSideEffects for Statement {
	43	fn has_side_effects(&self) -> bool {
	44	match self {
	45	Statement::Expr(expr) => expr.has_side_effects(),
	46	Statement::Decl(decl) => decl.init.has_side_effects(),
	47	Statement::Comment(_) => false,
	48	}
	49	}
	50	}
	51
	52	impl HasSideEffects for Block {
	53	fn has_side_effects(&self) -> bool {
	54	self.statements
	55	.iter()
	56	.any(\|statement\| statement.has_side_effects())
	57	}
	58	}
	59
	60	impl HasSideEffects for Expr {
	61	fn has_side_effects(&self) -> bool {
	62	self.data.has_side_effects()
	63	}
	64	}
	65
	66	impl HasSideEffects for If {
	67	fn has_side_effects(&self) -> bool {
	68	self.condition.has_side_effects()
	69	\|\| self.yes.has_side_effects()
	70	\|\| match &self.no {
	71	Some(no) => no.has_side_effects(),
	72	None => false,
	73	}
	74	}
	75	}
	76
	77	impl HasSideEffects for BlockOrIf {
	78	fn has_side_effects(&self) -> bool {
	79	match self {
	80	BlockOrIf::Block(block) => block.has_side_effects(),
	81	BlockOrIf::If(if_) => if_.has_side_effects(),
	82	}
	83	}
	84	}
	85
	86	impl HasSideEffects for BlockWithLoc {
	87	fn has_side_effects(&self) -> bool {
	88	self.block.has_side_effects()
	89	}
	90	}

crates/js_ast/hoist.rs created+614

...	...	@@ -0,0 +1,614 @@
	1	use std::collections::{HashMap, HashSet};
	2	use std::vec;
	3
	4	use ast::{BinaryOperator, BlockOrIf, BlockWithLoc, Expr, Statement};
	5	use tokenizer::Loc;
	6
	7	use crate::*;
	8
	9	pub struct HoistContext {
	10	pub statements: Vec<JsStatement>,
	11	pub ids: HashSet<String>,
	12	next_id: u32,
	13	scopes: Vec<HoistScope>,
	14	has_last_statement: bool,
	15	}
	16
	17	struct HoistScope {
	18	renames: HashMap<String, String>,
	19	}
	20
	21	pub struct TempId {
	22	unmapped_name: String,
	23	}
	24
	25	impl TempId {
	26	pub fn get(&self, hoist_context: &HoistContext) -> String {
	27	hoist_context.remap_identifier(&self.unmapped_name)
	28	}
	29	}
	30
	31	impl HoistContext {
	32	pub fn new() -> Self {
	33	Self {
	34	statements: Vec::new(),
	35	ids: HashSet::new(),
	36	next_id: 0,
	37	scopes: vec![HoistScope {
	38	renames: HashMap::new(),
	39	}],
	40	has_last_statement: false,
	41	}
	42	}
	43
	44	pub fn new_child(&self) -> Self {
	45	Self {
	46	statements: Vec::new(),
	47	ids: self.ids.clone(),
	48	next_id: 0,
	49	scopes: vec![HoistScope {
	50	renames: HashMap::new(),
	51	}],
	52	has_last_statement: false,
	53	}
	54	}
	55
	56	/// A virtual scope is a scope that exists in blight syntax
	57	/// but may or may not exist in blight. With these it is important to
	58	pub fn push_virtual_scope(&mut self) {
	59	self.scopes.push(HoistScope {
	60	renames: HashMap::new(),
	61	});
	62	}
	63
	64	/// See `push_virtual_scope` for more information on what this is.
	65	pub fn pop_virtual_scope(&mut self) {
	66	self.scopes.pop();
	67	}
	68
	69	pub fn generate_temp_id(&mut self, basename: &str) -> TempId {
	70	self.next_id += 1;
	71	loop {
	72	let name = format!("{}{}", basename, self.next_id);
	73	if !self.ids.contains(&name) {
	74	self.ids.insert(name.clone());
	75	return TempId {
	76	unmapped_name: name,
	77	};
	78	}
	79	self.next_id += 1;
	80	}
	81	}
	82
	83	/// This does some extra optimizations:
	84	/// - `let a = b;` will not be emitted but instad write a variable alias
	85	pub fn add_statement(&mut self, statement: JsStatement) {
	86	match statement.data {
	87	JsStatementData::Declaration(JSDeclaration {
	88	name,
	89	init:
	90	Some(JsExpr {
	91	data: JsExprData::Identifier(init_name),
	92	..
	93	}),
	94	is_constant: true,
	95	}) => {
	96	self.scopes
	97	.last_mut()
	98	.unwrap()
	99	.renames
	100	.insert(name, init_name);
	101	self.has_last_statement = false;
	102	return;
	103	}
	104	_ => {}
	105	}
	106	self.statements.push(statement);
	107	self.has_last_statement = true;
	108	}
	109
	110	/// Get the last statement if it exists.
	111	/// If the statement was optimized away, this returns None
	112	pub fn last_statement(&self) -> Option<&JsStatement> {
	113	if self.has_last_statement {
	114	Some(self.statements.last().unwrap())
	115	} else {
	116	None
	117	}
	118	}
	119
	120	pub fn generate_id(&mut self, basename: &String) -> TempId {
	121	let mut id = 0u32;
	122	loop {
	123	let name = match id {
	124	0 => basename.clone(),
	125	_ => format!("{}{}", basename, id),
	126	};
	127	if !self.ids.contains(&name) {
	128	self.ids.insert(name.clone());
	129	return TempId {
	130	unmapped_name: name,
	131	};
	132	}
	133	id += 1;
	134	}
	135	}
	136
	137	/// Given a blight identifier, remap it to a new identifier.
	138	pub fn remap_identifier(&self, name: &String) -> String {
	139	for scope in self.scopes.iter().rev() {
	140	if let Some(remapped) = scope.renames.get(name) {
	141	return remapped.clone();
	142	}
	143	}
	144	name.clone()
	145	}
	146
	147	/// Declare an identifier in the current scope.
	148	pub fn declare_identifier(&mut self, name: String) -> TempId {
	149	let remapped = self.generate_id(&name);
	150	let scope = self.scopes.last_mut().unwrap();
	151	self.ids.insert(remapped.unmapped_name.clone());
	152	scope.renames.insert(name, remapped.unmapped_name.clone());
	153	remapped
	154	}
	155
	156	pub fn process_statement(&mut self, statement: Statement) -> Option<()> {
	157	if statement.is_valid_js_expression() {
	158	let result = convert_valid_statement(self, statement);
	159	if !result.is_literal() {
	160	self.add_statement(result);
	161	}
	162	return Some(());
	163	}
	164
	165	match statement {
	166	Statement::Decl(decl) => {
	167	// visit init before name, this way allows for:
	168	//
	169	// let a = a;
	170	//
	171	// and `a` will be looked up before it is re-declared.
	172	if let Some(init) = self.hoist_expression_contents(decl.init) {
	173	let name = self.declare_identifier(decl.name);
	174
	175	self.add_statement(JsStatement::new(
	176	JsStatementData::Declaration(crate::JSDeclaration {
	177	name: name.get(&self),
	178	init: Some(init),
	179	is_constant: decl.is_constant,
	180	}),
	181	decl.loc,
	182	decl.comment.map(\|x\| x.into()),
	183	));
	184	Some(())
	185	} else {
	186	None
	187	}
	188	}
	189	Statement::Expr(expr) => {
	190	if let Some(expr) = self.hoist_expression_contents(expr) {
	191	let loc = expr.loc.clone();
	192	if !expr.is_literal() {
	193	self.add_statement(JsStatement::new(
	194	JsStatementData::Expression(expr),
	195	loc,
	196	None,
	197	))
	198	}
	199	Some(())
	200	} else {
	201	None
	202	}
	203	}
	204	Statement::Comment(_) => Some(()),
	205	}
	206	}
	207
	208	/// Pushes a decl with a generated identifier to the hoisted result
	209	/// and returns an identifier expression for it.
	210	pub fn hoist_expression_to_temporary(&mut self, expr: Expr) -> Option<JsExpr> {
	211	// TODO
	212	// if !expr.has_side_effects() {
	213	// return Some(self.hoist_expression_contents(expr)?);
	214	// }
	215
	216	let init = self.hoist_expression_contents(expr)?;
	217
	218	let id = self.generate_temp_id("_temp");
	219	let loc = init.loc.clone();
	220
	221	self.add_statement(JsStatement::new(
	222	JsStatementData::Declaration(JSDeclaration {
	223	name: id.get(&self),
	224	init: Some(init),
	225	is_constant: true,
	226	}),
	227	loc.clone(),
	228	None,
	229	));
	230
	231	Some(JsExpr::new(
	232	JsExprData::Identifier(id.get(&self)),
	233	loc,
	234	None,
	235	))
	236	}
	237
	238	// Returning `None` from this means that the expression is noreturn, and that
	239	// the rest of the statement never occurs and can be stripped.
	240	pub fn hoist_expression_contents(&mut self, expr: Expr) -> Option<JsExpr> {
	241	if expr.is_valid_js_expression() {
	242	return Some(convert_valid_expr(self, expr));
	243	}
	244
	245	match expr.data {
	246	ast::ExprData::Binary { op, left, right } => {
	247	let left = self.hoist_expression_to_temporary(*left)?;
	248	let right = self.hoist_expression_to_temporary(*right)?;
	249
	250	Some(JsExpr::new(
	251	JsExprData::Binary {
	252	op,
	253	left: Box::new(left),
	254	right: Box::new(right),
	255	},
	256	expr.loc,
	257	expr.comment,
	258	))
	259	}
	260
	261	ast::ExprData::Unary { op, value } => {
	262	let value = self.hoist_expression_to_temporary(*value)?;
	263	Some(JsExpr::new(
	264	JsExprData::Unary {
	265	op,
	266	value: Box::new(value),
	267	},
	268	expr.loc,
	269	expr.comment,
	270	))
	271	}
	272
	273	ast::ExprData::Field { base, field } => {
	274	let base = self.hoist_expression_to_temporary(*base)?;
	275	let field = self.hoist_expression_to_temporary(*field)?;
	276
	277	Some(JsExpr::new(
	278	match field.data {
	279	JsExprData::String(name) if is_js_identifier(name.as_str()) => {
	280	JsExprData::Dot {
	281	base: Box::new(base),
	282	name,
	283	}
	284	}
	285	_ => JsExprData::Index {
	286	base: Box::new(base),
	287	index: Box::new(field),
	288	},
	289	},
	290	expr.loc,
	291	expr.comment,
	292	))
	293	}
	294
	295	ast::ExprData::Block(block) => {
	296	let temporary = self.generate_temp_id("_block");
	297	let result = self.hoist_block_to_js_block_in_same_context(BlockWithLoc {
	298	block,
	299	loc: expr.loc.clone(),
	300	});
	301
	302	self.add_statement(JsStatement::new(
	303	JsStatementData::Declaration(crate::JSDeclaration {
	304	name: temporary.get(&self),
	305	init: Some(result),
	306	is_constant: false,
	307	}),
	308	expr.loc.clone(),
	309	None,
	310	));
	311
	312	Some(JsExpr::new(
	313	JsExprData::Identifier(temporary.get(&self)),
	314	expr.loc,
	315	expr.comment,
	316	))
	317	}
	318
	319	ast::ExprData::If(data) => {
	320	let temporary = self.generate_temp_id("_if");
	321
	322	let decl_index = self.statements.len();
	323	self.add_statement(JsStatement::new(
	324	JsStatementData::Declaration(crate::JSDeclaration {
	325	name: temporary.get(&self),
	326	init: None,
	327	is_constant: false,
	328	}),
	329	expr.loc.clone(),
	330	None,
	331	));
	332
	333	let condition = self.hoist_expression_to_temporary(*data.condition)?;
	334	let mut did_assign = false;
	335
	336	let js_if =
	337	self.hoist_js_if(data.yes, data.no, condition, &mut did_assign, &temporary)?;
	338
	339	self.add_statement(JsStatement::new(
	340	JsStatementData::If(js_if),
	341	expr.loc.clone(),
	342	None,
	343	));
	344
	345	if did_assign {
	346	Some(JsExpr::new(
	347	JsExprData::Identifier(temporary.get(&self)),
	348	expr.loc,
	349	expr.comment,
	350	))
	351	} else {
	352	self.statements.remove(decl_index);
	353	Some(JsExpr::new(JsExprData::Null, expr.loc, expr.comment))
	354	}
	355	}
	356
	357	ast::ExprData::Return(expr) => {
	358	let loc = expr.loc.clone();
	359	let return_value = self.hoist_expression_to_temporary(*expr)?;
	360	self.add_statement(JsStatement::new(
	361	JsStatementData::Return(return_value),
	362	loc,
	363	None,
	364	));
	365	None
	366	}
	367
	368	ast::ExprData::Call { base, arguments } => {
	369	let base = self.hoist_expression_to_temporary(*base)?;
	370	let arguments = arguments
	371	.into_iter()
	372	.map(\|x\| self.hoist_expression_to_temporary(x))
	373	.collect::<Option<Vec<_>>>()?;
	374
	375	Some(JsExpr::new(
	376	JsExprData::Call {
	377	base: Box::new(base),
	378	arguments,
	379	},
	380	expr.loc,
	381	expr.comment,
	382	))
	383	}
	384
	385	ast::ExprData::Fn { .. }
	386	\| ast::ExprData::Builtin { .. }
	387	\| ast::ExprData::Literal(_)
	388	\| ast::ExprData::Identifier(_) => {
	389	unreachable!("attempt to hoist an expression that does not require it")
	390	}
	391	}
	392	}
	393
	394	pub fn hoist_block_to_js_block(
	395	&mut self,
	396	b: BlockWithLoc,
	397	temporary: &String,
	398	did_assign: &mut bool,
	399	) -> Option<JsBlock> {
	400	let loc = b.loc.clone();
	401
	402	// A non-block will become a block with a single expression:
	403	// _temp = expr;
	404	//
	405	// We will also apply this transformation to blocks that are valid
	406	// expressions, because that is more concise.
	407	if b.is_valid_js_expression() {
	408	return Some(JsBlock::new(
	409	vec![new_js_assignment_statement(
	410	convert_valid_block(self, b),
	411	temporary.clone(),
	412	loc.clone(),
	413	)],
	414	loc,
	415	));
	416	}
	417
	418	let mut child = self.new_child();
	419
	420	for statement in b.block.statements {
	421	if let None = child.process_statement(statement) {
	422	return None;
	423	};
	424	}
	425
	426	// If the last statement is an expression, we need
	427	// to assign to the temporary variable.
	428	match child.statements.last().map(\|x\| &x.data) {
	429	Some(JsStatementData::Expression(_)) => {
	430	*did_assign = true;
	431
	432	let last = child.statements.pop().unwrap();
	433	let value = match last {
	434	JsStatement {
	435	data: JsStatementData::Expression(expr),
	436	..
	437	} => expr,
	438	_ => {
	439	self.statements.push(last);
	440	JsExpr::new(JsExprData::Null, loc.clone(), None)
	441	}
	442	};
	443	child.statements.push(new_js_assignment_statement(
	444	value,
	445	temporary.clone(),
	446	loc.clone(),
	447	));
	448	}
	449	_ => {}
	450	}
	451
	452	Some(JsBlock::new(child.statements, loc))
	453	}
	454
	455	pub fn hoist_block_to_arrow_fn_body(&mut self, b: BlockWithLoc) -> JsArrowBody {
	456	let loc = b.loc.clone();
	457
	458	// A block that is a valid expression will just become
	459	// that expression. We can convert multiple expressions
	460	// into a comma expression.
	461	if b.is_valid_js_expression() {
	462	return JsArrowBody::Expr(convert_valid_block(self, b));
	463	}
	464
	465	let mut child = self.new_child();
	466
	467	for statement in b.block.statements {
	468	if let None = child.process_statement(statement) {
	469	break;
	470	};
	471	}
	472
	473	// If the last statement is an expression,
	474	// we need to return that expression.
	475	match child.statements.last().map(\|x\| &x.data) {
	476	Some(JsStatementData::Expression(_)) => {
	477	let last = child.statements.pop().unwrap();
	478	let value = match last {
	479	JsStatement {
	480	data: JsStatementData::Expression(expr),
	481	..
	482	} => expr,
	483	_ => {
	484	self.statements.push(last);
	485	JsExpr::new(JsExprData::Null, loc.clone(), None)
	486	}
	487	};
	488	child.statements.push(JsStatement::new(
	489	JsStatementData::Return(value),
	490	loc.clone(),
	491	None,
	492	));
	493	}
	494	_ => {}
	495	}
	496
	497	JsArrowBody::Block(JsBlock::new(child.statements, loc))
	498	}
	499
	500	pub fn hoist_block_to_js_block_in_same_context(&mut self, b: BlockWithLoc) -> JsExpr {
	501	let loc = b.loc.clone();
	502	self.push_virtual_scope();
	503
	504	debug_assert!(!b.block.is_valid_js_expression());
	505
	506	for statement in b.block.statements {
	507	self.process_statement(statement);
	508	}
	509
	510	self.pop_virtual_scope();
	511
	512	// If the last statement is an expression, we need
	513	// to assign to the temporary variable.
	514	match self.last_statement().map(\|x\| &x.data) {
	515	Some(JsStatementData::Expression(_)) if b.block.has_implicit_return => {
	516	let last = self.statements.pop().unwrap();
	517	match last {
	518	JsStatement {
	519	data: JsStatementData::Expression(expr),
	520	..
	521	} => expr,
	522	_ => {
	523	self.statements.push(last);
	524	JsExpr::new(JsExprData::Null, loc.clone(), None)
	525	}
	526	}
	527	}
	528	_ => JsExpr::new(JsExprData::Null, loc.clone(), None),
	529	}
	530	}
	531
	532	pub fn hoist_js_if(
	533	&mut self,
	534	yes: BlockWithLoc,
	535	no: Option<BlockOrIf>,
	536	condition: JsExpr,
	537	did_assign: &mut bool,
	538	temporary: &TempId,
	539	) -> Option<JsIf> {
	540	let yes = self.hoist_block_to_js_block(yes, &temporary.get(&self), did_assign)?;
	541
	542	let no = match no {
	543	Some(BlockOrIf::Block(no)) => Some(JsElse::Block(self.hoist_block_to_js_block(
	544	no,
	545	&temporary.get(&self),
	546	did_assign,
	547	)?)),
	548	Some(BlockOrIf::If(else_if)) => {
	549	// If the condition of this block has a side effect, we need to
	550	// emit a block so that we can assign to the temporary variable.
	551	//
	552	// Otherwise, we can just emit the condition inline
	553	if else_if.condition.is_valid_js_expression() {
	554	let condition = convert_valid_expr(self, *else_if.condition.clone());
	555	Some(JsElse::If(Box::new(self.hoist_js_if(
	556	else_if.yes,
	557	else_if.no,
	558	condition,
	559	did_assign,
	560	temporary,
	561	)?)))
	562	} else {
	563	let mut temp_block = self.new_child();
	564	let condition =
	565	temp_block.hoist_expression_to_temporary(*else_if.condition.clone())?;
	566
	567	temp_block.add_statement(JsStatement::new(
	568	JsStatementData::If(self.hoist_js_if(
	569	else_if.yes,
	570	else_if.no,
	571	condition,
	572	did_assign,
	573	temporary,
	574	)?),
	575	Loc::EMPTY,
	576	None,
	577	));
	578
	579	Some(JsElse::Block(JsBlock::new(
	580	temp_block.statements,
	581	Loc::EMPTY,
	582	)))
	583	}
	584	}
	585	None => None,
	586	};
	587
	588	Some(JsIf { condition, yes, no })
	589	}
	590	}
	591
	592	pub fn new_js_assignment_statement(expr: JsExpr, target: String, loc: Loc) -> JsStatement {
	593	JsStatement::new(
	594	JsStatementData::Expression(new_js_assignment_expr(expr, target, loc.clone())),
	595	loc,
	596	None,
	597	)
	598	}
	599
	600	pub fn new_js_assignment_expr(expr: JsExpr, target: String, loc: Loc) -> JsExpr {
	601	JsExpr::new(
	602	JsExprData::Binary {
	603	op: BinaryOperator::Assign,
	604	left: Box::new(JsExpr::new(
	605	JsExprData::Identifier(target),
	606	loc.clone(),
	607	None,
	608	)),
	609	right: Box::new(expr),
	610	},
	611	loc,
	612	None,
	613	)
	614	}

crates/js_ast/is_valid_js_expression.rs created+85

...	...	@@ -0,0 +1,85 @@
	1	use ast::*;
	2
	3	pub trait IsValidJSExpression {
	4	/// Can this be lowered to a JS ast without hoisting items
	5	/// ex: a `return` expression is not a valid JS expression
	6	fn is_valid_js_expression(&self) -> bool;
	7	}
	8
	9	impl IsValidJSExpression for ExprData {
	10	fn is_valid_js_expression(&self) -> bool {
	11	match self {
	12	// A block containing just expressions can be an expression
	13	ExprData::Block(block) => block.is_valid_js_expression(),
	14	ExprData::If(expr) => expr.is_valid_js_expression(),
	15	ExprData::Return(_) => false,
	16	ExprData::Builtin { id, .. } => match id {
	17	BuiltinId::Raw => true,
	18	},
	19	ExprData::Binary { left, right, .. } => {
	20	left.is_valid_js_expression() && right.is_valid_js_expression()
	21	}
	22	ExprData::Unary { value, .. } => value.is_valid_js_expression(),
	23	ExprData::Fn { .. } => true,
	24	ExprData::Call { base, arguments } => {
	25	base.is_valid_js_expression()
	26	&& arguments
	27	.iter()
	28	.all(\|argument\| argument.is_valid_js_expression())
	29	}
	30	ExprData::Field { base, field } => {
	31	base.is_valid_js_expression() && field.is_valid_js_expression()
	32	}
	33	ExprData::Literal(_) \| ExprData::Identifier(_) => true,
	34	}
	35	}
	36	}
	37
	38	impl IsValidJSExpression for Block {
	39	fn is_valid_js_expression(&self) -> bool {
	40	self.statements
	41	.iter()
	42	.all(\|statement\| statement.is_valid_js_expression())
	43	}
	44	}
	45
	46	impl IsValidJSExpression for Expr {
	47	fn is_valid_js_expression(&self) -> bool {
	48	self.data.is_valid_js_expression()
	49	}
	50	}
	51
	52	impl IsValidJSExpression for If {
	53	fn is_valid_js_expression(&self) -> bool {
	54	self.condition.is_valid_js_expression()
	55	&& self.yes.is_valid_js_expression()
	56	&& match &self.no {
	57	Some(no) => no.is_valid_js_expression(),
	58	None => true,
	59	}
	60	}
	61	}
	62
	63	impl IsValidJSExpression for BlockOrIf {
	64	fn is_valid_js_expression(&self) -> bool {
	65	match self {
	66	BlockOrIf::Block(block) => block.is_valid_js_expression(),
	67	BlockOrIf::If(if_) => if_.is_valid_js_expression(),
	68	}
	69	}
	70	}
	71
	72	impl IsValidJSExpression for BlockWithLoc {
	73	fn is_valid_js_expression(&self) -> bool {
	74	self.block.is_valid_js_expression()
	75	}
	76	}
	77
	78	impl IsValidJSExpression for Statement {
	79	fn is_valid_js_expression(&self) -> bool {
	80	match self {
	81	Statement::Expr(expr) => expr.data.is_valid_js_expression(),
	82	_ => false,
	83	}
	84	}
	85	}

crates/js_ast/js_ast.rs+96-172

...	...	@@ -1,4 +1,4 @@
1		use ast::{BinaryOperator, BlockWithLoc, BuiltinId, Comment, Expr, UnaryOperator};
	1	use ast::{BinaryOperator, Comment, UnaryOperator};
2	2	use num::bigint::BigInt;
3	3	use tokenizer::Loc;
4	4
...	...	@@ -31,11 +31,11 @@ pub enum JsExprData {
31	31
32	32	Identifier(String),
33	33	Unary {
34		operator: UnaryOperator,
	34	op: UnaryOperator,
35	35	value: Box<JsExpr>,
36	36	},
37	37	Binary {
38		operator: BinaryOperator,
	38	op: BinaryOperator,
39	39	left: Box<JsExpr>,
40	40	right: Box<JsExpr>,
41	41	},
...	...	@@ -52,22 +52,23 @@ pub enum JsExprData {
52	52	arguments: Vec<JsExpr>,
53	53	},
54	54	Function {
55		name: String,
	55	name: Option<String>,
56	56	params: Vec<String>,
57		block: Vec<JsStatement>,
58		},
59		If {
60		condition: Box<JsExpr>,
61		yes: Vec<JsExpr>,
62		no: Option<Box<JsExpr>>,
	57	body: Box<JsExpr>,
63	58	},
64	59	ArrowFunction {
65	60	params: Vec<String>,
66		block: Vec<JsStatement>,
	61	body: Box<JsArrowBody>,
	62	},
	63	Ternary {
	64	condition: Box<JsExpr>,
	65	yes: Box<JsExpr>,
	66	no: Option<Box<JsExpr>>,
67	67	},
68	68	Comma {
69	69	items: Vec<JsExpr>,
70	70	},
	71	Void(Box<JsExpr>),
71	72	}
72	73
73	74	#[derive(Debug, PartialEq, Clone)]
...	...	@@ -77,33 +78,56 @@ pub struct JsStatement {
77	78	pub comment: Option<Comment>,
78	79	}
79	80
	81	#[derive(Debug, PartialEq, Clone)]
	82	pub struct JSFunctionDeclaration {
	83	pub name: String,
	84	pub params: Vec<String>,
	85	pub block: JsBlock,
	86	}
	87
	88	#[derive(Debug, PartialEq, Clone)]
	89	pub struct JSDeclaration {
	90	pub name: String,
	91	pub init: Option<JsExpr>,
	92	pub is_constant: bool,
	93	}
	94
	95	#[derive(Debug, PartialEq, Clone)]
	96	pub struct JsIf {
	97	pub condition: JsExpr,
	98	pub yes: JsBlock,
	99	pub no: Option<JsElse>,
	100	}
	101
	102	#[derive(Debug, PartialEq, Clone)]
	103	pub enum JsElse {
	104	Expr(JsExpr),
	105	Block(JsBlock),
	106	If(Box<JsIf>),
	107	}
	108
80	109	#[derive(Debug, PartialEq, Clone)]
81	110	pub enum JsStatementData {
82	111	Expression(JsExpr),
83	112	Return(JsExpr),
84	113	Export(JsExport),
85		Declaration {
86		name: String,
87		init: JsExpr,
88		is_constant: bool,
89		},
90		Function {
91		name: String,
92		params: Vec<String>,
93		block: JsBlock,
94		},
95		If {
96		condition: JsExpr,
97		yes: JsBlock,
98		no: Option<JsBlock>,
99		},
	114	Declaration(JSDeclaration),
	115	Function(JSFunctionDeclaration),
	116	If(JsIf),
100	117	}
101	118
102	119	#[derive(Debug, PartialEq, Clone)]
103	120	pub enum JsExport {
104		Named(Vec<String>),
	121	Named(Vec<JSNamedExport>),
	122	Function(Box<JSFunctionDeclaration>),
	123	Declaration(Box<JSDeclaration>),
105	124	Default(JsExpr),
106		Equals(JsExpr),
	125	}
	126
	127	#[derive(Debug, PartialEq, Clone)]
	128	pub struct JSNamedExport {
	129	pub identifier: String,
	130	pub export_name: String,
107	131	}
108	132
109	133	#[derive(Debug, PartialEq, Clone)]
...	...	@@ -113,156 +137,38 @@ pub struct JsBlock {
113	137	}
114	138
115	139	impl JsExpr {
116		pub fn from_blight(expr: &Expr) -> Option<JsExpr> {
117		use ast::ExprData::*;
118
119		let loc = &expr.loc;
120		let comment = &expr.comment;
121
122		match &expr.data {
123		Block(block) => Self::comma_expr_from_blight_block(&BlockWithLoc {
124		block: block.clone(),
125		loc: loc.clone(),
126		}),
127		Binary { op, left, right } => {
128		let left = Self::from_blight(&*left)?;
129		let right = Self::from_blight(&*right)?;
130		Some(JsExpr::new(
131		JsExprData::Binary {
132		operator: op.clone(),
133		left: Box::new(left),
134		right: Box::new(right),
135		},
136		loc.clone(),
137		comment.clone(),
138		))
139		}
140		Unary { op, value } => {
141		let value = Self::from_blight(&*value)?;
142		Some(JsExpr::new(
143		JsExprData::Unary {
144		operator: op.clone(),
145		value: Box::new(value),
146		},
147		loc.clone(),
148		comment.clone(),
149		))
150		}
151		Fn {
152		name,
153		params,
154		return_type,
155		block,
156		} => {
157		let block = Self::comma_expr_from_blight_block(&BlockWithLoc {
158		block: block.clone(),
159		loc: loc.clone(),
160		})?;
161		Some(JsExpr::new(
162		JsExprData::Function {
163		name: name.clone(),
164		params: vec![],
165		block: vec![JsStatement::new(
166		JsStatementData::Expression(block),
167		loc.clone(),
168		comment.clone(),
169		)],
170		},
171		loc.clone(),
172		comment.clone(),
173		))
174		}
175		Builtin { id, arguments } => match id {
176		BuiltinId::Raw => {
177		let mut args = arguments.iter();
178		let arg = args.next().unwrap();
179		let arg = match &arg.data {
180		Literal(ast::Literal::String(s)) => s,
181		_ => panic!("raw expects a string"),
182		};
183		Some(JsExpr::new(
184		JsExprData::Raw(arg.clone()),
185		loc.clone(),
186		comment.clone(),
187		))
188		}
189		},
190		If(expr) => {
191		let condition = Self::from_blight(&expr.condition)?;
192		let yes = Self::comma_expr_from_blight_block(&expr.yes)?;
193		let no = match &expr.no {
194		Some(no) => Some(Self::from_blight(&no)?),
195		None => None,
196		};
197		Some(JsExpr::new(
198		JsExprData::If {
199		condition: Box::new(condition),
200		yes: vec![yes],
201		no: no.map(Box::new),
202		},
203		loc.clone(),
204		comment.clone(),
205		))
206		}
207
208		Call { base, arguments } => todo!(),
209		Field { base, field } => todo!(),
210
211		Return(_) => None,
	140	pub fn new(data: JsExprData, loc: Loc, comment: Option<Comment>) -> Self {
	141	Self { data, loc, comment }
	142	}
212	143
213		Literal(_) => todo!(),
214		Identifier(_) => todo!(),
215		// Literal(ast::Literal::Number(n)) => {
216		// Some(JsExpr::new(JsExprData::Number(n), loc.clone(), comment))
217		// }
218		// Literal(ast::Literal::Bool(b)) => Some(JsExpr::new(JsExprData::Bool(b), loc, comment)),
219		// Literal(ast::Literal::String(s)) => {
220		// Some(JsExpr::new(JsExprData::String(s), loc, comment))
221		// }
222		// Literal(ast::Literal::Void) => Some(JsExpr::new(JsExprData::Null, loc, comment)),
223		// Identifier(id) => Some(JsExpr::new(JsExprData::Identifier(id), loc, comment)),
	144	pub fn is_literal(&self) -> bool {
	145	match &self.data {
	146	JsExprData::Number(_) => true,
	147	JsExprData::BigInt(_) => true,
	148	JsExprData::String(_) => true,
	149	JsExprData::Bool(_) => true,
	150	JsExprData::Null => true,
	151	JsExprData::Undefined => true,
	152	_ => false,
224	153	}
225	154	}
	155	}
226	156
227		/// Since blocks are not valid expressions in JS, this creates a
228		/// comma expression from a block with more than one statement.
229		///
230		/// Not possible to create an ast => None
231		pub fn comma_expr_from_blight_block(b: &ast::BlockWithLoc) -> Option<JsExpr> {
232		if !b.block.has_implicit_return {
233		return Some(JsExpr::new(JsExprData::Null, b.loc.clone(), None));
234		}
235		// a block expression with a single statement is just that statement
236		if b.block.statements.len() == 1 && b.block.has_implicit_return {
237		match b.block.statements.first().unwrap() {
238		ast::Statement::Expr(expr) => Self::from_blight(expr),
239		_ => return None,
240		};
241		}
242		let mut statements = Vec::new();
243		for statement in &b.block.statements {
244		match statement {
245		ast::Statement::Expr(expr) => {
246		if let Some(expr) = Self::from_blight(&expr) {
247		statements.push(expr);
248		}
249		}
250		_ => return None,
251		}
252		}
253		// a block without implicit return is to be type void, so we append null
254		if !b.block.has_implicit_return {
255		statements.push(JsExpr::new(JsExprData::Null, b.loc.clone(), None));
256		}
257		Some(JsExpr::new(
258		JsExprData::Comma { items: statements },
259		b.loc.clone(),
260		None,
261		))
	157	impl JsExprData {
	158	/// Helper for testing
	159	pub fn str(value: &str) -> Self {
	160	Self::String(value.to_string())
262	161	}
263	162
264		pub fn new(data: JsExprData, loc: Loc, comment: Option<Comment>) -> Self {
265		Self { data, loc, comment }
	163	/// Helper for testing
	164	pub fn str_id(value: &str) -> Self {
	165	Self::Identifier(value.to_string())
	166	}
	167	}
	168
	169	impl JsBlock {
	170	pub fn new(statements: Vec<JsStatement>, loc: Loc) -> Self {
	171	Self { statements, loc }
266	172	}
267	173	}
268	174
...	...	@@ -270,4 +176,22 @@ impl JsStatement {
270	176	pub fn new(data: JsStatementData, loc: Loc, comment: Option<Comment>) -> Self {
271	177	Self { data, loc, comment }
272	178	}
	179
	180	pub(crate) fn is_literal(&self) -> bool {
	181	match &self.data {
	182	JsStatementData::Expression(expr) => expr.is_literal(),
	183	_ => false,
	184	}
	185	}
	186	}
	187
	188	#[derive(Debug, PartialEq, Clone)]
	189	pub enum JsArrowBody {
	190	Block(JsBlock),
	191	Expr(JsExpr),
	192	}
	193
	194	#[derive(Debug, PartialEq, Clone)]
	195	pub struct JsModule {
	196	pub statements: Vec<JsStatement>,
273	197	}

crates/js_ast/lib.rs+19-402

...	...	@@ -1,411 +1,28 @@
1		use std::collections::HashMap;
2
3		use ast::{BlockWithLoc, Expr, ExprData, Literal, Module, Statement};
4		pub use js_ast::*;
5		use miette::Result;
6		use tokenizer::Loc;
7
	1	mod convert_valid_expr;
	2	mod has_side_effects;
	3	mod hoist;
	4	mod is_valid_js_expression;
8	5	mod js_ast;
	6	mod util;
9	7
10		struct StackContext {
11		statements: Vec<JsStatement>,
12		/// Variables that are declared in this context.
13		/// The number is the number of times the variable has been declared.
14		/// This is used to allow for shadowing
15		variable_map: HashMap<String, u32>,
16		block: u32,
17		}
18
19		impl StackContext {
20		fn new(block: u32) -> Self {
21		StackContext {
22		statements: Vec::new(),
23		variable_map: HashMap::new(),
24		block,
25		}
26		}
27		}
28
29		struct TranslateContext {
30		context: Vec<StackContext>,
31		internal_counter: u32,
32		block_counter: u32,
33		}
34
35		impl TranslateContext {
36		fn new() -> Self {
37		TranslateContext {
38		context: vec![StackContext::new(0)],
39		internal_counter: 0,
40		block_counter: 0,
41		}
42		}
43
44		fn push_context(self: &mut Self) {
45		self.block_counter += 1;
46		self.context.push(StackContext::new(self.block_counter));
47		}
48
49		fn pop_context(self: &mut Self) -> Vec<JsStatement> {
50		self.context.pop().unwrap().statements
51		}
52
53		fn push_internal_variable(self: &mut Self, value: JsExpr) -> JsExpr {
54		let name = format!("${}", self.internal_counter);
55		let loc = value.loc.clone();
56		self.context
57		.last_mut()
58		.unwrap()
59		.statements
60		.push(JsStatement::new(
61		JsStatementData::Declaration {
62		name: name.clone(),
63		init: value,
64		is_constant: false,
65		},
66		loc,
67		None,
68		));
69		self.internal_counter += 1;
70		JsExpr::new(JsExprData::Identifier(name), Loc::EMPTY, None)
71		}
72
73		fn push(self: &mut Self, statement: JsStatement) {
74		self.context.last_mut().unwrap().statements.push(statement);
75		}
76
77		// fn peek(self: &Self) -> &JsStatement {
78		// self.context.last().unwrap().statements.last().unwrap()
79		// }
80
81		fn declare_variable(self: &mut Self, name: String) -> String {
82		let context = self.context.last_mut().unwrap();
83		if let Some(value) = context.variable_map.get_mut(name.as_str()) {
84		*value += 1;
85		format!("{}${}${}", name, value, context.block)
86		} else {
87		context.variable_map.insert(name.to_string(), 0);
88		format!("{}$0${}", name, context.block)
89		}
90		}
	8	use ast::Module;
91	9
92		fn get_identifier(self: &Self, name: String) -> String {
93		for context in self.context.iter().rev() {
94		if let Some(value) = context.variable_map.get(name.as_str()) {
95		return format!("{}${}${}", name, value, context.block);
96		}
97		}
98		panic!("Variable {} not found", name);
99		}
100		}
101
102		fn get_block_handle(b: &ast::BlockWithLoc, ctx: &mut TranslateContext) -> JsExpr {
103		// :)
104		if b.is_valid_js_expression() {
105		return JsExpr::comma_expr_from_blight_block(b).unwrap();
106		}
107
108		ctx.push_context();
109		for (i, statement) in b.block.statements.iter().enumerate() {
110		if i == b.block.statements.len() - 1 {
111		break;
112		} else {
113		translate_statement(statement, ctx);
114		}
115		}
116		let result = match b.block.statements.last().unwrap() {
117		ast::Statement::Expr(expr) => get_expression_handle(expr, ctx),
118		stmt => {
119		translate_statement(stmt, ctx);
120		JsExpr::new(JsExprData::Null, Loc::EMPTY, None)
121		}
122		};
123		let value = ctx.pop_context();
124		value.iter().for_each(\|x\| ctx.push(x.clone()));
125		result
126		}
127
128		// basically, expressions are really hard to translate
129		// in blight, everything is an expression
130		// in js, very few things are expressions
131		// here, we recursively expand out expressions into statements
132		// we then store the result of all the expressions in a variable
133		// and then return the name of that variable
134		fn get_expression_handle(ast: &Expr, ctx: &mut TranslateContext) -> JsExpr {
135		match ast.data.clone() {
136		ExprData::Literal(Literal::String(value)) => JsExpr::new(
137		JsExprData::String(value),
138		ast.loc.clone(),
139		ast.comment.clone(),
140		),
141		ExprData::Literal(Literal::Number(value)) => JsExpr::new(
142		JsExprData::Number(value),
143		ast.loc.clone(),
144		ast.comment.clone(),
145		),
146		ExprData::Literal(Literal::Void) => {
147		JsExpr::new(JsExprData::Null, ast.loc.clone(), ast.comment.clone())
148		}
149		ExprData::Literal(Literal::Bool(b)) => {
150		JsExpr::new(JsExprData::Bool(b), ast.loc.clone(), ast.comment.clone())
151		}
152
153		ExprData::Identifier(value) => JsExpr::new(
154		JsExprData::Identifier(ctx.get_identifier(value.clone())),
155		ast.loc.clone(),
156		ast.comment.clone(),
157		),
158		ExprData::Call { base, arguments } => {
159		let mut handles = Vec::new();
160		for argument in arguments {
161		handles.push(get_expression_handle(&argument, ctx));
162		}
163		let base = get_expression_handle(&base, ctx);
164		ctx.push_internal_variable(JsExpr::new(
165		JsExprData::Call {
166		base: Box::new(base),
167		arguments: handles,
168		},
169		ast.loc.clone(),
170		ast.comment.clone(),
171		))
172		}
173		ExprData::Builtin { id, arguments } => match id {
174		ast::BuiltinId::Raw => {
175		if arguments.len() != 1 {
176		panic!("@raw expects exactly one argument");
177		}
178		match arguments.get(0).unwrap() {
179		Expr {
180		data: ExprData::Literal(Literal::String(value)),
181		..
182		} => JsExpr::new(
183		JsExprData::Raw(value.clone()),
184		ast.loc.clone(),
185		ast.comment.clone(),
186		),
187		_ => panic!("@raw expects a string literal"),
188		}
189		}
190		},
191		ExprData::Binary { op, left, right } => {
192		let left_handle = get_expression_handle(&left, ctx);
193		let right_handle = get_expression_handle(&right, ctx);
194		JsExpr::new(
195		JsExprData::Binary {
196		operator: op.clone(),
197		left: Box::new(left_handle),
198		right: Box::new(right_handle),
199		},
200		ast.loc.clone(),
201		ast.comment.clone(),
202		)
203		}
204
205		ExprData::Unary { op, value } => {
206		let value_handle = get_expression_handle(&value, ctx);
207		JsExpr::new(
208		JsExprData::Unary {
209		operator: op.clone(),
210		value: Box::new(value_handle),
211		},
212		ast.loc.clone(),
213		ast.comment.clone(),
214		)
215		}
216
217		ExprData::Field { base, field } => {
218		let base = Box::new(get_expression_handle(&base, ctx));
219
220		match field.data {
221		ExprData::Literal(Literal::String(name)) => JsExpr::new(
222		JsExprData::Dot { base, name },
223		ast.loc.clone(),
224		ast.comment.clone(),
225		),
226		_ => JsExpr::new(
227		JsExprData::Index {
228		base,
229		index: Box::new(get_expression_handle(&field, ctx)),
230		},
231		ast.loc.clone(),
232		ast.comment.clone(),
233		),
234		}
235		}
236
237		ExprData::Block(block) => get_block_handle(
238		&BlockWithLoc {
239		block,
240		loc: ast.loc.clone(),
241		},
242		ctx,
243		),
244
245		ExprData::If(expr) => {
246		let handle = get_expression_handle(&expr.condition, ctx);
247
248		let yes = get_block_handle(&expr.yes, ctx);
249		let no = match expr.no {
250		Some(block) => Some(get_expression_handle(&block, ctx)),
251		None => None,
252		};
253
254		// ctx.push(JsStatement::new(
255		// JsStatementData::If {
256		// condition: handle,
257		// yes,
258		// no,
259		// },
260		// ast.loc,
261		// ast.comment,
262		// ));
263
264		// panic!()
265		handle
266		}
267		ExprData::Fn {
268		name,
269		params,
270		return_type,
271		block,
272		} => todo!(),
273		ExprData::Return(_) => todo!(),
274		}
275		}
276
277		fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) {
278		match ast {
279		Statement::Decl(decl) => {
280		let handle = get_expression_handle(&decl.init, ctx);
281		let rename = ctx.declare_variable(decl.name.clone());
282		ctx.push(JsStatement::new(
283		JsStatementData::Declaration {
284		name: rename,
285		init: handle,
286		is_constant: decl.is_constant.clone(),
287		},
288		decl.loc.clone(),
289		None,
290		));
291		}
292		Statement::Expr(expr) => {
293		let h = get_expression_handle(expr, ctx);
294		ctx.push(JsStatement::new(
295		JsStatementData::Expression(h),
296		expr.loc.clone(),
297		None,
298		))
299		}
300		Statement::Comment(_) => {} // Ast::Function {
301		// name,
302		// arguments,
303		// return_type: _,
304		// block,
305		// } => {
306		// let rename = ctx.declare_variable(name.clone());
307		// let arguments = arguments
308		// .iter()
309		// .map(\|x\| ctx.declare_variable(x.name.clone()))
310		// .collect();
311
312		// ctx.push_context();
313		// for (i, statement) in block.value.iter().enumerate() {
314		// if block.implicit_return && i == block.value.len() - 1 {
315		// let handle = get_expression_handle(statement, ctx);
316		// ctx.push(JsStatement::Return(handle));
317		// } else {
318		// translate_statement(statement, ctx);
319		// }
320		// }
321		// let value = ctx.pop_context();
	10	pub use convert_valid_expr::*;
	11	pub use hoist::*;
	12	pub use is_valid_js_expression::*;
	13	pub use js_ast::*;
	14	pub use util::*;
322	15
323		// ctx.push(JsStatement::Function {
324		// name: rename,
325		// arguments,
326		// value,
327		// });
328		// }
329		// Ast::Public(value) => {
330		// translate_statement(value, ctx);
331		// // I'm sorry, I just don't know how to get the original name properly... my bad
332		// match ctx.peek() {
333		// JsStatement::Function { name, .. } \| JsStatement::Declaration { name, .. } => {
334		// let original_name = name.split("$").next().unwrap().to_string();
335		// ctx.push(JsStatement::Export(Box::new(JsStatement::Declaration {
336		// name: original_name,
337		// value: JsExpr::Identifier(name.clone()),
338		// is_constant: true,
339		// })));
340		// }
341		// _ => {
342		// panic!("Cannot make non-function public");
343		// }
344		// }
345		// }
346		// Ast::Return(value) => {
347		// let handle = get_expression_handle(value, ctx);
348		// ctx.push(JsStatement::Return(handle));
349		// }
350		// Ast::Block(block) => {
351		// ctx.push_context();
352		// for (i, statement) in block.value.iter().enumerate() {
353		// if block.implicit_return && i == block.value.len() - 1 {
354		// let handle = get_expression_handle(statement, ctx);
355		// ctx.push(JsStatement::Return(handle));
356		// } else {
357		// translate_statement(statement, ctx);
358		// }
359		// }
360		// let value = ctx.pop_context();
361		// value.iter().for_each(\|x\| ctx.push(x.clone()));
362		// }
	16	pub fn blight_to_js(blight: Module) -> JsModule {
	17	let mut root_context = HoistContext::new();
363	18
364		// Ast::FunctionCall { name, arguments } => {
365		// let mut handles = Vec::new();
366		// for argument in arguments {
367		// handles.push(get_expression_handle(argument, ctx));
368		// }
369		// match *name.clone() {
370		// Ast::Identifier(name) => {
371		// ctx.push(JsStatement::Expression(JsExpr::Call {
372		// name: Box::new(JsExpr::Identifier(ctx.get_identifier(name.clone()))),
373		// arguments: handles,
374		// }));
375		// }
376		// Ast::BuiltIn(name) => match name.as_str() {
377		// "@raw" => {
378		// if handles.len() != 1 {
379		// panic!("@raw expects exactly one argument");
380		// }
381		// match handles.get(0).unwrap() {
382		// JsExpr::String(value) => {
383		// ctx.push(JsStatement::Expression(JsExpr::Call {
384		// name: Box::new(JsExpr::Raw(value.clone())),
385		// arguments: handles,
386		// }));
387		// }
388		// _ => panic!("@raw expects a string argument"),
389		// }
390		// }
391		// _ => panic!("Unknown built-in function {}", name),
392		// },
393		// _ => panic!("Function name must be an identifier"),
394		// }
395		// }
396		// Ast::Comment(_) \| Ast::DocComment(_) => {}
397		// _ => {
398		// panic!("unimplemented: {:?}", ast);
399		// }
	19	for statement in blight.statements {
	20	if let None = root_context.process_statement(statement.to_statement()) {
	21	unreachable!("top level return is disallowed by the parser")
	22	};
400	23	}
401		}
402	24
403		pub fn translate(module: &Module) -> Result<Vec<JsStatement>> {
404		let mut ctx = TranslateContext::new();
405
406		for ast in module.statements.iter() {
407		translate_statement(&ast.clone().to_statement(), &mut ctx);
	25	JsModule {
	26	statements: root_context.statements,
408	27	}
409
410		Ok(ctx.pop_context())
411	28	}

crates/js_ast/util.rs created+17

...	...	@@ -0,0 +1,17 @@
	1	/// TODO(@paperdave): handle the rest of the identifiers
	2	pub fn is_js_identifier(s: &str) -> bool {
	3	if s.is_empty() {
	4	return false;
	5	}
	6	let mut chars = s.chars();
	7	let first = chars.next().unwrap();
	8	if !first.is_ascii_alphabetic() && first != '_' {
	9	return false;
	10	}
	11	for c in chars {
	12	if !c.is_ascii_alphanumeric() && c != '_' {
	13	return false;
	14	}
	15	}
	16	true
	17	}

crates/js_printer/Cargo.toml+1

...	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	4	edition = "2021"
5	5
6	6	[lib]
	7	path = "./lib.rs"
7	8
8	9	[dependencies]
9	10	miette.workspace = true

crates/js_printer/lib.rs created+358

...	...	@@ -0,0 +1,358 @@
	1	mod printer_tests;
	2
	3	use js_ast::*;
	4	use std::io::{Result, Write};
	5
	6	pub trait Print {
	7	fn print(&self, writer: &mut dyn Write) -> Result<()>;
	8
	9	fn print_to_string(&self) -> String {
	10	let mut buffer = Vec::new();
	11	self.print(&mut buffer).unwrap();
	12	String::from_utf8(buffer).unwrap()
	13	}
	14	}
	15
	16	// the printer:
	17
	18	fn print_module_clause_identifier(writer: &mut dyn Write, export_name: &str) -> Result<()> {
	19	// TODO(@paperdave): Escape invalid identifiers
	20	if js_ast::is_js_identifier(export_name) {
	21	write!(writer, "{}", export_name)
	22	} else {
	23	todo!("call JSON.stringify on identifier")
	24	}
	25	}
	26
	27	fn print_js_string(writer: &mut dyn Write, string: &str) -> Result<()> {
	28	write!(writer, "\"")?;
	29	// TODO(@paperdave): Instead of writing each character, this should write
	30	// chunks between the escapes instead of calling write! on each character individually.
	31	for c in string.chars() {
	32	match c {
	33	'\n' => write!(writer, "\\n")?,
	34	'\r' => write!(writer, "\\r")?,
	35	'\t' => write!(writer, "\\t")?,
	36	'"' => write!(writer, "\\\"")?,
	37	'\\' => write!(writer, "\\\\")?,
	38	'\u{0008}' => write!(writer, "\\b")?,
	39	'\u{000C}' => write!(writer, "\\f")?,
	40	'\u{0000}'..='\u{001F}' => write!(writer, "\\u{:04x}", c as u32)?,
	41	// TODO(@paperdave): I think there are some larger code points that need to be escaped?
	42	_ => write!(writer, "{}", c)?,
	43	}
	44	}
	45	write!(writer, "\"")
	46	}
	47
	48	impl Print for JsStatement {
	49	fn print(&self, w: &mut dyn Write) -> Result<()> {
	50	match &self.data {
	51	js_ast::JsStatementData::Expression(expr) => {
	52	expr.print(w)?;
	53	write!(w, ";")
	54	}
	55	js_ast::JsStatementData::Return(expr) => {
	56	write!(w, "return ")?;
	57	expr.print(w)
	58	}
	59	js_ast::JsStatementData::Export(export) => export.print(w),
	60	js_ast::JsStatementData::Declaration(decl) => decl.print(w),
	61	js_ast::JsStatementData::Function(function) => function.print(w),
	62	js_ast::JsStatementData::If(js_if) => js_if.print(w),
	63	}
	64	}
	65	}
	66
	67	impl Print for JsExport {
	68	fn print(&self, w: &mut dyn Write) -> Result<()> {
	69	match self {
	70	JsExport::Named(exports) => {
	71	write!(w, "export{{")?;
	72	for (i, export) in exports.iter().enumerate() {
	73	if i > 0 {
	74	write!(w, ",")?;
	75	}
	76	export.print(w)?;
	77	}
	78	Ok(())
	79	}
	80	JsExport::Function(function) => {
	81	write!(w, "export ")?;
	82	function.print(w)
	83	}
	84	JsExport::Declaration(declaration) => {
	85	write!(w, "export ")?;
	86	declaration.print(w)
	87	}
	88	JsExport::Default(expr) => {
	89	write!(w, "export default ")?;
	90	expr.print(w)
	91	}
	92	}
	93	}
	94	}
	95
	96	impl Print for JSNamedExport {
	97	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	98	if self.export_name == self.identifier {
	99	write!(writer, "{}", self.identifier)
	100	} else {
	101	write!(writer, "{} as ", self.identifier)?;
	102	print_module_clause_identifier(writer, &self.export_name)
	103	}
	104	}
	105	}
	106
	107	impl Print for JSDeclaration {
	108	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	109	write!(
	110	writer,
	111	"{} {}",
	112	if self.is_constant { "const" } else { "let" },
	113	self.name,
	114	)?;
	115	if let Some(init) = &self.init {
	116	write!(writer, " = ")?;
	117	init.print(writer)?;
	118	}
	119	write!(writer, ";")?;
	120	Ok(())
	121	}
	122	}
	123
	124	impl Print for JsExpr {
	125	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	126	self.data.print(writer)
	127	}
	128	}
	129
	130	impl Print for JsExprData {
	131	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	132	match &self {
	133	JsExprData::Raw(data) => write!(writer, "{}", data),
	134	JsExprData::Number(number) => write!(writer, "{}", number),
	135	JsExprData::BigInt(bigint) => write!(writer, "{}", bigint),
	136	JsExprData::String(string) => print_js_string(writer, string.as_str()),
	137	JsExprData::Bool(bool) => write!(writer, "{}", bool),
	138	JsExprData::Null => write!(writer, "null"),
	139	JsExprData::Undefined => write!(writer, "undefined"),
	140	JsExprData::Identifier(str) => write!(writer, "{}", str),
	141	JsExprData::Array(items) => {
	142	write!(writer, "[")?;
	143	for (i, item) in items.iter().enumerate() {
	144	if i > 0 {
	145	write!(writer, ",")?;
	146	}
	147	item.print(writer)?;
	148	}
	149	write!(writer, "]")
	150	}
	151	JsExprData::Object(items) => {
	152	write!(writer, "{{")?;
	153	for (i, kv) in items.iter().enumerate() {
	154	if i > 0 {
	155	write!(writer, ",")?;
	156	}
	157
	158	kv.print(writer)?;
	159	}
	160	write!(writer, "}}")
	161	}
	162	JsExprData::Unary { op, value } => {
	163	write!(writer, "{}", op.to_str())?;
	164	value.print(writer)
	165	}
	166	JsExprData::Binary { op, left, right } => {
	167	// TODO: figure out when we dont need to wrap
	168	let wrap = true;
	169	if wrap {
	170	write!(writer, "(")?;
	171	}
	172	left.print(writer)?;
	173	write!(writer, "{}", op.to_str())?;
	174	right.print(writer)?;
	175	if wrap {
	176	write!(writer, ")")?;
	177	}
	178	Ok(())
	179	}
	180	JsExprData::Dot { base, name } => {
	181	base.print(writer)?;
	182	write!(writer, ".{}", name)
	183	}
	184	JsExprData::Index { base, index } => {
	185	base.print(writer)?;
	186	write!(writer, "[")?;
	187	index.print(writer)?;
	188	write!(writer, "]")
	189	}
	190	JsExprData::Call { base, arguments } => {
	191	base.print(writer)?;
	192	write!(writer, "(")?;
	193	for (i, argument) in arguments.iter().enumerate() {
	194	if i > 0 {
	195	write!(writer, ",")?;
	196	}
	197	argument.print(writer)?;
	198	}
	199	write!(writer, ")")
	200	}
	201	JsExprData::Function { body, name, params } => {
	202	write!(writer, "function ")?;
	203	if let Some(name) = name {
	204	write!(writer, "{}", name)?;
	205	}
	206	write!(writer, "(")?;
	207	for (i, param) in params.iter().enumerate() {
	208	if i > 0 {
	209	write!(writer, ",")?;
	210	}
	211	write!(writer, "{}", param)?;
	212	}
	213	write!(writer, ")")?;
	214	body.print(writer)
	215	}
	216	JsExprData::ArrowFunction { body, params } => {
	217	let print_parens = params.len() != 1;
	218	if print_parens {
	219	write!(writer, "(")?;
	220	}
	221	for (i, param) in params.iter().enumerate() {
	222	if i > 0 {
	223	write!(writer, ",")?;
	224	}
	225	write!(writer, "{}", param)?;
	226	}
	227	if print_parens {
	228	write!(writer, ")")?;
	229	}
	230	write!(writer, "=>")?;
	231	body.print(writer)
	232	}
	233	JsExprData::Ternary { condition, yes, no } => {
	234	condition.print(writer)?;
	235	write!(writer, "?")?;
	236	yes.print(writer)?;
	237	write!(writer, ":")?;
	238	if let Some(no) = no {
	239	no.print(writer)?;
	240	} else {
	241	write!(writer, "null")?;
	242	}
	243	Ok(())
	244	}
	245	JsExprData::Comma { items } => {
	246	for (i, item) in items.iter().enumerate() {
	247	if i > 0 {
	248	write!(writer, ",")?;
	249	}
	250	item.print(writer)?;
	251	}
	252	Ok(())
	253	}
	254	JsExprData::Void(expr) => {
	255	write!(writer, "void ")?;
	256	expr.print(writer)
	257	}
	258	}
	259	}
	260	}
	261
	262	impl Print for JSFunctionDeclaration {
	263	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	264	write!(writer, "function {}", self.name)?;
	265	write!(writer, "(")?;
	266	for (i, param) in self.params.iter().enumerate() {
	267	if i > 0 {
	268	write!(writer, ",")?;
	269	}
	270	write!(writer, "{}", param)?;
	271	}
	272	write!(writer, ")")?;
	273	self.block.print(writer)
	274	}
	275	}
	276
	277	impl Print for JsBlock {
	278	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	279	write!(writer, "{{")?;
	280	for statement in self.statements.iter() {
	281	statement.print(writer)?;
	282	}
	283	write!(writer, "}}")
	284	}
	285	}
	286
	287	impl Print for JsIf {
	288	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	289	write!(writer, "if(")?;
	290	self.condition.print(writer)?;
	291	write!(writer, ")")?;
	292	self.yes.print(writer)?;
	293	if let Some(no) = &self.no {
	294	no.print(writer)?;
	295	}
	296	Ok(())
	297	}
	298	}
	299
	300	impl Print for JsElse {
	301	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	302	match self {
	303	JsElse::If(js_if) => {
	304	write!(writer, "else ")?;
	305	js_if.print(writer)
	306	}
	307	JsElse::Block(block) => {
	308	write!(writer, "else")?;
	309	block.print(writer)
	310	}
	311	JsElse::Expr(expr) => {
	312	write!(writer, "else ")?;
	313	expr.print(writer)
	314	}
	315	}
	316	}
	317	}
	318
	319	impl Print for JsKV {
	320	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	321	match &self.key.data {
	322	JsExprData::String(string) => {
	323	if is_js_identifier(string) {
	324	write!(writer, "{}", string)?;
	325	} else {
	326	print_js_string(writer, string.as_str())?;
	327	}
	328	}
	329	_ => {
	330	write!(writer, "[")?;
	331	self.key.print(writer)?;
	332	write!(writer, "]")?;
	333	}
	334	}
	335	write!(writer, ":")?;
	336	self.key.print(writer)?;
	337	write!(writer, ":")?;
	338	self.value.print(writer)
	339	}
	340	}
	341
	342	impl Print for JsModule {
	343	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	344	for statement in self.statements.iter() {
	345	statement.print(writer)?;
	346	}
	347	Ok(())
	348	}
	349	}
	350
	351	impl Print for JsArrowBody {
	352	fn print(&self, writer: &mut dyn Write) -> Result<()> {
	353	match self {
	354	JsArrowBody::Block(block) => block.print(writer),
	355	JsArrowBody::Expr(expr) => expr.print(writer),
	356	}
	357	}
	358	}

crates/js_printer/printer_tests.rs created+53

...	...	@@ -0,0 +1,53 @@
	1	#![cfg(test)]
	2	use crate::Print;
	3
	4	use js_ast::JsExprData as E;
	5	use num::{bigint::Sign, BigInt};
	6
	7	macro_rules! assert_printed {
	8	($value:expr, $expected:expr) => {
	9	assert_eq!($value.print_to_string(), $expected);
	10	};
	11	}
	12
	13	#[test]
	14	fn number() {
	15	assert_printed!(E::Number(1.0), "1");
	16	assert_printed!(E::Number(1.5), "1.5");
	17	assert_printed!(E::Number(1.5e10), "15000000000");
	18	assert_printed!(E::Number(1.5e-10), "0.00000000015");
	19	assert_printed!(E::Number(-1.0), "-1");
	20	assert_printed!(E::Number(-1.5), "-1.5");
	21	assert_printed!(E::Number(0f64), "0");
	22	}
	23
	24	#[test]
	25	fn bigint() {
	26	assert_printed!(E::BigInt(BigInt::new(Sign::Plus, vec![1])), "1n");
	27	assert_printed!(
	28	E::BigInt(BigInt::new(Sign::Plus, vec![10000000])),
	29	"10000000n"
	30	);
	31	assert_printed!(
	32	E::BigInt(BigInt::new(Sign::Minus, vec![10000000])),
	33	"-10000000n"
	34	);
	35	}
	36
	37	#[test]
	38	fn boolean() {
	39	assert_printed!(E::Bool(true), "true");
	40	assert_printed!(E::Bool(false), "false");
	41	}
	42
	43	#[test]
	44	fn string() {
	45	assert_printed!(E::str(""), "\"\"");
	46	assert_printed!(E::str("hello"), "\"hello\"");
	47	assert_printed!(E::str("hello\nworld"), "\"hello\\nworld\"");
	48	}
	49
	50	#[test]
	51	fn identifier() {
	52	assert_printed!(E::str_id("hello"), "hello");
	53	}

crates/js_printer/src/print_expression.rs+9-5

...	...	@@ -46,7 +46,7 @@ pub fn print_expression(expr: &JsExpr) -> String {
46	46	.join(",")
47	47	),
48	48	JsExprData::Binary {
49		operator,
	49	op: operator,
50	50	left,
51	51	right,
52	52	} => format!(
...	...	@@ -81,7 +81,10 @@ pub fn print_expression(expr: &JsExpr) -> String {
81	81	),
82	82	JsExprData::Null => todo!(),
83	83	JsExprData::Undefined => todo!(),
84		JsExprData::Unary { operator, value } => todo!(),
	84	JsExprData::Unary {
	85	op: operator,
	86	value,
	87	} => todo!(),
85	88	JsExprData::Dot { base, name } => todo!(),
86	89	JsExprData::Index { base, index } => todo!(),
87	90	JsExprData::Function {
...	...	@@ -89,9 +92,10 @@ pub fn print_expression(expr: &JsExpr) -> String {
89	92	params,
90	93	block,
91	94	} => todo!(),
92		JsExprData::If { condition, yes, no } => todo!(),
	95	JsExprData::Ternary { condition, yes, no } => todo!(),
93	96	JsExprData::ArrowFunction { params, block } => todo!(),
94	97	JsExprData::Comma { items } => todo!(),
	98	JsExprData::Void(_) => todo!(),
95	99	}
96	100	}
97	101
...	...	@@ -191,7 +195,7 @@ mod tests {
191	195	fn operation() {
192	196	assert_eq!(
193	197	print_expression(&JsExprData::Binary {
194		operator: AnyOperator::Add,
	198	op: AnyOperator::Add,
195	199	left: Box::new(JsExprData::Number(1.0)),
196	200	right: Box::new(JsExprData::Number(2.0)),
197	201	}),
...	...	@@ -199,7 +203,7 @@ mod tests {
199	203	);
200	204	assert_eq!(
201	205	print_expression(&JsExprData::Binary {
202		operator: AnyOperator::Sub,
	206	op: AnyOperator::Sub,
203	207	left: Box::new(JsExprData::Number(1.0)),
204	208	right: Box::new(JsExprData::Number(2.0)),
205	209	}),

crates/js_printer/src/print_statement.rs+6-6

...	...	@@ -1,4 +1,4 @@
1		use js_ast::{JsStatement, JsStatementData};
	1	use js_ast::{If, JSDeclaration, JSFunctionDeclaration, JsStatement, JsStatementData};
2	2
3	3	use crate::print_expression::print_expression;
4	4
...	...	@@ -8,11 +8,11 @@ pub fn print_statement(statement: &JsStatement) -> String {
8	8	// JsStatementData::Export(value) => format!("export {}", print_statement(value)),
9	9	JsStatementData::Export(value) => todo!(),
10	10	JsStatementData::Return(value) => format!("return {};", print_expression(&value)),
11		JsStatementData::Declaration {
	11	JsStatementData::Declaration(Declaration {
12	12	name,
13	13	init,
14	14	is_constant,
15		} => {
	15	}) => {
16	16	format!(
17	17	"{} {}={};",
18	18	match is_constant {
...	...	@@ -23,11 +23,11 @@ pub fn print_statement(statement: &JsStatement) -> String {
23	23	print_expression(&init)
24	24	)
25	25	}
26		JsStatementData::Function {
	26	JsStatementData::Function(JSFunctionDeclaration {
27	27	name,
28	28	block,
29	29	params,
30		} => format!(
	30	}) => format!(
31	31	"function {}({}){{{}}}",
32	32	name,
33	33	params.join(","),
...	...	@@ -38,7 +38,7 @@ pub fn print_statement(statement: &JsStatement) -> String {
38	38	.collect::<Vec<String>>()
39	39	.join("")
40	40	),
41		JsStatementData::If { condition, yes, no } => {
	41	JsStatementData::If(JsIf { condition, yes, no }) => {
42	42	let else_value = match no {
43	43	Some(value) => format!(
44	44	"else{{{}}}",

crates/parser/parse_block.rs+23-9

...	...	@@ -15,7 +15,11 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
15	15	let was_top_level = p.is_top_level;
16	16	p.is_top_level = false;
17	17
18		loop {
	18	'outer: loop {
	19	let is_statement_like_expression = p
	20	.peek()
	21	.map_or(false, \|x\| x.is_start_of_statement_like_expression());
	22
19	23	let statement = match parse_statement(p) {
20	24	Ok(statement) => statement,
21	25	Err(err) => {
...	...	@@ -25,8 +29,9 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
25	29	};
26	30	statements.push(statement.clone());
27	31
28		match p.next() {
	32	match p.peek() {
29	33	Some(Token::Semicolon) => loop {
	34	p.next();
30	35	match p.peek() {
31	36	Some(Token::ClosedBrace) => {
32	37	p.next();
...	...	@@ -36,14 +41,18 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
36	41	has_implicit_return: false,
37	42	});
38	43	}
39		_ => continue,
	44	_ => continue 'outer,
40	45	}
41	46	},
42	47	Some(Token::ClosedBrace) => {
43		if statement.requires_semicolon() {
	48	p.next();
	49	if !statement.is_expression()
	50	&& !statement.is_comment()
	51	&& is_statement_like_expression
	52	{
44	53	p.is_top_level = was_top_level;
45	54	return Err(p
46		.error("Expected semicolon after this statement".to_string())
	55	.error("Expected semicolon or '}' after statement".to_string())
47	56	.into());
48	57	}
49	58
...	...	@@ -54,10 +63,15 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
54	63	});
55	64	}
56	65	_ => {
57		p.is_top_level = was_top_level;
58		return Err(p
59		.error("Expected semicolon or '}' after statement".to_string())
60		.into());
	66	if !statement.is_expression()
	67	&& !statement.is_comment()
	68	&& is_statement_like_expression
	69	{
	70	p.is_top_level = was_top_level;
	71	return Err(p
	72	.error("Expected semicolon or '}' after statement".to_string())
	73	.into());
	74	}
61	75	}
62	76	};
63	77	}

crates/parser/parse_call_arguments.rs+5

...	...	@@ -8,6 +8,11 @@ use crate::{parse_expr::parse_expr, Parser};
8	8	pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
9	9	let mut args = Vec::new();
10	10
	11	if let Some(Token::ClosedParen) = p.peek() {
	12	_ = p.next();
	13	return Ok(args);
	14	}
	15
11	16	loop {
12	17	let arg = parse_expr(p)?;
13	18	args.push(arg);

crates/parser/parse_expr.rs+44-17

...	...	@@ -1,4 +1,5 @@
1		use ast::{get_binary_op_level, BinaryOperator, BinaryOperatorLevel, Expr, ExprData};
	1	use ast::Associativity::*;
	2	use ast::{BinaryOperator, Expr, ExprData, OperatorLevel};
2	3	use miette::Result;
3	4
4	5	use crate::parse_expr_basic::parse_expr_basic;
...	...	@@ -15,14 +16,27 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
15	16	// Parse operators
16	17	loop {
17	18	if let Some(token) = p.peek() {
18		if let Some(level) = get_binary_op_level(&token) {
	19	if let Some(level) = BinaryOperator::from_token(&token).map(\|o\| o.level()) {
19	20	_ = p.next();
20		expr = parse_expr_binary(
21		p,
22		BinaryOperator::from_token(&token).unwrap(),
23		expr.clone(),
24		level,
25		)?;
	21	match level.associativity() {
	22	RightToLeft => {
	23	expr = parse_expr_binary(
	24	p,
	25	BinaryOperator::from_token(&token).unwrap(),
	26	expr,
	27	level,
	28	)?;
	29	}
	30	LeftToRight => {
	31	// TODO(@paperdave): TODO
	32	expr = parse_expr_binary(
	33	p,
	34	BinaryOperator::from_token(&token).unwrap(),
	35	expr,
	36	level,
	37	)?;
	38	}
	39	}
26	40	continue;
27	41	}
28	42	}
...	...	@@ -37,22 +51,35 @@ fn parse_expr_binary(
37	51	p: &mut Parser,
38	52	op: BinaryOperator,
39	53	left: Expr,
40		level: BinaryOperatorLevel,
	54	level: OperatorLevel,
41	55	) -> Result<Expr> {
42	56	let loc = p.loc();
43	57	let mut right = parse_expr_basic(p)?;
44	58
45	59	loop {
46	60	if let Some(token) = p.peek() {
47		if let Some(next_level) = get_binary_op_level(&token) {
48		if next_level <= level {
	61	if let Some(new_level) = BinaryOperator::from_token(&token).map(\|o\| o.level()) {
	62	if new_level <= level {
49	63	_ = p.next();
50		right = parse_expr_binary(
51		p,
52		BinaryOperator::from_token(&token).unwrap(),
53		right,
54		next_level,
55		)?;
	64	match new_level.associativity() {
	65	RightToLeft => {
	66	right = parse_expr_binary(
	67	p,
	68	BinaryOperator::from_token(&token).unwrap(),
	69	right,
	70	new_level,
	71	)?;
	72	}
	73	LeftToRight => {
	74	// TODO(@paperdave): TODO
	75	right = parse_expr_binary(
	76	p,
	77	BinaryOperator::from_token(&token).unwrap(),
	78	right,
	79	new_level,
	80	)?;
	81	}
	82	}
56	83	continue;
57	84	}
58	85	}

crates/parser/parse_expr_if.rs+9-13

...	...	@@ -1,4 +1,4 @@
1		use ast::{BlockWithLoc, Expr, ExprData, If};
	1	use ast::{BlockOrIf, BlockWithLoc, If};
2	2	use miette::Result;
3	3	use tokenizer::Token;
4	4
...	...	@@ -48,21 +48,17 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
48	48	match p.peek() {
49	49	Some(Token::OpenBrace) => {
50	50	let loc = p.loc();
51		let comments = p.take_comments();
52		Some(Box::new(Expr::new(
53		ExprData::Block(parse_block(p)?),
	51	// let comments = p.take_comments();
	52	Some(BlockOrIf::Block(BlockWithLoc {
	53	block: parse_block(p)?,
54	54	loc,
55		comments,
56		)))
	55	}))
57	56	}
58	57	Some(Token::If) => {
59		let loc = p.loc();
60		let comments = p.take_comments();
61		Some(Box::new(Expr::new(
62		ExprData::If(parse_expr_if(p)?),
63		loc,
64		comments,
65		)))
	58	// TODO(@paperdave): Save location for the if
	59	// let loc = p.loc();
	60	// let comments = p.take_comments();
	61	Some(BlockOrIf::If(Box::new(parse_expr_if(p)?)))
66	62	}
67	63	_ => return Err(p.error("Expected a block after else".to_string()).into()),
68	64	}

crates/parser/parse_fn_declaration.rs+9-2

...	...	@@ -52,7 +52,10 @@ pub fn parse_fn_declaration(
52	52	break;
53	53	}
54	54	}
55		Some(Token::ClosedParen) => break,
	55	Some(Token::ClosedParen) => {
	56	p.next();
	57	break;
	58	}
56	59	_ => {
57	60	return Err(p
58	61	.error("Expected ',' or ')' after identifier".to_string())
...	...	@@ -108,8 +111,12 @@ pub fn parse_fn_declaration(
108	111	loc: loc,
109	112	})
110	113	} else {
	114	let actually = p.peek().map_or("EOF".to_string(), \|x\| x.to_string());
111	115	return Err(p
112		.error("Expected a block to follow function declaration".to_string())
	116	.error(format!(
	117	"Expected a block to follow function declaration, but got {}",
	118	actually,
	119	))
113	120	.into());
114	121	}
115	122	}

crates/parser/parse_module.rs+5-1

...	...	@@ -11,6 +11,10 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
11	11	let file_doc_comment = maybe_read_doc_comment(p);
12	12
13	13	loop {
	14	let is_statement_like_expression = p
	15	.peek()
	16	.map_or(false, \|x\| x.is_start_of_statement_like_expression());
	17
14	18	let statement = parse_top_level_statement(p)?;
15	19
16	20	match p.peek() {
...	...	@@ -20,7 +24,7 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
20	24	}
21	25	}
22	26	_ => {
23		if statement.requires_semicolon() {
	27	if !statement.is_comment() && is_statement_like_expression {
24	28	return Err(p
25	29	.error("Expected semicolon after statement".to_string())
26	30	.into());

crates/sema/Cargo.toml+1

...	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	4	edition = "2021"
5	5
6	6	[lib]
	7	path = "./sema.rs"
7	8
8	9	[dependencies]
9	10	miette.workspace = true

crates/sema/ast_type_coersion.rs created+54

...	...	@@ -0,0 +1,54 @@
	1	// use ast::{BinaryOperator, ExprData, Type};
	2
	3	// impl TypeCoerceable for ExprData {
	4	// fn to_type(&self) -> Option<Type> {
	5	// match self {
	6	// ExprData::Block(block) => {
	7	// if (block.has_implicit_return) {
	8	// block.statements.last().unwrap().to_type()
	9	// } else {
	10	// Some(Type::Void)
	11	// }
	12	// }
	13	// ExprData::Binary { op, left, right } => {
	14	// // TODO: this likely has to do further type checking
	15	// let left_type = left.to_type();
	16	// match op {
	17	// BinaryOperator::Add => left_type,
	18	// BinaryOperator::Sub => left_type,
	19	// BinaryOperator::Mul => left_type,
	20	// BinaryOperator::Div => left_type,
	21	// BinaryOperator::Pow => left_type,
	22	// BinaryOperator::Eq => Some(Type::Bool),
	23	// BinaryOperator::Gt => Some(Type::Bool),
	24	// BinaryOperator::Lt => Some(Type::Bool),
	25	// BinaryOperator::Gte => Some(Type::Bool),
	26	// BinaryOperator::Lte => Some(Type::Bool),
	27	// BinaryOperator::Lshift => Some(Type::Int),
	28	// BinaryOperator::Rshift => Some(Type::Int),
	29	// BinaryOperator::BitAnd => Some(Type::Int),
	30	// BinaryOperator::BitOr => Some(Type::Int),
	31	// BinaryOperator::BitXor => Some(Type::Int),
	32	// BinaryOperator::Mod => Some(Type::Int),
	33	// BinaryOperator::NotEq => Some(Type::Int),
	34
	35	// BinaryOperator::And => Some(Type::Bool),
	36	// BinaryOperator::Or => Some(Type::Bool),
	37	// }
	38	// }
	39	// ExprData::Unary { op, value } => todo!(),
	40	// ExprData::Function {
	41	// name,
	42	// params,
	43	// return_type,
	44	// block,
	45	// } => Some(return_type),
	46	// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
	47	// ExprData::Builtin { id, arguments } => todo!(),
	48	// ExprData::If(_) => todo!(),
	49	// ExprData::Return(_) => todo!(),
	50	// ExprData::Literal(_) => todo!(),
	51	// ExprData::Identifier(_) => todo!(),
	52	// }
	53	// }
	54	// }

crates/sema/sema.rs created+98

...	...	@@ -0,0 +1,98 @@
	1	use ast::Module;
	2	use miette::Result;
	3
	4	pub fn analyze_module(_block: &mut Module) -> Result<()> {
	5	//
	6	Ok(())
	7	}
	8
	9	// for statement in &block.statements {
	10	// check_expression(statement)?;
	11	// }
	12
	13	// Ok(())
	14	// }
	15
	16	// pub fn resolve_type(expr: &Ast) -> Result<Type> {
	17	// match expr {
	18	// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
	19	// Ast::Declaration {
	20	// name,
	21	// decl_type,
	22	// value,
	23	// is_constant,
	24	// } => Ok(Type::NoReturn),
	25	// Ast::Binary {
	26	// operator,
	27	// left,
	28	// right,
	29	// } => merge_types(resolve_type(left)?, resolve_type(right)?),
	30	// Ast::Function {
	31	// name,
	32	// arguments,
	33	// return_type,
	34	// block,
	35	// } => todo!(),
	36	// Ast::FunctionCall { name, arguments } => todo!(),
	37	// Ast::If {
	38	// condition,
	39	// value,
	40	// else_value,
	41	// } => todo!(),
	42	// Ast::Return(_) => Ok(Type::NoReturn),
	43	// Ast::Public(_) => todo!(), // TODO: remove Public
	44	// Ast::Block(_) => todo!(),
	45	// Ast::Bool(_) => Ok(Type::Bool),
	46	// Ast::String(_) => Ok(Type::String),
	47	// Ast::Identifier(_) => todo!(),
	48	// Ast::BuiltIn(_) => todo!(),
	49
	50	// Ast::Comment(_) => unreachable!(),
	51	// Ast::DocComment(_) => unreachable!(),
	52	// }
	53	// }
	54
	55	// pub fn check_expression(expr: &Ast) -> Result<()> {
	56	// panic!()
	57	// match expr {
	58	// Ast::Declaration {
	59	// name,
	60	// decl_type: return_type,
	61	// value,
	62	// is_constant,
	63	// } => {
	64	// check_expression(value)?;
	65	// // coerce_type(return_type.clone(), resolve_type(value)?)?;
	66	// }
	67	// _ => {}
	68	// // Ast::Operation {
	69	// // operator,
	70	// // left,
	71	// // right,
	72	// // } => Ok(()),
	73	// // Ast::Function {
	74	// // name,
	75	// // arguments,
	76	// // return_type,
	77	// // block,
	78	// // } => Ok(()),
	79	// // Ast::FunctionCall { name, arguments } => Ok(()),
	80	// // Ast::If {
	81	// // condition,
	82	// // value,
	83	// // else_value,
	84	// // } => Ok(()),
	85	// // Ast::Return(_) => Ok(()),
	86	// // Ast::Public(_) => Ok(()),
	87	// // Ast::Block(_) => Ok(()),
	88	// // Ast::Number(_) => Ok(()),
	89	// // Ast::Bool(_) => Ok(()),
	90	// // Ast::String(_) => Ok(()),
	91	// // Ast::Identifier(_) => Ok(()),
	92	// // Ast::BuiltIn(_) => Ok(()),
	93	// // Ast::Comment(_) => Ok(()),
	94	// // Ast::DocComment(_) => Ok(()),
	95	// }
	96
	97	// Ok(())
	98	// }

crates/sema/src/ast_type_coersion.rs deleted-54

...	...	@@ -1,54 +0,0 @@
1		// use ast::{BinaryOperator, ExprData, Type};
2
3		// impl TypeCoerceable for ExprData {
4		// fn to_type(&self) -> Option<Type> {
5		// match self {
6		// ExprData::Block(block) => {
7		// if (block.has_implicit_return) {
8		// block.statements.last().unwrap().to_type()
9		// } else {
10		// Some(Type::Void)
11		// }
12		// }
13		// ExprData::Binary { op, left, right } => {
14		// // TODO: this likely has to do further type checking
15		// let left_type = left.to_type();
16		// match op {
17		// BinaryOperator::Add => left_type,
18		// BinaryOperator::Sub => left_type,
19		// BinaryOperator::Mul => left_type,
20		// BinaryOperator::Div => left_type,
21		// BinaryOperator::Pow => left_type,
22		// BinaryOperator::Eq => Some(Type::Bool),
23		// BinaryOperator::Gt => Some(Type::Bool),
24		// BinaryOperator::Lt => Some(Type::Bool),
25		// BinaryOperator::Gte => Some(Type::Bool),
26		// BinaryOperator::Lte => Some(Type::Bool),
27		// BinaryOperator::Lshift => Some(Type::Int),
28		// BinaryOperator::Rshift => Some(Type::Int),
29		// BinaryOperator::BitAnd => Some(Type::Int),
30		// BinaryOperator::BitOr => Some(Type::Int),
31		// BinaryOperator::BitXor => Some(Type::Int),
32		// BinaryOperator::Mod => Some(Type::Int),
33		// BinaryOperator::NotEq => Some(Type::Int),
34
35		// BinaryOperator::And => Some(Type::Bool),
36		// BinaryOperator::Or => Some(Type::Bool),
37		// }
38		// }
39		// ExprData::Unary { op, value } => todo!(),
40		// ExprData::Function {
41		// name,
42		// params,
43		// return_type,
44		// block,
45		// } => Some(return_type),
46		// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47		// ExprData::Builtin { id, arguments } => todo!(),
48		// ExprData::If(_) => todo!(),
49		// ExprData::Return(_) => todo!(),
50		// ExprData::Literal(_) => todo!(),
51		// ExprData::Identifier(_) => todo!(),
52		// }
53		// }
54		// }

crates/sema/src/lib.rs deleted-96

...	...	@@ -1,96 +0,0 @@
1		// use ast::{ Module, Type};
2		// use miette::Result;
3
4		// mod ast_type_coersion;
5
6		// pub fn check_module(block: &Module) -> Result<()> {
7		// for statement in &block.statements {
8		// check_expression(statement)?;
9		// }
10
11		// Ok(())
12		// }
13
14		// pub fn resolve_type(expr: &Ast) -> Result<Type> {
15		// match expr {
16		// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
17		// Ast::Declaration {
18		// name,
19		// decl_type,
20		// value,
21		// is_constant,
22		// } => Ok(Type::NoReturn),
23		// Ast::Binary {
24		// operator,
25		// left,
26		// right,
27		// } => merge_types(resolve_type(left)?, resolve_type(right)?),
28		// Ast::Function {
29		// name,
30		// arguments,
31		// return_type,
32		// block,
33		// } => todo!(),
34		// Ast::FunctionCall { name, arguments } => todo!(),
35		// Ast::If {
36		// condition,
37		// value,
38		// else_value,
39		// } => todo!(),
40		// Ast::Return(_) => Ok(Type::NoReturn),
41		// Ast::Public(_) => todo!(), // TODO: remove Public
42		// Ast::Block(_) => todo!(),
43		// Ast::Bool(_) => Ok(Type::Bool),
44		// Ast::String(_) => Ok(Type::String),
45		// Ast::Identifier(_) => todo!(),
46		// Ast::BuiltIn(_) => todo!(),
47
48		// Ast::Comment(_) => unreachable!(),
49		// Ast::DocComment(_) => unreachable!(),
50		// }
51		// }
52
53		// pub fn check_expression(expr: &Ast) -> Result<()> {
54		// panic!()
55		// match expr {
56		// Ast::Declaration {
57		// name,
58		// decl_type: return_type,
59		// value,
60		// is_constant,
61		// } => {
62		// check_expression(value)?;
63		// // coerce_type(return_type.clone(), resolve_type(value)?)?;
64		// }
65		// _ => {}
66		// // Ast::Operation {
67		// // operator,
68		// // left,
69		// // right,
70		// // } => Ok(()),
71		// // Ast::Function {
72		// // name,
73		// // arguments,
74		// // return_type,
75		// // block,
76		// // } => Ok(()),
77		// // Ast::FunctionCall { name, arguments } => Ok(()),
78		// // Ast::If {
79		// // condition,
80		// // value,
81		// // else_value,
82		// // } => Ok(()),
83		// // Ast::Return(_) => Ok(()),
84		// // Ast::Public(_) => Ok(()),
85		// // Ast::Block(_) => Ok(()),
86		// // Ast::Number(_) => Ok(()),
87		// // Ast::Bool(_) => Ok(()),
88		// // Ast::String(_) => Ok(()),
89		// // Ast::Identifier(_) => Ok(()),
90		// // Ast::BuiltIn(_) => Ok(()),
91		// // Ast::Comment(_) => Ok(()),
92		// // Ast::DocComment(_) => Ok(()),
93		// }
94
95		// Ok(())
96		// }

crates/tokenizer/Cargo.toml+1

...	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	4	edition = "2021"
5	5
6	6	[lib]
	7	path="./tokenizer.rs"
7	8
8	9	[dependencies]
9	10	miette.workspace = true

crates/tokenizer/src/lib.rs deleted-393

...	...	@@ -1,393 +0,0 @@
1		use miette::{Diagnostic, Result};
2		use thiserror::Error;
3
4		#[derive(Debug, PartialEq, Clone)]
5		pub struct Loc {}
6
7		impl Loc {
8		pub const EMPTY: Loc = Loc {};
9
10		pub fn new() -> Self {
11		Self {}
12		}
13		}
14
15		#[derive(Error, Diagnostic, Debug)]
16		#[error("UnknownCharacter")]
17		struct UnknownCharacter {
18		#[help]
19		error: String,
20		}
21
22		#[derive(Debug, Clone)]
23		pub enum Token {
24		// Literals
25		Identifier(String),
26		Number(f64),
27		String(String),
28		// Comments
29		Comment(String),
30		DocComment(String),
31		// Keywords
32		Else,
33		False,
34		Fn,
35		If,
36		Let,
37		Pub,
38		Return,
39		True,
40		Var,
41		Void,
42		// Symbols
43		And, // &
44		AndAnd, // &&
45		Bang, // !
46		BangEqual, // !=
47		Caret, // ^
48		ClosedBrace, // }
49		ClosedBracket, // ]
50		ClosedParen, // )
51		Colon, // :
52		Comma, // ,
53		Equal, // =
54		EqualEqual, // ==
55		Gt, // >
56		GtEqual, // >=
57		GtGt, // >>
58		GtGtGt, // >>>
59		Lt, // <
60		LtEqual, // <=
61		LtLt, // <<
62		Minus, // -
63		OpenBrace, // {
64		OpenBracket, // [
65		OpenParen, // (
66		Or, // \|
67		OrOr, // \|\|
68		Percent, // %
69		Plus, // +
70		RightArrow, // ->
71		Semicolon, // ;
72		Slash, // /
73		Star, // *
74		StarStar, // **
75		Tilde, // ~
76		Dot, // .
77		}
78
79		impl std::fmt::Display for Token {
80		fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81		let token = match self {
82		Token::Identifier(_) => "identifier".to_string(),
83
84		Token::Number(n) => n.to_string(),
85		Token::String(s) => format!("\"{}\"", s),
86		Token::Comment(_) => "comment".to_string(),
87		Token::DocComment(_) => "documentation comment".to_string(),
88
89		Token::Else => "else".to_string(),
90		Token::False => "false".to_string(),
91		Token::Fn => "fn".to_string(),
92		Token::If => "if".to_string(),
93		Token::Let => "let".to_string(),
94		Token::Pub => "pub".to_string(),
95		Token::Return => "return".to_string(),
96		Token::True => "true".to_string(),
97		Token::Var => "var".to_string(),
98		Token::Void => "void".to_string(),
99
100		Token::And => "&".to_string(),
101		Token::AndAnd => "&&".to_string(),
102		Token::Bang => "!".to_string(),
103		Token::BangEqual => "!=".to_string(),
104		Token::Caret => "^".to_string(),
105		Token::ClosedBrace => "}".to_string(),
106		Token::ClosedBracket => "]".to_string(),
107		Token::ClosedParen => ")".to_string(),
108		Token::Colon => ":".to_string(),
109		Token::Comma => ",".to_string(),
110		Token::Equal => "=".to_string(),
111		Token::EqualEqual => "==".to_string(),
112		Token::Gt => ">".to_string(),
113		Token::GtEqual => ">=".to_string(),
114		Token::GtGt => ">>".to_string(),
115		Token::Lt => "<".to_string(),
116		Token::LtEqual => "<=".to_string(),
117		Token::LtLt => "<<".to_string(),
118		Token::Minus => "-".to_string(),
119		Token::OpenBrace => "{".to_string(),
120		Token::OpenBracket => "[".to_string(),
121		Token::OpenParen => "(".to_string(),
122		Token::Or => "\|".to_string(),
123		Token::OrOr => "\|\|".to_string(),
124		Token::Percent => "%".to_string(),
125		Token::Plus => "+".to_string(),
126		Token::RightArrow => "->".to_string(),
127		Token::Semicolon => ";".to_string(),
128		Token::Slash => "/".to_string(),
129		Token::Star => "*".to_string(),
130		Token::StarStar => "**".to_string(),
131		Token::Tilde => "~".to_string(),
132		Token::Dot => ".".to_string(),
133		Token::GtGtGt => ">>>".to_string(),
134		};
135
136		write!(f, "{}", token)
137		}
138		}
139
140		// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
141		pub fn tokenize(content: String) -> Result<Vec<Token>> {
142		let mut tokens: Vec<Token> = Vec::new();
143		let mut chars = content.chars().enumerate().peekable();
144
145		loop {
146		match chars.next() {
147		Some((_, c)) => match c {
148		'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
149		let mut current_token = String::new();
150		current_token.push(c);
151		loop {
152		let c = chars.peek();
153		if c.is_none() {
154		break;
155		}
156		let (_, c) = c.unwrap();
157		match c {
158		'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
159		current_token.push(*c);
160		chars.next();
161		}
162		_ => break,
163		}
164		}
165
166		match current_token.as_str() {
167		"else" => tokens.push(Token::Else),
168		"false" => tokens.push(Token::False),
169		"fn" => tokens.push(Token::Fn),
170		"if" => tokens.push(Token::If),
171		"let" => tokens.push(Token::Let),
172		"pub" => tokens.push(Token::Pub),
173		"return" => tokens.push(Token::Return),
174		"true" => tokens.push(Token::True),
175		"var" => tokens.push(Token::Var),
176		"void" => tokens.push(Token::Void),
177		_ => tokens.push(Token::Identifier(current_token)),
178		}
179		}
180		'"' => {
181		let mut current_token = String::new();
182		loop {
183		let c = chars.next();
184		if c.is_none() {
185		break;
186		}
187		let (_, c) = c.unwrap();
188		match c {
189		'"' => break,
190		_ => current_token.push(c),
191		}
192		}
193		tokens.push(Token::String(current_token));
194		}
195		'0'..='9' => {
196		let mut current_token = String::new();
197		current_token.push(c);
198		loop {
199		let c = chars.peek();
200		if c.is_none() {
201		break;
202		}
203		let (_, c) = c.unwrap();
204		match c {
205		'0'..='9' => {
206		current_token.push(*c);
207		chars.next();
208		}
209		_ => break,
210		}
211		}
212		match current_token.parse::<f64>() {
213		Ok(v) => tokens.push(Token::Number(v)),
214		Err(_) => {
215		return Err(UnknownCharacter {
216		error: format!("Unknown character: {}", current_token),
217		}
218		.into())
219		}
220		}
221		}
222		':' => tokens.push(Token::Colon),
223		'=' => match chars.peek() {
224		Some((_, '=')) => {
225		chars.next();
226		tokens.push(Token::EqualEqual);
227		}
228		_ => tokens.push(Token::Equal),
229		},
230		';' => tokens.push(Token::Semicolon),
231		'>' => match chars.peek() {
232		Some((_, '=')) => {
233		chars.next();
234		tokens.push(Token::GtEqual);
235		}
236		Some((_, '>')) => {
237		chars.next();
238		match chars.peek() {
239		Some((_, '>')) => {
240		chars.next();
241		tokens.push(Token::GtGtGt);
242		}
243		_ => tokens.push(Token::GtGt),
244		}
245		}
246		_ => tokens.push(Token::Gt),
247		},
248		'<' => match chars.peek() {
249		Some((_, '=')) => {
250		chars.next();
251		tokens.push(Token::LtEqual);
252		}
253		Some((_, '<')) => {
254		chars.next();
255		tokens.push(Token::LtLt);
256		}
257		_ => tokens.push(Token::Lt),
258		},
259		'+' => tokens.push(Token::Plus),
260		'-' => match chars.peek() {
261		Some((_, '>')) => {
262		chars.next();
263		tokens.push(Token::RightArrow);
264		}
265		_ => tokens.push(Token::Minus),
266		},
267		'*' => match chars.peek() {
268		Some((_, '*')) => {
269		chars.next();
270		tokens.push(Token::StarStar);
271		}
272		_ => tokens.push(Token::Star),
273		},
274		'/' => {
275		match chars.peek() {
276		Some((_, '/')) => {
277		chars.next(); // Consume the second slash
278
279		let is_doc_comment = match chars.peek() {
280		Some((_, '/')) => {
281		chars.next(); // Consume the third slash
282		true
283		}
284		_ => false,
285		};
286
287		let mut comment = String::new();
288		loop {
289		// skip leading whitespace
290		while let Some((_, c)) = chars.peek() {
291		if !c.is_whitespace() {
292		break;
293		}
294		chars.next();
295		}
296		// read the comment
297		while let Some((_, c)) = chars.next() {
298		if c == '\n' \|\| c == '\r' {
299		break;
300		}
301		comment.push(c);
302		}
303		// trim trailing whitespace on line
304		comment = comment.trim_end().to_string();
305		// skip to next non-whitespace character
306		while let Some((_, c)) = chars.peek() {
307		if !c.is_whitespace() {
308		break;
309		}
310		chars.next();
311		}
312		// check if we've hit another comment
313		let mut tmp = chars.clone();
314
315		let is_another_comment = match tmp.peek() {
316		Some((_, '/')) => {
317		tmp.next(); // Consume the second slash
318		match tmp.peek() {
319		Some((_, '/')) => {
320		tmp.next(); // Consume the third slash
321		is_doc_comment
322		}
323		_ => !is_doc_comment,
324		}
325		}
326		_ => false,
327		};
328
329		if is_another_comment {
330		// apply our temporary iterator to the main iterator
331		comment.push('\n');
332		chars = tmp;
333		chars.next();
334		} else {
335		break;
336		}
337		}
338
339		if is_doc_comment {
340		tokens.push(Token::DocComment(comment));
341		} else {
342		tokens.push(Token::Comment(comment));
343		}
344		}
345		_ => tokens.push(Token::Slash),
346		}
347		}
348		',' => tokens.push(Token::Comma),
349		'(' => tokens.push(Token::OpenParen),
350		')' => tokens.push(Token::ClosedParen),
351		'{' => tokens.push(Token::OpenBrace),
352		'}' => tokens.push(Token::ClosedBrace),
353		'[' => tokens.push(Token::OpenBracket),
354		']' => tokens.push(Token::ClosedBracket),
355		'&' => match chars.peek() {
356		Some((_, '&')) => {
357		chars.next();
358		tokens.push(Token::AndAnd);
359		}
360		_ => tokens.push(Token::And),
361		},
362		'\|' => match chars.peek() {
363		Some((_, '\|')) => {
364		chars.next();
365		tokens.push(Token::OrOr);
366		}
367		_ => tokens.push(Token::Or),
368		},
369		'!' => match chars.peek() {
370		Some((_, '=')) => {
371		chars.next();
372		tokens.push(Token::BangEqual);
373		}
374		_ => tokens.push(Token::Bang),
375		},
376		'^' => tokens.push(Token::Caret),
377		'%' => tokens.push(Token::Percent),
378		'~' => tokens.push(Token::Tilde),
379		'.' => tokens.push(Token::Dot),
380		s if s.is_whitespace() => (),
381		_ => {
382		return Err(UnknownCharacter {
383		error: format!("Unexpected '{}' while tokenizing", c),
384		}
385		.into())
386		}
387		},
388		None => break,
389		}
390		}
391
392		Ok(tokens)
393		}

crates/tokenizer/tokenizer.rs created+402

...	...	@@ -0,0 +1,402 @@
	1	use miette::{Diagnostic, Result};
	2	use thiserror::Error;
	3
	4	#[derive(Debug, PartialEq, Clone)]
	5	pub struct Loc {}
	6
	7	impl Loc {
	8	pub const EMPTY: Loc = Loc {};
	9
	10	pub fn new() -> Self {
	11	Self {}
	12	}
	13	}
	14
	15	#[derive(Error, Diagnostic, Debug)]
	16	#[error("UnknownCharacter")]
	17	struct UnknownCharacter {
	18	#[help]
	19	error: String,
	20	}
	21
	22	#[derive(Debug, Clone)]
	23	pub enum Token {
	24	// Literals
	25	Identifier(String),
	26	Number(f64),
	27	String(String),
	28	// Comments
	29	Comment(String),
	30	DocComment(String),
	31	// Keywords
	32	Else,
	33	False,
	34	Fn,
	35	If,
	36	Let,
	37	Pub,
	38	Return,
	39	True,
	40	Var,
	41	Void,
	42	// Symbols
	43	And, // &
	44	AndAnd, // &&
	45	Bang, // !
	46	BangEqual, // !=
	47	Caret, // ^
	48	ClosedBrace, // }
	49	ClosedBracket, // ]
	50	ClosedParen, // )
	51	Colon, // :
	52	Comma, // ,
	53	Equal, // =
	54	EqualEqual, // ==
	55	Gt, // >
	56	GtEqual, // >=
	57	GtGt, // >>
	58	GtGtGt, // >>>
	59	Lt, // <
	60	LtEqual, // <=
	61	LtLt, // <<
	62	Minus, // -
	63	OpenBrace, // {
	64	OpenBracket, // [
	65	OpenParen, // (
	66	Or, // \|
	67	OrOr, // \|\|
	68	Percent, // %
	69	Plus, // +
	70	RightArrow, // ->
	71	Semicolon, // ;
	72	Slash, // /
	73	Star, // *
	74	StarStar, // **
	75	Tilde, // ~
	76	Dot, // .
	77	}
	78
	79	impl Token {
	80	pub fn is_start_of_statement_like_expression(&self) -> bool {
	81	match &self {
	82	Token::If \| Token::OpenBrace \| Token::Fn => false,
	83	_ => true,
	84	}
	85	}
	86	}
	87
	88	impl std::fmt::Display for Token {
	89	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
	90	let token = match self {
	91	Token::Identifier(_) => "identifier".to_string(),
	92
	93	Token::Number(n) => n.to_string(),
	94	Token::String(s) => format!("\"{}\"", s),
	95	Token::Comment(_) => "comment".to_string(),
	96	Token::DocComment(_) => "documentation comment".to_string(),
	97
	98	Token::Else => "else".to_string(),
	99	Token::False => "false".to_string(),
	100	Token::Fn => "fn".to_string(),
	101	Token::If => "if".to_string(),
	102	Token::Let => "let".to_string(),
	103	Token::Pub => "pub".to_string(),
	104	Token::Return => "return".to_string(),
	105	Token::True => "true".to_string(),
	106	Token::Var => "var".to_string(),
	107	Token::Void => "void".to_string(),
	108
	109	Token::And => "&".to_string(),
	110	Token::AndAnd => "&&".to_string(),
	111	Token::Bang => "!".to_string(),
	112	Token::BangEqual => "!=".to_string(),
	113	Token::Caret => "^".to_string(),
	114	Token::ClosedBrace => "}".to_string(),
	115	Token::ClosedBracket => "]".to_string(),
	116	Token::ClosedParen => ")".to_string(),
	117	Token::Colon => ":".to_string(),
	118	Token::Comma => ",".to_string(),
	119	Token::Equal => "=".to_string(),
	120	Token::EqualEqual => "==".to_string(),
	121	Token::Gt => ">".to_string(),
	122	Token::GtEqual => ">=".to_string(),
	123	Token::GtGt => ">>".to_string(),
	124	Token::Lt => "<".to_string(),
	125	Token::LtEqual => "<=".to_string(),
	126	Token::LtLt => "<<".to_string(),
	127	Token::Minus => "-".to_string(),
	128	Token::OpenBrace => "{".to_string(),
	129	Token::OpenBracket => "[".to_string(),
	130	Token::OpenParen => "(".to_string(),
	131	Token::Or => "\|".to_string(),
	132	Token::OrOr => "\|\|".to_string(),
	133	Token::Percent => "%".to_string(),
	134	Token::Plus => "+".to_string(),
	135	Token::RightArrow => "->".to_string(),
	136	Token::Semicolon => ";".to_string(),
	137	Token::Slash => "/".to_string(),
	138	Token::Star => "*".to_string(),
	139	Token::StarStar => "**".to_string(),
	140	Token::Tilde => "~".to_string(),
	141	Token::Dot => ".".to_string(),
	142	Token::GtGtGt => ">>>".to_string(),
	143	};
	144
	145	write!(f, "{}", token)
	146	}
	147	}
	148
	149	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
	150	pub fn tokenize(content: String) -> Result<Vec<Token>> {
	151	let mut tokens: Vec<Token> = Vec::new();
	152	let mut chars = content.chars().enumerate().peekable();
	153
	154	loop {
	155	match chars.next() {
	156	Some((_, c)) => match c {
	157	'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
	158	let mut current_token = String::new();
	159	current_token.push(c);
	160	loop {
	161	let c = chars.peek();
	162	if c.is_none() {
	163	break;
	164	}
	165	let (_, c) = c.unwrap();
	166	match c {
	167	'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
	168	current_token.push(*c);
	169	chars.next();
	170	}
	171	_ => break,
	172	}
	173	}
	174
	175	match current_token.as_str() {
	176	"else" => tokens.push(Token::Else),
	177	"false" => tokens.push(Token::False),
	178	"fn" => tokens.push(Token::Fn),
	179	"if" => tokens.push(Token::If),
	180	"let" => tokens.push(Token::Let),
	181	"pub" => tokens.push(Token::Pub),
	182	"return" => tokens.push(Token::Return),
	183	"true" => tokens.push(Token::True),
	184	"var" => tokens.push(Token::Var),
	185	"void" => tokens.push(Token::Void),
	186	_ => tokens.push(Token::Identifier(current_token)),
	187	}
	188	}
	189	'"' => {
	190	let mut current_token = String::new();
	191	loop {
	192	let c = chars.next();
	193	if c.is_none() {
	194	break;
	195	}
	196	let (_, c) = c.unwrap();
	197	match c {
	198	'"' => break,
	199	_ => current_token.push(c),
	200	}
	201	}
	202	tokens.push(Token::String(current_token));
	203	}
	204	'0'..='9' => {
	205	let mut current_token = String::new();
	206	current_token.push(c);
	207	loop {
	208	let c = chars.peek();
	209	if c.is_none() {
	210	break;
	211	}
	212	let (_, c) = c.unwrap();
	213	match c {
	214	'0'..='9' => {
	215	current_token.push(*c);
	216	chars.next();
	217	}
	218	_ => break,
	219	}
	220	}
	221	match current_token.parse::<f64>() {
	222	Ok(v) => tokens.push(Token::Number(v)),
	223	Err(_) => {
	224	return Err(UnknownCharacter {
	225	error: format!("Unknown character: {}", current_token),
	226	}
	227	.into())
	228	}
	229	}
	230	}
	231	':' => tokens.push(Token::Colon),
	232	'=' => match chars.peek() {
	233	Some((_, '=')) => {
	234	chars.next();
	235	tokens.push(Token::EqualEqual);
	236	}
	237	_ => tokens.push(Token::Equal),
	238	},
	239	';' => tokens.push(Token::Semicolon),
	240	'>' => match chars.peek() {
	241	Some((_, '=')) => {
	242	chars.next();
	243	tokens.push(Token::GtEqual);
	244	}
	245	Some((_, '>')) => {
	246	chars.next();
	247	match chars.peek() {
	248	Some((_, '>')) => {
	249	chars.next();
	250	tokens.push(Token::GtGtGt);
	251	}
	252	_ => tokens.push(Token::GtGt),
	253	}
	254	}
	255	_ => tokens.push(Token::Gt),
	256	},
	257	'<' => match chars.peek() {
	258	Some((_, '=')) => {
	259	chars.next();
	260	tokens.push(Token::LtEqual);
	261	}
	262	Some((_, '<')) => {
	263	chars.next();
	264	tokens.push(Token::LtLt);
	265	}
	266	_ => tokens.push(Token::Lt),
	267	},
	268	'+' => tokens.push(Token::Plus),
	269	'-' => match chars.peek() {
	270	Some((_, '>')) => {
	271	chars.next();
	272	tokens.push(Token::RightArrow);
	273	}
	274	_ => tokens.push(Token::Minus),
	275	},
	276	'*' => match chars.peek() {
	277	Some((_, '*')) => {
	278	chars.next();
	279	tokens.push(Token::StarStar);
	280	}
	281	_ => tokens.push(Token::Star),
	282	},
	283	'/' => {
	284	match chars.peek() {
	285	Some((_, '/')) => {
	286	chars.next(); // Consume the second slash
	287
	288	let is_doc_comment = match chars.peek() {
	289	Some((_, '/')) => {
	290	chars.next(); // Consume the third slash
	291	true
	292	}
	293	_ => false,
	294	};
	295
	296	let mut comment = String::new();
	297	loop {
	298	// skip leading whitespace
	299	while let Some((_, c)) = chars.peek() {
	300	if !c.is_whitespace() {
	301	break;
	302	}
	303	chars.next();
	304	}
	305	// read the comment
	306	while let Some((_, c)) = chars.next() {
	307	if c == '\n' \|\| c == '\r' {
	308	break;
	309	}
	310	comment.push(c);
	311	}
	312	// trim trailing whitespace on line
	313	comment = comment.trim_end().to_string();
	314	// skip to next non-whitespace character
	315	while let Some((_, c)) = chars.peek() {
	316	if !c.is_whitespace() {
	317	break;
	318	}
	319	chars.next();
	320	}
	321	// check if we've hit another comment
	322	let mut tmp = chars.clone();
	323
	324	let is_another_comment = match tmp.peek() {
	325	Some((_, '/')) => {
	326	tmp.next(); // Consume the second slash
	327	match tmp.peek() {
	328	Some((_, '/')) => {
	329	tmp.next(); // Consume the third slash
	330	is_doc_comment
	331	}
	332	_ => !is_doc_comment,
	333	}
	334	}
	335	_ => false,
	336	};
	337
	338	if is_another_comment {
	339	// apply our temporary iterator to the main iterator
	340	comment.push('\n');
	341	chars = tmp;
	342	chars.next();
	343	} else {
	344	break;
	345	}
	346	}
	347
	348	if is_doc_comment {
	349	tokens.push(Token::DocComment(comment));
	350	} else {
	351	tokens.push(Token::Comment(comment));
	352	}
	353	}
	354	_ => tokens.push(Token::Slash),
	355	}
	356	}
	357	',' => tokens.push(Token::Comma),
	358	'(' => tokens.push(Token::OpenParen),
	359	')' => tokens.push(Token::ClosedParen),
	360	'{' => tokens.push(Token::OpenBrace),
	361	'}' => tokens.push(Token::ClosedBrace),
	362	'[' => tokens.push(Token::OpenBracket),
	363	']' => tokens.push(Token::ClosedBracket),
	364	'&' => match chars.peek() {
	365	Some((_, '&')) => {
	366	chars.next();
	367	tokens.push(Token::AndAnd);
	368	}
	369	_ => tokens.push(Token::And),
	370	},
	371	'\|' => match chars.peek() {
	372	Some((_, '\|')) => {
	373	chars.next();
	374	tokens.push(Token::OrOr);
	375	}
	376	_ => tokens.push(Token::Or),
	377	},
	378	'!' => match chars.peek() {
	379	Some((_, '=')) => {
	380	chars.next();
	381	tokens.push(Token::BangEqual);
	382	}
	383	_ => tokens.push(Token::Bang),
	384	},
	385	'^' => tokens.push(Token::Caret),
	386	'%' => tokens.push(Token::Percent),
	387	'~' => tokens.push(Token::Tilde),
	388	'.' => tokens.push(Token::Dot),
	389	s if s.is_whitespace() => (),
	390	_ => {
	391	return Err(UnknownCharacter {
	392	error: format!("Unexpected '{}' while tokenizing", c),
	393	}
	394	.into())
	395	}
	396	},
	397	None => break,
	398	}
	399	}
	400
	401	Ok(tokens)
	402	}