Commit 12864f7c49c005271c6c5610dfaa2cb71ff56ae7 | ix

Cargo.lock+1

...	@@ -239,6 +239,7 @@ dependencies = [	...	@@ -239,6 +239,7 @@ dependencies = [
239	"ast",	239	"ast",
240	"miette",	240	"miette",
241	"num",	241	"num",
		242	"parser",
242	"thiserror",	243	"thiserror",
243	"tokenizer",	244	"tokenizer",
244	]	245	]

Cargo.toml-1

...	@@ -1,6 +1,5 @@	...	@@ -1,6 +1,5 @@
1	[workspace.package]	1	[workspace.package]
2	version = "0.1.0"	2	version = "0.1.0"
3	name = "blight"
4	edition = "2023"	3	edition = "2023"
5	license = "MIT"	4	license = "MIT"
6	authors = ["Lino Le Van", "Dave Caruso"]	5	authors = ["Lino Le Van", "Dave Caruso"]

crates/ast/ast.rs+38-66

 @@ -74,6 +74,19 @@ impl Comment {
         }
         result
     }
+    pub fn to_doc_comment(self, loc: Loc) -> DocComment {
+        DocComment {
+            data: self.data,
+            loc,
+        }
+    }
+}
+impl Into<Comment> for DocComment {
+    fn into(self) -> Comment {
+        Comment { data: self.data }
+    }
 }
 impl DocComment {
 @@ -98,7 +111,13 @@ pub struct BlockWithLoc {
 pub struct If {
     pub condition: Box<Expr>,
     pub yes: BlockWithLoc,
-    pub no: Option<Box<Expr>>,
+    pub no: Option<BlockOrIf>,
+}
+#[derive(Debug, Clone)]
+pub enum BlockOrIf {
+    Block(BlockWithLoc),
+    If(Box<If>),
 }
 #[derive(Debug, Clone)]
 @@ -166,67 +185,10 @@ impl BuiltinId {
     }
 }
-impl ExprData {
-    pub fn is_valid_js_expression(&self) -> bool {
-        match self {
-            // A block containing just expressions can be an expression
-            ExprData::Block(block) => block.is_valid_js_expression(),
-            ExprData::If(expr) => expr.is_valid_js_expression(),
-            ExprData::Return(_) => false,
-            ExprData::Builtin { id, .. } => match id {
-                BuiltinId::Raw => true,
-            },
-            _ => true,
-        }
-    }
-}
-impl Block {
-    pub fn is_valid_js_expression(&self) -> bool {
-        self.statements
-            .iter()
-            .all(|statement| statement.is_valid_js_expression())
-    }
-}
-impl Expr {
-    pub fn is_valid_js_expression(&self) -> bool {
-        self.data.is_valid_js_expression()
-    }
-}
-impl If {
-    pub fn is_valid_js_expression(&self) -> bool {
-        self.condition.is_valid_js_expression()
-            && self.yes.is_valid_js_expression()
-            && match &self.no {
-                Some(no) => no.is_valid_js_expression(),
-                None => true,
-            }
-    }
-}
-impl BlockWithLoc {
-    pub fn is_valid_js_expression(&self) -> bool {
-        self.block.is_valid_js_expression()
-    }
-}
 impl Statement {
-    pub fn requires_semicolon(&self) -> bool {
+    pub fn is_expression(&self) -> bool {
         match &self {
-            Statement::Expr(expr) => match expr.data {
+            Statement::Expr(_) => true,
-                ExprData::Block(_) => false,
-                ExprData::If(_) => false,
-                _ => true,
-            },
-            _ => false,
-        }
-    }
-    pub fn is_valid_js_expression(&self) -> bool {
-        match self {
-            Statement::Expr(expr) => expr.data.is_valid_js_expression(),
             _ => false,
         }
     }
 @@ -238,16 +200,19 @@ impl Statement {
             Statement::Comment(comment) => TopLevelStatement::Comment(comment),
         }
     }
+    pub fn is_comment(&self) -> bool {
+        match self {
+            Statement::Comment(_) => true,
+            _ => false,
+        }
+    }
 }
 impl TopLevelStatement {
-    pub fn requires_semicolon(&self) -> bool {
+    pub fn is_expression(&self) -> bool {
         match &self {
-            TopLevelStatement::Expr(expr) => match expr.data {
+            TopLevelStatement::Expr(_) => true,
-                ExprData::Block(_) => false,
-                ExprData::If(_) => false,
-                _ => true,
-            },
             _ => false,
         }
     }
 @@ -259,4 +224,11 @@ impl TopLevelStatement {
             TopLevelStatement::Comment(comment) => Statement::Comment(comment),
         }
     }
+    pub fn is_comment(&self) -> bool {
+        match self {
+            TopLevelStatement::Comment(_) => true,
+            _ => false,
+        }
+    }
 }

crates/ast/operators.rs+133-23

...	@@ -24,6 +24,22 @@ pub enum BinaryOperator {	...	@@ -24,6 +24,22 @@ pub enum BinaryOperator {
24	NotEq,	24	NotEq,
25	And,	25	And,
26	Or,	26	Or,
		27
		28	Assign,
		29	AssignAdd,
		30	AssignSub,
		31	AssignMul,
		32	AssignDiv,
		33	AssignPow,
		34	AssignLshift,
		35	AssignRshift,
		36	AssignRshiftUnsigned,
		37	AssignBitAnd,
		38	AssignBitOr,
		39	AssignBitXor,
		40	AssignMod,
		41	AssignAnd,
		42	AssignOr,
27	}	43	}
28		44
29	#[derive(Debug, Clone, PartialEq)]	45	#[derive(Debug, Clone, PartialEq)]
...	@@ -34,8 +50,8 @@ pub enum UnaryOperator {	...	@@ -34,8 +50,8 @@ pub enum UnaryOperator {
34	}	50	}
35		51
36	impl BinaryOperator {	52	impl BinaryOperator {
37	pub fn to_string(&self) -> String {	53	pub fn to_str(&self) -> &'static str {
38	String::from(match self {	54	match self {
39	Self::Add => "+",	55	Self::Add => "+",
40	Self::Sub => "-",	56	Self::Sub => "-",
41	Self::Div => "/",	57	Self::Div => "/",
...	@@ -56,7 +72,26 @@ impl BinaryOperator {	...	@@ -56,7 +72,26 @@ impl BinaryOperator {
56	Self::NotEq => "!=",	72	Self::NotEq => "!=",
57	Self::And => "&&",	73	Self::And => "&&",
58	Self::Or => "\|\|",	74	Self::Or => "\|\|",
59	})	75	Self::AssignAdd => "+=",
		76	Self::AssignSub => "-=",
		77	Self::AssignDiv => "/=",
		78	Self::AssignMul => "*=",
		79	Self::AssignPow => "**=",
		80	Self::Assign => "=",
		81	Self::AssignLshift => "<<=",
		82	Self::AssignRshift => ">>=",
		83	Self::AssignRshiftUnsigned => ">>>=",
		84	Self::AssignBitAnd => "&=",
		85	Self::AssignBitOr => "\|=",
		86	Self::AssignBitXor => "^=",
		87	Self::AssignMod => "%=",
		88	Self::AssignAnd => "&&=",
		89	Self::AssignOr => "\|\|=",
		90	}
		91	}
		92
		93	pub fn to_string(&self) -> String {
		94	String::from(self.to_str())
60	}	95	}
61		96
62	pub fn from_token(token: &Token) -> Option<BinaryOperator> {	97	pub fn from_token(token: &Token) -> Option<BinaryOperator> {
...	@@ -85,6 +120,10 @@ impl BinaryOperator {	...	@@ -85,6 +120,10 @@ impl BinaryOperator {
85	}	120	}
86	}	121	}
87		122
		123	pub fn level(&self) -> OperatorLevel {
		124	OperatorLevel::from(self)
		125	}
		126
88	/// Error case of this means 'types do not allow this operation'	127	/// Error case of this means 'types do not allow this operation'
89	pub fn perform(&self, left: Literal, right: Literal) -> Result<Literal, ()> {	128	pub fn perform(&self, left: Literal, right: Literal) -> Result<Literal, ()> {
90	match self {	129	match self {
...	@@ -139,17 +178,24 @@ impl BinaryOperator {	...	@@ -139,17 +178,24 @@ impl BinaryOperator {
139	}),	178	}),
140	_ => Err(()),	179	_ => Err(()),
141	},	180	},
		181
		182	// Impossible condition doing assign-add to a literal
		183	_ => Err(()),
142	}	184	}
143	}	185	}
144	}	186	}
145		187
146	impl UnaryOperator {	188	impl UnaryOperator {
147	pub fn to_string(self) -> String {	189	pub fn to_str(&self) -> &'static str {
148	String::from(match self {	190	match self {
149	Self::Neg => "-",	191	Self::Neg => "-",
150	Self::Not => "!",	192	Self::Not => "!",
151	Self::BitNot => "~",	193	Self::BitNot => "~",
152	})	194	}
		195	}
		196
		197	pub fn to_string(&self) -> String {
		198	String::from(self.to_str())
153	}	199	}
154		200
155	pub fn perform(self, value: Literal) -> Result<Literal, ()> {	201	pub fn perform(self, value: Literal) -> Result<Literal, ()> {
...	@@ -171,8 +217,17 @@ impl UnaryOperator {	...	@@ -171,8 +217,17 @@ impl UnaryOperator {
171	}	217	}
172		218
173	/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table	219	/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
174	#[derive(PartialEq, PartialOrd)]	220	///
175	pub enum BinaryOperatorLevel {	221	/// Blight doesn't have all of these operators, but the
		222	/// printer uses it to determine where to put parens.
		223	#[derive(PartialEq, PartialOrd, Debug)]
		224	pub enum OperatorLevel {
		225	Accessor, // e. e[] e()
		226	New, // new
		227	Postfix, // expr++ expr--
		228	Prefix, // ++e, --e, !e, ~e, +e, -e, typeof, void, delete
		229
		230	// these are the binary operators
176	Exponentiation, // **	231	Exponentiation, // **
177	Multiplicative, // *, /, %	232	Multiplicative, // *, /, %
178	Additive, // +, -	233	Additive, // +, -
...	@@ -184,22 +239,77 @@ pub enum BinaryOperatorLevel {	...	@@ -184,22 +239,77 @@ pub enum BinaryOperatorLevel {
184	BitwiseOr, // \|	239	BitwiseOr, // \|
185	LogicalAnd, // &&	240	LogicalAnd, // &&
186	LogicalOr, // \|\|, ??	241	LogicalOr, // \|\|, ??
		242
		243	AssignmentAndOther, // =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
		244	Comma, // ,
187	}	245	}
188		246
189	pub fn get_binary_op_level(token: &Token) -> Option<BinaryOperatorLevel> {	247	pub enum Associativity {
190	use BinaryOperatorLevel::*;	248	LeftToRight,
191	match token {	249	RightToLeft,
192	Token::StarStar => Some(Exponentiation),	250	}
193	Token::Star \| Token::Slash \| Token::Percent => Some(Multiplicative),	251
194	Token::Plus \| Token::Minus => Some(Additive),	252	impl OperatorLevel {
195	Token::LtLt \| Token::GtGt \| Token::GtGtGt => Some(BitwiseShift),	253	pub fn associativity(&self) -> Associativity {
196	Token::Gt \| Token::GtEqual \| Token::Lt \| Token::LtEqual => Some(Relational),	254	use Associativity::*;
197	Token::EqualEqual \| Token::BangEqual => Some(Equality),	255	use OperatorLevel::*;
198	Token::And => Some(BitwiseAnd),	256
199	Token::Caret => Some(BitwiseXor),	257	match self {
200	Token::Or => Some(BitwiseOr),	258	Accessor => LeftToRight,
201	Token::AndAnd => Some(LogicalAnd),	259	Exponentiation => RightToLeft,
202	Token::OrOr => Some(LogicalOr),	260	Multiplicative => LeftToRight,
203	_ => None,	261	Additive => LeftToRight,
		262	BitwiseShift => LeftToRight,
		263	Relational => LeftToRight,
		264	Equality => LeftToRight,
		265	BitwiseAnd => LeftToRight,
		266	BitwiseXor => LeftToRight,
		267	BitwiseOr => LeftToRight,
		268	LogicalAnd => LeftToRight,
		269	LogicalOr => LeftToRight,
		270	AssignmentAndOther => RightToLeft,
		271	Comma => LeftToRight,
		272
		273	// These do not make sense to have an associativity
		274	// You will likely not run into this if you do
		275	// BinaryOperator::level(o).associativity()
		276	New \| Postfix \| Prefix => unreachable!("{:?} does not have an associativity", self),
		277	}
		278	}
		279	}
		280
		281	impl From<&BinaryOperator> for OperatorLevel {
		282	fn from(operator: &BinaryOperator) -> Self {
		283	use BinaryOperator::*;
		284	use OperatorLevel::*;
		285
		286	match operator {
		287	Add \| Sub => Additive,
		288	Mul \| Div \| Mod => Multiplicative,
		289	Pow => Exponentiation,
		290	Gt \| Lt \| Gte \| Lte => Relational,
		291	Lshift \| Rshift \| RshiftUnsigned => BitwiseShift,
		292	BitAnd => BitwiseAnd,
		293	BitOr => BitwiseOr,
		294	BitXor => BitwiseXor,
		295	Eq \| NotEq => Equality,
		296	And => LogicalAnd,
		297	Or => LogicalOr,
		298
		299	Assign \| AssignAdd \| AssignSub \| AssignMul \| AssignDiv \| AssignPow \| AssignLshift
		300	\| AssignRshift \| AssignRshiftUnsigned \| AssignBitAnd \| AssignBitOr \| AssignBitXor
		301	\| AssignMod \| AssignAnd \| AssignOr => AssignmentAndOther,
		302	}
		303	}
		304	}
		305
		306	impl From<UnaryOperator> for OperatorLevel {
		307	fn from(operator: UnaryOperator) -> Self {
		308	use OperatorLevel::*;
		309	use UnaryOperator::*;
		310
		311	match operator {
		312	Neg \| Not \| BitNot => Prefix,
		313	}
204	}	314	}
205	}	315	}

crates/cli/Cargo.lock deleted-532

...	@@ -1,532 +0,0 @@
1	# This file is automatically @generated by Cargo.
2	# It is not intended for manual editing.
3	version = 3
4
5	[[package]]
6	name = "addr2line"
7	version = "0.21.0"
8	source = "registry+https://github.com/rust-lang/crates.io-index"
9	checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
10	dependencies = [
11	"gimli",
12	]
13
14	[[package]]
15	name = "adler"
16	version = "1.0.2"
17	source = "registry+https://github.com/rust-lang/crates.io-index"
18	checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
19
20	[[package]]
21	name = "anstream"
22	version = "0.6.5"
23	source = "registry+https://github.com/rust-lang/crates.io-index"
24	checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
25	dependencies = [
26	"anstyle",
27	"anstyle-parse",
28	"anstyle-query",
29	"anstyle-wincon",
30	"colorchoice",
31	"utf8parse",
32	]
33
34	[[package]]
35	name = "anstyle"
36	version = "1.0.4"
37	source = "registry+https://github.com/rust-lang/crates.io-index"
38	checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
39
40	[[package]]
41	name = "anstyle-parse"
42	version = "0.2.3"
43	source = "registry+https://github.com/rust-lang/crates.io-index"
44	checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
45	dependencies = [
46	"utf8parse",
47	]
48
49	[[package]]
50	name = "anstyle-query"
51	version = "1.0.2"
52	source = "registry+https://github.com/rust-lang/crates.io-index"
53	checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
54	dependencies = [
55	"windows-sys",
56	]
57
58	[[package]]
59	name = "anstyle-wincon"
60	version = "3.0.2"
61	source = "registry+https://github.com/rust-lang/crates.io-index"
62	checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
63	dependencies = [
64	"anstyle",
65	"windows-sys",
66	]
67
68	[[package]]
69	name = "backtrace"
70	version = "0.3.69"
71	source = "registry+https://github.com/rust-lang/crates.io-index"
72	checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
73	dependencies = [
74	"addr2line",
75	"cc",
76	"cfg-if",
77	"libc",
78	"miniz_oxide",
79	"object",
80	"rustc-demangle",
81	]
82
83	[[package]]
84	name = "backtrace-ext"
85	version = "0.2.1"
86	source = "registry+https://github.com/rust-lang/crates.io-index"
87	checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
88	dependencies = [
89	"backtrace",
90	]
91
92	[[package]]
93	name = "bitflags"
94	version = "2.4.1"
95	source = "registry+https://github.com/rust-lang/crates.io-index"
96	checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
97
98	[[package]]
99	name = "blight"
100	version = "0.1.0"
101	dependencies = [
102	"clap",
103	"miette",
104	"thiserror",
105	]
106
107	[[package]]
108	name = "cc"
109	version = "1.0.83"
110	source = "registry+https://github.com/rust-lang/crates.io-index"
111	checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
112	dependencies = [
113	"libc",
114	]
115
116	[[package]]
117	name = "cfg-if"
118	version = "1.0.0"
119	source = "registry+https://github.com/rust-lang/crates.io-index"
120	checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
121
122	[[package]]
123	name = "clap"
124	version = "4.4.12"
125	source = "registry+https://github.com/rust-lang/crates.io-index"
126	checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d"
127	dependencies = [
128	"clap_builder",
129	"clap_derive",
130	]
131
132	[[package]]
133	name = "clap_builder"
134	version = "4.4.12"
135	source = "registry+https://github.com/rust-lang/crates.io-index"
136	checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
137	dependencies = [
138	"anstream",
139	"anstyle",
140	"clap_lex",
141	"strsim",
142	]
143
144	[[package]]
145	name = "clap_derive"
146	version = "4.4.7"
147	source = "registry+https://github.com/rust-lang/crates.io-index"
148	checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
149	dependencies = [
150	"heck",
151	"proc-macro2",
152	"quote",
153	"syn",
154	]
155
156	[[package]]
157	name = "clap_lex"
158	version = "0.6.0"
159	source = "registry+https://github.com/rust-lang/crates.io-index"
160	checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
161
162	[[package]]
163	name = "colorchoice"
164	version = "1.0.0"
165	source = "registry+https://github.com/rust-lang/crates.io-index"
166	checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
167
168	[[package]]
169	name = "errno"
170	version = "0.3.8"
171	source = "registry+https://github.com/rust-lang/crates.io-index"
172	checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
173	dependencies = [
174	"libc",
175	"windows-sys",
176	]
177
178	[[package]]
179	name = "gimli"
180	version = "0.28.1"
181	source = "registry+https://github.com/rust-lang/crates.io-index"
182	checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
183
184	[[package]]
185	name = "heck"
186	version = "0.4.1"
187	source = "registry+https://github.com/rust-lang/crates.io-index"
188	checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
189
190	[[package]]
191	name = "hermit-abi"
192	version = "0.3.3"
193	source = "registry+https://github.com/rust-lang/crates.io-index"
194	checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
195
196	[[package]]
197	name = "is-terminal"
198	version = "0.4.10"
199	source = "registry+https://github.com/rust-lang/crates.io-index"
200	checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
201	dependencies = [
202	"hermit-abi",
203	"rustix",
204	"windows-sys",
205	]
206
207	[[package]]
208	name = "is_ci"
209	version = "1.1.1"
210	source = "registry+https://github.com/rust-lang/crates.io-index"
211	checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
212
213	[[package]]
214	name = "libc"
215	version = "0.2.151"
216	source = "registry+https://github.com/rust-lang/crates.io-index"
217	checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
218
219	[[package]]
220	name = "linux-raw-sys"
221	version = "0.4.12"
222	source = "registry+https://github.com/rust-lang/crates.io-index"
223	checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
224
225	[[package]]
226	name = "memchr"
227	version = "2.7.1"
228	source = "registry+https://github.com/rust-lang/crates.io-index"
229	checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
230
231	[[package]]
232	name = "miette"
233	version = "5.10.0"
234	source = "registry+https://github.com/rust-lang/crates.io-index"
235	checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
236	dependencies = [
237	"backtrace",
238	"backtrace-ext",
239	"is-terminal",
240	"miette-derive",
241	"once_cell",
242	"owo-colors",
243	"supports-color",
244	"supports-hyperlinks",
245	"supports-unicode",
246	"terminal_size",
247	"textwrap",
248	"thiserror",
249	"unicode-width",
250	]
251
252	[[package]]
253	name = "miette-derive"
254	version = "5.10.0"
255	source = "registry+https://github.com/rust-lang/crates.io-index"
256	checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
257	dependencies = [
258	"proc-macro2",
259	"quote",
260	"syn",
261	]
262
263	[[package]]
264	name = "miniz_oxide"
265	version = "0.7.1"
266	source = "registry+https://github.com/rust-lang/crates.io-index"
267	checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
268	dependencies = [
269	"adler",
270	]
271
272	[[package]]
273	name = "object"
274	version = "0.32.2"
275	source = "registry+https://github.com/rust-lang/crates.io-index"
276	checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
277	dependencies = [
278	"memchr",
279	]
280
281	[[package]]
282	name = "once_cell"
283	version = "1.19.0"
284	source = "registry+https://github.com/rust-lang/crates.io-index"
285	checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
286
287	[[package]]
288	name = "owo-colors"
289	version = "3.5.0"
290	source = "registry+https://github.com/rust-lang/crates.io-index"
291	checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
292
293	[[package]]
294	name = "proc-macro2"
295	version = "1.0.73"
296	source = "registry+https://github.com/rust-lang/crates.io-index"
297	checksum = "2dd5e8a1f1029c43224ad5898e50140c2aebb1705f19e67c918ebf5b9e797fe1"
298	dependencies = [
299	"unicode-ident",
300	]
301
302	[[package]]
303	name = "quote"
304	version = "1.0.34"
305	source = "registry+https://github.com/rust-lang/crates.io-index"
306	checksum = "22a37c9326af5ed140c86a46655b5278de879853be5573c01df185b6f49a580a"
307	dependencies = [
308	"proc-macro2",
309	]
310
311	[[package]]
312	name = "rustc-demangle"
313	version = "0.1.23"
314	source = "registry+https://github.com/rust-lang/crates.io-index"
315	checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
316
317	[[package]]
318	name = "rustix"
319	version = "0.38.28"
320	source = "registry+https://github.com/rust-lang/crates.io-index"
321	checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
322	dependencies = [
323	"bitflags",
324	"errno",
325	"libc",
326	"linux-raw-sys",
327	"windows-sys",
328	]
329
330	[[package]]
331	name = "smawk"
332	version = "0.3.2"
333	source = "registry+https://github.com/rust-lang/crates.io-index"
334	checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
335
336	[[package]]
337	name = "strsim"
338	version = "0.10.0"
339	source = "registry+https://github.com/rust-lang/crates.io-index"
340	checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
341
342	[[package]]
343	name = "supports-color"
344	version = "2.1.0"
345	source = "registry+https://github.com/rust-lang/crates.io-index"
346	checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
347	dependencies = [
348	"is-terminal",
349	"is_ci",
350	]
351
352	[[package]]
353	name = "supports-hyperlinks"
354	version = "2.1.0"
355	source = "registry+https://github.com/rust-lang/crates.io-index"
356	checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d"
357	dependencies = [
358	"is-terminal",
359	]
360
361	[[package]]
362	name = "supports-unicode"
363	version = "2.0.0"
364	source = "registry+https://github.com/rust-lang/crates.io-index"
365	checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7"
366	dependencies = [
367	"is-terminal",
368	]
369
370	[[package]]
371	name = "syn"
372	version = "2.0.44"
373	source = "registry+https://github.com/rust-lang/crates.io-index"
374	checksum = "92d27c2c202598d05175a6dd3af46824b7f747f8d8e9b14c623f19fa5069735d"
375	dependencies = [
376	"proc-macro2",
377	"quote",
378	"unicode-ident",
379	]
380
381	[[package]]
382	name = "terminal_size"
383	version = "0.1.17"
384	source = "registry+https://github.com/rust-lang/crates.io-index"
385	checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
386	dependencies = [
387	"libc",
388	"winapi",
389	]
390
391	[[package]]
392	name = "textwrap"
393	version = "0.15.2"
394	source = "registry+https://github.com/rust-lang/crates.io-index"
395	checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d"
396	dependencies = [
397	"smawk",
398	"unicode-linebreak",
399	"unicode-width",
400	]
401
402	[[package]]
403	name = "thiserror"
404	version = "1.0.53"
405	source = "registry+https://github.com/rust-lang/crates.io-index"
406	checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09"
407	dependencies = [
408	"thiserror-impl",
409	]
410
411	[[package]]
412	name = "thiserror-impl"
413	version = "1.0.53"
414	source = "registry+https://github.com/rust-lang/crates.io-index"
415	checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19"
416	dependencies = [
417	"proc-macro2",
418	"quote",
419	"syn",
420	]
421
422	[[package]]
423	name = "unicode-ident"
424	version = "1.0.12"
425	source = "registry+https://github.com/rust-lang/crates.io-index"
426	checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
427
428	[[package]]
429	name = "unicode-linebreak"
430	version = "0.1.5"
431	source = "registry+https://github.com/rust-lang/crates.io-index"
432	checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
433
434	[[package]]
435	name = "unicode-width"
436	version = "0.1.11"
437	source = "registry+https://github.com/rust-lang/crates.io-index"
438	checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
439
440	[[package]]
441	name = "utf8parse"
442	version = "0.2.1"
443	source = "registry+https://github.com/rust-lang/crates.io-index"
444	checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
445
446	[[package]]
447	name = "winapi"
448	version = "0.3.9"
449	source = "registry+https://github.com/rust-lang/crates.io-index"
450	checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
451	dependencies = [
452	"winapi-i686-pc-windows-gnu",
453	"winapi-x86_64-pc-windows-gnu",
454	]
455
456	[[package]]
457	name = "winapi-i686-pc-windows-gnu"
458	version = "0.4.0"
459	source = "registry+https://github.com/rust-lang/crates.io-index"
460	checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
461
462	[[package]]
463	name = "winapi-x86_64-pc-windows-gnu"
464	version = "0.4.0"
465	source = "registry+https://github.com/rust-lang/crates.io-index"
466	checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
467
468	[[package]]
469	name = "windows-sys"
470	version = "0.52.0"
471	source = "registry+https://github.com/rust-lang/crates.io-index"
472	checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
473	dependencies = [
474	"windows-targets",
475	]
476
477	[[package]]
478	name = "windows-targets"
479	version = "0.52.0"
480	source = "registry+https://github.com/rust-lang/crates.io-index"
481	checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
482	dependencies = [
483	"windows_aarch64_gnullvm",
484	"windows_aarch64_msvc",
485	"windows_i686_gnu",
486	"windows_i686_msvc",
487	"windows_x86_64_gnu",
488	"windows_x86_64_gnullvm",
489	"windows_x86_64_msvc",
490	]
491
492	[[package]]
493	name = "windows_aarch64_gnullvm"
494	version = "0.52.0"
495	source = "registry+https://github.com/rust-lang/crates.io-index"
496	checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
497
498	[[package]]
499	name = "windows_aarch64_msvc"
500	version = "0.52.0"
501	source = "registry+https://github.com/rust-lang/crates.io-index"
502	checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
503
504	[[package]]
505	name = "windows_i686_gnu"
506	version = "0.52.0"
507	source = "registry+https://github.com/rust-lang/crates.io-index"
508	checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
509
510	[[package]]
511	name = "windows_i686_msvc"
512	version = "0.52.0"
513	source = "registry+https://github.com/rust-lang/crates.io-index"
514	checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
515
516	[[package]]
517	name = "windows_x86_64_gnu"
518	version = "0.52.0"
519	source = "registry+https://github.com/rust-lang/crates.io-index"
520	checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
521
522	[[package]]
523	name = "windows_x86_64_gnullvm"
524	version = "0.52.0"
525	source = "registry+https://github.com/rust-lang/crates.io-index"
526	checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
527
528	[[package]]
529	name = "windows_x86_64_msvc"
530	version = "0.52.0"
531	source = "registry+https://github.com/rust-lang/crates.io-index"
532	checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"

crates/cli/Cargo.toml+1-1

...	@@ -5,7 +5,7 @@ edition = "2021"	...	@@ -5,7 +5,7 @@ edition = "2021"
5		5
6	[[bin]]	6	[[bin]]
7	name = "blight"	7	name = "blight"
8	path = "./src/main.rs"	8	path = "./main.rs"
9		9
10	[dependencies]	10	[dependencies]
11	clap.workspace = true	11	clap.workspace = true

crates/cli/main.rs created+172

...	@@ -0,0 +1,172 @@
1	use std::{io::Write, path::PathBuf};
2
3	use clap::{Parser, Subcommand};
4	use miette::{diagnostic, Result};
5
6	use js_printer;
7	use tokenizer;
8
9	#[derive(Parser)]
10	#[command(author, version, about, long_about = None)]
11	struct Cli {
12	/// Turn debugging information on
13	#[arg(short, long, action = clap::ArgAction::Count)]
14	debug: u8,
15
16	#[command(subcommand)]
17	command: Commands,
18	}
19
20	#[derive(Subcommand)]
21	enum Commands {
22	/// Compiles a file into javascript
23	Compile {
24	file: PathBuf,
25	// Whether to output to stdout instead of a file
26	#[arg(long, default_value_t = false)]
27	stdout: bool,
28	/// What javascript runtime to compile to
29	#[arg(short, long, default_value = "deno")]
30	target: String,
31	/// Format the output javascript?
32	#[arg(long, default_value_t = true)]
33	format: bool,
34	},
35	Run {
36	file: PathBuf,
37	/// What javascript runtime to compile to and use
38	#[arg(short, long, default_value = "deno")]
39	target: String,
40	},
41	Astdump {
42	file: PathBuf,
43	},
44	}
45
46	fn get_runtime(runtime: &String) -> String {
47	match std::env::var("USER") {
48	Ok(user) if user == "dave" => String::from("bun"),
49	_ => runtime.clone(),
50	}
51	}
52
53	pub fn transpile(content: String, writer: &mut dyn Write) -> Result<()> {
54	// 1. Tokenize the input
55	let tokens = tokenizer::tokenize(content)?;
56	// 2. Parse the tokens into an AST
57	let mut blight_ast = parser::parse(tokens)?;
58	// 3. Semantic analysis.
59	sema::analyze_module(&mut blight_ast)?;
60	// 4. Optimize the AST
61	// optimize::optimize_module(&mut blight_ast);
62	// 5. Translate the AST into javascript
63	let js_ast = js_ast::blight_to_js(blight_ast);
64	// 6. Print the JS AST
65	// TODO(@paperdave): use std::io::Write instead of a string
66	js_printer::Print::print(&js_ast, writer)
67	.map_err(\|e\| diagnostic!("Error while printing: {}", e))?;
68	Ok(())
69	}
70
71	pub fn transpile_to_string(content: String) -> Result<String> {
72	let mut buffer = Vec::new();
73	transpile(content, &mut buffer)?;
74	Ok(String::from_utf8(buffer).unwrap())
75	}
76
77	pub fn astdump(content: String) -> Result<String> {
78	// 1. Tokenize the input
79	let tokens = tokenizer::tokenize(content)?;
80	// 2. Parse the tokens into an AST
81	let blight_ast = parser::parse(tokens)?;
82	return Ok(format!("{:#?}", blight_ast));
83	}
84
85	fn main() -> Result<()> {
86	let cli = Cli::parse();
87
88	match cli.command {
89	Commands::Compile {
90	file,
91	stdout,
92	target: _,
93	format,
94	} => {
95	let contents = match std::fs::read_to_string(file.clone()) {
96	Ok(contents) => contents,
97	Err(e) => {
98	eprintln!("Failed to read {}: {}", file.display(), e);
99	std::process::exit(1);
100	}
101	};
102
103	let mut result = transpile_to_string(contents.clone())?;
104
105	if format {
106	let mut formatter = std::process::Command::new("deno")
107	.stdin(std::process::Stdio::piped())
108	.stdout(std::process::Stdio::piped())
109	.stderr(std::process::Stdio::inherit())
110	.arg("fmt")
111	.arg("-")
112	.spawn()
113	.expect("Failed to execute 'deno fmt' on output");
114	let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
115	stdin
116	.write_all(result.as_bytes())
117	.expect("Failed to write to stdin");
118	stdin.flush().expect("Failed to flush stdin");
119	drop(stdin);
120	let out = formatter
121	.wait_with_output()
122	.expect("Failed to execute 'deno fmt' on output");
123	result = String::from_utf8(out.stdout).unwrap();
124	}
125
126	if stdout {
127	println!("{}", result);
128	} else {
129	std::fs::write(file.with_extension("js"), result).unwrap();
130	}
131	}
132	Commands::Run { file, target } => {
133	let contents = std::fs::read_to_string(file.clone()).unwrap();
134	let result = transpile_to_string(contents.clone())?;
135	let mut temp_file = std::env::temp_dir();
136	temp_file.push("temp.js");
137	std::fs::write(temp_file.clone(), result).unwrap();
138
139	let rt = get_runtime(&target);
140
141	match rt.as_str() {
142	"deno" => std::process::Command::new("deno")
143	.arg("run")
144	.arg("-A")
145	.arg(temp_file)
146	.spawn()
147	.expect("Failed to execute process"),
148	"node" => std::process::Command::new("node")
149	.arg(temp_file)
150	.spawn()
151	.expect("Failed to execute process"),
152	"bun" => std::process::Command::new("bun")
153	.arg("run")
154	.arg(temp_file)
155	.spawn()
156	.expect("Failed to execute process"),
157	_ => {
158	panic!("Unknown runtime: {}", target);
159	}
160	}
161	.wait()
162	.unwrap();
163	}
164	Commands::Astdump { file } => {
165	let contents = std::fs::read_to_string(file.clone()).unwrap();
166	let result = astdump(contents.clone())?;
167	println!("{}", result);
168	}
169	}
170
171	Ok(())
172	}

crates/cli/src/main.rs deleted-165

...	@@ -1,165 +0,0 @@
1	use std::{io::Write, path::PathBuf};
2
3	use clap::{Parser, Subcommand};
4	use miette::Result;
5
6	use js_printer;
7	use tokenizer;
8
9	#[derive(Parser)]
10	#[command(author, version, about, long_about = None)]
11	struct Cli {
12	/// Turn debugging information on
13	#[arg(short, long, action = clap::ArgAction::Count)]
14	debug: u8,
15
16	#[command(subcommand)]
17	command: Commands,
18	}
19
20	#[derive(Subcommand)]
21	enum Commands {
22	/// Compiles a file into javascript
23	Compile {
24	file: PathBuf,
25	// Whether to output to stdout instead of a file
26	#[arg(long, default_value_t = false)]
27	stdout: bool,
28	/// What javascript runtime to compile to
29	#[arg(short, long, default_value = "deno")]
30	target: String,
31	/// Format the output javascript?
32	#[arg(long, default_value_t = true)]
33	format: bool,
34	},
35	Run {
36	file: PathBuf,
37	/// What javascript runtime to compile to and use
38	#[arg(short, long, default_value = "deno")]
39	target: String,
40	},
41	Astdump {
42	file: PathBuf,
43	},
44	}
45
46	fn get_runtime(runtime: &String) -> String {
47	match std::env::var("USER") {
48	Ok(user) if user == "dave" => String::from("bun"),
49	_ => runtime.clone(),
50	}
51	}
52
53	pub fn transpile(content: String) -> Result<String> {
54	// 1. Tokenize the input
55	let tokens = tokenizer::tokenize(content)?;
56	// 2. Parse the tokens into an AST
57	let blight_ast = parser::parse(tokens)?;
58	// 3. Type check the AST
59	// sema::check_module(&blight_ast)?;
60	// 4. Optimize the AST
61	// optimize(tree.clone())?;
62	// 5. Translate the AST into javascript
63	let js_ast = js_ast::translate(&blight_ast)?;
64	// 6. Print the JS AST
65	let javascript = js_printer::print(js_ast);
66	// 7. Return the javascript
67	return Ok(javascript);
68	}
69
70	pub fn astdump(content: String) -> Result<String> {
71	// 1. Tokenize the input
72	let tokens = tokenizer::tokenize(content)?;
73	// 2. Parse the tokens into an AST
74	let blight_ast = parser::parse(tokens)?;
75	return Ok(format!("{:#?}", blight_ast));
76	}
77
78	fn main() -> Result<()> {
79	let cli = Cli::parse();
80
81	match cli.command {
82	Commands::Compile {
83	file,
84	stdout,
85	target: _,
86	format,
87	} => {
88	let contents = match std::fs::read_to_string(file.clone()) {
89	Ok(contents) => contents,
90	Err(e) => {
91	eprintln!("Failed to read {}: {}", file.display(), e);
92	std::process::exit(1);
93	}
94	};
95
96	let mut result = transpile(contents.clone())?;
97
98	if format {
99	let mut formatter = std::process::Command::new("deno")
100	.stdin(std::process::Stdio::piped())
101	.stdout(std::process::Stdio::piped())
102	.stderr(std::process::Stdio::inherit())
103	.arg("fmt")
104	.arg("-")
105	.spawn()
106	.expect("Failed to execute 'deno fmt' on output");
107	let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
108	stdin
109	.write_all(result.as_bytes())
110	.expect("Failed to write to stdin");
111	stdin.flush().expect("Failed to flush stdin");
112	drop(stdin);
113	let out = formatter
114	.wait_with_output()
115	.expect("Failed to execute 'deno fmt' on output");
116	result = String::from_utf8(out.stdout).unwrap();
117	}
118
119	if stdout {
120	println!("{}", result);
121	} else {
122	std::fs::write(file.with_extension("js"), result).unwrap();
123	}
124	}
125	Commands::Run { file, target } => {
126	let contents = std::fs::read_to_string(file.clone()).unwrap();
127	let result = transpile(contents.clone())?;
128	let mut temp_file = std::env::temp_dir();
129	temp_file.push("temp.js");
130	std::fs::write(temp_file.clone(), result).unwrap();
131
132	let rt = get_runtime(&target);
133
134	match rt.as_str() {
135	"deno" => std::process::Command::new("deno")
136	.arg("run")
137	.arg("-A")
138	.arg(temp_file)
139	.spawn()
140	.expect("Failed to execute process"),
141	"node" => std::process::Command::new("node")
142	.arg(temp_file)
143	.spawn()
144	.expect("Failed to execute process"),
145	"bun" => std::process::Command::new("bun")
146	.arg("run")
147	.arg(temp_file)
148	.spawn()
149	.expect("Failed to execute process"),
150	_ => {
151	panic!("Unknown runtime: {}", target);
152	}
153	}
154	.wait()
155	.unwrap();
156	}
157	Commands::Astdump { file } => {
158	let contents = std::fs::read_to_string(file.clone()).unwrap();
159	let result = astdump(contents.clone())?;
160	println!("{}", result);
161	}
162	}
163
164	Ok(())
165	}

crates/js_ast/Cargo.toml+1

...	@@ -13,3 +13,4 @@ num.workspace = true	...	@@ -13,3 +13,4 @@ num.workspace = true
13		13
14	ast = { path = "../ast" }	14	ast = { path = "../ast" }
15	tokenizer = { path = "../tokenizer" } # TODO: move Loc out of the tokenizer	15	tokenizer = { path = "../tokenizer" } # TODO: move Loc out of the tokenizer
		16	parser = { path = "../parser" } # TODO: move Loc out of the tokenizer

crates/js_ast/convert_valid_expr.rs created+204

...	@@ -0,0 +1,204 @@
1	//! "Valid" in this context means that the expression does not contain JS statements
2	//! Meaning conversion is a direct map from a blight as node into a JS one
3	//!
4	//! See `is_valid_js_expression` for more details on what qualifies
5	//!
6	//! For other nodes, it must hoist statements to the nearest JS block,
7	//! is done in the `hoist_*` functions in `host.rs`.
8
9	use ast::{BlockOrIf, Comment, ExprData, If};
10	use ast::{BlockWithLoc, BuiltinId, Expr, Literal, Statement};
11	use tokenizer::Loc;
12
13	use crate::is_valid_js_expression::IsValidJSExpression;
14	use crate::js_ast::*;
15	use crate::HoistContext;
16
17	/// Translate a blight ast expression that is a valid JS expression, to a JS expression
18	/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
19	pub fn convert_valid_expr(ctx: &mut HoistContext, expr: Expr) -> JsExpr {
20	// Assert may be skipped in release builds as all failures are
21	// annotated with a release-checked panic
22	debug_assert!(
23	expr.is_valid_js_expression(),
24	"must call convert_valid_expr with valid js expression (use hoist instead)"
25	);
26
27	let loc = expr.loc;
28	let comment = expr.comment;
29
30	match expr.data {
31	ExprData::Identifier(str) => JsExpr::new(
32	JsExprData::Identifier(ctx.remap_identifier(&str)),
33	loc,
34	comment,
35	),
36
37	ExprData::Literal(Literal::Bool(value)) => {
38	JsExpr::new(JsExprData::Bool(value), loc, comment)
39	}
40	ExprData::Literal(Literal::Number(value)) => {
41	JsExpr::new(JsExprData::Number(value), loc, comment)
42	}
43	ExprData::Literal(Literal::String(value)) => {
44	JsExpr::new(JsExprData::String(value), loc, comment)
45	}
46	ExprData::Literal(Literal::Void) => JsExpr::new(JsExprData::Null, loc, comment),
47
48	ExprData::Block(block) => convert_valid_block(ctx, BlockWithLoc { block, loc }),
49
50	ExprData::Binary { op, left, right } => {
51	let left = convert_valid_expr(ctx, *left);
52	let right = convert_valid_expr(ctx, *right);
53	JsExpr::new(
54	JsExprData::Binary {
55	op,
56	left: Box::new(left),
57	right: Box::new(right),
58	},
59	loc,
60	comment,
61	)
62	}
63	ExprData::Unary { op, value } => {
64	let value = convert_valid_expr(ctx, *value);
65	JsExpr::new(
66	JsExprData::Unary {
67	op,
68	value: Box::new(value),
69	},
70	loc,
71	comment,
72	)
73	}
74	ExprData::Fn { block, params, .. } => JsExpr::new(
75	JsExprData::ArrowFunction {
76	params: params.into_iter().map(\|x\| x.name).collect(),
77	body: Box::new(ctx.hoist_block_to_arrow_fn_body(BlockWithLoc {
78	block,
79	loc: loc.clone(),
80	})),
81	},
82	loc,
83	comment,
84	),
85	ExprData::Builtin { id, arguments } => match id {
86	BuiltinId::Raw => {
87	let first_arg = arguments.iter().next().unwrap();
88	// TODO(@paperdave): this string clone is stupid
89	let str = match first_arg.data.clone() {
90	ExprData::Literal(Literal::String(s)) => s,
91	_ => panic!("raw expects a string"),
92	};
93	JsExpr::new(JsExprData::Raw(str), loc, comment)
94	}
95	},
96	ExprData::If(expr) => convert_valid_if(ctx, expr, loc, comment),
97
98	ExprData::Call { base, arguments } => {
99	let base = convert_valid_expr(ctx, *base);
100	let arguments = arguments
101	.into_iter()
102	.map(\|x\| convert_valid_expr(ctx, x))
103	.collect();
104	JsExpr::new(
105	JsExprData::Call {
106	base: Box::new(base),
107	arguments,
108	},
109	loc,
110	comment,
111	)
112	}
113
114	ExprData::Field { base, field } => JsExpr::new(
115	match field.data {
116	ExprData::Literal(Literal::String(name)) => JsExprData::Dot {
117	base: Box::new(convert_valid_expr(ctx, *base)),
118	name,
119	},
120	_ => JsExprData::Index {
121	base: Box::new(convert_valid_expr(ctx, *base)),
122	index: Box::new(convert_valid_expr(ctx, *field)),
123	},
124	},
125	loc,
126	comment,
127	),
128
129	ExprData::Return(_) => unreachable!("convert_valid_expr does not support return"),
130	}
131	}
132
133	pub fn convert_valid_block(ctx: &mut HoistContext, b: BlockWithLoc) -> JsExpr {
134	debug_assert!(
135	b.is_valid_js_expression(),
136	"must call convert_valid_block with valid js expression"
137	);
138
139	let len = b.block.statements.len();
140	let loc = b.loc;
141	let block_expr = match len {
142	0 => JsExpr::new(JsExprData::Null, loc.clone(), None),
143	// a block expression with a single expression is just that expression
144	1 => convert_valid_statement_to_expr(ctx, b.block.statements.into_iter().next().unwrap()),
145	// Since blocks are not valid expressions in JS, we create a
146	// comma expression from a block with more than one statement.
147	2.. => {
148	let mut items = Vec::new();
149	for statement in b.block.statements.into_iter() {
150	items.push(convert_valid_statement_to_expr(ctx, statement));
151	}
152	JsExpr::new(JsExprData::Comma { items }, loc.clone(), None)
153	}
154	};
155
156	// a block without implicit return is to be type void, so we wrap in a void expression
157	if !b.block.has_implicit_return && len > 0 {
158	JsExpr::new(JsExprData::Void(Box::new(block_expr)), loc, None)
159	} else {
160	block_expr
161	}
162	}
163
164	pub fn convert_valid_statement_to_expr(ctx: &mut HoistContext, statement: Statement) -> JsExpr {
165	debug_assert!(
166	statement.is_valid_js_expression(),
167	"convert_valid_statement_to_expr on an invalid expression (use hoist instead)"
168	);
169
170	match statement {
171	ast::Statement::Expr(expr) => convert_valid_expr(ctx, expr),
172	_ => unreachable!("attempt to convert statement to js without hoisting"),
173	}
174	}
175
176	pub fn convert_valid_statement(ctx: &mut HoistContext, statement: Statement) -> JsStatement {
177	let expr = convert_valid_statement_to_expr(ctx, statement);
178	let loc = expr.loc.clone();
179	JsStatement::new(JsStatementData::Expression(expr), loc, None)
180	}
181
182	pub fn convert_valid_if(
183	ctx: &mut HoistContext,
184	expr: If,
185	loc: Loc,
186	comment: Option<Comment>,
187	) -> JsExpr {
188	let condition = convert_valid_expr(ctx, *expr.condition);
189	let yes = convert_valid_block(ctx, expr.yes);
190	let no = match expr.no {
191	Some(BlockOrIf::Block(block)) => Some(convert_valid_block(ctx, block)),
192	Some(BlockOrIf::If(else_if)) => Some(convert_valid_if(ctx, *else_if, Loc::EMPTY, None)),
193	None => None,
194	};
195	JsExpr::new(
196	JsExprData::Ternary {
197	condition: Box::new(condition),
198	yes: Box::new(yes),
199	no: no.map(Box::new),
200	},
201	loc,
202	comment,
203	)
204	}

crates/js_ast/has_side_effects.rs created+90

...	@@ -0,0 +1,90 @@
1	use ast::*;
2
3	pub trait HasSideEffects {
4	/// Can this item be removed from the AST without changing the program's behavior?
5	fn has_side_effects(&self) -> bool;
6	}
7
8	impl HasSideEffects for ExprData {
9	fn has_side_effects(&self) -> bool {
10	match self {
11	ExprData::Literal(_) => false,
12	ExprData::Identifier(_) => false,
13
14	ExprData::Block(block) => block.has_side_effects(),
15	ExprData::Binary { left, right, .. } => {
16	left.has_side_effects() \|\| right.has_side_effects()
17	}
18	ExprData::Unary { value, .. } => value.has_side_effects(),
19	ExprData::If(ifexpr) => ifexpr.has_side_effects(),
20
21	// note: a function declaration has no side effects, but the call can
22	ExprData::Fn { .. } => false,
23
24	// TODO(@paperdave): implement detection of pure functions
25	ExprData::Call { base, arguments } => {
26	base.has_side_effects() \|\| arguments.iter().any(\|x\| x.has_side_effects())
27	}
28
29	// note: we assume field access has side effects, ie: a getter doesn't throw
30	ExprData::Field { base, field } => base.has_side_effects() \|\| field.has_side_effects(),
31
32	// note: we assume the code in @raw always has side effects
33	ExprData::Builtin { id, .. } => match id {
34	BuiltinId::Raw => true,
35	},
36
37	ExprData::Return(_) => true,
38	}
39	}
40	}
41
42	impl HasSideEffects for Statement {
43	fn has_side_effects(&self) -> bool {
44	match self {
45	Statement::Expr(expr) => expr.has_side_effects(),
46	Statement::Decl(decl) => decl.init.has_side_effects(),
47	Statement::Comment(_) => false,
48	}
49	}
50	}
51
52	impl HasSideEffects for Block {
53	fn has_side_effects(&self) -> bool {
54	self.statements
55	.iter()
56	.any(\|statement\| statement.has_side_effects())
57	}
58	}
59
60	impl HasSideEffects for Expr {
61	fn has_side_effects(&self) -> bool {
62	self.data.has_side_effects()
63	}
64	}
65
66	impl HasSideEffects for If {
67	fn has_side_effects(&self) -> bool {
68	self.condition.has_side_effects()
69	\|\| self.yes.has_side_effects()
70	\|\| match &self.no {
71	Some(no) => no.has_side_effects(),
72	None => false,
73	}
74	}
75	}
76
77	impl HasSideEffects for BlockOrIf {
78	fn has_side_effects(&self) -> bool {
79	match self {
80	BlockOrIf::Block(block) => block.has_side_effects(),
81	BlockOrIf::If(if_) => if_.has_side_effects(),
82	}
83	}
84	}
85
86	impl HasSideEffects for BlockWithLoc {
87	fn has_side_effects(&self) -> bool {
88	self.block.has_side_effects()
89	}
90	}

crates/js_ast/hoist.rs created+614

...	@@ -0,0 +1,614 @@
1	use std::collections::{HashMap, HashSet};
2	use std::vec;
3
4	use ast::{BinaryOperator, BlockOrIf, BlockWithLoc, Expr, Statement};
5	use tokenizer::Loc;
6
7	use crate::*;
8
9	pub struct HoistContext {
10	pub statements: Vec<JsStatement>,
11	pub ids: HashSet<String>,
12	next_id: u32,
13	scopes: Vec<HoistScope>,
14	has_last_statement: bool,
15	}
16
17	struct HoistScope {
18	renames: HashMap<String, String>,
19	}
20
21	pub struct TempId {
22	unmapped_name: String,
23	}
24
25	impl TempId {
26	pub fn get(&self, hoist_context: &HoistContext) -> String {
27	hoist_context.remap_identifier(&self.unmapped_name)
28	}
29	}
30
31	impl HoistContext {
32	pub fn new() -> Self {
33	Self {
34	statements: Vec::new(),
35	ids: HashSet::new(),
36	next_id: 0,
37	scopes: vec![HoistScope {
38	renames: HashMap::new(),
39	}],
40	has_last_statement: false,
41	}
42	}
43
44	pub fn new_child(&self) -> Self {
45	Self {
46	statements: Vec::new(),
47	ids: self.ids.clone(),
48	next_id: 0,
49	scopes: vec![HoistScope {
50	renames: HashMap::new(),
51	}],
52	has_last_statement: false,
53	}
54	}
55
56	/// A virtual scope is a scope that exists in blight syntax
57	/// but may or may not exist in blight. With these it is important to
58	pub fn push_virtual_scope(&mut self) {
59	self.scopes.push(HoistScope {
60	renames: HashMap::new(),
61	});
62	}
63
64	/// See `push_virtual_scope` for more information on what this is.
65	pub fn pop_virtual_scope(&mut self) {
66	self.scopes.pop();
67	}
68
69	pub fn generate_temp_id(&mut self, basename: &str) -> TempId {
70	self.next_id += 1;
71	loop {
72	let name = format!("{}{}", basename, self.next_id);
73	if !self.ids.contains(&name) {
74	self.ids.insert(name.clone());
75	return TempId {
76	unmapped_name: name,
77	};
78	}
79	self.next_id += 1;
80	}
81	}
82
83	/// This does some extra optimizations:
84	/// - `let a = b;` will not be emitted but instad write a variable alias
85	pub fn add_statement(&mut self, statement: JsStatement) {
86	match statement.data {
87	JsStatementData::Declaration(JSDeclaration {
88	name,
89	init:
90	Some(JsExpr {
91	data: JsExprData::Identifier(init_name),
92	..
93	}),
94	is_constant: true,
95	}) => {
96	self.scopes
97	.last_mut()
98	.unwrap()
99	.renames
100	.insert(name, init_name);
101	self.has_last_statement = false;
102	return;
103	}
104	_ => {}
105	}
106	self.statements.push(statement);
107	self.has_last_statement = true;
108	}
109
110	/// Get the last statement if it exists.
111	/// If the statement was optimized away, this returns None
112	pub fn last_statement(&self) -> Option<&JsStatement> {
113	if self.has_last_statement {
114	Some(self.statements.last().unwrap())
115	} else {
116	None
117	}
118	}
119
120	pub fn generate_id(&mut self, basename: &String) -> TempId {
121	let mut id = 0u32;
122	loop {
123	let name = match id {
124	0 => basename.clone(),
125	_ => format!("{}{}", basename, id),
126	};
127	if !self.ids.contains(&name) {
128	self.ids.insert(name.clone());
129	return TempId {
130	unmapped_name: name,
131	};
132	}
133	id += 1;
134	}
135	}
136
137	/// Given a blight identifier, remap it to a new identifier.
138	pub fn remap_identifier(&self, name: &String) -> String {
139	for scope in self.scopes.iter().rev() {
140	if let Some(remapped) = scope.renames.get(name) {
141	return remapped.clone();
142	}
143	}
144	name.clone()
145	}
146
147	/// Declare an identifier in the current scope.
148	pub fn declare_identifier(&mut self, name: String) -> TempId {
149	let remapped = self.generate_id(&name);
150	let scope = self.scopes.last_mut().unwrap();
151	self.ids.insert(remapped.unmapped_name.clone());
152	scope.renames.insert(name, remapped.unmapped_name.clone());
153	remapped
154	}
155
156	pub fn process_statement(&mut self, statement: Statement) -> Option<()> {
157	if statement.is_valid_js_expression() {
158	let result = convert_valid_statement(self, statement);
159	if !result.is_literal() {
160	self.add_statement(result);
161	}
162	return Some(());
163	}
164
165	match statement {
166	Statement::Decl(decl) => {
167	// visit init before name, this way allows for:
168	//
169	// let a = a;
170	//
171	// and `a` will be looked up before it is re-declared.
172	if let Some(init) = self.hoist_expression_contents(decl.init) {
173	let name = self.declare_identifier(decl.name);
174
175	self.add_statement(JsStatement::new(
176	JsStatementData::Declaration(crate::JSDeclaration {
177	name: name.get(&self),
178	init: Some(init),
179	is_constant: decl.is_constant,
180	}),
181	decl.loc,
182	decl.comment.map(\|x\| x.into()),
183	));
184	Some(())
185	} else {
186	None
187	}
188	}
189	Statement::Expr(expr) => {
190	if let Some(expr) = self.hoist_expression_contents(expr) {
191	let loc = expr.loc.clone();
192	if !expr.is_literal() {
193	self.add_statement(JsStatement::new(
194	JsStatementData::Expression(expr),
195	loc,
196	None,
197	))
198	}
199	Some(())
200	} else {
201	None
202	}
203	}
204	Statement::Comment(_) => Some(()),
205	}
206	}
207
208	/// Pushes a decl with a generated identifier to the hoisted result
209	/// and returns an identifier expression for it.
210	pub fn hoist_expression_to_temporary(&mut self, expr: Expr) -> Option<JsExpr> {
211	// TODO
212	// if !expr.has_side_effects() {
213	// return Some(self.hoist_expression_contents(expr)?);
214	// }
215
216	let init = self.hoist_expression_contents(expr)?;
217
218	let id = self.generate_temp_id("_temp");
219	let loc = init.loc.clone();
220
221	self.add_statement(JsStatement::new(
222	JsStatementData::Declaration(JSDeclaration {
223	name: id.get(&self),
224	init: Some(init),
225	is_constant: true,
226	}),
227	loc.clone(),
228	None,
229	));
230
231	Some(JsExpr::new(
232	JsExprData::Identifier(id.get(&self)),
233	loc,
234	None,
235	))
236	}
237
238	// Returning `None` from this means that the expression is noreturn, and that
239	// the rest of the statement never occurs and can be stripped.
240	pub fn hoist_expression_contents(&mut self, expr: Expr) -> Option<JsExpr> {
241	if expr.is_valid_js_expression() {
242	return Some(convert_valid_expr(self, expr));
243	}
244
245	match expr.data {
246	ast::ExprData::Binary { op, left, right } => {
247	let left = self.hoist_expression_to_temporary(*left)?;
248	let right = self.hoist_expression_to_temporary(*right)?;
249
250	Some(JsExpr::new(
251	JsExprData::Binary {
252	op,
253	left: Box::new(left),
254	right: Box::new(right),
255	},
256	expr.loc,
257	expr.comment,
258	))
259	}
260
261	ast::ExprData::Unary { op, value } => {
262	let value = self.hoist_expression_to_temporary(*value)?;
263	Some(JsExpr::new(
264	JsExprData::Unary {
265	op,
266	value: Box::new(value),
267	},
268	expr.loc,
269	expr.comment,
270	))
271	}
272
273	ast::ExprData::Field { base, field } => {
274	let base = self.hoist_expression_to_temporary(*base)?;
275	let field = self.hoist_expression_to_temporary(*field)?;
276
277	Some(JsExpr::new(
278	match field.data {
279	JsExprData::String(name) if is_js_identifier(name.as_str()) => {
280	JsExprData::Dot {
281	base: Box::new(base),
282	name,
283	}
284	}
285	_ => JsExprData::Index {
286	base: Box::new(base),
287	index: Box::new(field),
288	},
289	},
290	expr.loc,
291	expr.comment,
292	))
293	}
294
295	ast::ExprData::Block(block) => {
296	let temporary = self.generate_temp_id("_block");
297	let result = self.hoist_block_to_js_block_in_same_context(BlockWithLoc {
298	block,
299	loc: expr.loc.clone(),
300	});
301
302	self.add_statement(JsStatement::new(
303	JsStatementData::Declaration(crate::JSDeclaration {
304	name: temporary.get(&self),
305	init: Some(result),
306	is_constant: false,
307	}),
308	expr.loc.clone(),
309	None,
310	));
311
312	Some(JsExpr::new(
313	JsExprData::Identifier(temporary.get(&self)),
314	expr.loc,
315	expr.comment,
316	))
317	}
318
319	ast::ExprData::If(data) => {
320	let temporary = self.generate_temp_id("_if");
321
322	let decl_index = self.statements.len();
323	self.add_statement(JsStatement::new(
324	JsStatementData::Declaration(crate::JSDeclaration {
325	name: temporary.get(&self),
326	init: None,
327	is_constant: false,
328	}),
329	expr.loc.clone(),
330	None,
331	));
332
333	let condition = self.hoist_expression_to_temporary(*data.condition)?;
334	let mut did_assign = false;
335
336	let js_if =
337	self.hoist_js_if(data.yes, data.no, condition, &mut did_assign, &temporary)?;
338
339	self.add_statement(JsStatement::new(
340	JsStatementData::If(js_if),
341	expr.loc.clone(),
342	None,
343	));
344
345	if did_assign {
346	Some(JsExpr::new(
347	JsExprData::Identifier(temporary.get(&self)),
348	expr.loc,
349	expr.comment,
350	))
351	} else {
352	self.statements.remove(decl_index);
353	Some(JsExpr::new(JsExprData::Null, expr.loc, expr.comment))
354	}
355	}
356
357	ast::ExprData::Return(expr) => {
358	let loc = expr.loc.clone();
359	let return_value = self.hoist_expression_to_temporary(*expr)?;
360	self.add_statement(JsStatement::new(
361	JsStatementData::Return(return_value),
362	loc,
363	None,
364	));
365	None
366	}
367
368	ast::ExprData::Call { base, arguments } => {
369	let base = self.hoist_expression_to_temporary(*base)?;
370	let arguments = arguments
371	.into_iter()
372	.map(\|x\| self.hoist_expression_to_temporary(x))
373	.collect::<Option<Vec<_>>>()?;
374
375	Some(JsExpr::new(
376	JsExprData::Call {
377	base: Box::new(base),
378	arguments,
379	},
380	expr.loc,
381	expr.comment,
382	))
383	}
384
385	ast::ExprData::Fn { .. }
386	\| ast::ExprData::Builtin { .. }
387	\| ast::ExprData::Literal(_)
388	\| ast::ExprData::Identifier(_) => {
389	unreachable!("attempt to hoist an expression that does not require it")
390	}
391	}
392	}
393
394	pub fn hoist_block_to_js_block(
395	&mut self,
396	b: BlockWithLoc,
397	temporary: &String,
398	did_assign: &mut bool,
399	) -> Option<JsBlock> {
400	let loc = b.loc.clone();
401
402	// A non-block will become a block with a single expression:
403	// _temp = expr;
404	//
405	// We will also apply this transformation to blocks that are valid
406	// expressions, because that is more concise.
407	if b.is_valid_js_expression() {
408	return Some(JsBlock::new(
409	vec![new_js_assignment_statement(
410	convert_valid_block(self, b),
411	temporary.clone(),
412	loc.clone(),
413	)],
414	loc,
415	));
416	}
417
418	let mut child = self.new_child();
419
420	for statement in b.block.statements {
421	if let None = child.process_statement(statement) {
422	return None;
423	};
424	}
425
426	// If the last statement is an expression, we need
427	// to assign to the temporary variable.
428	match child.statements.last().map(\|x\| &x.data) {
429	Some(JsStatementData::Expression(_)) => {
430	*did_assign = true;
431
432	let last = child.statements.pop().unwrap();
433	let value = match last {
434	JsStatement {
435	data: JsStatementData::Expression(expr),
436	..
437	} => expr,
438	_ => {
439	self.statements.push(last);
440	JsExpr::new(JsExprData::Null, loc.clone(), None)
441	}
442	};
443	child.statements.push(new_js_assignment_statement(
444	value,
445	temporary.clone(),
446	loc.clone(),
447	));
448	}
449	_ => {}
450	}
451
452	Some(JsBlock::new(child.statements, loc))
453	}
454
455	pub fn hoist_block_to_arrow_fn_body(&mut self, b: BlockWithLoc) -> JsArrowBody {
456	let loc = b.loc.clone();
457
458	// A block that is a valid expression will just become
459	// that expression. We can convert multiple expressions
460	// into a comma expression.
461	if b.is_valid_js_expression() {
462	return JsArrowBody::Expr(convert_valid_block(self, b));
463	}
464
465	let mut child = self.new_child();
466
467	for statement in b.block.statements {
468	if let None = child.process_statement(statement) {
469	break;
470	};
471	}
472
473	// If the last statement is an expression,
474	// we need to return that expression.
475	match child.statements.last().map(\|x\| &x.data) {
476	Some(JsStatementData::Expression(_)) => {
477	let last = child.statements.pop().unwrap();
478	let value = match last {
479	JsStatement {
480	data: JsStatementData::Expression(expr),
481	..
482	} => expr,
483	_ => {
484	self.statements.push(last);
485	JsExpr::new(JsExprData::Null, loc.clone(), None)
486	}
487	};
488	child.statements.push(JsStatement::new(
489	JsStatementData::Return(value),
490	loc.clone(),
491	None,
492	));
493	}
494	_ => {}
495	}
496
497	JsArrowBody::Block(JsBlock::new(child.statements, loc))
498	}
499
500	pub fn hoist_block_to_js_block_in_same_context(&mut self, b: BlockWithLoc) -> JsExpr {
501	let loc = b.loc.clone();
502	self.push_virtual_scope();
503
504	debug_assert!(!b.block.is_valid_js_expression());
505
506	for statement in b.block.statements {
507	self.process_statement(statement);
508	}
509
510	self.pop_virtual_scope();
511
512	// If the last statement is an expression, we need
513	// to assign to the temporary variable.
514	match self.last_statement().map(\|x\| &x.data) {
515	Some(JsStatementData::Expression(_)) if b.block.has_implicit_return => {
516	let last = self.statements.pop().unwrap();
517	match last {
518	JsStatement {
519	data: JsStatementData::Expression(expr),
520	..
521	} => expr,
522	_ => {
523	self.statements.push(last);
524	JsExpr::new(JsExprData::Null, loc.clone(), None)
525	}
526	}
527	}
528	_ => JsExpr::new(JsExprData::Null, loc.clone(), None),
529	}
530	}
531
532	pub fn hoist_js_if(
533	&mut self,
534	yes: BlockWithLoc,
535	no: Option<BlockOrIf>,
536	condition: JsExpr,
537	did_assign: &mut bool,
538	temporary: &TempId,
539	) -> Option<JsIf> {
540	let yes = self.hoist_block_to_js_block(yes, &temporary.get(&self), did_assign)?;
541
542	let no = match no {
543	Some(BlockOrIf::Block(no)) => Some(JsElse::Block(self.hoist_block_to_js_block(
544	no,
545	&temporary.get(&self),
546	did_assign,
547	)?)),
548	Some(BlockOrIf::If(else_if)) => {
549	// If the condition of this block has a side effect, we need to
550	// emit a block so that we can assign to the temporary variable.
551	//
552	// Otherwise, we can just emit the condition inline
553	if else_if.condition.is_valid_js_expression() {
554	let condition = convert_valid_expr(self, *else_if.condition.clone());
555	Some(JsElse::If(Box::new(self.hoist_js_if(
556	else_if.yes,
557	else_if.no,
558	condition,
559	did_assign,
560	temporary,
561	)?)))
562	} else {
563	let mut temp_block = self.new_child();
564	let condition =
565	temp_block.hoist_expression_to_temporary(*else_if.condition.clone())?;
566
567	temp_block.add_statement(JsStatement::new(
568	JsStatementData::If(self.hoist_js_if(
569	else_if.yes,
570	else_if.no,
571	condition,
572	did_assign,
573	temporary,
574	)?),
575	Loc::EMPTY,
576	None,
577	));
578
579	Some(JsElse::Block(JsBlock::new(
580	temp_block.statements,
581	Loc::EMPTY,
582	)))
583	}
584	}
585	None => None,
586	};
587
588	Some(JsIf { condition, yes, no })
589	}
590	}
591
592	pub fn new_js_assignment_statement(expr: JsExpr, target: String, loc: Loc) -> JsStatement {
593	JsStatement::new(
594	JsStatementData::Expression(new_js_assignment_expr(expr, target, loc.clone())),
595	loc,
596	None,
597	)
598	}
599
600	pub fn new_js_assignment_expr(expr: JsExpr, target: String, loc: Loc) -> JsExpr {
601	JsExpr::new(
602	JsExprData::Binary {
603	op: BinaryOperator::Assign,
604	left: Box::new(JsExpr::new(
605	JsExprData::Identifier(target),
606	loc.clone(),
607	None,
608	)),
609	right: Box::new(expr),
610	},
611	loc,
612	None,
613	)
614	}

crates/js_ast/is_valid_js_expression.rs created+85

...	@@ -0,0 +1,85 @@
1	use ast::*;
2
3	pub trait IsValidJSExpression {
4	/// Can this be lowered to a JS ast without hoisting items
5	/// ex: a `return` expression is not a valid JS expression
6	fn is_valid_js_expression(&self) -> bool;
7	}
8
9	impl IsValidJSExpression for ExprData {
10	fn is_valid_js_expression(&self) -> bool {
11	match self {
12	// A block containing just expressions can be an expression
13	ExprData::Block(block) => block.is_valid_js_expression(),
14	ExprData::If(expr) => expr.is_valid_js_expression(),
15	ExprData::Return(_) => false,
16	ExprData::Builtin { id, .. } => match id {
17	BuiltinId::Raw => true,
18	},
19	ExprData::Binary { left, right, .. } => {
20	left.is_valid_js_expression() && right.is_valid_js_expression()
21	}
22	ExprData::Unary { value, .. } => value.is_valid_js_expression(),
23	ExprData::Fn { .. } => true,
24	ExprData::Call { base, arguments } => {
25	base.is_valid_js_expression()
26	&& arguments
27	.iter()
28	.all(\|argument\| argument.is_valid_js_expression())
29	}
30	ExprData::Field { base, field } => {
31	base.is_valid_js_expression() && field.is_valid_js_expression()
32	}
33	ExprData::Literal(_) \| ExprData::Identifier(_) => true,
34	}
35	}
36	}
37
38	impl IsValidJSExpression for Block {
39	fn is_valid_js_expression(&self) -> bool {
40	self.statements
41	.iter()
42	.all(\|statement\| statement.is_valid_js_expression())
43	}
44	}
45
46	impl IsValidJSExpression for Expr {
47	fn is_valid_js_expression(&self) -> bool {
48	self.data.is_valid_js_expression()
49	}
50	}
51
52	impl IsValidJSExpression for If {
53	fn is_valid_js_expression(&self) -> bool {
54	self.condition.is_valid_js_expression()
55	&& self.yes.is_valid_js_expression()
56	&& match &self.no {
57	Some(no) => no.is_valid_js_expression(),
58	None => true,
59	}
60	}
61	}
62
63	impl IsValidJSExpression for BlockOrIf {
64	fn is_valid_js_expression(&self) -> bool {
65	match self {
66	BlockOrIf::Block(block) => block.is_valid_js_expression(),
67	BlockOrIf::If(if_) => if_.is_valid_js_expression(),
68	}
69	}
70	}
71
72	impl IsValidJSExpression for BlockWithLoc {
73	fn is_valid_js_expression(&self) -> bool {
74	self.block.is_valid_js_expression()
75	}
76	}
77
78	impl IsValidJSExpression for Statement {
79	fn is_valid_js_expression(&self) -> bool {
80	match self {
81	Statement::Expr(expr) => expr.data.is_valid_js_expression(),
82	_ => false,
83	}
84	}
85	}

crates/js_ast/js_ast.rs+96-172

...	@@ -1,4 +1,4 @@	...	@@ -1,4 +1,4 @@
1	use ast::{BinaryOperator, BlockWithLoc, BuiltinId, Comment, Expr, UnaryOperator};	1	use ast::{BinaryOperator, Comment, UnaryOperator};
2	use num::bigint::BigInt;	2	use num::bigint::BigInt;
3	use tokenizer::Loc;	3	use tokenizer::Loc;
4		4
...	@@ -31,11 +31,11 @@ pub enum JsExprData {	...	@@ -31,11 +31,11 @@ pub enum JsExprData {
31		31
32	Identifier(String),	32	Identifier(String),
33	Unary {	33	Unary {
34	operator: UnaryOperator,	34	op: UnaryOperator,
35	value: Box<JsExpr>,	35	value: Box<JsExpr>,
36	},	36	},
37	Binary {	37	Binary {
38	operator: BinaryOperator,	38	op: BinaryOperator,
39	left: Box<JsExpr>,	39	left: Box<JsExpr>,
40	right: Box<JsExpr>,	40	right: Box<JsExpr>,
41	},	41	},
...	@@ -52,22 +52,23 @@ pub enum JsExprData {	...	@@ -52,22 +52,23 @@ pub enum JsExprData {
52	arguments: Vec<JsExpr>,	52	arguments: Vec<JsExpr>,
53	},	53	},
54	Function {	54	Function {
55	name: String,	55	name: Option<String>,
56	params: Vec<String>,	56	params: Vec<String>,
57	block: Vec<JsStatement>,	57	body: Box<JsExpr>,
58	},
59	If {
60	condition: Box<JsExpr>,
61	yes: Vec<JsExpr>,
62	no: Option<Box<JsExpr>>,
63	},	58	},
64	ArrowFunction {	59	ArrowFunction {
65	params: Vec<String>,	60	params: Vec<String>,
66	block: Vec<JsStatement>,	61	body: Box<JsArrowBody>,
		62	},
		63	Ternary {
		64	condition: Box<JsExpr>,
		65	yes: Box<JsExpr>,
		66	no: Option<Box<JsExpr>>,
67	},	67	},
68	Comma {	68	Comma {
69	items: Vec<JsExpr>,	69	items: Vec<JsExpr>,
70	},	70	},
		71	Void(Box<JsExpr>),
71	}	72	}
72		73
73	#[derive(Debug, PartialEq, Clone)]	74	#[derive(Debug, PartialEq, Clone)]
...	@@ -77,33 +78,56 @@ pub struct JsStatement {	...	@@ -77,33 +78,56 @@ pub struct JsStatement {
77	pub comment: Option<Comment>,	78	pub comment: Option<Comment>,
78	}	79	}
79		80
		81	#[derive(Debug, PartialEq, Clone)]
		82	pub struct JSFunctionDeclaration {
		83	pub name: String,
		84	pub params: Vec<String>,
		85	pub block: JsBlock,
		86	}
		87
		88	#[derive(Debug, PartialEq, Clone)]
		89	pub struct JSDeclaration {
		90	pub name: String,
		91	pub init: Option<JsExpr>,
		92	pub is_constant: bool,
		93	}
		94
		95	#[derive(Debug, PartialEq, Clone)]
		96	pub struct JsIf {
		97	pub condition: JsExpr,
		98	pub yes: JsBlock,
		99	pub no: Option<JsElse>,
		100	}
		101
		102	#[derive(Debug, PartialEq, Clone)]
		103	pub enum JsElse {
		104	Expr(JsExpr),
		105	Block(JsBlock),
		106	If(Box<JsIf>),
		107	}
		108
80	#[derive(Debug, PartialEq, Clone)]	109	#[derive(Debug, PartialEq, Clone)]
81	pub enum JsStatementData {	110	pub enum JsStatementData {
82	Expression(JsExpr),	111	Expression(JsExpr),
83	Return(JsExpr),	112	Return(JsExpr),
84	Export(JsExport),	113	Export(JsExport),
85	Declaration {	114	Declaration(JSDeclaration),
86	name: String,	115	Function(JSFunctionDeclaration),
87	init: JsExpr,	116	If(JsIf),
88	is_constant: bool,
89	},
90	Function {
91	name: String,
92	params: Vec<String>,
93	block: JsBlock,
94	},
95	If {
96	condition: JsExpr,
97	yes: JsBlock,
98	no: Option<JsBlock>,
99	},
100	}	117	}
101		118
102	#[derive(Debug, PartialEq, Clone)]	119	#[derive(Debug, PartialEq, Clone)]
103	pub enum JsExport {	120	pub enum JsExport {
104	Named(Vec<String>),	121	Named(Vec<JSNamedExport>),
		122	Function(Box<JSFunctionDeclaration>),
		123	Declaration(Box<JSDeclaration>),
105	Default(JsExpr),	124	Default(JsExpr),
106	Equals(JsExpr),	125	}
		126
		127	#[derive(Debug, PartialEq, Clone)]
		128	pub struct JSNamedExport {
		129	pub identifier: String,
		130	pub export_name: String,
107	}	131	}
108		132
109	#[derive(Debug, PartialEq, Clone)]	133	#[derive(Debug, PartialEq, Clone)]
...	@@ -113,156 +137,38 @@ pub struct JsBlock {	...	@@ -113,156 +137,38 @@ pub struct JsBlock {
113	}	137	}
114		138
115	impl JsExpr {	139	impl JsExpr {
116	pub fn from_blight(expr: &Expr) -> Option<JsExpr> {	140	pub fn new(data: JsExprData, loc: Loc, comment: Option<Comment>) -> Self {
117	use ast::ExprData::*;	141	Self { data, loc, comment }
118		142	}
119	let loc = &expr.loc;
120	let comment = &expr.comment;
121
122	match &expr.data {
123	Block(block) => Self::comma_expr_from_blight_block(&BlockWithLoc {
124	block: block.clone(),
125	loc: loc.clone(),
126	}),
127	Binary { op, left, right } => {
128	let left = Self::from_blight(&*left)?;
129	let right = Self::from_blight(&*right)?;
130	Some(JsExpr::new(
131	JsExprData::Binary {
132	operator: op.clone(),
133	left: Box::new(left),
134	right: Box::new(right),
135	},
136	loc.clone(),
137	comment.clone(),
138	))
139	}
140	Unary { op, value } => {
141	let value = Self::from_blight(&*value)?;
142	Some(JsExpr::new(
143	JsExprData::Unary {
144	operator: op.clone(),
145	value: Box::new(value),
146	},
147	loc.clone(),
148	comment.clone(),
149	))
150	}
151	Fn {
152	name,
153	params,
154	return_type,
155	block,
156	} => {
157	let block = Self::comma_expr_from_blight_block(&BlockWithLoc {
158	block: block.clone(),
159	loc: loc.clone(),
160	})?;
161	Some(JsExpr::new(
162	JsExprData::Function {
163	name: name.clone(),
164	params: vec![],
165	block: vec![JsStatement::new(
166	JsStatementData::Expression(block),
167	loc.clone(),
168	comment.clone(),
169	)],
170	},
171	loc.clone(),
172	comment.clone(),
173	))
174	}
175	Builtin { id, arguments } => match id {
176	BuiltinId::Raw => {
177	let mut args = arguments.iter();
178	let arg = args.next().unwrap();
179	let arg = match &arg.data {
180	Literal(ast::Literal::String(s)) => s,
181	_ => panic!("raw expects a string"),
182	};
183	Some(JsExpr::new(
184	JsExprData::Raw(arg.clone()),
185	loc.clone(),
186	comment.clone(),
187	))
188	}
189	},
190	If(expr) => {
191	let condition = Self::from_blight(&expr.condition)?;
192	let yes = Self::comma_expr_from_blight_block(&expr.yes)?;
193	let no = match &expr.no {
194	Some(no) => Some(Self::from_blight(&no)?),
195	None => None,
196	};
197	Some(JsExpr::new(
198	JsExprData::If {
199	condition: Box::new(condition),
200	yes: vec![yes],
201	no: no.map(Box::new),
202	},
203	loc.clone(),
204	comment.clone(),
205	))
206	}
207
208	Call { base, arguments } => todo!(),
209	Field { base, field } => todo!(),
210
211	Return(_) => None,
212		143
213	Literal(_) => todo!(),	144	pub fn is_literal(&self) -> bool {
214	Identifier(_) => todo!(),	145	match &self.data {
215	// Literal(ast::Literal::Number(n)) => {	146	JsExprData::Number(_) => true,
216	// Some(JsExpr::new(JsExprData::Number(n), loc.clone(), comment))	147	JsExprData::BigInt(_) => true,
217	// }	148	JsExprData::String(_) => true,
218	// Literal(ast::Literal::Bool(b)) => Some(JsExpr::new(JsExprData::Bool(b), loc, comment)),	149	JsExprData::Bool(_) => true,
219	// Literal(ast::Literal::String(s)) => {	150	JsExprData::Null => true,
220	// Some(JsExpr::new(JsExprData::String(s), loc, comment))	151	JsExprData::Undefined => true,
221	// }	152	_ => false,
222	// Literal(ast::Literal::Void) => Some(JsExpr::new(JsExprData::Null, loc, comment)),
223	// Identifier(id) => Some(JsExpr::new(JsExprData::Identifier(id), loc, comment)),
224	}	153	}
225	}	154	}
		155	}
226		156
227	/// Since blocks are not valid expressions in JS, this creates a	157	impl JsExprData {
228	/// comma expression from a block with more than one statement.	158	/// Helper for testing
229	///	159	pub fn str(value: &str) -> Self {
230	/// Not possible to create an ast => None	160	Self::String(value.to_string())
231	pub fn comma_expr_from_blight_block(b: &ast::BlockWithLoc) -> Option<JsExpr> {
232	if !b.block.has_implicit_return {
233	return Some(JsExpr::new(JsExprData::Null, b.loc.clone(), None));
234	}
235	// a block expression with a single statement is just that statement
236	if b.block.statements.len() == 1 && b.block.has_implicit_return {
237	match b.block.statements.first().unwrap() {
238	ast::Statement::Expr(expr) => Self::from_blight(expr),
239	_ => return None,
240	};
241	}
242	let mut statements = Vec::new();
243	for statement in &b.block.statements {
244	match statement {
245	ast::Statement::Expr(expr) => {
246	if let Some(expr) = Self::from_blight(&expr) {
247	statements.push(expr);
248	}
249	}
250	_ => return None,
251	}
252	}
253	// a block without implicit return is to be type void, so we append null
254	if !b.block.has_implicit_return {
255	statements.push(JsExpr::new(JsExprData::Null, b.loc.clone(), None));
256	}
257	Some(JsExpr::new(
258	JsExprData::Comma { items: statements },
259	b.loc.clone(),
260	None,
261	))
262	}	161	}
263		162
264	pub fn new(data: JsExprData, loc: Loc, comment: Option<Comment>) -> Self {	163	/// Helper for testing
265	Self { data, loc, comment }	164	pub fn str_id(value: &str) -> Self {
		165	Self::Identifier(value.to_string())
		166	}
		167	}
		168
		169	impl JsBlock {
		170	pub fn new(statements: Vec<JsStatement>, loc: Loc) -> Self {
		171	Self { statements, loc }
266	}	172	}
267	}	173	}
268		174
...	@@ -270,4 +176,22 @@ impl JsStatement {	...	@@ -270,4 +176,22 @@ impl JsStatement {
270	pub fn new(data: JsStatementData, loc: Loc, comment: Option<Comment>) -> Self {	176	pub fn new(data: JsStatementData, loc: Loc, comment: Option<Comment>) -> Self {
271	Self { data, loc, comment }	177	Self { data, loc, comment }
272	}	178	}
		179
		180	pub(crate) fn is_literal(&self) -> bool {
		181	match &self.data {
		182	JsStatementData::Expression(expr) => expr.is_literal(),
		183	_ => false,
		184	}
		185	}
		186	}
		187
		188	#[derive(Debug, PartialEq, Clone)]
		189	pub enum JsArrowBody {
		190	Block(JsBlock),
		191	Expr(JsExpr),
		192	}
		193
		194	#[derive(Debug, PartialEq, Clone)]
		195	pub struct JsModule {
		196	pub statements: Vec<JsStatement>,
273	}	197	}

crates/js_ast/lib.rs+19-402

...	@@ -1,411 +1,28 @@	...	@@ -1,411 +1,28 @@
1	use std::collections::HashMap;	1	mod convert_valid_expr;
2		2	mod has_side_effects;
3	use ast::{BlockWithLoc, Expr, ExprData, Literal, Module, Statement};	3	mod hoist;
4	pub use js_ast::*;	4	mod is_valid_js_expression;
5	use miette::Result;
6	use tokenizer::Loc;
7
8	mod js_ast;	5	mod js_ast;
		6	mod util;
9		7
10	struct StackContext {	8	use ast::Module;
11	statements: Vec<JsStatement>,
12	/// Variables that are declared in this context.
13	/// The number is the number of times the variable has been declared.
14	/// This is used to allow for shadowing
15	variable_map: HashMap<String, u32>,
16	block: u32,
17	}
18
19	impl StackContext {
20	fn new(block: u32) -> Self {
21	StackContext {
22	statements: Vec::new(),
23	variable_map: HashMap::new(),
24	block,
25	}
26	}
27	}
28
29	struct TranslateContext {
30	context: Vec<StackContext>,
31	internal_counter: u32,
32	block_counter: u32,
33	}
34
35	impl TranslateContext {
36	fn new() -> Self {
37	TranslateContext {
38	context: vec![StackContext::new(0)],
39	internal_counter: 0,
40	block_counter: 0,
41	}
42	}
43
44	fn push_context(self: &mut Self) {
45	self.block_counter += 1;
46	self.context.push(StackContext::new(self.block_counter));
47	}
48
49	fn pop_context(self: &mut Self) -> Vec<JsStatement> {
50	self.context.pop().unwrap().statements
51	}
52
53	fn push_internal_variable(self: &mut Self, value: JsExpr) -> JsExpr {
54	let name = format!("${}", self.internal_counter);
55	let loc = value.loc.clone();
56	self.context
57	.last_mut()
58	.unwrap()
59	.statements
60	.push(JsStatement::new(
61	JsStatementData::Declaration {
62	name: name.clone(),
63	init: value,
64	is_constant: false,
65	},
66	loc,
67	None,
68	));
69	self.internal_counter += 1;
70	JsExpr::new(JsExprData::Identifier(name), Loc::EMPTY, None)
71	}
72
73	fn push(self: &mut Self, statement: JsStatement) {
74	self.context.last_mut().unwrap().statements.push(statement);
75	}
76
77	// fn peek(self: &Self) -> &JsStatement {
78	// self.context.last().unwrap().statements.last().unwrap()
79	// }
80
81	fn declare_variable(self: &mut Self, name: String) -> String {
82	let context = self.context.last_mut().unwrap();
83	if let Some(value) = context.variable_map.get_mut(name.as_str()) {
84	*value += 1;
85	format!("{}${}${}", name, value, context.block)
86	} else {
87	context.variable_map.insert(name.to_string(), 0);
88	format!("{}$0${}", name, context.block)
89	}
90	}
91		9
92	fn get_identifier(self: &Self, name: String) -> String {	10	pub use convert_valid_expr::*;
93	for context in self.context.iter().rev() {	11	pub use hoist::*;
94	if let Some(value) = context.variable_map.get(name.as_str()) {	12	pub use is_valid_js_expression::*;
95	return format!("{}${}${}", name, value, context.block);	13	pub use js_ast::*;
96	}	14	pub use util::*;
97	}
98	panic!("Variable {} not found", name);
99	}
100	}
101
102	fn get_block_handle(b: &ast::BlockWithLoc, ctx: &mut TranslateContext) -> JsExpr {
103	// :)
104	if b.is_valid_js_expression() {
105	return JsExpr::comma_expr_from_blight_block(b).unwrap();
106	}
107
108	ctx.push_context();
109	for (i, statement) in b.block.statements.iter().enumerate() {
110	if i == b.block.statements.len() - 1 {
111	break;
112	} else {
113	translate_statement(statement, ctx);
114	}
115	}
116	let result = match b.block.statements.last().unwrap() {
117	ast::Statement::Expr(expr) => get_expression_handle(expr, ctx),
118	stmt => {
119	translate_statement(stmt, ctx);
120	JsExpr::new(JsExprData::Null, Loc::EMPTY, None)
121	}
122	};
123	let value = ctx.pop_context();
124	value.iter().for_each(\|x\| ctx.push(x.clone()));
125	result
126	}
127
128	// basically, expressions are really hard to translate
129	// in blight, everything is an expression
130	// in js, very few things are expressions
131	// here, we recursively expand out expressions into statements
132	// we then store the result of all the expressions in a variable
133	// and then return the name of that variable
134	fn get_expression_handle(ast: &Expr, ctx: &mut TranslateContext) -> JsExpr {
135	match ast.data.clone() {
136	ExprData::Literal(Literal::String(value)) => JsExpr::new(
137	JsExprData::String(value),
138	ast.loc.clone(),
139	ast.comment.clone(),
140	),
141	ExprData::Literal(Literal::Number(value)) => JsExpr::new(
142	JsExprData::Number(value),
143	ast.loc.clone(),
144	ast.comment.clone(),
145	),
146	ExprData::Literal(Literal::Void) => {
147	JsExpr::new(JsExprData::Null, ast.loc.clone(), ast.comment.clone())
148	}
149	ExprData::Literal(Literal::Bool(b)) => {
150	JsExpr::new(JsExprData::Bool(b), ast.loc.clone(), ast.comment.clone())
151	}
152
153	ExprData::Identifier(value) => JsExpr::new(
154	JsExprData::Identifier(ctx.get_identifier(value.clone())),
155	ast.loc.clone(),
156	ast.comment.clone(),
157	),
158	ExprData::Call { base, arguments } => {
159	let mut handles = Vec::new();
160	for argument in arguments {
161	handles.push(get_expression_handle(&argument, ctx));
162	}
163	let base = get_expression_handle(&base, ctx);
164	ctx.push_internal_variable(JsExpr::new(
165	JsExprData::Call {
166	base: Box::new(base),
167	arguments: handles,
168	},
169	ast.loc.clone(),
170	ast.comment.clone(),
171	))
172	}
173	ExprData::Builtin { id, arguments } => match id {
174	ast::BuiltinId::Raw => {
175	if arguments.len() != 1 {
176	panic!("@raw expects exactly one argument");
177	}
178	match arguments.get(0).unwrap() {
179	Expr {
180	data: ExprData::Literal(Literal::String(value)),
181	..
182	} => JsExpr::new(
183	JsExprData::Raw(value.clone()),
184	ast.loc.clone(),
185	ast.comment.clone(),
186	),
187	_ => panic!("@raw expects a string literal"),
188	}
189	}
190	},
191	ExprData::Binary { op, left, right } => {
192	let left_handle = get_expression_handle(&left, ctx);
193	let right_handle = get_expression_handle(&right, ctx);
194	JsExpr::new(
195	JsExprData::Binary {
196	operator: op.clone(),
197	left: Box::new(left_handle),
198	right: Box::new(right_handle),
199	},
200	ast.loc.clone(),
201	ast.comment.clone(),
202	)
203	}
204
205	ExprData::Unary { op, value } => {
206	let value_handle = get_expression_handle(&value, ctx);
207	JsExpr::new(
208	JsExprData::Unary {
209	operator: op.clone(),
210	value: Box::new(value_handle),
211	},
212	ast.loc.clone(),
213	ast.comment.clone(),
214	)
215	}
216
217	ExprData::Field { base, field } => {
218	let base = Box::new(get_expression_handle(&base, ctx));
219
220	match field.data {
221	ExprData::Literal(Literal::String(name)) => JsExpr::new(
222	JsExprData::Dot { base, name },
223	ast.loc.clone(),
224	ast.comment.clone(),
225	),
226	_ => JsExpr::new(
227	JsExprData::Index {
228	base,
229	index: Box::new(get_expression_handle(&field, ctx)),
230	},
231	ast.loc.clone(),
232	ast.comment.clone(),
233	),
234	}
235	}
236
237	ExprData::Block(block) => get_block_handle(
238	&BlockWithLoc {
239	block,
240	loc: ast.loc.clone(),
241	},
242	ctx,
243	),
244
245	ExprData::If(expr) => {
246	let handle = get_expression_handle(&expr.condition, ctx);
247
248	let yes = get_block_handle(&expr.yes, ctx);
249	let no = match expr.no {
250	Some(block) => Some(get_expression_handle(&block, ctx)),
251	None => None,
252	};
253
254	// ctx.push(JsStatement::new(
255	// JsStatementData::If {
256	// condition: handle,
257	// yes,
258	// no,
259	// },
260	// ast.loc,
261	// ast.comment,
262	// ));
263
264	// panic!()
265	handle
266	}
267	ExprData::Fn {
268	name,
269	params,
270	return_type,
271	block,
272	} => todo!(),
273	ExprData::Return(_) => todo!(),
274	}
275	}
276
277	fn translate_statement(ast: &Statement, ctx: &mut TranslateContext) {
278	match ast {
279	Statement::Decl(decl) => {
280	let handle = get_expression_handle(&decl.init, ctx);
281	let rename = ctx.declare_variable(decl.name.clone());
282	ctx.push(JsStatement::new(
283	JsStatementData::Declaration {
284	name: rename,
285	init: handle,
286	is_constant: decl.is_constant.clone(),
287	},
288	decl.loc.clone(),
289	None,
290	));
291	}
292	Statement::Expr(expr) => {
293	let h = get_expression_handle(expr, ctx);
294	ctx.push(JsStatement::new(
295	JsStatementData::Expression(h),
296	expr.loc.clone(),
297	None,
298	))
299	}
300	Statement::Comment(_) => {} // Ast::Function {
301	// name,
302	// arguments,
303	// return_type: _,
304	// block,
305	// } => {
306	// let rename = ctx.declare_variable(name.clone());
307	// let arguments = arguments
308	// .iter()
309	// .map(\|x\| ctx.declare_variable(x.name.clone()))
310	// .collect();
311
312	// ctx.push_context();
313	// for (i, statement) in block.value.iter().enumerate() {
314	// if block.implicit_return && i == block.value.len() - 1 {
315	// let handle = get_expression_handle(statement, ctx);
316	// ctx.push(JsStatement::Return(handle));
317	// } else {
318	// translate_statement(statement, ctx);
319	// }
320	// }
321	// let value = ctx.pop_context();
322		15
323	// ctx.push(JsStatement::Function {	16	pub fn blight_to_js(blight: Module) -> JsModule {
324	// name: rename,	17	let mut root_context = HoistContext::new();
325	// arguments,
326	// value,
327	// });
328	// }
329	// Ast::Public(value) => {
330	// translate_statement(value, ctx);
331	// // I'm sorry, I just don't know how to get the original name properly... my bad
332	// match ctx.peek() {
333	// JsStatement::Function { name, .. } \| JsStatement::Declaration { name, .. } => {
334	// let original_name = name.split("$").next().unwrap().to_string();
335	// ctx.push(JsStatement::Export(Box::new(JsStatement::Declaration {
336	// name: original_name,
337	// value: JsExpr::Identifier(name.clone()),
338	// is_constant: true,
339	// })));
340	// }
341	// _ => {
342	// panic!("Cannot make non-function public");
343	// }
344	// }
345	// }
346	// Ast::Return(value) => {
347	// let handle = get_expression_handle(value, ctx);
348	// ctx.push(JsStatement::Return(handle));
349	// }
350	// Ast::Block(block) => {
351	// ctx.push_context();
352	// for (i, statement) in block.value.iter().enumerate() {
353	// if block.implicit_return && i == block.value.len() - 1 {
354	// let handle = get_expression_handle(statement, ctx);
355	// ctx.push(JsStatement::Return(handle));
356	// } else {
357	// translate_statement(statement, ctx);
358	// }
359	// }
360	// let value = ctx.pop_context();
361	// value.iter().for_each(\|x\| ctx.push(x.clone()));
362	// }
363		18
364	// Ast::FunctionCall { name, arguments } => {	19	for statement in blight.statements {
365	// let mut handles = Vec::new();	20	if let None = root_context.process_statement(statement.to_statement()) {
366	// for argument in arguments {	21	unreachable!("top level return is disallowed by the parser")
367	// handles.push(get_expression_handle(argument, ctx));	22	};
368	// }
369	// match *name.clone() {
370	// Ast::Identifier(name) => {
371	// ctx.push(JsStatement::Expression(JsExpr::Call {
372	// name: Box::new(JsExpr::Identifier(ctx.get_identifier(name.clone()))),
373	// arguments: handles,
374	// }));
375	// }
376	// Ast::BuiltIn(name) => match name.as_str() {
377	// "@raw" => {
378	// if handles.len() != 1 {
379	// panic!("@raw expects exactly one argument");
380	// }
381	// match handles.get(0).unwrap() {
382	// JsExpr::String(value) => {
383	// ctx.push(JsStatement::Expression(JsExpr::Call {
384	// name: Box::new(JsExpr::Raw(value.clone())),
385	// arguments: handles,
386	// }));
387	// }
388	// _ => panic!("@raw expects a string argument"),
389	// }
390	// }
391	// _ => panic!("Unknown built-in function {}", name),
392	// },
393	// _ => panic!("Function name must be an identifier"),
394	// }
395	// }
396	// Ast::Comment(_) \| Ast::DocComment(_) => {}
397	// _ => {
398	// panic!("unimplemented: {:?}", ast);
399	// }
400	}	23	}
401	}
402		24
403	pub fn translate(module: &Module) -> Result<Vec<JsStatement>> {	25	JsModule {
404	let mut ctx = TranslateContext::new();	26	statements: root_context.statements,
405
406	for ast in module.statements.iter() {
407	translate_statement(&ast.clone().to_statement(), &mut ctx);
408	}	27	}
409
410	Ok(ctx.pop_context())
411	}	28	}

crates/js_ast/util.rs created+17

...	@@ -0,0 +1,17 @@
1	/// TODO(@paperdave): handle the rest of the identifiers
2	pub fn is_js_identifier(s: &str) -> bool {
3	if s.is_empty() {
4	return false;
5	}
6	let mut chars = s.chars();
7	let first = chars.next().unwrap();
8	if !first.is_ascii_alphabetic() && first != '_' {
9	return false;
10	}
11	for c in chars {
12	if !c.is_ascii_alphanumeric() && c != '_' {
13	return false;
14	}
15	}
16	true
17	}

crates/js_printer/Cargo.toml+1

...	@@ -4,6 +4,7 @@ version = "0.1.0"	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	edition = "2021"	4	edition = "2021"
5		5
6	[lib]	6	[lib]
		7	path = "./lib.rs"
7		8
8	[dependencies]	9	[dependencies]
9	miette.workspace = true	10	miette.workspace = true

crates/js_printer/lib.rs created+358

...	@@ -0,0 +1,358 @@
1	mod printer_tests;
2
3	use js_ast::*;
4	use std::io::{Result, Write};
5
6	pub trait Print {
7	fn print(&self, writer: &mut dyn Write) -> Result<()>;
8
9	fn print_to_string(&self) -> String {
10	let mut buffer = Vec::new();
11	self.print(&mut buffer).unwrap();
12	String::from_utf8(buffer).unwrap()
13	}
14	}
15
16	// the printer:
17
18	fn print_module_clause_identifier(writer: &mut dyn Write, export_name: &str) -> Result<()> {
19	// TODO(@paperdave): Escape invalid identifiers
20	if js_ast::is_js_identifier(export_name) {
21	write!(writer, "{}", export_name)
22	} else {
23	todo!("call JSON.stringify on identifier")
24	}
25	}
26
27	fn print_js_string(writer: &mut dyn Write, string: &str) -> Result<()> {
28	write!(writer, "\"")?;
29	// TODO(@paperdave): Instead of writing each character, this should write
30	// chunks between the escapes instead of calling write! on each character individually.
31	for c in string.chars() {
32	match c {
33	'\n' => write!(writer, "\\n")?,
34	'\r' => write!(writer, "\\r")?,
35	'\t' => write!(writer, "\\t")?,
36	'"' => write!(writer, "\\\"")?,
37	'\\' => write!(writer, "\\\\")?,
38	'\u{0008}' => write!(writer, "\\b")?,
39	'\u{000C}' => write!(writer, "\\f")?,
40	'\u{0000}'..='\u{001F}' => write!(writer, "\\u{:04x}", c as u32)?,
41	// TODO(@paperdave): I think there are some larger code points that need to be escaped?
42	_ => write!(writer, "{}", c)?,
43	}
44	}
45	write!(writer, "\"")
46	}
47
48	impl Print for JsStatement {
49	fn print(&self, w: &mut dyn Write) -> Result<()> {
50	match &self.data {
51	js_ast::JsStatementData::Expression(expr) => {
52	expr.print(w)?;
53	write!(w, ";")
54	}
55	js_ast::JsStatementData::Return(expr) => {
56	write!(w, "return ")?;
57	expr.print(w)
58	}
59	js_ast::JsStatementData::Export(export) => export.print(w),
60	js_ast::JsStatementData::Declaration(decl) => decl.print(w),
61	js_ast::JsStatementData::Function(function) => function.print(w),
62	js_ast::JsStatementData::If(js_if) => js_if.print(w),
63	}
64	}
65	}
66
67	impl Print for JsExport {
68	fn print(&self, w: &mut dyn Write) -> Result<()> {
69	match self {
70	JsExport::Named(exports) => {
71	write!(w, "export{{")?;
72	for (i, export) in exports.iter().enumerate() {
73	if i > 0 {
74	write!(w, ",")?;
75	}
76	export.print(w)?;
77	}
78	Ok(())
79	}
80	JsExport::Function(function) => {
81	write!(w, "export ")?;
82	function.print(w)
83	}
84	JsExport::Declaration(declaration) => {
85	write!(w, "export ")?;
86	declaration.print(w)
87	}
88	JsExport::Default(expr) => {
89	write!(w, "export default ")?;
90	expr.print(w)
91	}
92	}
93	}
94	}
95
96	impl Print for JSNamedExport {
97	fn print(&self, writer: &mut dyn Write) -> Result<()> {
98	if self.export_name == self.identifier {
99	write!(writer, "{}", self.identifier)
100	} else {
101	write!(writer, "{} as ", self.identifier)?;
102	print_module_clause_identifier(writer, &self.export_name)
103	}
104	}
105	}
106
107	impl Print for JSDeclaration {
108	fn print(&self, writer: &mut dyn Write) -> Result<()> {
109	write!(
110	writer,
111	"{} {}",
112	if self.is_constant { "const" } else { "let" },
113	self.name,
114	)?;
115	if let Some(init) = &self.init {
116	write!(writer, " = ")?;
117	init.print(writer)?;
118	}
119	write!(writer, ";")?;
120	Ok(())
121	}
122	}
123
124	impl Print for JsExpr {
125	fn print(&self, writer: &mut dyn Write) -> Result<()> {
126	self.data.print(writer)
127	}
128	}
129
130	impl Print for JsExprData {
131	fn print(&self, writer: &mut dyn Write) -> Result<()> {
132	match &self {
133	JsExprData::Raw(data) => write!(writer, "{}", data),
134	JsExprData::Number(number) => write!(writer, "{}", number),
135	JsExprData::BigInt(bigint) => write!(writer, "{}", bigint),
136	JsExprData::String(string) => print_js_string(writer, string.as_str()),
137	JsExprData::Bool(bool) => write!(writer, "{}", bool),
138	JsExprData::Null => write!(writer, "null"),
139	JsExprData::Undefined => write!(writer, "undefined"),
140	JsExprData::Identifier(str) => write!(writer, "{}", str),
141	JsExprData::Array(items) => {
142	write!(writer, "[")?;
143	for (i, item) in items.iter().enumerate() {
144	if i > 0 {
145	write!(writer, ",")?;
146	}
147	item.print(writer)?;
148	}
149	write!(writer, "]")
150	}
151	JsExprData::Object(items) => {
152	write!(writer, "{{")?;
153	for (i, kv) in items.iter().enumerate() {
154	if i > 0 {
155	write!(writer, ",")?;
156	}
157
158	kv.print(writer)?;
159	}
160	write!(writer, "}}")
161	}
162	JsExprData::Unary { op, value } => {
163	write!(writer, "{}", op.to_str())?;
164	value.print(writer)
165	}
166	JsExprData::Binary { op, left, right } => {
167	// TODO: figure out when we dont need to wrap
168	let wrap = true;
169	if wrap {
170	write!(writer, "(")?;
171	}
172	left.print(writer)?;
173	write!(writer, "{}", op.to_str())?;
174	right.print(writer)?;
175	if wrap {
176	write!(writer, ")")?;
177	}
178	Ok(())
179	}
180	JsExprData::Dot { base, name } => {
181	base.print(writer)?;
182	write!(writer, ".{}", name)
183	}
184	JsExprData::Index { base, index } => {
185	base.print(writer)?;
186	write!(writer, "[")?;
187	index.print(writer)?;
188	write!(writer, "]")
189	}
190	JsExprData::Call { base, arguments } => {
191	base.print(writer)?;
192	write!(writer, "(")?;
193	for (i, argument) in arguments.iter().enumerate() {
194	if i > 0 {
195	write!(writer, ",")?;
196	}
197	argument.print(writer)?;
198	}
199	write!(writer, ")")
200	}
201	JsExprData::Function { body, name, params } => {
202	write!(writer, "function ")?;
203	if let Some(name) = name {
204	write!(writer, "{}", name)?;
205	}
206	write!(writer, "(")?;
207	for (i, param) in params.iter().enumerate() {
208	if i > 0 {
209	write!(writer, ",")?;
210	}
211	write!(writer, "{}", param)?;
212	}
213	write!(writer, ")")?;
214	body.print(writer)
215	}
216	JsExprData::ArrowFunction { body, params } => {
217	let print_parens = params.len() != 1;
218	if print_parens {
219	write!(writer, "(")?;
220	}
221	for (i, param) in params.iter().enumerate() {
222	if i > 0 {
223	write!(writer, ",")?;
224	}
225	write!(writer, "{}", param)?;
226	}
227	if print_parens {
228	write!(writer, ")")?;
229	}
230	write!(writer, "=>")?;
231	body.print(writer)
232	}
233	JsExprData::Ternary { condition, yes, no } => {
234	condition.print(writer)?;
235	write!(writer, "?")?;
236	yes.print(writer)?;
237	write!(writer, ":")?;
238	if let Some(no) = no {
239	no.print(writer)?;
240	} else {
241	write!(writer, "null")?;
242	}
243	Ok(())
244	}
245	JsExprData::Comma { items } => {
246	for (i, item) in items.iter().enumerate() {
247	if i > 0 {
248	write!(writer, ",")?;
249	}
250	item.print(writer)?;
251	}
252	Ok(())
253	}
254	JsExprData::Void(expr) => {
255	write!(writer, "void ")?;
256	expr.print(writer)
257	}
258	}
259	}
260	}
261
262	impl Print for JSFunctionDeclaration {
263	fn print(&self, writer: &mut dyn Write) -> Result<()> {
264	write!(writer, "function {}", self.name)?;
265	write!(writer, "(")?;
266	for (i, param) in self.params.iter().enumerate() {
267	if i > 0 {
268	write!(writer, ",")?;
269	}
270	write!(writer, "{}", param)?;
271	}
272	write!(writer, ")")?;
273	self.block.print(writer)
274	}
275	}
276
277	impl Print for JsBlock {
278	fn print(&self, writer: &mut dyn Write) -> Result<()> {
279	write!(writer, "{{")?;
280	for statement in self.statements.iter() {
281	statement.print(writer)?;
282	}
283	write!(writer, "}}")
284	}
285	}
286
287	impl Print for JsIf {
288	fn print(&self, writer: &mut dyn Write) -> Result<()> {
289	write!(writer, "if(")?;
290	self.condition.print(writer)?;
291	write!(writer, ")")?;
292	self.yes.print(writer)?;
293	if let Some(no) = &self.no {
294	no.print(writer)?;
295	}
296	Ok(())
297	}
298	}
299
300	impl Print for JsElse {
301	fn print(&self, writer: &mut dyn Write) -> Result<()> {
302	match self {
303	JsElse::If(js_if) => {
304	write!(writer, "else ")?;
305	js_if.print(writer)
306	}
307	JsElse::Block(block) => {
308	write!(writer, "else")?;
309	block.print(writer)
310	}
311	JsElse::Expr(expr) => {
312	write!(writer, "else ")?;
313	expr.print(writer)
314	}
315	}
316	}
317	}
318
319	impl Print for JsKV {
320	fn print(&self, writer: &mut dyn Write) -> Result<()> {
321	match &self.key.data {
322	JsExprData::String(string) => {
323	if is_js_identifier(string) {
324	write!(writer, "{}", string)?;
325	} else {
326	print_js_string(writer, string.as_str())?;
327	}
328	}
329	_ => {
330	write!(writer, "[")?;
331	self.key.print(writer)?;
332	write!(writer, "]")?;
333	}
334	}
335	write!(writer, ":")?;
336	self.key.print(writer)?;
337	write!(writer, ":")?;
338	self.value.print(writer)
339	}
340	}
341
342	impl Print for JsModule {
343	fn print(&self, writer: &mut dyn Write) -> Result<()> {
344	for statement in self.statements.iter() {
345	statement.print(writer)?;
346	}
347	Ok(())
348	}
349	}
350
351	impl Print for JsArrowBody {
352	fn print(&self, writer: &mut dyn Write) -> Result<()> {
353	match self {
354	JsArrowBody::Block(block) => block.print(writer),
355	JsArrowBody::Expr(expr) => expr.print(writer),
356	}
357	}
358	}

crates/js_printer/printer_tests.rs created+53

...	@@ -0,0 +1,53 @@
1	#![cfg(test)]
2	use crate::Print;
3
4	use js_ast::JsExprData as E;
5	use num::{bigint::Sign, BigInt};
6
7	macro_rules! assert_printed {
8	($value:expr, $expected:expr) => {
9	assert_eq!($value.print_to_string(), $expected);
10	};
11	}
12
13	#[test]
14	fn number() {
15	assert_printed!(E::Number(1.0), "1");
16	assert_printed!(E::Number(1.5), "1.5");
17	assert_printed!(E::Number(1.5e10), "15000000000");
18	assert_printed!(E::Number(1.5e-10), "0.00000000015");
19	assert_printed!(E::Number(-1.0), "-1");
20	assert_printed!(E::Number(-1.5), "-1.5");
21	assert_printed!(E::Number(0f64), "0");
22	}
23
24	#[test]
25	fn bigint() {
26	assert_printed!(E::BigInt(BigInt::new(Sign::Plus, vec![1])), "1n");
27	assert_printed!(
28	E::BigInt(BigInt::new(Sign::Plus, vec![10000000])),
29	"10000000n"
30	);
31	assert_printed!(
32	E::BigInt(BigInt::new(Sign::Minus, vec![10000000])),
33	"-10000000n"
34	);
35	}
36
37	#[test]
38	fn boolean() {
39	assert_printed!(E::Bool(true), "true");
40	assert_printed!(E::Bool(false), "false");
41	}
42
43	#[test]
44	fn string() {
45	assert_printed!(E::str(""), "\"\"");
46	assert_printed!(E::str("hello"), "\"hello\"");
47	assert_printed!(E::str("hello\nworld"), "\"hello\\nworld\"");
48	}
49
50	#[test]
51	fn identifier() {
52	assert_printed!(E::str_id("hello"), "hello");
53	}

crates/js_printer/src/print_expression.rs+9-5

...	@@ -46,7 +46,7 @@ pub fn print_expression(expr: &JsExpr) -> String {	...	@@ -46,7 +46,7 @@ pub fn print_expression(expr: &JsExpr) -> String {
46	.join(",")	46	.join(",")
47	),	47	),
48	JsExprData::Binary {	48	JsExprData::Binary {
49	operator,	49	op: operator,
50	left,	50	left,
51	right,	51	right,
52	} => format!(	52	} => format!(
...	@@ -81,7 +81,10 @@ pub fn print_expression(expr: &JsExpr) -> String {	...	@@ -81,7 +81,10 @@ pub fn print_expression(expr: &JsExpr) -> String {
81	),	81	),
82	JsExprData::Null => todo!(),	82	JsExprData::Null => todo!(),
83	JsExprData::Undefined => todo!(),	83	JsExprData::Undefined => todo!(),
84	JsExprData::Unary { operator, value } => todo!(),	84	JsExprData::Unary {
		85	op: operator,
		86	value,
		87	} => todo!(),
85	JsExprData::Dot { base, name } => todo!(),	88	JsExprData::Dot { base, name } => todo!(),
86	JsExprData::Index { base, index } => todo!(),	89	JsExprData::Index { base, index } => todo!(),
87	JsExprData::Function {	90	JsExprData::Function {
...	@@ -89,9 +92,10 @@ pub fn print_expression(expr: &JsExpr) -> String {	...	@@ -89,9 +92,10 @@ pub fn print_expression(expr: &JsExpr) -> String {
89	params,	92	params,
90	block,	93	block,
91	} => todo!(),	94	} => todo!(),
92	JsExprData::If { condition, yes, no } => todo!(),	95	JsExprData::Ternary { condition, yes, no } => todo!(),
93	JsExprData::ArrowFunction { params, block } => todo!(),	96	JsExprData::ArrowFunction { params, block } => todo!(),
94	JsExprData::Comma { items } => todo!(),	97	JsExprData::Comma { items } => todo!(),
		98	JsExprData::Void(_) => todo!(),
95	}	99	}
96	}	100	}
97		101
...	@@ -191,7 +195,7 @@ mod tests {	...	@@ -191,7 +195,7 @@ mod tests {
191	fn operation() {	195	fn operation() {
192	assert_eq!(	196	assert_eq!(
193	print_expression(&JsExprData::Binary {	197	print_expression(&JsExprData::Binary {
194	operator: AnyOperator::Add,	198	op: AnyOperator::Add,
195	left: Box::new(JsExprData::Number(1.0)),	199	left: Box::new(JsExprData::Number(1.0)),
196	right: Box::new(JsExprData::Number(2.0)),	200	right: Box::new(JsExprData::Number(2.0)),
197	}),	201	}),
...	@@ -199,7 +203,7 @@ mod tests {	...	@@ -199,7 +203,7 @@ mod tests {
199	);	203	);
200	assert_eq!(	204	assert_eq!(
201	print_expression(&JsExprData::Binary {	205	print_expression(&JsExprData::Binary {
202	operator: AnyOperator::Sub,	206	op: AnyOperator::Sub,
203	left: Box::new(JsExprData::Number(1.0)),	207	left: Box::new(JsExprData::Number(1.0)),
204	right: Box::new(JsExprData::Number(2.0)),	208	right: Box::new(JsExprData::Number(2.0)),
205	}),	209	}),

crates/js_printer/src/print_statement.rs+6-6

...	@@ -1,4 +1,4 @@	...	@@ -1,4 +1,4 @@
1	use js_ast::{JsStatement, JsStatementData};	1	use js_ast::{If, JSDeclaration, JSFunctionDeclaration, JsStatement, JsStatementData};
2		2
3	use crate::print_expression::print_expression;	3	use crate::print_expression::print_expression;
4		4
...	@@ -8,11 +8,11 @@ pub fn print_statement(statement: &JsStatement) -> String {	...	@@ -8,11 +8,11 @@ pub fn print_statement(statement: &JsStatement) -> String {
8	// JsStatementData::Export(value) => format!("export {}", print_statement(value)),	8	// JsStatementData::Export(value) => format!("export {}", print_statement(value)),
9	JsStatementData::Export(value) => todo!(),	9	JsStatementData::Export(value) => todo!(),
10	JsStatementData::Return(value) => format!("return {};", print_expression(&value)),	10	JsStatementData::Return(value) => format!("return {};", print_expression(&value)),
11	JsStatementData::Declaration {	11	JsStatementData::Declaration(Declaration {
12	name,	12	name,
13	init,	13	init,
14	is_constant,	14	is_constant,
15	} => {	15	}) => {
16	format!(	16	format!(
17	"{} {}={};",	17	"{} {}={};",
18	match is_constant {	18	match is_constant {
...	@@ -23,11 +23,11 @@ pub fn print_statement(statement: &JsStatement) -> String {	...	@@ -23,11 +23,11 @@ pub fn print_statement(statement: &JsStatement) -> String {
23	print_expression(&init)	23	print_expression(&init)
24	)	24	)
25	}	25	}
26	JsStatementData::Function {	26	JsStatementData::Function(JSFunctionDeclaration {
27	name,	27	name,
28	block,	28	block,
29	params,	29	params,
30	} => format!(	30	}) => format!(
31	"function {}({}){{{}}}",	31	"function {}({}){{{}}}",
32	name,	32	name,
33	params.join(","),	33	params.join(","),
...	@@ -38,7 +38,7 @@ pub fn print_statement(statement: &JsStatement) -> String {	...	@@ -38,7 +38,7 @@ pub fn print_statement(statement: &JsStatement) -> String {
38	.collect::<Vec<String>>()	38	.collect::<Vec<String>>()
39	.join("")	39	.join("")
40	),	40	),
41	JsStatementData::If { condition, yes, no } => {	41	JsStatementData::If(JsIf { condition, yes, no }) => {
42	let else_value = match no {	42	let else_value = match no {
43	Some(value) => format!(	43	Some(value) => format!(
44	"else{{{}}}",	44	"else{{{}}}",

crates/parser/parse_block.rs+23-9

 @@ -15,7 +15,11 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
     let was_top_level = p.is_top_level;
     p.is_top_level = false;
-    loop {
+    'outer: loop {
+        let is_statement_like_expression = p
+            .peek()
+            .map_or(false, |x| x.is_start_of_statement_like_expression());
         let statement = match parse_statement(p) {
             Ok(statement) => statement,
             Err(err) => {
 @@ -25,8 +29,9 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
         };
         statements.push(statement.clone());
-        match p.next() {
+        match p.peek() {
             Some(Token::Semicolon) => loop {
+                p.next();
                 match p.peek() {
                     Some(Token::ClosedBrace) => {
                         p.next();
 @@ -36,14 +41,18 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
                             has_implicit_return: false,
                         });
                     }
-                    _ => continue,
+                    _ => continue 'outer,
                 }
             },
             Some(Token::ClosedBrace) => {
-                if statement.requires_semicolon() {
+                p.next();
+                if !statement.is_expression()
+                    && !statement.is_comment()
+                    && is_statement_like_expression
+                {
                     p.is_top_level = was_top_level;
                     return Err(p
-                        .error("Expected semicolon after this statement".to_string())
+                        .error("Expected semicolon or '}' after statement".to_string())
                         .into());
                 }
 @@ -54,10 +63,15 @@ pub fn parse_block(p: &mut Parser) -> Result<Block> {
                 });
             }
             _ => {
-                p.is_top_level = was_top_level;
+                if !statement.is_expression()
-                return Err(p
+                    && !statement.is_comment()
-                    .error("Expected semicolon or '}' after statement".to_string())
+                    && is_statement_like_expression
-                    .into());
+                {
+                    p.is_top_level = was_top_level;
+                    return Err(p
+                        .error("Expected semicolon or '}' after statement".to_string())
+                        .into());
+                }
             }
         };
     }

crates/parser/parse_call_arguments.rs+5

 @@ -8,6 +8,11 @@ use crate::{parse_expr::parse_expr, Parser};
 pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
     let mut args = Vec::new();
+    if let Some(Token::ClosedParen) = p.peek() {
+        _ = p.next();
+        return Ok(args);
+    }
     loop {
         let arg = parse_expr(p)?;
         args.push(arg);

crates/parser/parse_expr.rs+44-17

 @@ -1,4 +1,5 @@
-use ast::{get_binary_op_level, BinaryOperator, BinaryOperatorLevel, Expr, ExprData};
+use ast::Associativity::*;
+use ast::{BinaryOperator, Expr, ExprData, OperatorLevel};
 use miette::Result;
 use crate::parse_expr_basic::parse_expr_basic;
 @@ -15,14 +16,27 @@ pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
     // Parse operators
     loop {
         if let Some(token) = p.peek() {
-            if let Some(level) = get_binary_op_level(&token) {
+            if let Some(level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
                 _ = p.next();
-                expr = parse_expr_binary(
+                match level.associativity() {
-                    p,
+                    RightToLeft => {
-                    BinaryOperator::from_token(&token).unwrap(),
+                        expr = parse_expr_binary(
-                    expr.clone(),
+                            p,
-                    level,
+                            BinaryOperator::from_token(&token).unwrap(),
-                )?;
+                            expr,
+                            level,
+                        )?;
+                    }
+                    LeftToRight => {
+                        // TODO(@paperdave): TODO
+                        expr = parse_expr_binary(
+                            p,
+                            BinaryOperator::from_token(&token).unwrap(),
+                            expr,
+                            level,
+                        )?;
+                    }
+                }
                 continue;
             }
         }
 @@ -37,22 +51,35 @@ fn parse_expr_binary(
     p: &mut Parser,
     op: BinaryOperator,
     left: Expr,
-    level: BinaryOperatorLevel,
+    level: OperatorLevel,
 ) -> Result<Expr> {
     let loc = p.loc();
     let mut right = parse_expr_basic(p)?;
     loop {
         if let Some(token) = p.peek() {
-            if let Some(next_level) = get_binary_op_level(&token) {
+            if let Some(new_level) = BinaryOperator::from_token(&token).map(|o| o.level()) {
-                if next_level <= level {
+                if new_level <= level {
                     _ = p.next();
-                    right = parse_expr_binary(
+                    match new_level.associativity() {
-                        p,
+                        RightToLeft => {
-                        BinaryOperator::from_token(&token).unwrap(),
+                            right = parse_expr_binary(
-                        right,
+                                p,
-                        next_level,
+                                BinaryOperator::from_token(&token).unwrap(),
-                    )?;
+                                right,
+                                new_level,
+                            )?;
+                        }
+                        LeftToRight => {
+                            // TODO(@paperdave): TODO
+                            right = parse_expr_binary(
+                                p,
+                                BinaryOperator::from_token(&token).unwrap(),
+                                right,
+                                new_level,
+                            )?;
+                        }
+                    }
                     continue;
                 }
             }

crates/parser/parse_expr_if.rs+9-13

 @@ -1,4 +1,4 @@
-use ast::{BlockWithLoc, Expr, ExprData, If};
+use ast::{BlockOrIf, BlockWithLoc, If};
 use miette::Result;
 use tokenizer::Token;
 @@ -48,21 +48,17 @@ pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
             match p.peek() {
                 Some(Token::OpenBrace) => {
                     let loc = p.loc();
-                    let comments = p.take_comments();
+                    // let comments = p.take_comments();
-                    Some(Box::new(Expr::new(
+                    Some(BlockOrIf::Block(BlockWithLoc {
-                        ExprData::Block(parse_block(p)?),
+                        block: parse_block(p)?,
                         loc,
-                        comments,
+                    }))
-                    )))
                 }
                 Some(Token::If) => {
-                    let loc = p.loc();
+                    // TODO(@paperdave): Save location for the if
-                    let comments = p.take_comments();
+                    // let loc = p.loc();
-                    Some(Box::new(Expr::new(
+                    // let comments = p.take_comments();
-                        ExprData::If(parse_expr_if(p)?),
+                    Some(BlockOrIf::If(Box::new(parse_expr_if(p)?)))
-                        loc,
-                        comments,
-                    )))
                 }
                 _ => return Err(p.error("Expected a block after else".to_string()).into()),
             }

crates/parser/parse_fn_declaration.rs+9-2

 @@ -52,7 +52,10 @@ pub fn parse_fn_declaration(
                             break;
                         }
                     }
-                    Some(Token::ClosedParen) => break,
+                    Some(Token::ClosedParen) => {
+                        p.next();
+                        break;
+                    }
                     _ => {
                         return Err(p
                             .error("Expected ',' or ')' after identifier".to_string())
 @@ -108,8 +111,12 @@ pub fn parse_fn_declaration(
             loc: loc,
         })
     } else {
+        let actually = p.peek().map_or("EOF".to_string(), |x| x.to_string());
         return Err(p
-            .error("Expected a block to follow function declaration".to_string())
+            .error(format!(
+                "Expected a block to follow function declaration, but got {}",
+                actually,
+            ))
             .into());
     }
 }

crates/parser/parse_module.rs+5-1

...	@@ -11,6 +11,10 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {	...	@@ -11,6 +11,10 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
11	let file_doc_comment = maybe_read_doc_comment(p);	11	let file_doc_comment = maybe_read_doc_comment(p);
12		12
13	loop {	13	loop {
		14	let is_statement_like_expression = p
		15	.peek()
		16	.map_or(false, \|x\| x.is_start_of_statement_like_expression());
		17
14	let statement = parse_top_level_statement(p)?;	18	let statement = parse_top_level_statement(p)?;
15		19
16	match p.peek() {	20	match p.peek() {
...	@@ -20,7 +24,7 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {	...	@@ -20,7 +24,7 @@ pub fn parse_module(p: &mut Parser) -> Result<Module> {
20	}	24	}
21	}	25	}
22	_ => {	26	_ => {
23	if statement.requires_semicolon() {	27	if !statement.is_comment() && is_statement_like_expression {
24	return Err(p	28	return Err(p
25	.error("Expected semicolon after statement".to_string())	29	.error("Expected semicolon after statement".to_string())
26	.into());	30	.into());

crates/sema/Cargo.toml+1

...	@@ -4,6 +4,7 @@ version = "0.1.0"	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	edition = "2021"	4	edition = "2021"
5		5
6	[lib]	6	[lib]
		7	path = "./sema.rs"
7		8
8	[dependencies]	9	[dependencies]
9	miette.workspace = true	10	miette.workspace = true

crates/sema/ast_type_coersion.rs created+54

...	@@ -0,0 +1,54 @@
1	// use ast::{BinaryOperator, ExprData, Type};
2
3	// impl TypeCoerceable for ExprData {
4	// fn to_type(&self) -> Option<Type> {
5	// match self {
6	// ExprData::Block(block) => {
7	// if (block.has_implicit_return) {
8	// block.statements.last().unwrap().to_type()
9	// } else {
10	// Some(Type::Void)
11	// }
12	// }
13	// ExprData::Binary { op, left, right } => {
14	// // TODO: this likely has to do further type checking
15	// let left_type = left.to_type();
16	// match op {
17	// BinaryOperator::Add => left_type,
18	// BinaryOperator::Sub => left_type,
19	// BinaryOperator::Mul => left_type,
20	// BinaryOperator::Div => left_type,
21	// BinaryOperator::Pow => left_type,
22	// BinaryOperator::Eq => Some(Type::Bool),
23	// BinaryOperator::Gt => Some(Type::Bool),
24	// BinaryOperator::Lt => Some(Type::Bool),
25	// BinaryOperator::Gte => Some(Type::Bool),
26	// BinaryOperator::Lte => Some(Type::Bool),
27	// BinaryOperator::Lshift => Some(Type::Int),
28	// BinaryOperator::Rshift => Some(Type::Int),
29	// BinaryOperator::BitAnd => Some(Type::Int),
30	// BinaryOperator::BitOr => Some(Type::Int),
31	// BinaryOperator::BitXor => Some(Type::Int),
32	// BinaryOperator::Mod => Some(Type::Int),
33	// BinaryOperator::NotEq => Some(Type::Int),
34
35	// BinaryOperator::And => Some(Type::Bool),
36	// BinaryOperator::Or => Some(Type::Bool),
37	// }
38	// }
39	// ExprData::Unary { op, value } => todo!(),
40	// ExprData::Function {
41	// name,
42	// params,
43	// return_type,
44	// block,
45	// } => Some(return_type),
46	// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47	// ExprData::Builtin { id, arguments } => todo!(),
48	// ExprData::If(_) => todo!(),
49	// ExprData::Return(_) => todo!(),
50	// ExprData::Literal(_) => todo!(),
51	// ExprData::Identifier(_) => todo!(),
52	// }
53	// }
54	// }

crates/sema/sema.rs created+98

...	@@ -0,0 +1,98 @@
1	use ast::Module;
2	use miette::Result;
3
4	pub fn analyze_module(_block: &mut Module) -> Result<()> {
5	//
6	Ok(())
7	}
8
9	// for statement in &block.statements {
10	// check_expression(statement)?;
11	// }
12
13	// Ok(())
14	// }
15
16	// pub fn resolve_type(expr: &Ast) -> Result<Type> {
17	// match expr {
18	// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
19	// Ast::Declaration {
20	// name,
21	// decl_type,
22	// value,
23	// is_constant,
24	// } => Ok(Type::NoReturn),
25	// Ast::Binary {
26	// operator,
27	// left,
28	// right,
29	// } => merge_types(resolve_type(left)?, resolve_type(right)?),
30	// Ast::Function {
31	// name,
32	// arguments,
33	// return_type,
34	// block,
35	// } => todo!(),
36	// Ast::FunctionCall { name, arguments } => todo!(),
37	// Ast::If {
38	// condition,
39	// value,
40	// else_value,
41	// } => todo!(),
42	// Ast::Return(_) => Ok(Type::NoReturn),
43	// Ast::Public(_) => todo!(), // TODO: remove Public
44	// Ast::Block(_) => todo!(),
45	// Ast::Bool(_) => Ok(Type::Bool),
46	// Ast::String(_) => Ok(Type::String),
47	// Ast::Identifier(_) => todo!(),
48	// Ast::BuiltIn(_) => todo!(),
49
50	// Ast::Comment(_) => unreachable!(),
51	// Ast::DocComment(_) => unreachable!(),
52	// }
53	// }
54
55	// pub fn check_expression(expr: &Ast) -> Result<()> {
56	// panic!()
57	// match expr {
58	// Ast::Declaration {
59	// name,
60	// decl_type: return_type,
61	// value,
62	// is_constant,
63	// } => {
64	// check_expression(value)?;
65	// // coerce_type(return_type.clone(), resolve_type(value)?)?;
66	// }
67	// _ => {}
68	// // Ast::Operation {
69	// // operator,
70	// // left,
71	// // right,
72	// // } => Ok(()),
73	// // Ast::Function {
74	// // name,
75	// // arguments,
76	// // return_type,
77	// // block,
78	// // } => Ok(()),
79	// // Ast::FunctionCall { name, arguments } => Ok(()),
80	// // Ast::If {
81	// // condition,
82	// // value,
83	// // else_value,
84	// // } => Ok(()),
85	// // Ast::Return(_) => Ok(()),
86	// // Ast::Public(_) => Ok(()),
87	// // Ast::Block(_) => Ok(()),
88	// // Ast::Number(_) => Ok(()),
89	// // Ast::Bool(_) => Ok(()),
90	// // Ast::String(_) => Ok(()),
91	// // Ast::Identifier(_) => Ok(()),
92	// // Ast::BuiltIn(_) => Ok(()),
93	// // Ast::Comment(_) => Ok(()),
94	// // Ast::DocComment(_) => Ok(()),
95	// }
96
97	// Ok(())
98	// }

crates/sema/src/ast_type_coersion.rs deleted-54

...	@@ -1,54 +0,0 @@
1	// use ast::{BinaryOperator, ExprData, Type};
2
3	// impl TypeCoerceable for ExprData {
4	// fn to_type(&self) -> Option<Type> {
5	// match self {
6	// ExprData::Block(block) => {
7	// if (block.has_implicit_return) {
8	// block.statements.last().unwrap().to_type()
9	// } else {
10	// Some(Type::Void)
11	// }
12	// }
13	// ExprData::Binary { op, left, right } => {
14	// // TODO: this likely has to do further type checking
15	// let left_type = left.to_type();
16	// match op {
17	// BinaryOperator::Add => left_type,
18	// BinaryOperator::Sub => left_type,
19	// BinaryOperator::Mul => left_type,
20	// BinaryOperator::Div => left_type,
21	// BinaryOperator::Pow => left_type,
22	// BinaryOperator::Eq => Some(Type::Bool),
23	// BinaryOperator::Gt => Some(Type::Bool),
24	// BinaryOperator::Lt => Some(Type::Bool),
25	// BinaryOperator::Gte => Some(Type::Bool),
26	// BinaryOperator::Lte => Some(Type::Bool),
27	// BinaryOperator::Lshift => Some(Type::Int),
28	// BinaryOperator::Rshift => Some(Type::Int),
29	// BinaryOperator::BitAnd => Some(Type::Int),
30	// BinaryOperator::BitOr => Some(Type::Int),
31	// BinaryOperator::BitXor => Some(Type::Int),
32	// BinaryOperator::Mod => Some(Type::Int),
33	// BinaryOperator::NotEq => Some(Type::Int),
34
35	// BinaryOperator::And => Some(Type::Bool),
36	// BinaryOperator::Or => Some(Type::Bool),
37	// }
38	// }
39	// ExprData::Unary { op, value } => todo!(),
40	// ExprData::Function {
41	// name,
42	// params,
43	// return_type,
44	// block,
45	// } => Some(return_type),
46	// ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47	// ExprData::Builtin { id, arguments } => todo!(),
48	// ExprData::If(_) => todo!(),
49	// ExprData::Return(_) => todo!(),
50	// ExprData::Literal(_) => todo!(),
51	// ExprData::Identifier(_) => todo!(),
52	// }
53	// }
54	// }

crates/sema/src/lib.rs deleted-96

...	@@ -1,96 +0,0 @@
1	// use ast::{ Module, Type};
2	// use miette::Result;
3
4	// mod ast_type_coersion;
5
6	// pub fn check_module(block: &Module) -> Result<()> {
7	// for statement in &block.statements {
8	// check_expression(statement)?;
9	// }
10
11	// Ok(())
12	// }
13
14	// pub fn resolve_type(expr: &Ast) -> Result<Type> {
15	// match expr {
16	// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
17	// Ast::Declaration {
18	// name,
19	// decl_type,
20	// value,
21	// is_constant,
22	// } => Ok(Type::NoReturn),
23	// Ast::Binary {
24	// operator,
25	// left,
26	// right,
27	// } => merge_types(resolve_type(left)?, resolve_type(right)?),
28	// Ast::Function {
29	// name,
30	// arguments,
31	// return_type,
32	// block,
33	// } => todo!(),
34	// Ast::FunctionCall { name, arguments } => todo!(),
35	// Ast::If {
36	// condition,
37	// value,
38	// else_value,
39	// } => todo!(),
40	// Ast::Return(_) => Ok(Type::NoReturn),
41	// Ast::Public(_) => todo!(), // TODO: remove Public
42	// Ast::Block(_) => todo!(),
43	// Ast::Bool(_) => Ok(Type::Bool),
44	// Ast::String(_) => Ok(Type::String),
45	// Ast::Identifier(_) => todo!(),
46	// Ast::BuiltIn(_) => todo!(),
47
48	// Ast::Comment(_) => unreachable!(),
49	// Ast::DocComment(_) => unreachable!(),
50	// }
51	// }
52
53	// pub fn check_expression(expr: &Ast) -> Result<()> {
54	// panic!()
55	// match expr {
56	// Ast::Declaration {
57	// name,
58	// decl_type: return_type,
59	// value,
60	// is_constant,
61	// } => {
62	// check_expression(value)?;
63	// // coerce_type(return_type.clone(), resolve_type(value)?)?;
64	// }
65	// _ => {}
66	// // Ast::Operation {
67	// // operator,
68	// // left,
69	// // right,
70	// // } => Ok(()),
71	// // Ast::Function {
72	// // name,
73	// // arguments,
74	// // return_type,
75	// // block,
76	// // } => Ok(()),
77	// // Ast::FunctionCall { name, arguments } => Ok(()),
78	// // Ast::If {
79	// // condition,
80	// // value,
81	// // else_value,
82	// // } => Ok(()),
83	// // Ast::Return(_) => Ok(()),
84	// // Ast::Public(_) => Ok(()),
85	// // Ast::Block(_) => Ok(()),
86	// // Ast::Number(_) => Ok(()),
87	// // Ast::Bool(_) => Ok(()),
88	// // Ast::String(_) => Ok(()),
89	// // Ast::Identifier(_) => Ok(()),
90	// // Ast::BuiltIn(_) => Ok(()),
91	// // Ast::Comment(_) => Ok(()),
92	// // Ast::DocComment(_) => Ok(()),
93	// }
94
95	// Ok(())
96	// }

crates/tokenizer/Cargo.toml+1

...	@@ -4,6 +4,7 @@ version = "0.1.0"	...	@@ -4,6 +4,7 @@ version = "0.1.0"
4	edition = "2021"	4	edition = "2021"
5		5
6	[lib]	6	[lib]
		7	path="./tokenizer.rs"
7		8
8	[dependencies]	9	[dependencies]
9	miette.workspace = true	10	miette.workspace = true

crates/tokenizer/src/lib.rs deleted-393

...	@@ -1,393 +0,0 @@
1	use miette::{Diagnostic, Result};
2	use thiserror::Error;
3
4	#[derive(Debug, PartialEq, Clone)]
5	pub struct Loc {}
6
7	impl Loc {
8	pub const EMPTY: Loc = Loc {};
9
10	pub fn new() -> Self {
11	Self {}
12	}
13	}
14
15	#[derive(Error, Diagnostic, Debug)]
16	#[error("UnknownCharacter")]
17	struct UnknownCharacter {
18	#[help]
19	error: String,
20	}
21
22	#[derive(Debug, Clone)]
23	pub enum Token {
24	// Literals
25	Identifier(String),
26	Number(f64),
27	String(String),
28	// Comments
29	Comment(String),
30	DocComment(String),
31	// Keywords
32	Else,
33	False,
34	Fn,
35	If,
36	Let,
37	Pub,
38	Return,
39	True,
40	Var,
41	Void,
42	// Symbols
43	And, // &
44	AndAnd, // &&
45	Bang, // !
46	BangEqual, // !=
47	Caret, // ^
48	ClosedBrace, // }
49	ClosedBracket, // ]
50	ClosedParen, // )
51	Colon, // :
52	Comma, // ,
53	Equal, // =
54	EqualEqual, // ==
55	Gt, // >
56	GtEqual, // >=
57	GtGt, // >>
58	GtGtGt, // >>>
59	Lt, // <
60	LtEqual, // <=
61	LtLt, // <<
62	Minus, // -
63	OpenBrace, // {
64	OpenBracket, // [
65	OpenParen, // (
66	Or, // \|
67	OrOr, // \|\|
68	Percent, // %
69	Plus, // +
70	RightArrow, // ->
71	Semicolon, // ;
72	Slash, // /
73	Star, // *
74	StarStar, // **
75	Tilde, // ~
76	Dot, // .
77	}
78
79	impl std::fmt::Display for Token {
80	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81	let token = match self {
82	Token::Identifier(_) => "identifier".to_string(),
83
84	Token::Number(n) => n.to_string(),
85	Token::String(s) => format!("\"{}\"", s),
86	Token::Comment(_) => "comment".to_string(),
87	Token::DocComment(_) => "documentation comment".to_string(),
88
89	Token::Else => "else".to_string(),
90	Token::False => "false".to_string(),
91	Token::Fn => "fn".to_string(),
92	Token::If => "if".to_string(),
93	Token::Let => "let".to_string(),
94	Token::Pub => "pub".to_string(),
95	Token::Return => "return".to_string(),
96	Token::True => "true".to_string(),
97	Token::Var => "var".to_string(),
98	Token::Void => "void".to_string(),
99
100	Token::And => "&".to_string(),
101	Token::AndAnd => "&&".to_string(),
102	Token::Bang => "!".to_string(),
103	Token::BangEqual => "!=".to_string(),
104	Token::Caret => "^".to_string(),
105	Token::ClosedBrace => "}".to_string(),
106	Token::ClosedBracket => "]".to_string(),
107	Token::ClosedParen => ")".to_string(),
108	Token::Colon => ":".to_string(),
109	Token::Comma => ",".to_string(),
110	Token::Equal => "=".to_string(),
111	Token::EqualEqual => "==".to_string(),
112	Token::Gt => ">".to_string(),
113	Token::GtEqual => ">=".to_string(),
114	Token::GtGt => ">>".to_string(),
115	Token::Lt => "<".to_string(),
116	Token::LtEqual => "<=".to_string(),
117	Token::LtLt => "<<".to_string(),
118	Token::Minus => "-".to_string(),
119	Token::OpenBrace => "{".to_string(),
120	Token::OpenBracket => "[".to_string(),
121	Token::OpenParen => "(".to_string(),
122	Token::Or => "\|".to_string(),
123	Token::OrOr => "\|\|".to_string(),
124	Token::Percent => "%".to_string(),
125	Token::Plus => "+".to_string(),
126	Token::RightArrow => "->".to_string(),
127	Token::Semicolon => ";".to_string(),
128	Token::Slash => "/".to_string(),
129	Token::Star => "*".to_string(),
130	Token::StarStar => "**".to_string(),
131	Token::Tilde => "~".to_string(),
132	Token::Dot => ".".to_string(),
133	Token::GtGtGt => ">>>".to_string(),
134	};
135
136	write!(f, "{}", token)
137	}
138	}
139
140	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
141	pub fn tokenize(content: String) -> Result<Vec<Token>> {
142	let mut tokens: Vec<Token> = Vec::new();
143	let mut chars = content.chars().enumerate().peekable();
144
145	loop {
146	match chars.next() {
147	Some((_, c)) => match c {
148	'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
149	let mut current_token = String::new();
150	current_token.push(c);
151	loop {
152	let c = chars.peek();
153	if c.is_none() {
154	break;
155	}
156	let (_, c) = c.unwrap();
157	match c {
158	'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
159	current_token.push(*c);
160	chars.next();
161	}
162	_ => break,
163	}
164	}
165
166	match current_token.as_str() {
167	"else" => tokens.push(Token::Else),
168	"false" => tokens.push(Token::False),
169	"fn" => tokens.push(Token::Fn),
170	"if" => tokens.push(Token::If),
171	"let" => tokens.push(Token::Let),
172	"pub" => tokens.push(Token::Pub),
173	"return" => tokens.push(Token::Return),
174	"true" => tokens.push(Token::True),
175	"var" => tokens.push(Token::Var),
176	"void" => tokens.push(Token::Void),
177	_ => tokens.push(Token::Identifier(current_token)),
178	}
179	}
180	'"' => {
181	let mut current_token = String::new();
182	loop {
183	let c = chars.next();
184	if c.is_none() {
185	break;
186	}
187	let (_, c) = c.unwrap();
188	match c {
189	'"' => break,
190	_ => current_token.push(c),
191	}
192	}
193	tokens.push(Token::String(current_token));
194	}
195	'0'..='9' => {
196	let mut current_token = String::new();
197	current_token.push(c);
198	loop {
199	let c = chars.peek();
200	if c.is_none() {
201	break;
202	}
203	let (_, c) = c.unwrap();
204	match c {
205	'0'..='9' => {
206	current_token.push(*c);
207	chars.next();
208	}
209	_ => break,
210	}
211	}
212	match current_token.parse::<f64>() {
213	Ok(v) => tokens.push(Token::Number(v)),
214	Err(_) => {
215	return Err(UnknownCharacter {
216	error: format!("Unknown character: {}", current_token),
217	}
218	.into())
219	}
220	}
221	}
222	':' => tokens.push(Token::Colon),
223	'=' => match chars.peek() {
224	Some((_, '=')) => {
225	chars.next();
226	tokens.push(Token::EqualEqual);
227	}
228	_ => tokens.push(Token::Equal),
229	},
230	';' => tokens.push(Token::Semicolon),
231	'>' => match chars.peek() {
232	Some((_, '=')) => {
233	chars.next();
234	tokens.push(Token::GtEqual);
235	}
236	Some((_, '>')) => {
237	chars.next();
238	match chars.peek() {
239	Some((_, '>')) => {
240	chars.next();
241	tokens.push(Token::GtGtGt);
242	}
243	_ => tokens.push(Token::GtGt),
244	}
245	}
246	_ => tokens.push(Token::Gt),
247	},
248	'<' => match chars.peek() {
249	Some((_, '=')) => {
250	chars.next();
251	tokens.push(Token::LtEqual);
252	}
253	Some((_, '<')) => {
254	chars.next();
255	tokens.push(Token::LtLt);
256	}
257	_ => tokens.push(Token::Lt),
258	},
259	'+' => tokens.push(Token::Plus),
260	'-' => match chars.peek() {
261	Some((_, '>')) => {
262	chars.next();
263	tokens.push(Token::RightArrow);
264	}
265	_ => tokens.push(Token::Minus),
266	},
267	'*' => match chars.peek() {
268	Some((_, '*')) => {
269	chars.next();
270	tokens.push(Token::StarStar);
271	}
272	_ => tokens.push(Token::Star),
273	},
274	'/' => {
275	match chars.peek() {
276	Some((_, '/')) => {
277	chars.next(); // Consume the second slash
278
279	let is_doc_comment = match chars.peek() {
280	Some((_, '/')) => {
281	chars.next(); // Consume the third slash
282	true
283	}
284	_ => false,
285	};
286
287	let mut comment = String::new();
288	loop {
289	// skip leading whitespace
290	while let Some((_, c)) = chars.peek() {
291	if !c.is_whitespace() {
292	break;
293	}
294	chars.next();
295	}
296	// read the comment
297	while let Some((_, c)) = chars.next() {
298	if c == '\n' \|\| c == '\r' {
299	break;
300	}
301	comment.push(c);
302	}
303	// trim trailing whitespace on line
304	comment = comment.trim_end().to_string();
305	// skip to next non-whitespace character
306	while let Some((_, c)) = chars.peek() {
307	if !c.is_whitespace() {
308	break;
309	}
310	chars.next();
311	}
312	// check if we've hit another comment
313	let mut tmp = chars.clone();
314
315	let is_another_comment = match tmp.peek() {
316	Some((_, '/')) => {
317	tmp.next(); // Consume the second slash
318	match tmp.peek() {
319	Some((_, '/')) => {
320	tmp.next(); // Consume the third slash
321	is_doc_comment
322	}
323	_ => !is_doc_comment,
324	}
325	}
326	_ => false,
327	};
328
329	if is_another_comment {
330	// apply our temporary iterator to the main iterator
331	comment.push('\n');
332	chars = tmp;
333	chars.next();
334	} else {
335	break;
336	}
337	}
338
339	if is_doc_comment {
340	tokens.push(Token::DocComment(comment));
341	} else {
342	tokens.push(Token::Comment(comment));
343	}
344	}
345	_ => tokens.push(Token::Slash),
346	}
347	}
348	',' => tokens.push(Token::Comma),
349	'(' => tokens.push(Token::OpenParen),
350	')' => tokens.push(Token::ClosedParen),
351	'{' => tokens.push(Token::OpenBrace),
352	'}' => tokens.push(Token::ClosedBrace),
353	'[' => tokens.push(Token::OpenBracket),
354	']' => tokens.push(Token::ClosedBracket),
355	'&' => match chars.peek() {
356	Some((_, '&')) => {
357	chars.next();
358	tokens.push(Token::AndAnd);
359	}
360	_ => tokens.push(Token::And),
361	},
362	'\|' => match chars.peek() {
363	Some((_, '\|')) => {
364	chars.next();
365	tokens.push(Token::OrOr);
366	}
367	_ => tokens.push(Token::Or),
368	},
369	'!' => match chars.peek() {
370	Some((_, '=')) => {
371	chars.next();
372	tokens.push(Token::BangEqual);
373	}
374	_ => tokens.push(Token::Bang),
375	},
376	'^' => tokens.push(Token::Caret),
377	'%' => tokens.push(Token::Percent),
378	'~' => tokens.push(Token::Tilde),
379	'.' => tokens.push(Token::Dot),
380	s if s.is_whitespace() => (),
381	_ => {
382	return Err(UnknownCharacter {
383	error: format!("Unexpected '{}' while tokenizing", c),
384	}
385	.into())
386	}
387	},
388	None => break,
389	}
390	}
391
392	Ok(tokens)
393	}

crates/tokenizer/tokenizer.rs created+402

...	@@ -0,0 +1,402 @@
1	use miette::{Diagnostic, Result};
2	use thiserror::Error;
3
4	#[derive(Debug, PartialEq, Clone)]
5	pub struct Loc {}
6
7	impl Loc {
8	pub const EMPTY: Loc = Loc {};
9
10	pub fn new() -> Self {
11	Self {}
12	}
13	}
14
15	#[derive(Error, Diagnostic, Debug)]
16	#[error("UnknownCharacter")]
17	struct UnknownCharacter {
18	#[help]
19	error: String,
20	}
21
22	#[derive(Debug, Clone)]
23	pub enum Token {
24	// Literals
25	Identifier(String),
26	Number(f64),
27	String(String),
28	// Comments
29	Comment(String),
30	DocComment(String),
31	// Keywords
32	Else,
33	False,
34	Fn,
35	If,
36	Let,
37	Pub,
38	Return,
39	True,
40	Var,
41	Void,
42	// Symbols
43	And, // &
44	AndAnd, // &&
45	Bang, // !
46	BangEqual, // !=
47	Caret, // ^
48	ClosedBrace, // }
49	ClosedBracket, // ]
50	ClosedParen, // )
51	Colon, // :
52	Comma, // ,
53	Equal, // =
54	EqualEqual, // ==
55	Gt, // >
56	GtEqual, // >=
57	GtGt, // >>
58	GtGtGt, // >>>
59	Lt, // <
60	LtEqual, // <=
61	LtLt, // <<
62	Minus, // -
63	OpenBrace, // {
64	OpenBracket, // [
65	OpenParen, // (
66	Or, // \|
67	OrOr, // \|\|
68	Percent, // %
69	Plus, // +
70	RightArrow, // ->
71	Semicolon, // ;
72	Slash, // /
73	Star, // *
74	StarStar, // **
75	Tilde, // ~
76	Dot, // .
77	}
78
79	impl Token {
80	pub fn is_start_of_statement_like_expression(&self) -> bool {
81	match &self {
82	Token::If \| Token::OpenBrace \| Token::Fn => false,
83	_ => true,
84	}
85	}
86	}
87
88	impl std::fmt::Display for Token {
89	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
90	let token = match self {
91	Token::Identifier(_) => "identifier".to_string(),
92
93	Token::Number(n) => n.to_string(),
94	Token::String(s) => format!("\"{}\"", s),
95	Token::Comment(_) => "comment".to_string(),
96	Token::DocComment(_) => "documentation comment".to_string(),
97
98	Token::Else => "else".to_string(),
99	Token::False => "false".to_string(),
100	Token::Fn => "fn".to_string(),
101	Token::If => "if".to_string(),
102	Token::Let => "let".to_string(),
103	Token::Pub => "pub".to_string(),
104	Token::Return => "return".to_string(),
105	Token::True => "true".to_string(),
106	Token::Var => "var".to_string(),
107	Token::Void => "void".to_string(),
108
109	Token::And => "&".to_string(),
110	Token::AndAnd => "&&".to_string(),
111	Token::Bang => "!".to_string(),
112	Token::BangEqual => "!=".to_string(),
113	Token::Caret => "^".to_string(),
114	Token::ClosedBrace => "}".to_string(),
115	Token::ClosedBracket => "]".to_string(),
116	Token::ClosedParen => ")".to_string(),
117	Token::Colon => ":".to_string(),
118	Token::Comma => ",".to_string(),
119	Token::Equal => "=".to_string(),
120	Token::EqualEqual => "==".to_string(),
121	Token::Gt => ">".to_string(),
122	Token::GtEqual => ">=".to_string(),
123	Token::GtGt => ">>".to_string(),
124	Token::Lt => "<".to_string(),
125	Token::LtEqual => "<=".to_string(),
126	Token::LtLt => "<<".to_string(),
127	Token::Minus => "-".to_string(),
128	Token::OpenBrace => "{".to_string(),
129	Token::OpenBracket => "[".to_string(),
130	Token::OpenParen => "(".to_string(),
131	Token::Or => "\|".to_string(),
132	Token::OrOr => "\|\|".to_string(),
133	Token::Percent => "%".to_string(),
134	Token::Plus => "+".to_string(),
135	Token::RightArrow => "->".to_string(),
136	Token::Semicolon => ";".to_string(),
137	Token::Slash => "/".to_string(),
138	Token::Star => "*".to_string(),
139	Token::StarStar => "**".to_string(),
140	Token::Tilde => "~".to_string(),
141	Token::Dot => ".".to_string(),
142	Token::GtGtGt => ">>>".to_string(),
143	};
144
145	write!(f, "{}", token)
146	}
147	}
148
149	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Loc
150	pub fn tokenize(content: String) -> Result<Vec<Token>> {
151	let mut tokens: Vec<Token> = Vec::new();
152	let mut chars = content.chars().enumerate().peekable();
153
154	loop {
155	match chars.next() {
156	Some((_, c)) => match c {
157	'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
158	let mut current_token = String::new();
159	current_token.push(c);
160	loop {
161	let c = chars.peek();
162	if c.is_none() {
163	break;
164	}
165	let (_, c) = c.unwrap();
166	match c {
167	'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
168	current_token.push(*c);
169	chars.next();
170	}
171	_ => break,
172	}
173	}
174
175	match current_token.as_str() {
176	"else" => tokens.push(Token::Else),
177	"false" => tokens.push(Token::False),
178	"fn" => tokens.push(Token::Fn),
179	"if" => tokens.push(Token::If),
180	"let" => tokens.push(Token::Let),
181	"pub" => tokens.push(Token::Pub),
182	"return" => tokens.push(Token::Return),
183	"true" => tokens.push(Token::True),
184	"var" => tokens.push(Token::Var),
185	"void" => tokens.push(Token::Void),
186	_ => tokens.push(Token::Identifier(current_token)),
187	}
188	}
189	'"' => {
190	let mut current_token = String::new();
191	loop {
192	let c = chars.next();
193	if c.is_none() {
194	break;
195	}
196	let (_, c) = c.unwrap();
197	match c {
198	'"' => break,
199	_ => current_token.push(c),
200	}
201	}
202	tokens.push(Token::String(current_token));
203	}
204	'0'..='9' => {
205	let mut current_token = String::new();
206	current_token.push(c);
207	loop {
208	let c = chars.peek();
209	if c.is_none() {
210	break;
211	}
212	let (_, c) = c.unwrap();
213	match c {
214	'0'..='9' => {
215	current_token.push(*c);
216	chars.next();
217	}
218	_ => break,
219	}
220	}
221	match current_token.parse::<f64>() {
222	Ok(v) => tokens.push(Token::Number(v)),
223	Err(_) => {
224	return Err(UnknownCharacter {
225	error: format!("Unknown character: {}", current_token),
226	}
227	.into())
228	}
229	}
230	}
231	':' => tokens.push(Token::Colon),
232	'=' => match chars.peek() {
233	Some((_, '=')) => {
234	chars.next();
235	tokens.push(Token::EqualEqual);
236	}
237	_ => tokens.push(Token::Equal),
238	},
239	';' => tokens.push(Token::Semicolon),
240	'>' => match chars.peek() {
241	Some((_, '=')) => {
242	chars.next();
243	tokens.push(Token::GtEqual);
244	}
245	Some((_, '>')) => {
246	chars.next();
247	match chars.peek() {
248	Some((_, '>')) => {
249	chars.next();
250	tokens.push(Token::GtGtGt);
251	}
252	_ => tokens.push(Token::GtGt),
253	}
254	}
255	_ => tokens.push(Token::Gt),
256	},
257	'<' => match chars.peek() {
258	Some((_, '=')) => {
259	chars.next();
260	tokens.push(Token::LtEqual);
261	}
262	Some((_, '<')) => {
263	chars.next();
264	tokens.push(Token::LtLt);
265	}
266	_ => tokens.push(Token::Lt),
267	},
268	'+' => tokens.push(Token::Plus),
269	'-' => match chars.peek() {
270	Some((_, '>')) => {
271	chars.next();
272	tokens.push(Token::RightArrow);
273	}
274	_ => tokens.push(Token::Minus),
275	},
276	'*' => match chars.peek() {
277	Some((_, '*')) => {
278	chars.next();
279	tokens.push(Token::StarStar);
280	}
281	_ => tokens.push(Token::Star),
282	},
283	'/' => {
284	match chars.peek() {
285	Some((_, '/')) => {
286	chars.next(); // Consume the second slash
287
288	let is_doc_comment = match chars.peek() {
289	Some((_, '/')) => {
290	chars.next(); // Consume the third slash
291	true
292	}
293	_ => false,
294	};
295
296	let mut comment = String::new();
297	loop {
298	// skip leading whitespace
299	while let Some((_, c)) = chars.peek() {
300	if !c.is_whitespace() {
301	break;
302	}
303	chars.next();
304	}
305	// read the comment
306	while let Some((_, c)) = chars.next() {
307	if c == '\n' \|\| c == '\r' {
308	break;
309	}
310	comment.push(c);
311	}
312	// trim trailing whitespace on line
313	comment = comment.trim_end().to_string();
314	// skip to next non-whitespace character
315	while let Some((_, c)) = chars.peek() {
316	if !c.is_whitespace() {
317	break;
318	}
319	chars.next();
320	}
321	// check if we've hit another comment
322	let mut tmp = chars.clone();
323
324	let is_another_comment = match tmp.peek() {
325	Some((_, '/')) => {
326	tmp.next(); // Consume the second slash
327	match tmp.peek() {
328	Some((_, '/')) => {
329	tmp.next(); // Consume the third slash
330	is_doc_comment
331	}
332	_ => !is_doc_comment,
333	}
334	}
335	_ => false,
336	};
337
338	if is_another_comment {
339	// apply our temporary iterator to the main iterator
340	comment.push('\n');
341	chars = tmp;
342	chars.next();
343	} else {
344	break;
345	}
346	}
347
348	if is_doc_comment {
349	tokens.push(Token::DocComment(comment));
350	} else {
351	tokens.push(Token::Comment(comment));
352	}
353	}
354	_ => tokens.push(Token::Slash),
355	}
356	}
357	',' => tokens.push(Token::Comma),
358	'(' => tokens.push(Token::OpenParen),
359	')' => tokens.push(Token::ClosedParen),
360	'{' => tokens.push(Token::OpenBrace),
361	'}' => tokens.push(Token::ClosedBrace),
362	'[' => tokens.push(Token::OpenBracket),
363	']' => tokens.push(Token::ClosedBracket),
364	'&' => match chars.peek() {
365	Some((_, '&')) => {
366	chars.next();
367	tokens.push(Token::AndAnd);
368	}
369	_ => tokens.push(Token::And),
370	},
371	'\|' => match chars.peek() {
372	Some((_, '\|')) => {
373	chars.next();
374	tokens.push(Token::OrOr);
375	}
376	_ => tokens.push(Token::Or),
377	},
378	'!' => match chars.peek() {
379	Some((_, '=')) => {
380	chars.next();
381	tokens.push(Token::BangEqual);
382	}
383	_ => tokens.push(Token::Bang),
384	},
385	'^' => tokens.push(Token::Caret),
386	'%' => tokens.push(Token::Percent),
387	'~' => tokens.push(Token::Tilde),
388	'.' => tokens.push(Token::Dot),
389	s if s.is_whitespace() => (),
390	_ => {
391	return Err(UnknownCharacter {
392	error: format!("Unexpected '{}' while tokenizing", c),
393	}
394	.into())
395	}
396	},
397	None => break,
398	}
399	}
400
401	Ok(tokens)
402	}

author	chloe caruso <git@paperclover.net> 2024-01-22 00:32:12-08:00
committer	chloe caruso <git@paperclover.net> 2024-01-22 00:32:12-08:00
log	12864f7c49c005271c6c5610dfaa2cb71ff56ae7
tree	d4f9793c94bcab05e4855cf894dee62e17476bd3
parent	b9cad6ad285b99192b686be1cea8b1b8cfcdd6c1

finish rewriting the project in rust