Commit 634a70376c0023f46a6aae6611ed672397eae557 | ix

.github/logo.png deleted

Binary files a/.github/logo.png and /dev/null differ

.gitignore+10-2

...	@@ -1,2 +1,10 @@	...	@@ -1,2 +1,10 @@
1	target	1	# If you have OS-specific or editor-specific files to ignore,
2	*.js	2	# such as *.swp or .DS_Store, put those in your global
		3	# ~/.gitignore and put this in your ~/.gitconfig:
		4	#
		5	# [core]
		6	# excludesfile = ~/.gitignore
		7	#
		8
		9	zig-out
		10	zig-cache

Cargo.lock deleted-759

...	@@ -1,759 +0,0 @@
1	# This file is automatically @generated by Cargo.
2	# It is not intended for manual editing.
3	version = 3
4
5	[[package]]
6	name = "addr2line"
7	version = "0.21.0"
8	source = "registry+https://github.com/rust-lang/crates.io-index"
9	checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
10	dependencies = [
11	"gimli",
12	]
13
14	[[package]]
15	name = "adler"
16	version = "1.0.2"
17	source = "registry+https://github.com/rust-lang/crates.io-index"
18	checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
19
20	[[package]]
21	name = "anstream"
22	version = "0.6.5"
23	source = "registry+https://github.com/rust-lang/crates.io-index"
24	checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
25	dependencies = [
26	"anstyle",
27	"anstyle-parse",
28	"anstyle-query",
29	"anstyle-wincon",
30	"colorchoice",
31	"utf8parse",
32	]
33
34	[[package]]
35	name = "anstyle"
36	version = "1.0.4"
37	source = "registry+https://github.com/rust-lang/crates.io-index"
38	checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
39
40	[[package]]
41	name = "anstyle-parse"
42	version = "0.2.3"
43	source = "registry+https://github.com/rust-lang/crates.io-index"
44	checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
45	dependencies = [
46	"utf8parse",
47	]
48
49	[[package]]
50	name = "anstyle-query"
51	version = "1.0.2"
52	source = "registry+https://github.com/rust-lang/crates.io-index"
53	checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
54	dependencies = [
55	"windows-sys",
56	]
57
58	[[package]]
59	name = "anstyle-wincon"
60	version = "3.0.2"
61	source = "registry+https://github.com/rust-lang/crates.io-index"
62	checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
63	dependencies = [
64	"anstyle",
65	"windows-sys",
66	]
67
68	[[package]]
69	name = "ast"
70	version = "0.1.0"
71	dependencies = [
72	"log",
73	"miette",
74	"num",
75	"thiserror",
76	"tokenizer",
77	]
78
79	[[package]]
80	name = "autocfg"
81	version = "1.1.0"
82	source = "registry+https://github.com/rust-lang/crates.io-index"
83	checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
84
85	[[package]]
86	name = "backtrace"
87	version = "0.3.69"
88	source = "registry+https://github.com/rust-lang/crates.io-index"
89	checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
90	dependencies = [
91	"addr2line",
92	"cc",
93	"cfg-if",
94	"libc",
95	"miniz_oxide",
96	"object",
97	"rustc-demangle",
98	]
99
100	[[package]]
101	name = "backtrace-ext"
102	version = "0.2.1"
103	source = "registry+https://github.com/rust-lang/crates.io-index"
104	checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50"
105	dependencies = [
106	"backtrace",
107	]
108
109	[[package]]
110	name = "bitflags"
111	version = "2.4.1"
112	source = "registry+https://github.com/rust-lang/crates.io-index"
113	checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
114
115	[[package]]
116	name = "cc"
117	version = "1.0.83"
118	source = "registry+https://github.com/rust-lang/crates.io-index"
119	checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
120	dependencies = [
121	"libc",
122	]
123
124	[[package]]
125	name = "cfg-if"
126	version = "1.0.0"
127	source = "registry+https://github.com/rust-lang/crates.io-index"
128	checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
129
130	[[package]]
131	name = "clap"
132	version = "4.4.14"
133	source = "registry+https://github.com/rust-lang/crates.io-index"
134	checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2"
135	dependencies = [
136	"clap_builder",
137	"clap_derive",
138	]
139
140	[[package]]
141	name = "clap_builder"
142	version = "4.4.14"
143	source = "registry+https://github.com/rust-lang/crates.io-index"
144	checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370"
145	dependencies = [
146	"anstream",
147	"anstyle",
148	"clap_lex",
149	"strsim",
150	]
151
152	[[package]]
153	name = "clap_derive"
154	version = "4.4.7"
155	source = "registry+https://github.com/rust-lang/crates.io-index"
156	checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
157	dependencies = [
158	"heck",
159	"proc-macro2",
160	"quote",
161	"syn",
162	]
163
164	[[package]]
165	name = "clap_lex"
166	version = "0.6.0"
167	source = "registry+https://github.com/rust-lang/crates.io-index"
168	checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
169
170	[[package]]
171	name = "cli"
172	version = "0.1.0"
173	dependencies = [
174	"ast",
175	"clap",
176	"js_ast",
177	"js_printer",
178	"log",
179	"miette",
180	"parser",
181	"sema",
182	"thiserror",
183	"tokenizer",
184	]
185
186	[[package]]
187	name = "colorchoice"
188	version = "1.0.0"
189	source = "registry+https://github.com/rust-lang/crates.io-index"
190	checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
191
192	[[package]]
193	name = "errno"
194	version = "0.3.8"
195	source = "registry+https://github.com/rust-lang/crates.io-index"
196	checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
197	dependencies = [
198	"libc",
199	"windows-sys",
200	]
201
202	[[package]]
203	name = "form_urlencoded"
204	version = "1.2.1"
205	source = "registry+https://github.com/rust-lang/crates.io-index"
206	checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
207	dependencies = [
208	"percent-encoding",
209	]
210
211	[[package]]
212	name = "gimli"
213	version = "0.28.1"
214	source = "registry+https://github.com/rust-lang/crates.io-index"
215	checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
216
217	[[package]]
218	name = "heck"
219	version = "0.4.1"
220	source = "registry+https://github.com/rust-lang/crates.io-index"
221	checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
222
223	[[package]]
224	name = "hermit-abi"
225	version = "0.3.3"
226	source = "registry+https://github.com/rust-lang/crates.io-index"
227	checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
228
229	[[package]]
230	name = "idna"
231	version = "0.5.0"
232	source = "registry+https://github.com/rust-lang/crates.io-index"
233	checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
234	dependencies = [
235	"unicode-bidi",
236	"unicode-normalization",
237	]
238
239	[[package]]
240	name = "is-terminal"
241	version = "0.4.10"
242	source = "registry+https://github.com/rust-lang/crates.io-index"
243	checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
244	dependencies = [
245	"hermit-abi",
246	"rustix",
247	"windows-sys",
248	]
249
250	[[package]]
251	name = "is_ci"
252	version = "1.1.1"
253	source = "registry+https://github.com/rust-lang/crates.io-index"
254	checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
255
256	[[package]]
257	name = "js_ast"
258	version = "0.1.0"
259	dependencies = [
260	"ast",
261	"log",
262	"miette",
263	"num",
264	"thiserror",
265	]
266
267	[[package]]
268	name = "js_printer"
269	version = "0.1.0"
270	dependencies = [
271	"js_ast",
272	"miette",
273	"num",
274	"thiserror",
275	]
276
277	[[package]]
278	name = "libc"
279	version = "0.2.152"
280	source = "registry+https://github.com/rust-lang/crates.io-index"
281	checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
282
283	[[package]]
284	name = "linux-raw-sys"
285	version = "0.4.12"
286	source = "registry+https://github.com/rust-lang/crates.io-index"
287	checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
288
289	[[package]]
290	name = "log"
291	version = "0.0.0"
292	dependencies = [
293	"miette",
294	"thiserror",
295	"url",
296	]
297
298	[[package]]
299	name = "memchr"
300	version = "2.7.1"
301	source = "registry+https://github.com/rust-lang/crates.io-index"
302	checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
303
304	[[package]]
305	name = "miette"
306	version = "5.10.0"
307	source = "registry+https://github.com/rust-lang/crates.io-index"
308	checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
309	dependencies = [
310	"backtrace",
311	"backtrace-ext",
312	"is-terminal",
313	"miette-derive",
314	"once_cell",
315	"owo-colors",
316	"supports-color",
317	"supports-hyperlinks",
318	"supports-unicode",
319	"terminal_size",
320	"textwrap",
321	"thiserror",
322	"unicode-width",
323	]
324
325	[[package]]
326	name = "miette-derive"
327	version = "5.10.0"
328	source = "registry+https://github.com/rust-lang/crates.io-index"
329	checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
330	dependencies = [
331	"proc-macro2",
332	"quote",
333	"syn",
334	]
335
336	[[package]]
337	name = "miniz_oxide"
338	version = "0.7.1"
339	source = "registry+https://github.com/rust-lang/crates.io-index"
340	checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
341	dependencies = [
342	"adler",
343	]
344
345	[[package]]
346	name = "num"
347	version = "0.4.1"
348	source = "registry+https://github.com/rust-lang/crates.io-index"
349	checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
350	dependencies = [
351	"num-bigint",
352	"num-complex",
353	"num-integer",
354	"num-iter",
355	"num-rational",
356	"num-traits",
357	]
358
359	[[package]]
360	name = "num-bigint"
361	version = "0.4.4"
362	source = "registry+https://github.com/rust-lang/crates.io-index"
363	checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
364	dependencies = [
365	"autocfg",
366	"num-integer",
367	"num-traits",
368	]
369
370	[[package]]
371	name = "num-complex"
372	version = "0.4.4"
373	source = "registry+https://github.com/rust-lang/crates.io-index"
374	checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
375	dependencies = [
376	"num-traits",
377	]
378
379	[[package]]
380	name = "num-integer"
381	version = "0.1.45"
382	source = "registry+https://github.com/rust-lang/crates.io-index"
383	checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
384	dependencies = [
385	"autocfg",
386	"num-traits",
387	]
388
389	[[package]]
390	name = "num-iter"
391	version = "0.1.43"
392	source = "registry+https://github.com/rust-lang/crates.io-index"
393	checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
394	dependencies = [
395	"autocfg",
396	"num-integer",
397	"num-traits",
398	]
399
400	[[package]]
401	name = "num-rational"
402	version = "0.4.1"
403	source = "registry+https://github.com/rust-lang/crates.io-index"
404	checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
405	dependencies = [
406	"autocfg",
407	"num-bigint",
408	"num-integer",
409	"num-traits",
410	]
411
412	[[package]]
413	name = "num-traits"
414	version = "0.2.17"
415	source = "registry+https://github.com/rust-lang/crates.io-index"
416	checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
417	dependencies = [
418	"autocfg",
419	]
420
421	[[package]]
422	name = "object"
423	version = "0.32.2"
424	source = "registry+https://github.com/rust-lang/crates.io-index"
425	checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
426	dependencies = [
427	"memchr",
428	]
429
430	[[package]]
431	name = "once_cell"
432	version = "1.19.0"
433	source = "registry+https://github.com/rust-lang/crates.io-index"
434	checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
435
436	[[package]]
437	name = "owo-colors"
438	version = "3.5.0"
439	source = "registry+https://github.com/rust-lang/crates.io-index"
440	checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
441
442	[[package]]
443	name = "parser"
444	version = "0.1.0"
445	dependencies = [
446	"ast",
447	"log",
448	"miette",
449	"num",
450	"thiserror",
451	"tokenizer",
452	]
453
454	[[package]]
455	name = "percent-encoding"
456	version = "2.3.1"
457	source = "registry+https://github.com/rust-lang/crates.io-index"
458	checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
459
460	[[package]]
461	name = "proc-macro2"
462	version = "1.0.76"
463	source = "registry+https://github.com/rust-lang/crates.io-index"
464	checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
465	dependencies = [
466	"unicode-ident",
467	]
468
469	[[package]]
470	name = "quote"
471	version = "1.0.35"
472	source = "registry+https://github.com/rust-lang/crates.io-index"
473	checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
474	dependencies = [
475	"proc-macro2",
476	]
477
478	[[package]]
479	name = "rustc-demangle"
480	version = "0.1.23"
481	source = "registry+https://github.com/rust-lang/crates.io-index"
482	checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
483
484	[[package]]
485	name = "rustix"
486	version = "0.38.28"
487	source = "registry+https://github.com/rust-lang/crates.io-index"
488	checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
489	dependencies = [
490	"bitflags",
491	"errno",
492	"libc",
493	"linux-raw-sys",
494	"windows-sys",
495	]
496
497	[[package]]
498	name = "sema"
499	version = "0.1.0"
500	dependencies = [
501	"ast",
502	"miette",
503	"num",
504	"thiserror",
505	]
506
507	[[package]]
508	name = "smawk"
509	version = "0.3.2"
510	source = "registry+https://github.com/rust-lang/crates.io-index"
511	checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
512
513	[[package]]
514	name = "strsim"
515	version = "0.10.0"
516	source = "registry+https://github.com/rust-lang/crates.io-index"
517	checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
518
519	[[package]]
520	name = "supports-color"
521	version = "2.1.0"
522	source = "registry+https://github.com/rust-lang/crates.io-index"
523	checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
524	dependencies = [
525	"is-terminal",
526	"is_ci",
527	]
528
529	[[package]]
530	name = "supports-hyperlinks"
531	version = "2.1.0"
532	source = "registry+https://github.com/rust-lang/crates.io-index"
533	checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d"
534	dependencies = [
535	"is-terminal",
536	]
537
538	[[package]]
539	name = "supports-unicode"
540	version = "2.0.0"
541	source = "registry+https://github.com/rust-lang/crates.io-index"
542	checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7"
543	dependencies = [
544	"is-terminal",
545	]
546
547	[[package]]
548	name = "syn"
549	version = "2.0.48"
550	source = "registry+https://github.com/rust-lang/crates.io-index"
551	checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
552	dependencies = [
553	"proc-macro2",
554	"quote",
555	"unicode-ident",
556	]
557
558	[[package]]
559	name = "terminal_size"
560	version = "0.1.17"
561	source = "registry+https://github.com/rust-lang/crates.io-index"
562	checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df"
563	dependencies = [
564	"libc",
565	"winapi",
566	]
567
568	[[package]]
569	name = "textwrap"
570	version = "0.15.2"
571	source = "registry+https://github.com/rust-lang/crates.io-index"
572	checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d"
573	dependencies = [
574	"smawk",
575	"unicode-linebreak",
576	"unicode-width",
577	]
578
579	[[package]]
580	name = "thiserror"
581	version = "1.0.56"
582	source = "registry+https://github.com/rust-lang/crates.io-index"
583	checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
584	dependencies = [
585	"thiserror-impl",
586	]
587
588	[[package]]
589	name = "thiserror-impl"
590	version = "1.0.56"
591	source = "registry+https://github.com/rust-lang/crates.io-index"
592	checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
593	dependencies = [
594	"proc-macro2",
595	"quote",
596	"syn",
597	]
598
599	[[package]]
600	name = "tinyvec"
601	version = "1.6.0"
602	source = "registry+https://github.com/rust-lang/crates.io-index"
603	checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
604	dependencies = [
605	"tinyvec_macros",
606	]
607
608	[[package]]
609	name = "tinyvec_macros"
610	version = "0.1.1"
611	source = "registry+https://github.com/rust-lang/crates.io-index"
612	checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
613
614	[[package]]
615	name = "tokenizer"
616	version = "0.1.0"
617	dependencies = [
618	"log",
619	"miette",
620	"thiserror",
621	]
622
623	[[package]]
624	name = "unicode-bidi"
625	version = "0.3.15"
626	source = "registry+https://github.com/rust-lang/crates.io-index"
627	checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
628
629	[[package]]
630	name = "unicode-ident"
631	version = "1.0.12"
632	source = "registry+https://github.com/rust-lang/crates.io-index"
633	checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
634
635	[[package]]
636	name = "unicode-linebreak"
637	version = "0.1.5"
638	source = "registry+https://github.com/rust-lang/crates.io-index"
639	checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
640
641	[[package]]
642	name = "unicode-normalization"
643	version = "0.1.22"
644	source = "registry+https://github.com/rust-lang/crates.io-index"
645	checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
646	dependencies = [
647	"tinyvec",
648	]
649
650	[[package]]
651	name = "unicode-width"
652	version = "0.1.11"
653	source = "registry+https://github.com/rust-lang/crates.io-index"
654	checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
655
656	[[package]]
657	name = "url"
658	version = "2.5.0"
659	source = "registry+https://github.com/rust-lang/crates.io-index"
660	checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
661	dependencies = [
662	"form_urlencoded",
663	"idna",
664	"percent-encoding",
665	]
666
667	[[package]]
668	name = "utf8parse"
669	version = "0.2.1"
670	source = "registry+https://github.com/rust-lang/crates.io-index"
671	checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
672
673	[[package]]
674	name = "winapi"
675	version = "0.3.9"
676	source = "registry+https://github.com/rust-lang/crates.io-index"
677	checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
678	dependencies = [
679	"winapi-i686-pc-windows-gnu",
680	"winapi-x86_64-pc-windows-gnu",
681	]
682
683	[[package]]
684	name = "winapi-i686-pc-windows-gnu"
685	version = "0.4.0"
686	source = "registry+https://github.com/rust-lang/crates.io-index"
687	checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
688
689	[[package]]
690	name = "winapi-x86_64-pc-windows-gnu"
691	version = "0.4.0"
692	source = "registry+https://github.com/rust-lang/crates.io-index"
693	checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
694
695	[[package]]
696	name = "windows-sys"
697	version = "0.52.0"
698	source = "registry+https://github.com/rust-lang/crates.io-index"
699	checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
700	dependencies = [
701	"windows-targets",
702	]
703
704	[[package]]
705	name = "windows-targets"
706	version = "0.52.0"
707	source = "registry+https://github.com/rust-lang/crates.io-index"
708	checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
709	dependencies = [
710	"windows_aarch64_gnullvm",
711	"windows_aarch64_msvc",
712	"windows_i686_gnu",
713	"windows_i686_msvc",
714	"windows_x86_64_gnu",
715	"windows_x86_64_gnullvm",
716	"windows_x86_64_msvc",
717	]
718
719	[[package]]
720	name = "windows_aarch64_gnullvm"
721	version = "0.52.0"
722	source = "registry+https://github.com/rust-lang/crates.io-index"
723	checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
724
725	[[package]]
726	name = "windows_aarch64_msvc"
727	version = "0.52.0"
728	source = "registry+https://github.com/rust-lang/crates.io-index"
729	checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
730
731	[[package]]
732	name = "windows_i686_gnu"
733	version = "0.52.0"
734	source = "registry+https://github.com/rust-lang/crates.io-index"
735	checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
736
737	[[package]]
738	name = "windows_i686_msvc"
739	version = "0.52.0"
740	source = "registry+https://github.com/rust-lang/crates.io-index"
741	checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
742
743	[[package]]
744	name = "windows_x86_64_gnu"
745	version = "0.52.0"
746	source = "registry+https://github.com/rust-lang/crates.io-index"
747	checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
748
749	[[package]]
750	name = "windows_x86_64_gnullvm"
751	version = "0.52.0"
752	source = "registry+https://github.com/rust-lang/crates.io-index"
753	checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
754
755	[[package]]
756	name = "windows_x86_64_msvc"
757	version = "0.52.0"
758	source = "registry+https://github.com/rust-lang/crates.io-index"
759	checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"

Cargo.toml deleted-26

...	@@ -1,26 +0,0 @@
1	[workspace.package]
2	version = "0.1.0"
3	edition = "2021"
4	license = "MIT"
5	authors = ["Lino Le Van", "Dave Caruso"]
6	repository = "https://github.com/lino-levan/blight"
7
8	[workspace]
9	resolver = "2"
10	members = [
11	"crates/ast",
12	"crates/cli",
13	"crates/js_ast",
14	"crates/js_printer",
15	"crates/log",
16	"crates/parser",
17	"crates/sema",
18	"crates/tokenizer",
19	]
20
21	[workspace.dependencies]
22	clap = { version = "4.4.12", features = ["derive"] }
23	miette = { version = "5.10.0", features = ["fancy"] }
24	thiserror = "1.0.53"
25	num = "0.4.1"
26	url = "2.2.2"

README.md+70-10

...	@@ -1,19 +1,79 @@	...	@@ -1,19 +1,79 @@
1	<img src="/.github/logo.png" width="256">	1	# Compiler
2		2
3	# Blight	3	This language is a mix of features from JavaScript, Rust, and Zig. Some notable features carried over from each include:
4		4
5	Blight sure is a programming language.	5	- Rust - Expressive control flow that is easy to write.
		6	- Zig - Execute any code at compile time, everything is an expression.
		7	- JavaScript - Garbage collected values with a versitile type system.
6		8
7	## Developing	9	## Standard Library
8		10
9	Compiling a file:	11	Currently, there is no standard library. One will be added later.
10		12
11	```bash	13	## Hello World
12	cargo run compile --stdout ./examples/basics/fib.blight	14
		15	```
		16	let print = @js("console").log;
		17	print("Hello, world!");
		18	```
		19
		20	The `print` function is implemented by getting an external reference to the `console` JavaScript global, and then retriving the `log` property from it.
		21
		22	## Primative Data Types
		23
		24	\| name \| ts equivilent \| description \|
		25	\| ---------- \| --------------------- \| ---------------------------------------------- \|
		26	\| `int` \| `number` \| a number in signed 32-bit range \|
		27	\| `float` \| `number` \| a 64-bit floating point number \|
		28	\| `bool` \| `boolean` \| either `true` or `false` \|
		29	\| `string` \| `string` \| a sequence of text characters \|
		30	\| `void` \| `undefined` or `null` \| detectable lack of a value \|
		31	\| `noreturn` \| `never` \| control flow prevents this value from existing \|
		32
		33	> note: `bigint` is planned but not added
		34
		35	## Variables
		36
		37	There are two types of variables, `let` and `var`.
		38
		39	`let` bindings cannot be overridden, and should be thought of as aliases of the expression within. Referring to these identifiers is equivilent to inlining it's where it was used, aside from side effect duplication.
		40
		41	```
		42	let hello: int = 1;
		43	let world = hello + 2;
		44	let hello = hello * world;
		45
		46	@js("console").log(hello);
13	```	47	```
14		48
15	Running the compiler on a file:	49	`var` bindings create mutable memory locations at runtime. This means they can be re-assigned to in arbitrary ways.
16		50
17	```bash	51	```
18	cargo run run ./examples/basics/fib.blight	52	var message = "first message";
		53
		54	if @js("Math").random() > 0.5 {
		55	message = "second message"
		56	}
		57
		58	@js("console").log(message);
		59	```
		60
		61	`let` is preferred whenever possible, as it will lead to more readable code, and the compiler is able to optimize it better. The above can be rewritten as:
		62
		63	```
		64	let message = if @js("Math").random() > 0.5 {
		65	"second message"
		66	} else {
		67	"first message"
		68	};
		69
		70	@js("console").log(message);
		71	```
		72
		73	The initializer for a `let` and `var` are always required, but the type annotations can be omitted when the data type can be inferred. For `let`, this is always the case. `var`, being runtime-mutable memory, only supports runtime representable data types. This means for numerical literals, an explicit type annotation is needed to determine weather to store an integer or a floating point number.
		74
		75	```
		76	var a = 5; // error
		77	var b: int = 5; // ok
		78	var c: float = 5; // ok
19	```	79	```

build.zig created+96

...	@@ -0,0 +1,96 @@
1	const std = @import("std");
2
3	pub fn build(b: *std.Build) void {
4	b.reference_trace = 20;
5
6	const target = b.standardTargetOptions(.{});
7	const optimize = b.standardOptimizeOption(.{});
8
9	const dep_opts = .{ .target = target, .optimize = optimize };
10
11	const zg = b.dependency("zg", dep_opts);
12	const tracer = b.dependency("tracer", dep_opts);
13	// const ada = b.dependency("ada", dep_opts);
14
15	const exe = b.addExecutable(.{
16	.name = "compiler",
17	.root_source_file = b.path("src/compiler/main.zig"),
18	.target = target,
19	.optimize = optimize,
20	});
21	b.installArtifact(exe);
22
23	exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
24	exe.root_module.addImport("tracer", tracer.module("tracer"));
25	// if (!target.result.isWasm()) {
26	// exe.root_module.addImport("ada", ada.module("ada"));
27	// }
28
29	{
30	const run_cmd = b.addRunArtifact(exe);
31	run_cmd.step.dependOn(b.getInstallStep());
32	if (b.args) \|args\| {
33	run_cmd.addArgs(args);
34	}
35	const run_step = b.step("compiler", "Run compiler CLI");
36	run_step.dependOn(&run_cmd.step);
37	}
38
39	// const wasm_api = b.addExecutable(.{
40	// .name = "compiler",
41	// .root_source_file = b.path("src/wasm_api.zig"),
42	// .target = b.resolveTargetQuery(.{
43	// .cpu_arch = .wasm32,
44	// .os_tag = .freestanding,
45	// }),
46	// .optimize = optimize,
47	// });
48	// wasm_api.root_module.export_symbol_names = &[_][]const u8{
49	// "iterateTokens",
50	// };
51	// wasm_api.entry = .disabled;
52	// const wasm_step = b.step("wasm", "Build wasm api");
53	// b.installArtifact(wasm_api);
54	// wasm_step.dependOn(&wasm_api.step);
55	// wasm_api.root_module.addImport("zg:PropsData", zg.module("PropsData"));
56
57	const release = b.step("release", "Package releases");
58	inline for (.{
59	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .linux, .abi = .musl }),
60	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .linux, .abi = .musl }),
61	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .macos }),
62	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .macos }),
63	b.resolveTargetQuery(.{ .cpu_arch = .x86_64, .os_tag = .windows }),
64	b.resolveTargetQuery(.{ .cpu_arch = .aarch64, .os_tag = .windows }),
65	}) \|cross_target\| {
66	const name = std.fmt.allocPrint(
67	b.allocator,
68	"{s}-{s}",
69	.{
70	@tagName(cross_target.result.cpu.arch),
71	@tagName(cross_target.result.os.tag),
72	},
73	) catch @panic("OOM");
74
75	const cross_exe = b.addExecutable(.{
76	.name = "compiler",
77	.root_source_file = b.path("src/compiler/main.zig"),
78	.target = cross_target,
79	.optimize = .ReleaseSafe,
80	.strip = true,
81	});
82	cross_exe.root_module.addImport("zg:PropsData", zg.module("PropsData"));
83	cross_exe.root_module.addImport("tracer", tracer.module("tracer"));
84
85	release.dependOn(&b.addInstallFile(
86	cross_exe.getEmittedBin(),
87	b.pathJoin(&.{
88	name,
89	std.mem.concat(b.allocator, u8, &.{
90	"compiler",
91	cross_target.result.exeFileExt(),
92	}) catch @panic("OOM"),
93	}),
94	).step);
95	}
96	}

build.zig.zon created+22

...	@@ -0,0 +1,22 @@
1	.{
2	.name = "",
3	.version = "0.0.0",
4
5	.minimum_zig_version = "0.12.0",
6
7	.dependencies = .{
8	.zg = .{
9	.url = "https://codeberg.org/dude_the_builder/zg/archive/16735685fcc3410de361ba3411788ad1fb4fe188.tar.gz",
10	.hash = "1220fe9ac5cdb41833d327a78745614e67d472469f8666567bd8cf9f5847a52b1c51",
11	},
12	.tracer = .{
13	.url = "https://github.com/paperdave/zig-tracer/archive/d4c2eff040e9e69b9f08d2175b315f3055ad7ba3.tar.gz",
14	.hash = "12202d60766a46ceb4074ce5651a58a11f2f6b32927476a55c6889805073e8e3d261",
15	},
16	},
17	.paths = .{
18	"build.zig",
19	"build.zig.zon",
20	"src",
21	},
22	}

crates/ast/Cargo.toml deleted-15

...	@@ -1,15 +0,0 @@
1	[package]
2	name = "ast"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path = "./lib.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12	num.workspace = true
13
14	tokenizer = { path = "../tokenizer" } # Do not depend on the tokenizer
15	log = { path = "../log" }

crates/ast/ast.rs deleted-211

...	@@ -1,211 +0,0 @@
1	use log::Range;
2
3	use crate::types::{Literal, Type};
4	use crate::{BinaryOperator, UnaryOperator};
5
6	#[derive(Debug, Clone)]
7	pub enum BuiltinId {
8	Raw,
9	}
10
11	#[derive(Debug, Clone)]
12	pub struct Module {
13	pub comment: Option<DocComment>,
14	pub statements: Vec<Statement>,
15	}
16
17	#[derive(Debug, Clone)]
18	pub struct Block {
19	pub statements: Vec<Statement>,
20	pub has_implicit_return: bool,
21	}
22
23	#[derive(Debug, Clone)]
24	pub struct Param {
25	pub name: String,
26	pub param_type: Type,
27	}
28
29	#[derive(Debug, Clone)]
30
31	pub struct VarDecl {
32	pub name: String,
33	pub data_type: Type,
34	pub init: Expr,
35	pub is_constant: bool,
36	pub is_pub: bool,
37	pub range: Range,
38	pub comment: Option<DocComment>,
39	}
40
41	#[derive(Debug, Clone)]
42	pub struct FnDecl {
43	pub name: String,
44	pub return_type: Type,
45	pub params: Vec<Param>,
46	pub body: Expr,
47	pub is_pub: bool,
48	pub range: Range,
49	pub comment: Option<DocComment>,
50	}
51
52	#[derive(Debug, Clone)]
53	pub struct Expr {
54	pub data: ExprData,
55	pub range: Range,
56	pub comment: Option<Comment>,
57	}
58
59	impl Expr {
60	pub fn new(data: ExprData, range: Range, comment: Option<Comment>) -> Self {
61	Self {
62	data,
63	range,
64	comment,
65	}
66	}
67	}
68
69	#[derive(Debug, Clone)]
70	pub struct DocComment {
71	pub data: String,
72	pub range: Range,
73	}
74
75	#[derive(Debug, PartialEq, Clone)]
76	pub struct Comment {
77	pub data: String,
78	}
79
80	impl Comment {
81	pub fn format(&self) -> String {
82	let mut lines = self.data.lines();
83	let first_line = lines.next().unwrap();
84	let mut result = format!("// {}", first_line);
85	for line in lines {
86	result.push_str(&format!("\n// {}", line));
87	}
88	result
89	}
90
91	pub fn to_doc_comment(self, range: Range) -> DocComment {
92	DocComment {
93	data: self.data,
94	range,
95	}
96	}
97	}
98
99	impl Into<Comment> for DocComment {
100	fn into(self) -> Comment {
101	Comment { data: self.data }
102	}
103	}
104
105	impl DocComment {
106	pub fn format(&self) -> String {
107	let mut lines = self.data.lines();
108	let first_line = lines.next().unwrap();
109	let mut result = format!("/// {}", first_line);
110	for line in lines {
111	result.push_str(&format!("\n/// {}", line));
112	}
113	result
114	}
115	}
116
117	#[derive(Debug, Clone)]
118	pub struct BlockWithRange {
119	pub block: Block,
120	pub range: Range,
121	}
122
123	#[derive(Debug, Clone)]
124	pub struct If {
125	pub condition: Box<Expr>,
126	pub yes: BlockWithRange,
127	pub no: Option<BlockOrIf>,
128	}
129
130	#[derive(Debug, Clone)]
131	pub enum BlockOrIf {
132	Block(BlockWithRange),
133	If(Box<If>),
134	}
135
136	#[derive(Debug, Clone)]
137	pub enum ExprData {
138	Block(Block),
139	Binary {
140	op: BinaryOperator,
141	left: Box<Expr>,
142	right: Box<Expr>,
143	},
144	Unary {
145	op: UnaryOperator,
146	value: Box<Expr>,
147	},
148	Fn {
149	name: String,
150	params: Vec<Param>,
151	return_type: Type,
152	body: Box<Expr>,
153	},
154	Call {
155	base: Box<Expr>,
156	arguments: Vec<Expr>,
157	},
158	Field {
159	base: Box<Expr>,
160	field: Box<Expr>,
161	},
162	Builtin {
163	id: BuiltinId,
164	arguments: Vec<Expr>,
165	},
166	If(If),
167	Return(Option<Box<Expr>>),
168	Literal(Literal),
169	Identifier(String),
170	}
171
172	#[derive(Debug, Clone)]
173	pub enum Statement {
174	Expr(Expr),
175	VarDecl(VarDecl),
176	FnDecl(FnDecl),
177	Comment(Comment),
178	}
179
180	impl BuiltinId {
181	pub fn display_name(&self) -> &'static str {
182	match self {
183	Self::Raw => "@raw",
184	}
185	}
186
187	pub fn from_str(name: &str) -> Option<Self> {
188	match name {
189	"@raw" => Some(Self::Raw),
190	_ => None,
191	}
192	}
193	}
194
195	impl Statement {
196	/// This is stupid function im sorry
197	pub fn require_semicolon(&self, semicolon_hint: bool) -> bool {
198	match &self {
199	Statement::Expr(_) => !semicolon_hint,
200	Statement::VarDecl(_) => true,
201	_ => false,
202	}
203	}
204
205	pub fn is_expression(&self) -> bool {
206	match &self {
207	Statement::Expr(_) => true,
208	_ => false,
209	}
210	}
211	}

crates/ast/lib.rs deleted-7

...	@@ -1,7 +0,0 @@
1	mod ast;
2	mod operators;
3	mod types;
4
5	pub use ast::*;
6	pub use operators::*;
7	pub use types::*;

crates/ast/operators.rs deleted-316

...	@@ -1,316 +0,0 @@
1	use tokenizer::TokenType;
2
3	use crate::Literal;
4
5	#[derive(Debug, Clone, PartialEq)]
6	pub enum BinaryOperator {
7	Add,
8	Sub,
9	Mul,
10	Div,
11	Pow,
12	Eq,
13	Gt,
14	Lt,
15	Gte,
16	Lte,
17	Lshift,
18	Rshift,
19	RshiftUnsigned,
20	BitAnd,
21	BitOr,
22	BitXor,
23	Mod,
24	NotEq,
25	And,
26	Or,
27
28	Assign,
29	AssignAdd,
30	AssignSub,
31	AssignMul,
32	AssignDiv,
33	AssignPow,
34	AssignLshift,
35	AssignRshift,
36	AssignRshiftUnsigned,
37	AssignBitAnd,
38	AssignBitOr,
39	AssignBitXor,
40	AssignMod,
41	AssignAnd,
42	AssignOr,
43	}
44
45	#[derive(Debug, Clone, PartialEq)]
46	pub enum UnaryOperator {
47	Neg,
48	Not,
49	BitNot,
50	}
51
52	impl BinaryOperator {
53	pub fn to_str(&self) -> &'static str {
54	match self {
55	Self::Add => "+",
56	Self::Sub => "-",
57	Self::Div => "/",
58	Self::Mul => "*",
59	Self::Pow => "**",
60	Self::Eq => "==",
61	Self::Gt => ">",
62	Self::Lt => "<",
63	Self::Gte => ">=",
64	Self::Lte => "<=",
65	Self::Lshift => "<<",
66	Self::Rshift => ">>",
67	Self::RshiftUnsigned => ">>>",
68	Self::BitAnd => "&",
69	Self::BitOr => "\|",
70	Self::BitXor => "^",
71	Self::Mod => "%",
72	Self::NotEq => "!=",
73	Self::And => "&&",
74	Self::Or => "\|\|",
75	Self::AssignAdd => "+=",
76	Self::AssignSub => "-=",
77	Self::AssignDiv => "/=",
78	Self::AssignMul => "*=",
79	Self::AssignPow => "**=",
80	Self::Assign => "=",
81	Self::AssignLshift => "<<=",
82	Self::AssignRshift => ">>=",
83	Self::AssignRshiftUnsigned => ">>>=",
84	Self::AssignBitAnd => "&=",
85	Self::AssignBitOr => "\|=",
86	Self::AssignBitXor => "^=",
87	Self::AssignMod => "%=",
88	Self::AssignAnd => "&&=",
89	Self::AssignOr => "\|\|=",
90	}
91	}
92
93	pub fn to_string(&self) -> String {
94	String::from(self.to_str())
95	}
96
97	pub fn from_token(token: &TokenType) -> Option<BinaryOperator> {
98	match &token {
99	TokenType::And => Some(Self::BitAnd),
100	TokenType::AndAnd => Some(Self::And),
101	TokenType::BangEqual => Some(Self::NotEq),
102	TokenType::Caret => Some(Self::BitXor),
103	TokenType::EqualEqual => Some(Self::Eq),
104	TokenType::Gt => Some(Self::Gt),
105	TokenType::GtEqual => Some(Self::Gte),
106	TokenType::GtGt => Some(Self::Rshift),
107	TokenType::GtGtGt => Some(Self::RshiftUnsigned),
108	TokenType::Lt => Some(Self::Lt),
109	TokenType::LtEqual => Some(Self::Lte),
110	TokenType::LtLt => Some(Self::Lshift),
111	TokenType::Minus => Some(Self::Sub),
112	TokenType::Or => Some(Self::BitOr),
113	TokenType::OrOr => Some(Self::Or),
114	TokenType::Percent => Some(Self::Mod),
115	TokenType::Plus => Some(Self::Add),
116	TokenType::Slash => Some(Self::Div),
117	TokenType::Star => Some(Self::Mul),
118	TokenType::StarStar => Some(Self::Pow),
119	_ => None,
120	}
121	}
122
123	pub fn level(&self) -> OperatorLevel {
124	OperatorLevel::from(self)
125	}
126
127	/// Error case of this means 'types do not allow this operation'
128	pub fn perform(&self, left: Literal, right: Literal) -> Result<Literal, ()> {
129	match self {
130	BinaryOperator::Eq => Ok(Literal::Bool(left == right)),
131	BinaryOperator::NotEq => Ok(Literal::Bool(left != right)),
132
133	BinaryOperator::Or \| BinaryOperator::And => match (left, right) {
134	(Literal::Bool(a), Literal::Bool(b)) => Ok(Literal::Bool(match self {
135	BinaryOperator::Or => a \|\| b,
136	BinaryOperator::And => a && b,
137	_ => unreachable!(),
138	})),
139	_ => Err(()),
140	},
141
142	BinaryOperator::Add
143	\| BinaryOperator::Sub
144	\| BinaryOperator::Mul
145	\| BinaryOperator::Div
146	\| BinaryOperator::Pow
147	\| BinaryOperator::Gt
148	\| BinaryOperator::Lt
149	\| BinaryOperator::Gte
150	\| BinaryOperator::Lte
151	\| BinaryOperator::Lshift
152	\| BinaryOperator::Rshift
153	\| BinaryOperator::RshiftUnsigned
154	\| BinaryOperator::BitAnd
155	\| BinaryOperator::BitOr
156	\| BinaryOperator::BitXor
157	\| BinaryOperator::Mod => match (left, right) {
158	(Literal::Number(a), Literal::Number(b)) => Ok(match self {
159	BinaryOperator::Add => Literal::Number(a + b),
160	BinaryOperator::Sub => Literal::Number(a - b),
161	BinaryOperator::Mul => Literal::Number(a * b),
162	BinaryOperator::Div => Literal::Number(a / b),
163	BinaryOperator::Pow => Literal::Number(a.powf(b)),
164	BinaryOperator::Gt => Literal::Bool(a > b),
165	BinaryOperator::Lt => Literal::Bool(a < b),
166	BinaryOperator::Gte => Literal::Bool(a >= b),
167	BinaryOperator::Lte => Literal::Bool(a <= b),
168	BinaryOperator::Lshift => Literal::Number(((a as i32) << (b as i32)).into()),
169	BinaryOperator::Rshift => Literal::Number(((a as i32) >> (b as i32)).into()),
170	BinaryOperator::RshiftUnsigned => {
171	Literal::Number(((a as u32) >> (b as u32)).into())
172	}
173	BinaryOperator::BitAnd => Literal::Number(((a as i32) & (b as i32)).into()),
174	BinaryOperator::BitOr => Literal::Number(((a as i32) \| (b as i32)).into()),
175	BinaryOperator::BitXor => Literal::Number(((a as i32) ^ (b as i32)).into()),
176	BinaryOperator::Mod => Literal::Number(a % b),
177	_ => unreachable!(),
178	}),
179	_ => Err(()),
180	},
181
182	// Impossible condition doing assign-add to a literal
183	_ => Err(()),
184	}
185	}
186	}
187
188	impl UnaryOperator {
189	pub fn to_str(&self) -> &'static str {
190	match self {
191	Self::Neg => "-",
192	Self::Not => "!",
193	Self::BitNot => "~",
194	}
195	}
196
197	pub fn to_string(&self) -> String {
198	String::from(self.to_str())
199	}
200
201	pub fn perform(self, value: Literal) -> Result<Literal, ()> {
202	match self {
203	UnaryOperator::Neg => match value {
204	Literal::Number(a) => Ok(Literal::Number(-a)),
205	_ => Err(()),
206	},
207	UnaryOperator::Not => match value {
208	Literal::Bool(a) => Ok(Literal::Bool(!a)),
209	_ => Err(()),
210	},
211	UnaryOperator::BitNot => match value {
212	Literal::Number(a) => Ok(Literal::Number((!(a as i32)).into())),
213	_ => Err(()),
214	},
215	}
216	}
217	}
218
219	/// Taken from from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence#table
220	///
221	/// Blight doesn't have all of these operators, but the
222	/// printer uses it to determine where to put parens.
223	#[derive(PartialEq, PartialOrd, Debug)]
224	pub enum OperatorLevel {
225	Accessor, // e. e[] e()
226	New, // new
227	Postfix, // expr++ expr--
228	Prefix, // ++e, --e, !e, ~e, +e, -e, typeof, void, delete
229
230	// these are the binary operators
231	Exponentiation, // **
232	Multiplicative, // *, /, %
233	Additive, // +, -
234	BitwiseShift, // <<, >>, >>>
235	Relational, // <, <=, >, >=
236	Equality, // ==, !=
237	BitwiseAnd, // &
238	BitwiseXor, // ^
239	BitwiseOr, // \|
240	LogicalAnd, // &&
241	LogicalOr, // \|\|, ??
242
243	AssignmentAndOther, // =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
244	Comma, // ,
245	}
246
247	#[derive(PartialEq)]
248	pub enum Associativity {
249	LeftToRight,
250	RightToLeft,
251	}
252
253	impl OperatorLevel {
254	pub fn associativity(&self) -> Associativity {
255	use Associativity::*;
256	use OperatorLevel::*;
257
258	match self {
259	Accessor => LeftToRight,
260	Exponentiation => RightToLeft,
261	Multiplicative => LeftToRight,
262	Additive => LeftToRight,
263	BitwiseShift => LeftToRight,
264	Relational => LeftToRight,
265	Equality => LeftToRight,
266	BitwiseAnd => LeftToRight,
267	BitwiseXor => LeftToRight,
268	BitwiseOr => LeftToRight,
269	LogicalAnd => LeftToRight,
270	LogicalOr => LeftToRight,
271	AssignmentAndOther => RightToLeft,
272	Comma => LeftToRight,
273
274	// These do not make sense to have an associativity
275	// You will likely not run into this if you do
276	// BinaryOperator::level(o).associativity()
277	New \| Postfix \| Prefix => unreachable!("{:?} does not have an associativity", self),
278	}
279	}
280	}
281
282	impl From<&BinaryOperator> for OperatorLevel {
283	fn from(operator: &BinaryOperator) -> Self {
284	use BinaryOperator::*;
285	use OperatorLevel::*;
286
287	match operator {
288	Add \| Sub => Additive,
289	Mul \| Div \| Mod => Multiplicative,
290	Pow => Exponentiation,
291	Gt \| Lt \| Gte \| Lte => Relational,
292	Lshift \| Rshift \| RshiftUnsigned => BitwiseShift,
293	BitAnd => BitwiseAnd,
294	BitOr => BitwiseOr,
295	BitXor => BitwiseXor,
296	Eq \| NotEq => Equality,
297	And => LogicalAnd,
298	Or => LogicalOr,
299
300	Assign \| AssignAdd \| AssignSub \| AssignMul \| AssignDiv \| AssignPow \| AssignLshift
301	\| AssignRshift \| AssignRshiftUnsigned \| AssignBitAnd \| AssignBitOr \| AssignBitXor
302	\| AssignMod \| AssignAnd \| AssignOr => AssignmentAndOther,
303	}
304	}
305	}
306
307	impl From<UnaryOperator> for OperatorLevel {
308	fn from(operator: UnaryOperator) -> Self {
309	use OperatorLevel::*;
310	use UnaryOperator::*;
311
312	match operator {
313	Neg \| Not \| BitNot => Prefix,
314	}
315	}
316	}

crates/ast/types.rs deleted-134

...	@@ -1,134 +0,0 @@
1	use miette::{Diagnostic, Result};
2	use thiserror::Error;
3
4	/// A value that is known at compile-time
5	#[derive(Debug, Clone, PartialEq)]
6	pub enum Literal {
7	Number(f64),
8	Bool(bool),
9	String(String),
10	Void,
11	}
12
13	impl std::fmt::Display for Literal {
14	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
15	match self {
16	Literal::Number(value) => write!(f, "{}", value),
17	Literal::Bool(value) => write!(f, "{}", value),
18	Literal::String(value) => write!(f, "{}", value),
19	Literal::Void => write!(f, "void"),
20	}
21	}
22	}
23
24	#[derive(Debug, Clone, PartialEq)]
25	pub enum Type {
26	/// Variables without an explicit type are assigned this until
27	/// the type inferencer figures out what this is. There is not a
28	/// keyword for this.
29	Infer,
30	/// The type assigned to literals. This allows integer literals
31	/// to coerce to integer or float depending on the context, but
32	/// prevents implicit casting of runtime values.
33	Literal(Literal),
34	/// Return value of `@raw`. All type checks are disabled.
35	Any,
36	/// The type of `void`. This is the return type of functions
37	/// that do not return a value.
38	Void,
39	/// noreturn is the type of 'return', aka a value that never exists.
40	/// unlike void, this includes functions that do not return at all.
41	NoReturn,
42	/// 32 signed integer
43	Int,
44	/// 64 signed JS float
45	Float,
46	/// Boolean
47	Bool,
48	/// JS BigInt
49	BigInt,
50	/// JS String
51	String,
52	/// A union of multiple possible types. Might be removed
53	Union(Vec<Type>),
54	}
55
56	pub trait TypeCoerceable {
57	fn to_type(&self) -> Option<Type>;
58	}
59
60	impl TypeCoerceable for Literal {
61	fn to_type(&self) -> Option<Type> {
62	Some(match self {
63	Literal::Number(_) => Type::Float,
64	Literal::Bool(_) => Type::Bool,
65	Literal::String(_) => Type::String,
66	Literal::Void => Type::Void,
67	})
68	}
69	}
70
71	impl TypeCoerceable for Type {
72	fn to_type(&self) -> Option<Type> {
73	Some(self.clone())
74	}
75	}
76
77	impl std::fmt::Display for Type {
78	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
79	match self {
80	Type::Infer => write!(f, "infered type"),
81	Type::Any => write!(f, "opaque js value"),
82	Type::Literal(value) => write!(f, "{}", value),
83	Type::Int => write!(f, "int"),
84	Type::Float => write!(f, "float"),
85	Type::BigInt => write!(f, "bigint"),
86	Type::Bool => write!(f, "bool"),
87	Type::String => write!(f, "string"),
88	Type::NoReturn => write!(f, "noreturn"),
89	Type::Void => write!(f, "void"),
90	Type::Union(of) => {
91	for (i, t) in of.iter().enumerate() {
92	if i != 0 {
93	write!(f, " \| ")?;
94	}
95	write!(f, "{}", t)?;
96	}
97	Ok(())
98	}
99	}
100	}
101	}
102
103	pub fn merge_types(a: Type, b: Type) -> Result<Type> {
104	match a {
105	Type::Infer => return Ok(b),
106	Type::Any => return Ok(Type::Any),
107	_ => {}
108	}
109
110	match b {
111	Type::Infer => return Ok(a),
112	Type::Any => return Ok(Type::Any),
113	_ => {}
114	}
115
116	if a == b {
117	return Ok(a);
118	}
119
120	Err(TypeError::new(format!("TODO: Implement type merging for {} and {}", a, b)).into())
121	}
122
123	#[derive(Error, Diagnostic, Debug)]
124	#[error("TypeError")]
125	pub struct TypeError {
126	#[help]
127	error: String,
128	}
129
130	impl TypeError {
131	pub fn new(error: String) -> Self {
132	Self { error }
133	}
134	}

crates/cli/Cargo.toml deleted-21

...	@@ -1,21 +0,0 @@
1	[package]
2	name = "cli"
3	version = "0.1.0"
4	edition = "2021"
5
6	[[bin]]
7	name = "blight"
8	path = "./main.rs"
9
10	[dependencies]
11	clap.workspace = true
12	miette.workspace = true
13	thiserror.workspace = true
14
15	tokenizer = { path = "../tokenizer" }
16	ast = { path = "../ast" }
17	parser = { path = "../parser" }
18	js_ast = { path = "../js_ast" }
19	js_printer = { path = "../js_printer" }
20	sema = { path = "../sema" }
21	log = { path = "../log" }

crates/cli/main.rs deleted-172

...	@@ -1,172 +0,0 @@
1	use std::{io::Write, path::PathBuf};
2
3	use clap::{Parser, Subcommand};
4	use miette::{diagnostic, Result};
5
6	use js_printer;
7	use tokenizer;
8
9	#[derive(Parser)]
10	#[command(author, version, about, long_about = None)]
11	struct Cli {
12	/// Turn debugging information on
13	#[arg(short, long, action = clap::ArgAction::Count)]
14	debug: u8,
15
16	#[command(subcommand)]
17	command: Commands,
18	}
19
20	#[derive(Subcommand)]
21	enum Commands {
22	/// Compiles a file into javascript
23	Compile {
24	file: PathBuf,
25	// Whether to output to stdout instead of a file
26	#[arg(long, default_value_t = false)]
27	stdout: bool,
28	/// What javascript runtime to compile to
29	#[arg(short, long, default_value = "deno")]
30	target: String,
31	/// Format the output javascript?
32	#[arg(long, default_value_t = true)]
33	format: bool,
34	},
35	Run {
36	file: PathBuf,
37	/// What javascript runtime to compile to and use
38	#[arg(short, long, default_value = "deno")]
39	target: String,
40	},
41	Astdump {
42	file: PathBuf,
43	},
44	}
45
46	fn get_runtime(runtime: &String) -> String {
47	match std::env::var("USER") {
48	Ok(user) if user == "dave" => String::from("bun"),
49	_ => runtime.clone(),
50	}
51	}
52
53	pub fn transpile(content: String, writer: &mut dyn Write) -> Result<()> {
54	// 1. Tokenize the input
55	let tokens = tokenizer::tokenize(content)?;
56	// 2. Parse the tokens into an AST
57	let mut blight_ast = parser::parse(tokens)?;
58	// 3. Semantic analysis.
59	sema::analyze_module(&mut blight_ast)?;
60	// 4. Optimize the AST
61	// optimize::optimize_module(&mut blight_ast);
62	// 5. Translate the AST into javascript
63	let js_ast = js_ast::blight_to_js(blight_ast);
64	// 6. Print the JS AST
65	// TODO(@paperdave): use std::io::Write instead of a string
66	js_printer::Print::print(&js_ast, writer)
67	.map_err(\|e\| diagnostic!("Error while printing: {}", e))?;
68	Ok(())
69	}
70
71	pub fn transpile_to_string(content: String) -> Result<String> {
72	let mut buffer = Vec::new();
73	transpile(content, &mut buffer)?;
74	Ok(String::from_utf8(buffer).unwrap())
75	}
76
77	pub fn astdump(content: String) -> Result<String> {
78	// 1. Tokenize the input
79	let tokens = tokenizer::tokenize(content)?;
80	// 2. Parse the tokens into an AST
81	let blight_ast = parser::parse(tokens)?;
82	return Ok(format!("{:#?}", blight_ast));
83	}
84
85	fn main() -> Result<()> {
86	let cli = Cli::parse();
87
88	match cli.command {
89	Commands::Compile {
90	file,
91	stdout,
92	target: _,
93	format,
94	} => {
95	let contents = match std::fs::read_to_string(file.clone()) {
96	Ok(contents) => contents,
97	Err(e) => {
98	eprintln!("Failed to read {}: {}", file.display(), e);
99	std::process::exit(1);
100	}
101	};
102
103	let mut result = transpile_to_string(contents.clone())?;
104
105	if format {
106	let mut formatter = std::process::Command::new("deno")
107	.stdin(std::process::Stdio::piped())
108	.stdout(std::process::Stdio::piped())
109	.stderr(std::process::Stdio::inherit())
110	.arg("fmt")
111	.arg("-")
112	.spawn()
113	.expect("Failed to execute 'deno fmt' on output");
114	let mut stdin = formatter.stdin.take().expect("Failed to open stdin");
115	stdin
116	.write_all(result.as_bytes())
117	.expect("Failed to write to stdin");
118	stdin.flush().expect("Failed to flush stdin");
119	drop(stdin);
120	let out = formatter
121	.wait_with_output()
122	.expect("Failed to execute 'deno fmt' on output");
123	result = String::from_utf8(out.stdout).unwrap();
124	}
125
126	if stdout {
127	println!("{}", result);
128	} else {
129	std::fs::write(file.with_extension("js"), result).unwrap();
130	}
131	}
132	Commands::Run { file, target } => {
133	let contents = std::fs::read_to_string(file.clone()).unwrap();
134	let result = transpile_to_string(contents.clone())?;
135	let mut temp_file = std::env::temp_dir();
136	temp_file.push("temp.js");
137	std::fs::write(temp_file.clone(), result).unwrap();
138
139	let rt = get_runtime(&target);
140
141	match rt.as_str() {
142	"deno" => std::process::Command::new("deno")
143	.arg("run")
144	.arg("-A")
145	.arg(temp_file)
146	.spawn()
147	.expect("Failed to execute process"),
148	"node" => std::process::Command::new("node")
149	.arg(temp_file)
150	.spawn()
151	.expect("Failed to execute process"),
152	"bun" => std::process::Command::new("bun")
153	.arg("run")
154	.arg(temp_file)
155	.spawn()
156	.expect("Failed to execute process"),
157	_ => {
158	panic!("Unknown runtime: {}", target);
159	}
160	}
161	.wait()
162	.unwrap();
163	}
164	Commands::Astdump { file } => {
165	let contents = std::fs::read_to_string(file.clone()).unwrap();
166	let result = astdump(contents.clone())?;
167	println!("{}", result);
168	}
169	}
170
171	Ok(())
172	}

crates/js_ast/Cargo.toml deleted-15

...	@@ -1,15 +0,0 @@
1	[package]
2	name = "js_ast"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path = "./lib.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12	num.workspace = true
13
14	ast = { path = "../ast" }
15	log = { path = "../log" }

crates/js_ast/convert/block_convert_context.rs deleted-244

...	@@ -1,244 +0,0 @@
1	use crate::js_ast::*;
2	use ast::*;
3	use log::Range;
4	use std::collections::{HashMap, HashSet};
5	use std::vec;
6
7	use super::tree_shake_statement_list;
8
9	/// A block convert context keeps track of state while converting non-valid JS expressions to JS
10	pub struct BlockConvertContext {
11	/// Statements in this block context
12	/// This may be the merged contents of multiple blocks
13	statements: Vec<JsStatement>,
14	ids: HashSet<String>,
15	next_id: u32,
16	scopes: Vec<VirtualScope>,
17	/// If the last item in `statements` is actually valid
18	has_last_statement: bool,
19	}
20
21	struct VirtualScope {
22	renames: HashMap<String, String>,
23	}
24
25	pub struct TempId {
26	unmapped_name: String,
27	}
28
29	pub struct StatementHandle {
30	index: Option<u32>,
31	}
32
33	impl StatementHandle {
34	pub fn delete(self, ctx: &mut BlockConvertContext) {
35	if let Some(index) = self.index {
36	ctx.statements.remove(index as usize);
37	}
38	}
39	}
40
41	impl TempId {
42	pub fn get(&self, hoist_context: &BlockConvertContext) -> String {
43	hoist_context.remap_identifier(&self.unmapped_name)
44	}
45	}
46
47	impl BlockConvertContext {
48	pub fn new() -> Self {
49	Self {
50	statements: Vec::new(),
51	ids: HashSet::new(),
52	next_id: 0,
53	scopes: vec![VirtualScope {
54	renames: HashMap::new(),
55	}],
56	has_last_statement: false,
57	}
58	}
59
60	pub fn new_child(&self) -> Self {
61	Self {
62	statements: Vec::new(),
63	ids: self.ids.clone(),
64	next_id: 0,
65	scopes: vec![VirtualScope {
66	renames: HashMap::new(),
67	}],
68	has_last_statement: false,
69	}
70	}
71
72	/// A virtual scope is a scope that exists in blight syntax
73	/// but may or may not exist in blight. With these it is important to
74	pub fn push_virtual_scope(&mut self) {
75	self.scopes.push(VirtualScope {
76	renames: HashMap::new(),
77	});
78	}
79
80	/// See `push_virtual_scope` for more information on what this is.
81	pub fn pop_virtual_scope(&mut self) {
82	self.scopes.pop();
83	debug_assert!(self.scopes.len() > 0);
84	}
85
86	pub fn generate_temp_id(&mut self, basename: &str) -> TempId {
87	self.next_id += 1;
88	loop {
89	let name = format!("{}{}", basename, self.next_id);
90	if !self.ids.contains(&name) {
91	self.ids.insert(name.clone());
92	return TempId {
93	unmapped_name: name,
94	};
95	}
96	self.next_id += 1;
97	}
98	}
99
100	/// This does some extra optimizations:
101	/// - `let a = b;` will not be emitted but instad write a variable alias
102	pub fn add_statement(&mut self, statement: JsStatement) -> StatementHandle {
103	match statement.data {
104	JsStatementData::Declaration(JSDeclaration {
105	name,
106	init:
107	Some(JsExpr {
108	data: JsExprData::Identifier(init_name),
109	..
110	}),
111	is_constant: true,
112	}) => {
113	self.scopes
114	.last_mut()
115	.unwrap()
116	.renames
117	.insert(name, init_name);
118	self.has_last_statement = false;
119	return StatementHandle { index: None };
120	}
121	_ => {}
122	}
123	self.statements.push(statement);
124	self.has_last_statement = true;
125	StatementHandle {
126	index: Some(self.statements.len() as u32 - 1),
127	}
128	}
129
130	/// Get the last statement if it exists.
131	/// If the statement was optimized away, this returns None
132	pub fn last_statement(&self) -> Option<&JsStatement> {
133	if self.has_last_statement {
134	Some(self.statements.last().unwrap())
135	} else {
136	None
137	}
138	}
139
140	pub fn generate_id(&mut self, basename: &String) -> TempId {
141	let mut id = 0u32;
142	loop {
143	let name = match id {
144	0 => basename.clone(),
145	_ => format!("{}{}", basename, id),
146	};
147	if !self.ids.contains(&name) {
148	self.ids.insert(name.clone());
149	return TempId {
150	unmapped_name: name,
151	};
152	}
153	id += 1;
154	}
155	}
156
157	/// Given a blight identifier, remap it to a new identifier.
158	pub fn remap_identifier(&self, name: &String) -> String {
159	for scope in self.scopes.iter().rev() {
160	if let Some(remapped) = scope.renames.get(name) {
161	return remapped.clone();
162	}
163	}
164	name.clone()
165	}
166
167	/// Declare an identifier in the current scope.
168	pub fn declare_identifier(&mut self, name: String) -> TempId {
169	let remapped = self.generate_id(&name);
170	let scope = self.scopes.last_mut().unwrap();
171	self.ids.insert(remapped.unmapped_name.clone());
172	scope.renames.insert(name, remapped.unmapped_name.clone());
173	remapped
174	}
175
176	pub fn take_last_statement(&mut self) -> Option<JsStatement> {
177	if !self.has_last_statement {
178	return None;
179	}
180	self.has_last_statement = false;
181	Some(self.statements.pop().unwrap())
182	}
183
184	pub fn take_last_statement_if_expr(&mut self) -> Option<JsExpr> {
185	if !self.has_last_statement {
186	return None;
187	}
188	match self.statements.last().unwrap().data {
189	JsStatementData::Expression(_) => {
190	self.has_last_statement = false;
191	match self.statements.pop().unwrap().data {
192	JsStatementData::Expression(expr) => Some(expr),
193	_ => unreachable!(),
194	}
195	}
196	_ => return None,
197	}
198	}
199	}
200
201	pub fn new_js_assignment_statement(expr: JsExpr, target: String, range: Range) -> JsStatement {
202	JsStatement::new(
203	JsStatementData::Expression(new_js_assignment_expr(expr, target, range.clone())),
204	range,
205	None,
206	)
207	}
208
209	pub fn new_js_assignment_expr(expr: JsExpr, target: String, range: Range) -> JsExpr {
210	JsExpr::new(
211	JsExprData::Binary {
212	op: BinaryOperator::Assign,
213	left: Box::new(JsExpr::new(
214	JsExprData::Identifier(target),
215	range.clone(),
216	None,
217	)),
218	right: Box::new(expr),
219	},
220	range,
221	None,
222	)
223	}
224
225	impl Into<JsBlock> for BlockConvertContext {
226	fn into(self) -> JsBlock {
227	JsBlock::new(self.into(), Range::EMPTY)
228	}
229	}
230
231	impl Into<JsModule> for BlockConvertContext {
232	fn into(self) -> JsModule {
233	JsModule {
234	statements: self.into(),
235	}
236	}
237	}
238
239	impl Into<Vec<JsStatement>> for BlockConvertContext {
240	fn into(mut self) -> Vec<JsStatement> {
241	tree_shake_statement_list(&mut self.statements);
242	self.statements
243	}
244	}

crates/js_ast/convert/convert_valid_block.rs deleted-37

...	@@ -1,37 +0,0 @@
1	use super::convert_valid_statement_to_expr;
2	use super::BlockConvertContext;
3	use super::IsValidJSExpression;
4	use crate::js_ast::*;
5	use ast::*;
6	use log::Range;
7
8	pub fn convert_valid_block(ctx: &mut BlockConvertContext, block: Block, range: Range) -> JsExpr {
9	debug_assert!(
10	block.is_valid_js_expression(),
11	"must call convert_valid_block with valid js expression"
12	);
13
14	let len = block.statements.len();
15	let range = range;
16	let block_expr = match len {
17	0 => JsExpr::new(JsExprData::Null, range.clone(), None),
18	// a block expression with a single expression is just that expression
19	1 => convert_valid_statement_to_expr(ctx, block.statements.into_iter().next().unwrap()),
20	// Since blocks are not valid expressions in JS, we create a
21	// comma expression from a block with more than one statement.
22	2.. => {
23	let mut items = Vec::new();
24	for statement in block.statements.into_iter() {
25	items.push(convert_valid_statement_to_expr(ctx, statement));
26	}
27	JsExpr::new(JsExprData::Comma { items }, range.clone(), None)
28	}
29	};
30
31	// a block without implicit return is to be type void, so we wrap in a void expression
32	if !block.has_implicit_return && len > 0 {
33	JsExpr::new(JsExprData::Void(Box::new(block_expr)), range, None)
34	} else {
35	block_expr
36	}
37	}

crates/js_ast/convert/convert_valid_expr.rs deleted-115

...	@@ -1,115 +0,0 @@
1	use super::convert_valid_block;
2	use super::BlockConvertContext;
3	use crate::convert::convert_valid_if;
4	use crate::convert::process_function::process_function_to_arrow;
5	use crate::convert::IsValidJSExpression;
6	use crate::js_ast::*;
7	use ast::*;
8
9	/// Translate a blight ast expression that is a valid JS expression, to a JS expression
10	/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
11	pub fn convert_valid_expr(ctx: &mut BlockConvertContext, expr: Expr) -> JsExpr {
12	// Assert may be skipped in release builds as all failures are
13	// annotated with a release-checked panic
14	debug_assert!(
15	expr.is_valid_js_expression(),
16	"must call convert_valid_expr with valid js expression (use hoist instead)"
17	);
18
19	let range = expr.range;
20	let comment = expr.comment;
21
22	match expr.data {
23	ExprData::Identifier(str) => JsExpr::new(
24	JsExprData::Identifier(ctx.remap_identifier(&str)),
25	range,
26	comment,
27	),
28
29	ExprData::Literal(Literal::Bool(value)) => {
30	JsExpr::new(JsExprData::Bool(value), range, comment)
31	}
32	ExprData::Literal(Literal::Number(value)) => {
33	JsExpr::new(JsExprData::Number(value), range, comment)
34	}
35	ExprData::Literal(Literal::String(value)) => {
36	JsExpr::new(JsExprData::String(value), range, comment)
37	}
38	ExprData::Literal(Literal::Void) => JsExpr::new(JsExprData::Null, range, comment),
39
40	ExprData::Block(block) => convert_valid_block(ctx, block, range),
41
42	ExprData::Binary { op, left, right } => {
43	let left = convert_valid_expr(ctx, *left);
44	let right = convert_valid_expr(ctx, *right);
45	JsExpr::new(
46	JsExprData::Binary {
47	op,
48	left: Box::new(left),
49	right: Box::new(right),
50	},
51	range,
52	comment,
53	)
54	}
55	ExprData::Unary { op, value } => {
56	let value = convert_valid_expr(ctx, *value);
57	JsExpr::new(
58	JsExprData::Unary {
59	op,
60	value: Box::new(value),
61	},
62	range,
63	comment,
64	)
65	}
66	ExprData::Fn { body, params, .. } => {
67	process_function_to_arrow(ctx, *body, params, range, comment)
68	}
69	ExprData::Builtin { id, arguments } => match id {
70	BuiltinId::Raw => {
71	let first_arg = arguments.iter().next().unwrap();
72	// TODO(@paperdave): this string clone is stupid
73	let str = match first_arg.data.clone() {
74	ExprData::Literal(Literal::String(s)) => s,
75	_ => panic!("raw expects a string"),
76	};
77	JsExpr::new(JsExprData::Raw(str), range, comment)
78	}
79	},
80	ExprData::If(expr) => convert_valid_if(ctx, expr, range, comment),
81
82	ExprData::Call { base, arguments } => {
83	let base = convert_valid_expr(ctx, *base);
84	let arguments = arguments
85	.into_iter()
86	.map(\|x\| convert_valid_expr(ctx, x))
87	.collect();
88	JsExpr::new(
89	JsExprData::Call {
90	base: Box::new(base),
91	arguments,
92	},
93	range,
94	comment,
95	)
96	}
97
98	ExprData::Field { base, field } => JsExpr::new(
99	match field.data {
100	ExprData::Literal(Literal::String(name)) => JsExprData::Dot {
101	base: Box::new(convert_valid_expr(ctx, *base)),
102	name,
103	},
104	_ => JsExprData::Index {
105	base: Box::new(convert_valid_expr(ctx, *base)),
106	index: Box::new(convert_valid_expr(ctx, *field)),
107	},
108	},
109	range,
110	comment,
111	),
112
113	ExprData::Return(_) => unreachable!("convert_valid_expr called on return"),
114	}
115	}

crates/js_ast/convert/convert_valid_if.rs deleted-38

...	@@ -1,38 +0,0 @@
1	use super::convert_valid_block;
2	use super::convert_valid_expr;
3	use super::BlockConvertContext;
4	use crate::convert::IsValidJSExpression;
5	use crate::js_ast::*;
6	use ast::*;
7	use log::Range;
8
9	/// Convert a valid blight if statement to a JS expression
10	pub fn convert_valid_if(
11	ctx: &mut BlockConvertContext,
12	expr: If,
13	range: Range,
14	comment: Option<Comment>,
15	) -> JsExpr {
16	debug_assert!(
17	expr.is_valid_js_expression(),
18	"convert_valid_if on an invalid expression (use hoist instead)"
19	);
20	let condition = convert_valid_expr(ctx, *expr.condition);
21
22	let yes = convert_valid_block(ctx, expr.yes.block, expr.yes.range);
23	let no = match expr.no {
24	Some(BlockOrIf::Block(b)) => Some(convert_valid_block(ctx, b.block, b.range)),
25	Some(BlockOrIf::If(else_if)) => Some(convert_valid_if(ctx, *else_if, Range::EMPTY, None)),
26	None => None,
27	};
28
29	JsExpr::new(
30	JsExprData::Ternary {
31	condition: Box::new(condition),
32	yes: Box::new(yes),
33	no: no.map(Box::new),
34	},
35	range,
36	comment,
37	)
38	}

crates/js_ast/convert/convert_valid_statement.rs deleted-32

...	@@ -1,32 +0,0 @@
1	use super::convert_valid_expr;
2	use super::is_valid_js_expression::*;
3	use super::BlockConvertContext;
4	use crate::js_ast::*;
5	use ast::*;
6
7	/// Translate a blight ast statement that is a valid JS expression, to a JS expression
8	/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
9	pub fn convert_valid_statement_to_expr(
10	ctx: &mut BlockConvertContext,
11	statement: Statement,
12	) -> JsExpr {
13	debug_assert!(
14	statement.is_valid_js_expression(),
15	"convert_valid_statement_to_expr on an invalid expression (use hoist instead)"
16	);
17
18	match statement {
19	ast::Statement::Expr(expr) => convert_valid_expr(ctx, expr),
20	_ => unreachable!("attempt to convert statement to js without hoisting"),
21	}
22	}
23
24	/// Translate a blight ast statement that is a valid JS expression, to a JS statement
25	/// Precondition: it is a valid JS expression, check this with `is_valid_js_expression`
26	///
27	/// TODO(@paperdave): Replace this with Into<JsStatement> for JsExpr
28	pub fn convert_valid_statement(ctx: &mut BlockConvertContext, statement: Statement) -> JsStatement {
29	let expr = convert_valid_statement_to_expr(ctx, statement);
30	let range = expr.range.clone();
31	JsStatement::new(JsStatementData::Expression(expr), range, None)
32	}

crates/js_ast/convert/hoist_block_to_expr.rs deleted-78

...	@@ -1,78 +0,0 @@
1	use super::is_valid_js_expression::*;
2	use super::process_statement;
3	use super::BlockConvertContext;
4	use crate::js_ast::*;
5	use ast::*;
6	use log::Range;
7
8	/// This block hoist is used when we have a block in an expression context.
9	///
10	/// All the expressions in the block are lifted above the currently
11	/// processed expression, and then a temporary variable is returned.
12	/// (or the last expression might get inlined as the return value)
13	///
14	/// A "virtual scope" is created for this block, which handles renaming
15	/// situations like:
16	///
17	/// let x = 1;
18	/// {
19	/// let x = 1;
20	/// }
21	/// call(x)
22	///
23	/// The output JS of the above would be:
24	///
25	/// const x = 1;
26	/// const x2 = 1;
27	/// call(x)
28	///
29	/// For expressions that have branches, use `hoist_block_to_js_block`
30	pub fn hoist_block_to_expr(
31	ctx: &mut BlockConvertContext,
32	block: Block,
33	range: Range,
34	comment: Option<Comment>,
35	) -> Option<JsExpr> {
36	debug_assert!(
37	!block.is_valid_js_expression(),
38	"hoist_block on a valid expression, should call convert_valid_block instead"
39	);
40
41	let temporary = ctx.generate_temp_id("_block");
42
43	ctx.push_virtual_scope();
44
45	for statement in block.statements {
46	process_statement(ctx, statement);
47	}
48
49	ctx.pop_virtual_scope();
50
51	// If the last statement is an expression, we need
52	// to assign to the temporary variable.
53	let result = match ctx.last_statement().map(\|x\| &x.data) {
54	Some(JsStatementData::Expression(_)) if block.has_implicit_return => {
55	match ctx.take_last_statement().unwrap().data {
56	JsStatementData::Expression(expr) => expr,
57	_ => unreachable!(),
58	}
59	}
60	_ => JsExpr::new(JsExprData::Null, range.clone(), None),
61	};
62
63	ctx.add_statement(JsStatement::new(
64	JsStatementData::Declaration(JSDeclaration {
65	name: temporary.get(&ctx),
66	init: Some(result),
67	is_constant: false,
68	}),
69	range.clone(),
70	comment,
71	));
72
73	Some(JsExpr::new(
74	JsExprData::Identifier(temporary.get(&ctx)),
75	range,
76	None,
77	))
78	}

crates/js_ast/convert/hoist_block_to_js_block.rs deleted-70

...	@@ -1,70 +0,0 @@
1	use crate::js_ast::*;
2	use ast::*;
3	use log::Range;
4
5	use super::*;
6
7	/// This block hoist is used when we actually need to create a JsBlock, which
8	/// is needed if there is an invalid expression within a branch of code.
9	///
10	/// For example, `if` has two branches, the 'yes' and 'no' paths.
11	/// When we cannot lower it as a ternary, we have to create two JsBlocks
12	///
13	/// The `target` param is a shared temporary variable that will be assigned
14	/// to the result of the block.
15	///
16	/// Since blocks can be void or noreturn, this target is not always
17	/// assigned to. The `did_assign` flag is used to indicate that.
18	///
19	/// For expressions that do not have branches, use `hoist_block_to_expr`
20	///
21	/// TODO(@paperdave): move `did_assign` into the return value
22	pub fn hoist_block_to_js_block(
23	ctx: &mut BlockConvertContext,
24	block: Block,
25	range: Range,
26	target: &TempId,
27	did_assign: &mut bool,
28	) -> JsBlock {
29	// A non-block will become a block with a single expression:
30	// _temp = expr;
31	//
32	// We will also apply this transformation to blocks that are valid
33	// expressions, because that is more concise.
34	if block.is_valid_js_expression() {
35	return JsBlock::new(
36	vec![new_js_assignment_statement(
37	convert_valid_block(ctx, block, range.clone()),
38	target.get(ctx),
39	range.clone(),
40	)],
41	range,
42	);
43	}
44
45	let mut child = ctx.new_child();
46
47	for statement in block.statements {
48	if let None = process_statement(&mut child, statement) {
49	return child.into();
50	};
51	}
52
53	// If the last statement is an expression, we need
54	// to assign to the temporary variable.
55	if block.has_implicit_return {
56	match child.take_last_statement_if_expr() {
57	Some(expr) => {
58	*did_assign = true;
59	child.add_statement(new_js_assignment_statement(
60	expr,
61	target.get(ctx),
62	range.clone(),
63	));
64	}
65	_ => {}
66	}
67	}
68
69	child.into()
70	}

crates/js_ast/convert/hoist_expr.rs deleted-139

...	@@ -1,139 +0,0 @@
1	use super::convert_valid_expr;
2	use super::hoist_block_to_expr;
3	use super::hoist_if_expr;
4	use super::is_valid_js_expression::*;
5	use super::BlockConvertContext;
6	// use super::HasSideEffects;
7	use crate::js_ast::*;
8	use crate::util::is_js_identifier;
9	use ast::*;
10
11	/// Pushes a decl with a generated identifier to the hoisted result
12	/// and returns an identifier expression for it.
13	pub fn hoist_expr_if_needed(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
14	// if !expr.has_side_effects() {
15	// return Some(ctx.hoist_expression_contents(expr)?);
16	// }
17
18	let init = hoist_expr_contents(ctx, expr)?;
19
20	let id = ctx.generate_temp_id("_");
21	let range = init.range.clone();
22
23	ctx.add_statement(JsStatement::new(
24	JsStatementData::Declaration(JSDeclaration {
25	name: id.get(&ctx),
26	init: Some(init),
27	is_constant: true,
28	}),
29	range.clone(),
30	None,
31	));
32
33	Some(JsExpr::new(
34	JsExprData::Identifier(id.get(&ctx)),
35	range,
36	None,
37	))
38	}
39
40	/// Returning `None` from this means that the expression is noreturn, and that
41	/// the rest of the statement never occurs and can be stripped.
42	pub fn hoist_expr_contents(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
43	if expr.is_valid_js_expression() {
44	return Some(convert_valid_expr(ctx, expr));
45	}
46
47	match expr.data {
48	ExprData::Binary { op, left, right } => {
49	let left = hoist_expr_if_needed(ctx, *left)?;
50	let right = hoist_expr_if_needed(ctx, *right)?;
51
52	Some(JsExpr::new(
53	JsExprData::Binary {
54	op,
55	left: Box::new(left),
56	right: Box::new(right),
57	},
58	expr.range,
59	expr.comment,
60	))
61	}
62
63	ExprData::Unary { op, value } => {
64	let value = hoist_expr_if_needed(ctx, *value)?;
65	Some(JsExpr::new(
66	JsExprData::Unary {
67	op,
68	value: Box::new(value),
69	},
70	expr.range,
71	expr.comment,
72	))
73	}
74
75	ExprData::Field { base, field } => {
76	let base = hoist_expr_if_needed(ctx, *base)?;
77	let field = hoist_expr_if_needed(ctx, *field)?;
78
79	Some(JsExpr::new(
80	match field.data {
81	JsExprData::String(name) if is_js_identifier(name.as_str()) => {
82	JsExprData::Dot {
83	base: Box::new(base),
84	name,
85	}
86	}
87	_ => JsExprData::Index {
88	base: Box::new(base),
89	index: Box::new(field),
90	},
91	},
92	expr.range,
93	expr.comment,
94	))
95	}
96
97	ExprData::Block(block) => hoist_block_to_expr(ctx, block, expr.range, expr.comment),
98
99	ExprData::If(_) => hoist_if_expr(ctx, expr),
100
101	ExprData::Return(maybe_expr) => {
102	let range = expr.range.clone();
103	let return_value = match maybe_expr {
104	Some(expr) => Some(hoist_expr_if_needed(ctx, *expr)?),
105	None => None,
106	};
107	ctx.add_statement(JsStatement::new(
108	JsStatementData::Return(return_value),
109	range,
110	None,
111	));
112	None
113	}
114
115	ExprData::Call { base, arguments } => {
116	let base = hoist_expr_if_needed(ctx, *base)?;
117	let arguments = arguments
118	.into_iter()
119	.map(\|x\| hoist_expr_if_needed(ctx, x))
120	.collect::<Option<Vec<_>>>()?;
121
122	Some(JsExpr::new(
123	JsExprData::Call {
124	base: Box::new(base),
125	arguments,
126	},
127	expr.range,
128	expr.comment,
129	))
130	}
131
132	ExprData::Fn { .. }
133	\| ExprData::Builtin { .. }
134	\| ExprData::Literal(_)
135	\| ExprData::Identifier(_) => {
136	unreachable!("attempt to hoist an expression that does not require it")
137	}
138	}
139	}

crates/js_ast/convert/hoist_if.rs deleted-113

...	@@ -1,113 +0,0 @@
1	use super::convert_valid_expr;
2	use super::hoist_block_to_js_block;
3	use super::hoist_expr_if_needed;
4	use super::BlockConvertContext;
5	use super::IsValidJSExpression;
6	use super::TempId;
7	use crate::js_ast::*;
8	use ast::*;
9	use log::Range;
10
11	pub fn hoist_if_expr(ctx: &mut BlockConvertContext, expr: Expr) -> Option<JsExpr> {
12	let data = match expr.data {
13	ExprData::If(data) => data,
14	_ => unreachable!("hoist_if_expr called on non-if expression"),
15	};
16
17	let temporary = ctx.generate_temp_id("_if");
18
19	let decl = ctx.add_statement(JsStatement::new(
20	JsStatementData::Declaration(JSDeclaration {
21	name: temporary.get(&ctx),
22	init: None,
23	is_constant: false,
24	}),
25	expr.range.clone(),
26	None,
27	));
28
29	let condition = hoist_expr_if_needed(ctx, *data.condition)?;
30	let mut did_assign = false;
31
32	let js_if = hoist_if_branch(
33	ctx,
34	data.yes,
35	data.no,
36	condition,
37	&mut did_assign,
38	&temporary,
39	);
40
41	ctx.add_statement(JsStatement::new(
42	JsStatementData::If(js_if),
43	expr.range.clone(),
44	None,
45	));
46
47	if did_assign {
48	Some(JsExpr::new(
49	JsExprData::Identifier(temporary.get(&ctx)),
50	expr.range,
51	expr.comment,
52	))
53	} else {
54	decl.delete(ctx);
55	Some(JsExpr::new(JsExprData::Null, expr.range, expr.comment))
56	}
57	}
58
59	pub fn hoist_if_branch(
60	ctx: &mut BlockConvertContext,
61	yes: BlockWithRange,
62	no: Option<BlockOrIf>,
63	condition: JsExpr,
64	did_assign: &mut bool,
65	temporary: &TempId,
66	) -> JsIf {
67	let yes = hoist_block_to_js_block(ctx, yes.block, yes.range, &temporary, did_assign);
68
69	let no = match no {
70	Some(BlockOrIf::Block(no)) => Some(JsElse::Block(hoist_block_to_js_block(
71	ctx, no.block, no.range, &temporary, did_assign,
72	))),
73	Some(BlockOrIf::If(else_if)) => {
74	// If the condition of this block has a side effect, we need to
75	// emit a block so that we can assign to the temporary variable.
76	//
77	// Otherwise, we can just emit the condition inline
78	if else_if.condition.is_valid_js_expression() {
79	let condition = convert_valid_expr(ctx, *else_if.condition.clone());
80	Some(JsElse::If(Box::new(hoist_if_branch(
81	ctx,
82	else_if.yes,
83	else_if.no,
84	condition,
85	did_assign,
86	temporary,
87	))))
88	} else {
89	let mut temp_block = ctx.new_child();
90	let condition =
91	hoist_expr_if_needed(&mut temp_block, *else_if.condition.clone()).unwrap();
92
93	temp_block.add_statement(JsStatement::new(
94	JsStatementData::If(hoist_if_branch(
95	ctx,
96	else_if.yes,
97	else_if.no,
98	condition,
99	did_assign,
100	temporary,
101	)),
102	Range::EMPTY,
103	None,
104	));
105
106	Some(JsElse::Block(temp_block.into()))
107	}
108	}
109	None => None,
110	};
111
112	JsIf { condition, yes, no }
113	}

crates/js_ast/convert/is_valid_js_expression.rs deleted-91

...	@@ -1,91 +0,0 @@
1	use ast::*;
2
3	/// "Valid" in this context means that the expression does not contain JS statements
4	/// Meaning conversion is a direct map from a blight as node into a JS one
5	///
6	/// A valid expression should not go through any hoist logic, but if an expression
7	/// isn't valid, then it means that it must be split into multiple JS statements
8	/// inside of a block.
9	pub trait IsValidJSExpression {
10	/// Can this be lowered to a JS ast without hoisting items
11	/// ex: a `return` expression is not a valid JS expression
12	fn is_valid_js_expression(&self) -> bool;
13	}
14
15	impl IsValidJSExpression for ExprData {
16	fn is_valid_js_expression(&self) -> bool {
17	match self {
18	// A block containing just expressions can be an expression
19	ExprData::Block(block) => block.is_valid_js_expression(),
20	ExprData::If(expr) => expr.is_valid_js_expression(),
21	ExprData::Return(_) => false,
22	ExprData::Builtin { id, .. } => match id {
23	BuiltinId::Raw => true,
24	},
25	ExprData::Binary { left, right, .. } => {
26	left.is_valid_js_expression() && right.is_valid_js_expression()
27	}
28	ExprData::Unary { value, .. } => value.is_valid_js_expression(),
29	ExprData::Fn { .. } => true,
30	ExprData::Call { base, arguments } => {
31	base.is_valid_js_expression()
32	&& arguments
33	.iter()
34	.all(\|argument\| argument.is_valid_js_expression())
35	}
36	ExprData::Field { base, field } => {
37	base.is_valid_js_expression() && field.is_valid_js_expression()
38	}
39	ExprData::Literal(_) \| ExprData::Identifier(_) => true,
40	}
41	}
42	}
43
44	impl IsValidJSExpression for Block {
45	fn is_valid_js_expression(&self) -> bool {
46	self.statements
47	.iter()
48	.all(\|statement\| statement.is_valid_js_expression())
49	}
50	}
51
52	impl IsValidJSExpression for Expr {
53	fn is_valid_js_expression(&self) -> bool {
54	self.data.is_valid_js_expression()
55	}
56	}
57
58	impl IsValidJSExpression for If {
59	fn is_valid_js_expression(&self) -> bool {
60	self.condition.is_valid_js_expression()
61	&& self.yes.is_valid_js_expression()
62	&& match &self.no {
63	Some(no) => no.is_valid_js_expression(),
64	None => true,
65	}
66	}
67	}
68
69	impl IsValidJSExpression for BlockOrIf {
70	fn is_valid_js_expression(&self) -> bool {
71	match self {
72	BlockOrIf::Block(block) => block.is_valid_js_expression(),
73	BlockOrIf::If(if_) => if_.is_valid_js_expression(),
74	}
75	}
76	}
77
78	impl IsValidJSExpression for BlockWithRange {
79	fn is_valid_js_expression(&self) -> bool {
80	self.block.is_valid_js_expression()
81	}
82	}
83
84	impl IsValidJSExpression for Statement {
85	fn is_valid_js_expression(&self) -> bool {
86	match self {
87	Statement::Expr(expr) => expr.data.is_valid_js_expression(),
88	_ => false,
89	}
90	}
91	}

crates/js_ast/convert/mod.rs deleted-42

...	@@ -1,42 +0,0 @@
1	mod block_convert_context;
2	mod convert_valid_block;
3	mod convert_valid_expr;
4	mod convert_valid_if;
5	mod convert_valid_statement;
6	mod hoist_block_to_expr;
7	mod hoist_block_to_js_block;
8	mod hoist_expr;
9	mod hoist_if;
10	mod is_valid_js_expression;
11	mod process_function;
12	mod process_statement;
13	mod side_effects;
14	mod tree_shake_statement_list;
15
16	pub use block_convert_context::*;
17	pub use convert_valid_block::*;
18	pub use convert_valid_expr::*;
19	pub use convert_valid_if::*;
20	pub use convert_valid_statement::*;
21	pub use hoist_block_to_expr::*;
22	pub use hoist_block_to_js_block::*;
23	pub use hoist_expr::*;
24	pub use hoist_if::*;
25	pub use is_valid_js_expression::*;
26	pub use process_function::*;
27	pub use process_statement::*;
28	// pub use side_effects::*;
29	pub use tree_shake_statement_list::*;
30
31	use crate::js_ast::JsModule;
32	use ast::Module;
33
34	pub fn blight_to_js(blight: Module) -> JsModule {
35	let mut root_context = BlockConvertContext::new();
36
37	for statement in blight.statements {
38	process_statement(&mut root_context, statement).unwrap();
39	}
40
41	root_context.into()
42	}

crates/js_ast/convert/process_function.rs deleted-65

...	@@ -1,65 +0,0 @@
1	use super::convert_valid_expr;
2	use super::hoist_expr_if_needed;
3	use super::process_statement;
4	use super::BlockConvertContext;
5	use super::IsValidJSExpression;
6	use crate::js_ast::*;
7	use ast::*;
8	use log::Range;
9
10	pub fn process_function_to_arrow(
11	ctx: &mut BlockConvertContext,
12	body: Expr,
13	params: Vec<Param>,
14	range: Range,
15	comment: Option<Comment>,
16	) -> JsExpr {
17	// A block that is a valid expression will just become
18	// that expression. We can convert multiple expressions
19	// into a comma expression.
20	let body = if body.is_valid_js_expression() {
21	JsArrowBody::Expr(convert_valid_expr(ctx, body))
22	} else {
23	let mut child = ctx.new_child();
24
25	match body.data {
26	ExprData::Block(block) => {
27	for statement in block.statements {
28	if let None = process_statement(&mut child, statement) {
29	break;
30	};
31	}
32
33	if block.has_implicit_return {
34	if let Some(expr) = child.take_last_statement_if_expr() {
35	child.add_statement(JsStatement::new(
36	JsStatementData::Return(Some(expr)),
37	range.clone(),
38	None,
39	));
40	}
41	}
42	}
43	_ => {
44	if let Some(expr) = hoist_expr_if_needed(&mut child, body) {
45	child.add_statement(JsStatement::new(
46	JsStatementData::Return(Some(expr)),
47	range.clone(),
48	None,
49	));
50	}
51	}
52	}
53
54	JsArrowBody::Block(child.into())
55	};
56
57	JsExpr::new(
58	JsExprData::ArrowFunction {
59	params: params.into_iter().map(\|x\| x.name).collect(),
60	body: Box::new(body),
61	},
62	range.clone(),
63	comment.map(\|x\| x.into()),
64	)
65	}

crates/js_ast/convert/process_statement.rs deleted-98

...	@@ -1,98 +0,0 @@
1	use super::convert_valid_statement;
2	use super::hoist_expr_contents;
3	use super::process_function_to_arrow;
4	use super::BlockConvertContext;
5	use super::IsValidJSExpression;
6	use crate::js_ast::*;
7	use ast::*;
8
9	pub fn process_statement(ctx: &mut BlockConvertContext, statement: Statement) -> Option<()> {
10	if statement.is_valid_js_expression() {
11	let result = convert_valid_statement(ctx, statement);
12	ctx.add_statement(result);
13	return Some(());
14	}
15
16	match statement {
17	Statement::VarDecl(decl) => {
18	// visit init before name, this way allows for:
19	//
20	// let a = a;
21	//
22	// and `a` will be looked up before it is re-declared.
23	// we will also return in the noreturn case
24	let init = hoist_expr_contents(ctx, decl.init)?;
25	let name = ctx.declare_identifier(decl.name);
26
27	let decl_data = JSDeclaration {
28	name: name.get(&ctx),
29	init: Some(init),
30	is_constant: decl.is_constant,
31	};
32	let stmt = JsStatement::new(
33	if decl.is_pub {
34	JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
35	} else {
36	JsStatementData::Declaration(decl_data)
37	},
38	decl.range,
39	decl.comment.map(\|x\| x.into()),
40	);
41
42	ctx.add_statement(stmt);
43
44	Some(())
45	}
46	Statement::Expr(expr) => {
47	if let Some(expr) = hoist_expr_contents(ctx, expr) {
48	let range = expr.range.clone();
49	ctx.add_statement(JsStatement::new(
50	JsStatementData::Expression(expr),
51	range,
52	None,
53	));
54	Some(())
55	} else {
56	None
57	}
58	}
59	Statement::FnDecl(FnDecl {
60	name,
61	params,
62	body,
63	range,
64	comment,
65	is_pub,
66	..
67	}) => {
68	let name = ctx.declare_identifier(name);
69
70	let processed = Some(process_function_to_arrow(
71	ctx,
72	body,
73	params,
74	range.clone(),
75	None,
76	));
77
78	let decl_data = JSDeclaration {
79	name: name.get(&ctx),
80	init: processed,
81	is_constant: true,
82	};
83
84	ctx.add_statement(JsStatement::new(
85	if is_pub {
86	JsStatementData::Export(JsExport::Declaration(Box::new(decl_data)))
87	} else {
88	JsStatementData::Declaration(decl_data)
89	},
90	range,
91	comment.map(\|x\| x.into()),
92	));
93
94	Some(())
95	}
96	Statement::Comment(_) => Some(()),
97	}
98	}

crates/js_ast/convert/side_effects.rs deleted-91

...	@@ -1,91 +0,0 @@
1	use ast::*;
2
3	pub trait HasSideEffects {
4	/// Can this item be removed from the AST without changing the program's behavior?
5	fn has_side_effects(&self) -> bool;
6	}
7
8	impl HasSideEffects for ExprData {
9	fn has_side_effects(&self) -> bool {
10	match self {
11	ExprData::Literal(_) => false,
12	ExprData::Identifier(_) => false,
13
14	ExprData::Block(block) => block.has_side_effects(),
15	ExprData::Binary { left, right, .. } => {
16	left.has_side_effects() \|\| right.has_side_effects()
17	}
18	ExprData::Unary { value, .. } => value.has_side_effects(),
19	ExprData::If(ifexpr) => ifexpr.has_side_effects(),
20
21	// note: a function declaration has no side effects, but the call can
22	ExprData::Fn { .. } => false,
23
24	// TODO(@paperdave): implement detection of pure functions
25	ExprData::Call { base, arguments } => {
26	base.has_side_effects() \|\| arguments.iter().any(\|x\| x.has_side_effects())
27	}
28
29	// note: we assume field access has side effects, ie: a getter doesn't throw
30	ExprData::Field { base, field } => base.has_side_effects() \|\| field.has_side_effects(),
31
32	// note: we assume the code in @raw always has side effects
33	ExprData::Builtin { id, .. } => match id {
34	BuiltinId::Raw => true,
35	},
36
37	ExprData::Return(_) => true,
38	}
39	}
40	}
41
42	impl HasSideEffects for Statement {
43	fn has_side_effects(&self) -> bool {
44	match self {
45	Statement::Expr(expr) => expr.has_side_effects(),
46	Statement::VarDecl(decl) => decl.init.has_side_effects(),
47	Statement::Comment(_) => false,
48	Statement::FnDecl(_) => false,
49	}
50	}
51	}
52
53	impl HasSideEffects for Block {
54	fn has_side_effects(&self) -> bool {
55	self.statements
56	.iter()
57	.any(\|statement\| statement.has_side_effects())
58	}
59	}
60
61	impl HasSideEffects for Expr {
62	fn has_side_effects(&self) -> bool {
63	self.data.has_side_effects()
64	}
65	}
66
67	impl HasSideEffects for If {
68	fn has_side_effects(&self) -> bool {
69	self.condition.has_side_effects()
70	\|\| self.yes.has_side_effects()
71	\|\| match &self.no {
72	Some(no) => no.has_side_effects(),
73	None => false,
74	}
75	}
76	}
77
78	impl HasSideEffects for BlockOrIf {
79	fn has_side_effects(&self) -> bool {
80	match self {
81	BlockOrIf::Block(block) => block.has_side_effects(),
82	BlockOrIf::If(if_) => if_.has_side_effects(),
83	}
84	}
85	}
86
87	impl HasSideEffects for BlockWithRange {
88	fn has_side_effects(&self) -> bool {
89	self.block.has_side_effects()
90	}
91	}

crates/js_ast/convert/tree_shake_statement_list.rs deleted-31

...	@@ -1,31 +0,0 @@
1	use crate::js_ast::*;
2
3	/// Removes statements that are not needed from a JS block. Very limited.
4	///
5	/// BlockConvertContext will automatically execute this when converting
6	/// itself into a Statement list or other data structure.
7	///
8	/// What this does:
9	/// - Remove discarded literals `null` -> ``
10	pub fn tree_shake_statement_list(list: &mut Vec<JsStatement>) {
11	let mut set_i = 0;
12	for i in 0..list.len() {
13	let stmt = list.get(i).unwrap();
14	let should_keep = match &stmt.data {
15	JsStatementData::Expression(expr) => !expr.is_literal(),
16	JsStatementData::Return(_) => true,
17	JsStatementData::Export(_) => true,
18	JsStatementData::Declaration(_) => true,
19	JsStatementData::Function(_) => true,
20	JsStatementData::If(_) => true,
21	};
22
23	if should_keep {
24	if set_i != i {
25	list.swap(i, set_i)
26	}
27	set_i += 1;
28	}
29	}
30	list.truncate(set_i)
31	}

crates/js_ast/js_ast.rs deleted-198

...	@@ -1,198 +0,0 @@
1	use ast::{BinaryOperator, Comment, UnaryOperator};
2	use log::Range;
3	use num::bigint::BigInt;
4
5	#[derive(Debug, PartialEq, Clone)]
6	pub struct JsExpr {
7	pub data: JsExprData,
8	pub range: Range,
9	pub comment: Option<Comment>,
10	}
11
12	#[derive(Debug, PartialEq, Clone)]
13	pub struct JsKV {
14	pub key: JsExpr,
15	pub value: JsExpr,
16	}
17
18	#[derive(Debug, PartialEq, Clone)]
19	pub enum JsExprData {
20	/// node emitted by the '@raw' building
21	Raw(String),
22	// Litearls
23	Number(f64),
24	BigInt(BigInt),
25	String(String),
26	Bool(bool),
27	Array(Vec<JsExpr>),
28	Object(Vec<JsKV>),
29	Null,
30	Undefined,
31
32	Identifier(String),
33	Unary {
34	op: UnaryOperator,
35	value: Box<JsExpr>,
36	},
37	Binary {
38	op: BinaryOperator,
39	left: Box<JsExpr>,
40	right: Box<JsExpr>,
41	},
42	Dot {
43	base: Box<JsExpr>,
44	name: String,
45	},
46	Index {
47	base: Box<JsExpr>,
48	index: Box<JsExpr>,
49	},
50	Call {
51	base: Box<JsExpr>,
52	arguments: Vec<JsExpr>,
53	},
54	Function {
55	name: Option<String>,
56	params: Vec<String>,
57	body: Box<JsExpr>,
58	},
59	ArrowFunction {
60	params: Vec<String>,
61	body: Box<JsArrowBody>,
62	},
63	Ternary {
64	condition: Box<JsExpr>,
65	yes: Box<JsExpr>,
66	no: Option<Box<JsExpr>>,
67	},
68	Comma {
69	items: Vec<JsExpr>,
70	},
71	Void(Box<JsExpr>),
72	}
73
74	#[derive(Debug, PartialEq, Clone)]
75	pub struct JsStatement {
76	pub data: JsStatementData,
77	pub range: Range,
78	pub comment: Option<Comment>,
79	}
80
81	#[derive(Debug, PartialEq, Clone)]
82	pub struct JSFunctionDeclaration {
83	pub name: String,
84	pub params: Vec<String>,
85	pub block: JsBlock,
86	}
87
88	#[derive(Debug, PartialEq, Clone)]
89	pub struct JSDeclaration {
90	pub name: String,
91	pub init: Option<JsExpr>,
92	pub is_constant: bool,
93	}
94
95	#[derive(Debug, PartialEq, Clone)]
96	pub struct JsIf {
97	pub condition: JsExpr,
98	pub yes: JsBlock,
99	pub no: Option<JsElse>,
100	}
101
102	#[derive(Debug, PartialEq, Clone)]
103	pub enum JsElse {
104	Expr(JsExpr),
105	Block(JsBlock),
106	If(Box<JsIf>),
107	}
108
109	#[derive(Debug, PartialEq, Clone)]
110	pub enum JsStatementData {
111	Expression(JsExpr),
112	Return(Option<JsExpr>),
113	Export(JsExport),
114	Declaration(JSDeclaration),
115	Function(JSFunctionDeclaration),
116	If(JsIf),
117	}
118
119	#[derive(Debug, PartialEq, Clone)]
120	pub enum JsExport {
121	Named(Vec<JSNamedExport>),
122	Function(Box<JSFunctionDeclaration>),
123	Declaration(Box<JSDeclaration>),
124	Default(JsExpr),
125	}
126
127	#[derive(Debug, PartialEq, Clone)]
128	pub struct JSNamedExport {
129	pub identifier: String,
130	pub export_name: String,
131	}
132
133	#[derive(Debug, PartialEq, Clone)]
134	pub struct JsBlock {
135	pub statements: Vec<JsStatement>,
136	pub range: Range,
137	}
138
139	impl JsExpr {
140	pub fn new(data: JsExprData, range: Range, comment: Option<Comment>) -> Self {
141	Self {
142	data,
143	range,
144	comment,
145	}
146	}
147
148	pub fn is_literal(&self) -> bool {
149	match &self.data {
150	JsExprData::Number(_) => true,
151	JsExprData::BigInt(_) => true,
152	JsExprData::String(_) => true,
153	JsExprData::Bool(_) => true,
154	JsExprData::Null => true,
155	JsExprData::Undefined => true,
156	_ => false,
157	}
158	}
159	}
160
161	impl JsExprData {
162	/// Helper for testing
163	pub fn str(value: &str) -> Self {
164	Self::String(value.to_string())
165	}
166
167	/// Helper for testing
168	pub fn str_id(value: &str) -> Self {
169	Self::Identifier(value.to_string())
170	}
171	}
172
173	impl JsBlock {
174	pub fn new(statements: Vec<JsStatement>, range: Range) -> Self {
175	Self { statements, range }
176	}
177	}
178
179	impl JsStatement {
180	pub fn new(data: JsStatementData, range: Range, comment: Option<Comment>) -> Self {
181	Self {
182	data,
183	range,
184	comment,
185	}
186	}
187	}
188
189	#[derive(Debug, PartialEq, Clone)]
190	pub enum JsArrowBody {
191	Block(JsBlock),
192	Expr(JsExpr),
193	}
194
195	#[derive(Debug, PartialEq, Clone)]
196	pub struct JsModule {
197	pub statements: Vec<JsStatement>,
198	}

crates/js_ast/lib.rs deleted-7

...	@@ -1,7 +0,0 @@
1	mod convert;
2	mod js_ast;
3	mod util;
4
5	pub use convert::*;
6	pub use js_ast::*;
7	pub use util::*;

crates/js_ast/util.rs deleted-17

...	@@ -1,17 +0,0 @@
1	/// TODO(@paperdave): handle the rest of the identifiers
2	pub fn is_js_identifier(s: &str) -> bool {
3	if s.is_empty() {
4	return false;
5	}
6	let mut chars = s.chars();
7	let first = chars.next().unwrap();
8	if !first.is_ascii_alphabetic() && first != '_' {
9	return false;
10	}
11	for c in chars {
12	if !c.is_ascii_alphanumeric() && c != '_' {
13	return false;
14	}
15	}
16	true
17	}

crates/js_printer/Cargo.toml deleted-14

...	@@ -1,14 +0,0 @@
1	[package]
2	name = "js_printer"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path = "./lib.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12	num.workspace = true
13
14	js_ast = { path = "../js_ast" }

crates/js_printer/lib.rs deleted-377

...	@@ -1,377 +0,0 @@
1	mod printer_tests;
2
3	use js_ast::*;
4	use std::io::{Result, Write};
5
6	pub trait Print {
7	fn print(&self, writer: &mut dyn Write) -> Result<()>;
8
9	fn print_to_string(&self) -> String {
10	let mut buffer = Vec::new();
11	self.print(&mut buffer).unwrap();
12	String::from_utf8(buffer).unwrap()
13	}
14	}
15
16	// the printer:
17
18	fn print_module_clause_identifier(writer: &mut dyn Write, export_name: &str) -> Result<()> {
19	// TODO(@paperdave): Escape invalid identifiers
20	if js_ast::is_js_identifier(export_name) {
21	write!(writer, "{}", export_name)
22	} else {
23	todo!("call JSON.stringify on identifier")
24	}
25	}
26
27	fn print_js_string(writer: &mut dyn Write, string: &str) -> Result<()> {
28	write!(writer, "\"")?;
29	// TODO(@paperdave): Instead of writing each character, this should write
30	// chunks between the escapes instead of calling write! on each character individually.
31	for c in string.chars() {
32	match c {
33	'\n' => write!(writer, "\\n")?,
34	'\r' => write!(writer, "\\r")?,
35	'\t' => write!(writer, "\\t")?,
36	'"' => write!(writer, "\\\"")?,
37	'\\' => write!(writer, "\\\\")?,
38	'\u{0008}' => write!(writer, "\\b")?,
39	'\u{000C}' => write!(writer, "\\f")?,
40	'\u{0000}'..='\u{001F}' => write!(writer, "\\u{:04x}", c as u32)?,
41	// TODO(@paperdave): I think there are some larger code points that need to be escaped?
42	_ => write!(writer, "{}", c)?,
43	}
44	}
45	write!(writer, "\"")
46	}
47
48	impl Print for JsStatement {
49	fn print(&self, w: &mut dyn Write) -> Result<()> {
50	match &self.data {
51	js_ast::JsStatementData::Expression(expr) => {
52	expr.print(w)?;
53	write!(w, ";")
54	}
55	js_ast::JsStatementData::Return(expr) => {
56	write!(w, "return")?;
57	if let Some(expr) = expr {
58	write!(w, " ")?;
59	expr.print(w)?;
60	}
61	write!(w, ";")?;
62	Ok(())
63	}
64	js_ast::JsStatementData::Export(export) => export.print(w),
65	js_ast::JsStatementData::Declaration(decl) => decl.print(w),
66	js_ast::JsStatementData::Function(function) => function.print(w),
67	js_ast::JsStatementData::If(js_if) => js_if.print(w),
68	}
69	}
70	}
71
72	impl Print for JsExport {
73	fn print(&self, w: &mut dyn Write) -> Result<()> {
74	match self {
75	JsExport::Named(exports) => {
76	write!(w, "export{{")?;
77	for (i, export) in exports.iter().enumerate() {
78	if i > 0 {
79	write!(w, ",")?;
80	}
81	export.print(w)?;
82	}
83	Ok(())
84	}
85	JsExport::Function(function) => {
86	write!(w, "export ")?;
87	function.print(w)
88	}
89	JsExport::Declaration(declaration) => {
90	write!(w, "export ")?;
91	declaration.print(w)
92	}
93	JsExport::Default(expr) => {
94	write!(w, "export default ")?;
95	expr.print(w)
96	}
97	}
98	}
99	}
100
101	impl Print for JSNamedExport {
102	fn print(&self, writer: &mut dyn Write) -> Result<()> {
103	if self.export_name == self.identifier {
104	write!(writer, "{}", self.identifier)
105	} else {
106	write!(writer, "{} as ", self.identifier)?;
107	print_module_clause_identifier(writer, &self.export_name)
108	}
109	}
110	}
111
112	impl Print for JSDeclaration {
113	fn print(&self, writer: &mut dyn Write) -> Result<()> {
114	write!(
115	writer,
116	"{} {}",
117	if self.is_constant { "const" } else { "let" },
118	self.name,
119	)?;
120	if let Some(init) = &self.init {
121	write!(writer, " = ")?;
122	init.print(writer)?;
123	}
124	write!(writer, ";")?;
125	Ok(())
126	}
127	}
128
129	impl Print for JsExpr {
130	fn print(&self, writer: &mut dyn Write) -> Result<()> {
131	self.data.print(writer)
132	}
133	}
134
135	impl Print for JsExprData {
136	fn print(&self, writer: &mut dyn Write) -> Result<()> {
137	match &self {
138	JsExprData::Raw(data) => write!(writer, "{}", data),
139	JsExprData::Number(number) => write!(writer, "{}", number),
140	JsExprData::BigInt(bigint) => write!(writer, "{}", bigint),
141	JsExprData::String(string) => print_js_string(writer, string.as_str()),
142	JsExprData::Bool(bool) => write!(writer, "{}", bool),
143	JsExprData::Null => write!(writer, "null"),
144	JsExprData::Undefined => write!(writer, "undefined"),
145	JsExprData::Identifier(str) => write!(writer, "{}", str),
146	JsExprData::Array(items) => {
147	write!(writer, "[")?;
148	for (i, item) in items.iter().enumerate() {
149	if i > 0 {
150	write!(writer, ",")?;
151	}
152	item.print(writer)?;
153	}
154	write!(writer, "]")
155	}
156	JsExprData::Object(items) => {
157	write!(writer, "{{")?;
158	for (i, kv) in items.iter().enumerate() {
159	if i > 0 {
160	write!(writer, ",")?;
161	}
162
163	kv.print(writer)?;
164	}
165	write!(writer, "}}")
166	}
167	JsExprData::Unary { op, value } => {
168	write!(writer, "{}", op.to_str())?;
169	value.print(writer)
170	}
171	JsExprData::Binary { op, left, right } => {
172	// TODO: figure out when we dont need to wrap
173	let wrap = true;
174	if wrap {
175	write!(writer, "(")?;
176	}
177	left.print(writer)?;
178	write!(writer, "{}", op.to_str())?;
179	right.print(writer)?;
180	if wrap {
181	write!(writer, ")")?;
182	}
183	Ok(())
184	}
185	JsExprData::Dot { base, name } => {
186	base.print(writer)?;
187	write!(writer, ".{}", name)
188	}
189	JsExprData::Index { base, index } => {
190	base.print(writer)?;
191	write!(writer, "[")?;
192	index.print(writer)?;
193	write!(writer, "]")
194	}
195	JsExprData::Call { base, arguments } => {
196	base.print(writer)?;
197	write!(writer, "(")?;
198	for (i, argument) in arguments.iter().enumerate() {
199	if i > 0 {
200	write!(writer, ",")?;
201	}
202	argument.print(writer)?;
203	}
204	write!(writer, ")")
205	}
206	JsExprData::Function { body, name, params } => {
207	write!(writer, "function ")?;
208	if let Some(name) = name {
209	write!(writer, "{}", name)?;
210	}
211	write!(writer, "(")?;
212	for (i, param) in params.iter().enumerate() {
213	if i > 0 {
214	write!(writer, ",")?;
215	}
216	write!(writer, "{}", param)?;
217	}
218	write!(writer, ")")?;
219	body.print(writer)
220	}
221	JsExprData::ArrowFunction { body, params } => {
222	let print_parens = params.len() != 1;
223	if print_parens {
224	write!(writer, "(")?;
225	}
226	for (i, param) in params.iter().enumerate() {
227	if i > 0 {
228	write!(writer, ",")?;
229	}
230	write!(writer, "{}", param)?;
231	}
232	if print_parens {
233	write!(writer, ")")?;
234	}
235	write!(writer, "=>")?;
236	body.print(writer)
237	}
238	JsExprData::Ternary { condition, yes, no } => {
239	let wrap = false;
240	if wrap {
241	write!(writer, "(")?;
242	}
243	condition.print(writer)?;
244	write!(writer, "?")?;
245	yes.print(writer)?;
246	write!(writer, ":")?;
247	if let Some(no) = no {
248	no.print(writer)?;
249	} else {
250	write!(writer, "null")?;
251	}
252	if wrap {
253	write!(writer, ")")?;
254	}
255	Ok(())
256	}
257	JsExprData::Comma { items } => {
258	let wrap = true;
259	if wrap {
260	write!(writer, "(")?;
261	}
262	for (i, item) in items.iter().enumerate() {
263	if i > 0 {
264	write!(writer, ",")?;
265	}
266	item.print(writer)?;
267	}
268	if wrap {
269	write!(writer, ")")?;
270	}
271	Ok(())
272	}
273	JsExprData::Void(expr) => {
274	write!(writer, "void ")?;
275	expr.print(writer)
276	}
277	}
278	}
279	}
280
281	impl Print for JSFunctionDeclaration {
282	fn print(&self, writer: &mut dyn Write) -> Result<()> {
283	write!(writer, "function {}", self.name)?;
284	write!(writer, "(")?;
285	for (i, param) in self.params.iter().enumerate() {
286	if i > 0 {
287	write!(writer, ",")?;
288	}
289	write!(writer, "{}", param)?;
290	}
291	write!(writer, ")")?;
292	self.block.print(writer)
293	}
294	}
295
296	impl Print for JsBlock {
297	fn print(&self, writer: &mut dyn Write) -> Result<()> {
298	write!(writer, "{{")?;
299	for statement in self.statements.iter() {
300	statement.print(writer)?;
301	}
302	write!(writer, "}}")
303	}
304	}
305
306	impl Print for JsIf {
307	fn print(&self, writer: &mut dyn Write) -> Result<()> {
308	write!(writer, "if(")?;
309	self.condition.print(writer)?;
310	write!(writer, ")")?;
311	self.yes.print(writer)?;
312	if let Some(no) = &self.no {
313	no.print(writer)?;
314	}
315	Ok(())
316	}
317	}
318
319	impl Print for JsElse {
320	fn print(&self, writer: &mut dyn Write) -> Result<()> {
321	match self {
322	JsElse::If(js_if) => {
323	write!(writer, "else ")?;
324	js_if.print(writer)
325	}
326	JsElse::Block(block) => {
327	write!(writer, "else")?;
328	block.print(writer)
329	}
330	JsElse::Expr(expr) => {
331	write!(writer, "else ")?;
332	expr.print(writer)
333	}
334	}
335	}
336	}
337
338	impl Print for JsKV {
339	fn print(&self, writer: &mut dyn Write) -> Result<()> {
340	match &self.key.data {
341	JsExprData::String(string) => {
342	if is_js_identifier(string) {
343	write!(writer, "{}", string)?;
344	} else {
345	print_js_string(writer, string.as_str())?;
346	}
347	}
348	_ => {
349	write!(writer, "[")?;
350	self.key.print(writer)?;
351	write!(writer, "]")?;
352	}
353	}
354	write!(writer, ":")?;
355	self.key.print(writer)?;
356	write!(writer, ":")?;
357	self.value.print(writer)
358	}
359	}
360
361	impl Print for JsModule {
362	fn print(&self, writer: &mut dyn Write) -> Result<()> {
363	for statement in self.statements.iter() {
364	statement.print(writer)?;
365	}
366	Ok(())
367	}
368	}
369
370	impl Print for JsArrowBody {
371	fn print(&self, writer: &mut dyn Write) -> Result<()> {
372	match self {
373	JsArrowBody::Block(block) => block.print(writer),
374	JsArrowBody::Expr(expr) => expr.print(writer),
375	}
376	}
377	}

crates/js_printer/printer_tests.rs deleted-53

...	@@ -1,53 +0,0 @@
1	#![cfg(test)]
2	use crate::Print;
3
4	use js_ast::JsExprData as E;
5	use num::{bigint::Sign, BigInt};
6
7	macro_rules! assert_printed {
8	($value:expr, $expected:expr) => {
9	assert_eq!($value.print_to_string(), $expected);
10	};
11	}
12
13	#[test]
14	fn number() {
15	assert_printed!(E::Number(1.0), "1");
16	assert_printed!(E::Number(1.5), "1.5");
17	assert_printed!(E::Number(1.5e10), "15000000000");
18	assert_printed!(E::Number(1.5e-10), "0.00000000015");
19	assert_printed!(E::Number(-1.0), "-1");
20	assert_printed!(E::Number(-1.5), "-1.5");
21	assert_printed!(E::Number(0f64), "0");
22	}
23
24	#[test]
25	fn bigint() {
26	assert_printed!(E::BigInt(BigInt::new(Sign::Plus, vec![1])), "1n");
27	assert_printed!(
28	E::BigInt(BigInt::new(Sign::Plus, vec![10000000])),
29	"10000000n"
30	);
31	assert_printed!(
32	E::BigInt(BigInt::new(Sign::Minus, vec![10000000])),
33	"-10000000n"
34	);
35	}
36
37	#[test]
38	fn boolean() {
39	assert_printed!(E::Bool(true), "true");
40	assert_printed!(E::Bool(false), "false");
41	}
42
43	#[test]
44	fn string() {
45	assert_printed!(E::str(""), "\"\"");
46	assert_printed!(E::str("hello"), "\"hello\"");
47	assert_printed!(E::str("hello\nworld"), "\"hello\\nworld\"");
48	}
49
50	#[test]
51	fn identifier() {
52	assert_printed!(E::str_id("hello"), "hello");
53	}

crates/log/Cargo.toml deleted-11

...	@@ -1,11 +0,0 @@
1	[package]
2	name = "log"
3	edition = "2021"
4
5	[lib]
6	path="./log.rs"
7
8	[dependencies]
9	miette.workspace = true
10	thiserror.workspace = true
11	url.workspace = true

crates/log/log.rs deleted-88

...	@@ -1,88 +0,0 @@
1	use std::{fs, path::Path, rc::Rc};
2
3	use miette::{IntoDiagnostic, Result};
4	use url::Url;
5
6	#[derive(Debug, PartialEq, Clone, Copy)]
7	pub struct Range {
8	pub start: usize,
9	pub length: usize,
10	}
11
12	impl Range {
13	/// TODO(@paperdave): Remove this. every usage of this is incorrect.
14	pub const EMPTY: Range = Range {
15	start: 0,
16	length: 0,
17	};
18
19	pub fn new(start: usize, length: usize) -> Self {
20	Self { start, length }
21	}
22
23	pub fn len(&self) -> usize {
24	self.length
25	}
26	}
27
28	// TODO(@lino-levan): make this loggable nicely
29	#[derive(Debug, Clone)]
30	pub struct Message<'a> {
31	pub source: &'a Source,
32	pub range: Range,
33	pub message: String,
34	pub notes: Vec<Note<'a>>,
35	}
36
37	#[derive(Debug, Clone)]
38	pub struct Note<'a> {
39	pub location: Option<RangeAndSource<'a>>,
40	pub message: String,
41	}
42
43	#[derive(Debug, Clone)]
44	pub struct RangeAndSource<'a> {
45	pub range: Range,
46	pub source: &'a Source,
47	}
48
49	#[derive(Debug)]
50	pub struct Source {
51	pub url: Url,
52	pub code: String,
53	}
54
55	impl Source {
56	pub fn new(url: Url, source_code: String) -> Rc<Self> {
57	Rc::new(Source {
58	url,
59	code: source_code,
60	})
61	}
62
63	pub fn from_file(path: &Path) -> Result<Rc<Self>> {
64	let source_code = fs::read_to_string(path).into_diagnostic()?;
65	let url = Url::from_file_path(path).unwrap();
66	Ok(Rc::new(Source {
67	code: source_code,
68	url,
69	}))
70	}
71
72	pub fn from_url(url: Url) -> Result<Rc<Self>> {
73	match url.scheme() {
74	"file" => {
75	// TODO: how to return this error, into_diagnostic() doesn't exist
76	let file_path = url.to_file_path().unwrap();
77	let source_code = fs::read_to_string(file_path).into_diagnostic()?;
78	Ok(Rc::new(Source {
79	code: source_code,
80	url,
81	}))
82	}
83	_ => {
84	todo!("Fetch from URL: {}", url)
85	}
86	}
87	}
88	}

crates/parser/Cargo.toml deleted-16

...	@@ -1,16 +0,0 @@
1	[package]
2	name = "parser"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path = "./parser.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12	num.workspace = true
13
14	ast = { path = "../ast" }
15	tokenizer = { path = "../tokenizer" }
16	log = { path = "../log" }

crates/parser/parse_block.rs deleted-84

...	@@ -1,84 +0,0 @@
1	use crate::{parse_statement::parse_statement, Parser};
2	use ast::Block;
3	use miette::Result;
4	use tokenizer::TokenType;
5
6	/// Precondition: The first token exists and is '{'
7	pub fn parse_block(p: &mut Parser) -> Result<Block> {
8	match p.next() {
9	Some(TokenType::OpenBrace) => (),
10	_ => unreachable!(),
11	}
12
13	let mut statements = Vec::new();
14
15	let was_top_level = p.is_top_level;
16	p.is_top_level = false;
17
18	'outer: loop {
19	let is_statement_like_expression = p
20	.peek()
21	.map_or(false, \|x\| x.is_start_of_statement_like_expression());
22
23	match p.peek() {
24	Some(TokenType::ClosedBrace) => {
25	p.next();
26	p.is_top_level = was_top_level;
27	return Ok(Block {
28	statements,
29	has_implicit_return: false,
30	});
31	}
32	_ => (),
33	}
34
35	let statement = match parse_statement(p) {
36	Ok(statement) => statement,
37	Err(err) => {
38	p.is_top_level = was_top_level;
39	return Err(err);
40	}
41	};
42	statements.push(statement.clone());
43
44	match p.peek() {
45	Some(TokenType::Semicolon) => loop {
46	p.next();
47	match p.peek() {
48	Some(TokenType::ClosedBrace) => {
49	p.next();
50	p.is_top_level = was_top_level;
51	return Ok(Block {
52	statements,
53	has_implicit_return: false,
54	});
55	}
56	_ => continue 'outer,
57	}
58	},
59	Some(TokenType::ClosedBrace) => {
60	p.next();
61	p.is_top_level = was_top_level;
62
63	if !statement.is_expression() {
64	return Err(p
65	.error("Expected semicolon or '}' after statement".to_string())
66	.into());
67	}
68
69	return Ok(Block {
70	statements,
71	has_implicit_return: true,
72	});
73	}
74	_ => {
75	if statement.require_semicolon(is_statement_like_expression) {
76	p.is_top_level = was_top_level;
77	return Err(p
78	.error("Expected semicolon or '}' after statement".to_string())
79	.into());
80	}
81	}
82	};
83	}
84	}

crates/parser/parse_call_arguments.rs deleted-36

...	@@ -1,36 +0,0 @@
1	use ast::Expr;
2	use miette::Result;
3	use tokenizer::TokenType;
4
5	use crate::{parse_expr::parse_expr, Parser};
6
7	/// Precondition: The '(' was already consumed
8	pub fn parse_call_arguments(p: &mut Parser) -> Result<Vec<Expr>> {
9	let mut args = Vec::new();
10
11	if let Some(TokenType::ClosedParen) = p.peek() {
12	_ = p.next();
13	return Ok(args);
14	}
15
16	loop {
17	let arg = parse_expr(p)?;
18	args.push(arg);
19
20	match p.peek() {
21	Some(TokenType::Comma) => {
22	_ = p.next();
23	continue;
24	}
25	Some(TokenType::ClosedParen) => {
26	_ = p.next();
27	return Ok(args);
28	}
29	_ => {
30	return Err(p
31	.error("Expected ',' or ')' after argument".to_string())
32	.into())
33	}
34	}
35	}
36	}

crates/parser/parse_declaration.rs deleted-51

...	@@ -1,51 +0,0 @@
1	use ast::{DocComment, Type, VarDecl};
2	use log::Range;
3	use miette::Result;
4	use tokenizer::TokenType;
5
6	use crate::parse_expr::parse_expr;
7	use crate::parse_type::parse_type;
8	use crate::Parser;
9
10	/// Precondition: The first token exists and is either 'var' or 'let'
11	pub fn parse_declaration(
12	p: &mut Parser,
13	is_pub: bool,
14	comment: Option<DocComment>,
15	) -> Result<VarDecl> {
16	let start = p.index;
17	let is_constant = match p.next() {
18	Some(TokenType::Var) => false,
19	Some(TokenType::Let) => true,
20	_ => unreachable!(),
21	};
22	let name = match p.next() {
23	Some(TokenType::Identifier(name)) => name,
24	Some(tok) => return Err(p.error(format!("Unexpected {}", tok)).into()),
25	None => return Err(p.error(format!("Unexpected end of file")).into()),
26	};
27	let return_type = match p.peek() {
28	Some(TokenType::Colon) => {
29	p.next();
30	parse_type(p)?
31	}
32	_ => Type::Infer,
33	};
34	let init = match p.next() {
35	Some(TokenType::Equal) => parse_expr(p)?,
36	_ => {
37	return Err(p
38	.error("Variable must have an initializer".to_string())
39	.into())
40	}
41	};
42	Ok(VarDecl {
43	name: name.clone(),
44	data_type: return_type,
45	init,
46	is_constant,
47	is_pub,
48	comment,
49	range: Range::new(start, p.index - start),
50	})
51	}

crates/parser/parse_doc_comment.rs deleted-19

...	@@ -1,19 +0,0 @@
1	use ast::DocComment;
2	use log::Range;
3	use tokenizer::TokenType;
4
5	use crate::Parser;
6
7	pub fn maybe_read_doc_comment(p: &mut Parser) -> Option<DocComment> {
8	match p.peek() {
9	Some(TokenType::DocComment(data)) => {
10	let start = p.index;
11	p.next();
12	Some(DocComment {
13	data,
14	range: Range::new(start, p.index - start),
15	})
16	}
17	_ => None,
18	}
19	}

crates/parser/parse_expr.rs deleted-71

...	@@ -1,71 +0,0 @@
1	use ast::{Associativity, BinaryOperator, Expr, ExprData, OperatorLevel};
2	use log::Range;
3	use miette::Result;
4
5	use crate::parse_expr_basic::parse_expr_basic;
6	use crate::Parser;
7
8	pub fn parse_expr(p: &mut Parser) -> Result<Expr> {
9	let mut expr = parse_expr_basic(p)?;
10
11	// Edge case for return. This has already parsed all the way to the end of the expression
12	if let ExprData::Return(_) = expr.data {
13	return Ok(expr);
14	}
15
16	// Parse operators
17	loop {
18	if let Some(token) = p.peek() {
19	if let Some(level) = BinaryOperator::from_token(&token).map(\|o\| o.level()) {
20	_ = p.next();
21	expr =
22	parse_expr_binary(p, BinaryOperator::from_token(&token).unwrap(), expr, level)?;
23	continue;
24	}
25	}
26	break;
27	}
28
29	Ok(expr)
30	}
31
32	/// Prerequisite: `p.peek()` is an operator
33	fn parse_expr_binary(
34	p: &mut Parser,
35	op: BinaryOperator,
36	left: Expr,
37	level: OperatorLevel,
38	) -> Result<Expr> {
39	let start = p.index;
40	let mut right = parse_expr_basic(p)?;
41
42	loop {
43	if let Some(token) = p.peek() {
44	if let Some(new_level) = BinaryOperator::from_token(&token).map(\|o\| o.level()) {
45	if (level.associativity() == Associativity::LeftToRight && new_level < level)
46	\|\| (level.associativity() == Associativity::RightToLeft && new_level <= level)
47	{
48	_ = p.next();
49	right = parse_expr_binary(
50	p,
51	BinaryOperator::from_token(&token).unwrap(),
52	right,
53	new_level,
54	)?;
55	continue;
56	}
57	}
58	}
59	break;
60	}
61
62	Ok(Expr::new(
63	ExprData::Binary {
64	op,
65	left: Box::new(left),
66	right: Box::new(right),
67	},
68	Range::new(start, p.index - start),
69	None,
70	))
71	}

crates/parser/parse_expr_basic.rs deleted-230

...	@@ -1,230 +0,0 @@
1	use ast::{BuiltinId, Expr, ExprData, Literal, UnaryOperator};
2	use log::Range;
3	use miette::Result;
4	use tokenizer::TokenType;
5
6	use crate::{
7	parse_block::parse_block, parse_call_arguments::parse_call_arguments, parse_expr::parse_expr,
8	parse_expr_if::parse_expr_if, Parser,
9	};
10
11	pub fn maybe_parse_expr_basic(p: &mut Parser) -> Result<Option<Expr>> {
12	let start = p.index;
13	let comments = p.take_comments();
14	let mut expr = match p.peek() {
15	Some(TokenType::Number(number)) => {
16	_ = p.next();
17	Expr::new(
18	ExprData::Literal(Literal::Number(number.clone())),
19	Range::new(start, p.index - start),
20	comments,
21	)
22	}
23	Some(TokenType::String(string)) => {
24	_ = p.next();
25	Expr::new(
26	ExprData::Literal(Literal::String(string.clone())),
27	Range::new(start, p.index - start),
28	comments,
29	)
30	}
31	Some(TokenType::True) => {
32	_ = p.next();
33	Expr::new(
34	ExprData::Literal(Literal::Bool(true)),
35	Range::new(start, p.index - start),
36	comments,
37	)
38	}
39	Some(TokenType::False) => {
40	_ = p.next();
41	Expr::new(
42	ExprData::Literal(Literal::Bool(false)),
43	Range::new(start, p.index - start),
44	comments,
45	)
46	}
47	Some(TokenType::Void) => {
48	_ = p.next();
49	Expr::new(
50	ExprData::Literal(Literal::Void),
51	Range::new(start, p.index - start),
52	comments,
53	)
54	}
55	Some(TokenType::Identifier(name)) => {
56	let identifier_start = p.index;
57	_ = p.next();
58	if name.starts_with("@") {
59	if let Some(id) = BuiltinId::from_str(name.as_str()) {
60	if let Some(TokenType::OpenParen) = p.next() {
61	let arguments = parse_call_arguments(p)?;
62	Expr::new(
63	ExprData::Builtin { id, arguments },
64	Range::new(identifier_start, p.index - identifier_start),
65	comments,
66	)
67	} else {
68	return Err(p
69	.error_at(
70	format!("Built-in function '{}' must be called", name),
71	Range::new(identifier_start, p.index - identifier_start),
72	)
73	.into());
74	}
75	} else {
76	return Err(p
77	.error_at(
78	format!("Non-existant built-in: '{}'", name),
79	Range::new(identifier_start, p.index - identifier_start),
80	)
81	.into());
82	}
83	} else {
84	Expr::new(
85	ExprData::Identifier(name),
86	Range::new(identifier_start, p.index - identifier_start),
87	comments,
88	)
89	}
90	}
91
92	Some(TokenType::Return) => {
93	if p.is_top_level {
94	return Err(p
95	.error("'return' cannot be used at top level".to_string())
96	.into());
97	} else {
98	_ = p.next();
99	let inner = maybe_parse_expr_basic(p)?.map(Box::new);
100
101	return Ok(Some(Expr::new(
102	ExprData::Return(inner),
103	Range::new(start, p.index - start),
104	comments,
105	)));
106	}
107	}
108
109	Some(TokenType::If) => Expr::new(
110	ExprData::If(parse_expr_if(p)?),
111	Range::new(start, p.index - start),
112	comments,
113	),
114
115	Some(TokenType::OpenBrace) => Expr::new(
116	ExprData::Block(parse_block(p)?),
117	Range::new(start, p.index - start),
118	comments,
119	),
120
121	Some(TokenType::OpenParen) => {
122	_ = p.next();
123	let expr = parse_expr(p)?;
124	match p.next() {
125	Some(TokenType::ClosedParen) => (),
126	Some(token) => {
127	return Err(p.error(format!("Expected ')', but got {:?}", token)).into())
128	}
129	None => return Err(p.error("Unexpected end of file".to_string()).into()),
130	}
131	expr
132	}
133
134	Some(t @ TokenType::Bang) \| Some(t @ TokenType::Minus) \| Some(t @ TokenType::Tilde) => {
135	_ = p.next();
136	Expr::new(
137	ExprData::Unary {
138	op: match t {
139	TokenType::Bang => UnaryOperator::Not,
140	TokenType::Minus => UnaryOperator::Neg,
141	TokenType::Tilde => UnaryOperator::BitNot,
142	_ => unreachable!(),
143	},
144	value: Box::new(parse_expr_basic(p)?),
145	},
146	Range::new(start, p.index - start),
147	comments,
148	)
149	}
150
151	Some(_) \| None => return Ok(None),
152	};
153
154	// Parse function calls and field accesses
155	loop {
156	match p.peek() {
157	Some(TokenType::Dot) => {
158	let field_start = p.index;
159	_ = p.next();
160	let field = match p.peek() {
161	Some(TokenType::Identifier(name)) => {
162	let field_start = p.index;
163	p.next();
164	Expr::new(
165	ExprData::Literal(Literal::String(name)),
166	Range::new(field_start, p.index - field_start),
167	None,
168	)
169	}
170	_ => {
171	return Err(p.error("Expected identifier after dot".to_string()).into());
172	}
173	};
174	expr = Expr::new(
175	ExprData::Field {
176	base: Box::new(expr),
177	field: Box::new(field),
178	},
179	Range::new(field_start, p.index - field_start),
180	None,
181	);
182	}
183	Some(TokenType::OpenParen) => {
184	let expr_start = p.index;
185	_ = p.next();
186	let arguments = parse_call_arguments(p)?;
187	expr = Expr::new(
188	ExprData::Call {
189	base: Box::new(expr),
190	arguments,
191	},
192	Range::new(expr_start, p.index - expr_start),
193	None,
194	);
195	}
196	Some(TokenType::OpenBracket) => {
197	let expr_start = p.index;
198	_ = p.next();
199	let index = parse_expr(p)?;
200	if let Some(TokenType::ClosedBracket) = p.next() {
201	expr = Expr::new(
202	ExprData::Field {
203	base: Box::new(expr),
204	field: Box::new(index),
205	},
206	Range::new(expr_start, p.index - expr_start),
207	None,
208	);
209	} else {
210	return Err(p
211	.error("Expected closing bracket after index".to_string())
212	.into());
213	}
214	}
215	_ => break,
216	}
217	}
218
219	Ok(Some(expr))
220	}
221
222	pub fn parse_expr_basic(p: &mut Parser) -> Result<Expr> {
223	match maybe_parse_expr_basic(p)? {
224	Some(expr) => Ok(expr),
225	None => match p.peek() {
226	Some(token) => Err(p.error(format!("Unexpected '{}'", token)).into()),
227	None => Err(p.error("Unexpected end of file".to_string()).into()),
228	},
229	}
230	}

crates/parser/parse_expr_if.rs deleted-71

...	@@ -1,71 +0,0 @@
1	use ast::{BlockOrIf, BlockWithRange, If};
2	use log::Range;
3	use miette::Result;
4	use tokenizer::TokenType;
5
6	use crate::parse_block::parse_block;
7	use crate::parse_expr::parse_expr;
8	use crate::Parser;
9
10	/// Precondition: the first token exists and is 'if'
11	pub fn parse_expr_if(p: &mut Parser) -> Result<If> {
12	match p.next() {
13	Some(TokenType::If) => (),
14	_ => unreachable!(),
15	}
16
17	match p.peek() {
18	Some(TokenType::OpenBrace) => {
19	return Err(p
20	.error(
21	"To use a block as the condition for an if statement, wrap it in parentheses"
22	.to_string(),
23	)
24	.into());
25	}
26	_ => (),
27	}
28
29	let condition = Box::new(parse_expr(p)?);
30
31	let yes = match p.peek() {
32	Some(TokenType::OpenBrace) => {
33	let block_start = p.index;
34	BlockWithRange {
35	block: parse_block(p)?,
36	range: Range::new(block_start, p.index - block_start),
37	}
38	}
39	_ => {
40	return Err(p
41	.error("Expected a block after condition".to_string())
42	.into())
43	}
44	};
45
46	let no = match p.peek() {
47	Some(TokenType::Else) => {
48	_ = p.next();
49	match p.peek() {
50	Some(TokenType::OpenBrace) => {
51	let block_start = p.index;
52	// let comments = p.take_comments();
53	Some(BlockOrIf::Block(BlockWithRange {
54	block: parse_block(p)?,
55	range: Range::new(block_start, p.index - block_start),
56	}))
57	}
58	Some(TokenType::If) => {
59	// TODO(@paperdave): Save location for the if
60	// let range = p.range();
61	// let comments = p.take_comments();
62	Some(BlockOrIf::If(Box::new(parse_expr_if(p)?)))
63	}
64	_ => return Err(p.error("Expected a block after else".to_string()).into()),
65	}
66	}
67	_ => None,
68	};
69
70	Ok(If { condition, yes, no })
71	}

crates/parser/parse_fn_declaration.rs deleted-127

...	@@ -1,127 +0,0 @@
1	use ast::{DocComment, Expr, ExprData, FnDecl, Type};
2	use log::Range;
3	use miette::Result;
4	use tokenizer::TokenType;
5
6	use crate::parse_block::parse_block;
7	use crate::parse_expr::parse_expr;
8	use crate::parse_param::parse_param;
9	use crate::parse_type::parse_type;
10	use crate::Parser;
11
12	/// Precondition:
13	pub fn parse_fn_declaration(
14	p: &mut Parser,
15	is_pub: bool,
16	comment: Option<DocComment>,
17	) -> Result<FnDecl> {
18	let start = p.index;
19	match p.next() {
20	Some(TokenType::Fn) => (),
21	_ => unreachable!(),
22	}
23
24	let name = match p.next() {
25	Some(TokenType::Identifier(name)) => name,
26	_ => return Err(p.error("Expected identifier after 'fn'".to_string()).into()),
27	};
28
29	match p.peek() {
30	Some(TokenType::OpenParen) => p.next(),
31	_ => {
32	return Err(p
33	.error("Expected '(' after function name".to_string())
34	.into())
35	}
36	};
37
38	let mut params = Vec::new();
39
40	loop {
41	match p.peek() {
42	Some(TokenType::ClosedParen) => {
43	_ = p.next();
44	break;
45	}
46	Some(TokenType::Identifier(_)) => {
47	params.push(parse_param(p)?);
48
49	match p.peek() {
50	Some(TokenType::Comma) => {
51	p.next();
52	if let Some(TokenType::ClosedParen) = p.peek() {
53	p.next();
54	break;
55	}
56	}
57	Some(TokenType::ClosedParen) => {
58	p.next();
59	break;
60	}
61	_ => {
62	let actual = p.peek().map_or("EOF".to_string(), \|x\| format!("{:?}", x));
63	return Err(p
64	.error(format!(
65	"Expected ',' or ')' after identifier, got {}",
66	actual
67	))
68	.into());
69	}
70	};
71	}
72	_ => {
73	return Err(p
74	.error("Expected ')' or identifier after '('".to_string())
75	.into())
76	}
77	}
78	}
79
80	let return_type = match p.peek() {
81	Some(TokenType::MinusLt) => {
82	_ = p.next();
83	parse_type(p)?
84	}
85	_ => Type::Infer,
86	};
87
88	let was_in_fn = p.is_in_fn;
89	p.is_in_fn = true;
90
91	let body = match p.peek() {
92	Some(TokenType::OpenBrace) => {
93	let block_start = p.index;
94	let block = parse_block(p)?;
95	Expr::new(
96	ExprData::Block(block),
97	Range::new(block_start, p.index - block_start),
98	None,
99	)
100	}
101	Some(TokenType::EqualLt) => {
102	_ = p.next();
103	parse_expr(p)?
104	}
105	_ => {
106	let actually = p.peek().map_or("EOF".to_string(), \|x\| format!("{:?}", x));
107	return Err(p
108	.error(format!(
109	"Expected a block or '=>' to follow function declaration, but got {}",
110	actually,
111	))
112	.into());
113	}
114	};
115
116	p.is_in_fn = was_in_fn;
117
118	Ok(FnDecl {
119	name,
120	return_type,
121	params,
122	body,
123	is_pub,
124	comment,
125	range: Range::new(start, p.index - start),
126	})
127	}

crates/parser/parse_module.rs deleted-44

...	@@ -1,44 +0,0 @@
1	use crate::parse_doc_comment::maybe_read_doc_comment;
2	use crate::parse_statement::parse_statement;
3	use crate::Parser;
4	use ast::Module;
5	use miette::Result;
6	use tokenizer::TokenType;
7
8	pub fn parse_module(p: &mut Parser) -> Result<Module> {
9	let mut statements = Vec::new();
10
11	let file_doc_comment = maybe_read_doc_comment(p);
12
13	loop {
14	let is_statement_like_expression = p
15	.peek()
16	.map_or(false, \|x\| x.is_start_of_statement_like_expression());
17
18	let statement = parse_statement(p)?;
19
20	match p.peek() {
21	Some(TokenType::Semicolon) => {
22	while let Some(TokenType::Semicolon) = p.peek() {
23	p.next();
24	}
25	}
26	_ => {
27	if statement.require_semicolon(is_statement_like_expression) {
28	return Err(p
29	.error("Expected semicolon after statement".to_string())
30	.into());
31	}
32	}
33	};
34
35	statements.push(statement);
36
37	if p.empty() {
38	return Ok(Module {
39	statements,
40	comment: file_doc_comment,
41	});
42	}
43	}
44	}

crates/parser/parse_param.rs deleted-30

...	@@ -1,30 +0,0 @@
1	use ast::Param;
2	use miette::Result;
3	use tokenizer::TokenType;
4
5	use crate::{parse_type::parse_type, Parser};
6
7	/// Precondition: A token exists and is an identifier
8	pub fn parse_param(p: &mut Parser) -> Result<Param> {
9	let name = match p.next() {
10	Some(TokenType::Identifier(name)) => name.clone(),
11	_ => unreachable!(),
12	};
13
14	let param_type = match p.peek() {
15	Some(TokenType::Colon) => {
16	_ = p.next();
17	parse_type(p)?
18	}
19	_ => {
20	return Err(p
21	.error("Function parameters require type annotations".to_string())
22	.into())
23	}
24	};
25
26	Ok(Param {
27	name: name.clone(),
28	param_type,
29	})
30	}

crates/parser/parse_statement.rs deleted-88

...	@@ -1,88 +0,0 @@
1	use ast::Statement;
2	use miette::Result;
3	use tokenizer::TokenType;
4
5	use crate::parse_declaration::parse_declaration;
6	use crate::parse_doc_comment::maybe_read_doc_comment;
7	use crate::parse_expr::parse_expr;
8	use crate::parse_fn_declaration::parse_fn_declaration;
9	use crate::Parser;
10
11	pub fn parse_statement(p: &mut Parser) -> Result<Statement> {
12	if let Some(comments) = p.take_comments() {
13	return Ok(Statement::Comment(comments));
14	}
15
16	let m_doc_comment = maybe_read_doc_comment(p);
17	if let Some(doc_comment) = m_doc_comment.clone() {
18	if p.has_comments() {
19	// this implies that comments are immediately after the doc
20	return Err(p
21	.error_at(format!("Unattached doc comment"), doc_comment.range)
22	.into());
23	}
24	}
25
26	match p.peek() {
27	Some(TokenType::DocComment(_)) => unreachable!(),
28	Some(TokenType::Pub) => {
29	if !p.is_top_level {
30	return Err(p
31	.error("Public declarations are only allowed at the top level".to_string())
32	.into());
33	}
34	_ = p.next();
35	match p.peek() {
36	Some(TokenType::Var) \| Some(TokenType::Let) => Ok(Statement::VarDecl(
37	parse_declaration(p, true, m_doc_comment)?,
38	)),
39	Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
40	p,
41	true,
42	m_doc_comment,
43	)?)),
44	Some(_) => Err(p.error("Expected a declaration".to_string()).into()),
45	None => Err(p.error("Unexpected end of file".to_string()).into()),
46	}
47	}
48	Some(TokenType::Var) \| Some(TokenType::Let) => Ok(Statement::VarDecl(parse_declaration(
49	p,
50	false,
51	m_doc_comment,
52	)?)),
53	Some(TokenType::Fn) => Ok(Statement::FnDecl(parse_fn_declaration(
54	p,
55	false,
56	m_doc_comment,
57	)?)),
58	Some(TokenType::Defer) => {
59	let label = p.peek().unwrap();
60	Err(p
61	.error(format!("'{}' is not yet implemented", label))
62	.into())
63	}
64	Some(TokenType::Struct \| TokenType::Enum \| TokenType::Trait \| TokenType::Impl) => {
65	let label = p.peek().unwrap();
66	if !p.is_top_level {
67	return Err(p
68	.error(format!("'{}' is only allowed at the top level", label))
69	.into());
70	}
71
72	Err(p
73	.error(format!("'{}' is not yet implemented", label))
74	.into())
75	}
76
77	Some(_) => {
78	if let Some(doc_comment) = m_doc_comment {
79	Err(p
80	.error_at(format!("Unattached doc comment"), doc_comment.range)
81	.into())
82	} else {
83	Ok(Statement::Expr(parse_expr(p)?))
84	}
85	}
86	None => Err(p.error("Unexpected end of file".to_string()).into()),
87	}
88	}

crates/parser/parse_type.rs deleted-26

...	@@ -1,26 +0,0 @@
1	use ast::Type;
2	use miette::Result;
3
4	use tokenizer::TokenType;
5
6	use crate::Parser;
7
8	pub fn parse_type(p: &mut Parser) -> Result<Type> {
9	match p.next() {
10	Some(TokenType::Identifier(id)) => match id.as_str() {
11	"int" => Ok(Type::Int),
12	"float" => Ok(Type::Float),
13	"bigint" => Ok(Type::BigInt),
14	"bool" => Ok(Type::Bool),
15	"string" => Ok(Type::String),
16	invalid_type => Err(p
17	.error(format!(
18	"Invalid type '{}' (TODO: add custom type support)",
19	invalid_type
20	))
21	.into()),
22	},
23	Some(unexpected) => Err(p.error(format!("Expected type, got {}", unexpected)).into()),
24	None => Err(p.error("Expected type, got EOF".to_string()).into()),
25	}
26	}

crates/parser/parser.rs deleted-150

...	@@ -1,150 +0,0 @@
1	mod parse_block;
2	mod parse_call_arguments;
3	mod parse_declaration;
4	mod parse_doc_comment;
5	mod parse_expr;
6	mod parse_expr_basic;
7	mod parse_expr_if;
8	mod parse_fn_declaration;
9	mod parse_module;
10	mod parse_param;
11	mod parse_statement;
12	mod parse_type;
13
14	use ast::Comment;
15	use ast::Module;
16	use log::Range;
17	use miette::Diagnostic;
18	use miette::Result;
19	use parse_module::parse_module;
20	use thiserror::Error;
21	use tokenizer::tokenize;
22	use tokenizer::Token;
23	use tokenizer::TokenType;
24
25	pub fn parse(tokens: Vec<Token>) -> Result<Module> {
26	let mut p = Parser::new(tokens);
27	parse_module(&mut p)
28	}
29
30	pub fn parse_from_string(string: String) -> Result<Module> {
31	let tokens = tokenize(string)?;
32	parse(tokens)
33	}
34
35	#[derive(Error, Diagnostic, Debug)]
36	#[error("ParseError")]
37	pub struct ParseError {
38	#[help]
39	error: String,
40	range: Option<Range>,
41	}
42
43	impl ParseError {
44	pub fn new(error: String, range: Option<Range>) -> ParseError {
45	ParseError { error, range }
46	}
47	}
48
49	#[derive(Debug)]
50	pub struct Parser {
51	/// TODO(@paperdave): switch to streaming tokenizer + one that can
52	/// report line and column info
53	tokens: Vec<Token>,
54	cursor: usize,
55
56	/// Current index in the string
57	pub index: usize,
58
59	/// Whether or not the parser is at the top level.
60	/// Currently controls the validity of `pub` keywords.
61	pub is_top_level: bool,
62	/// Whether or not the parser is inside a function.
63	/// Controls the validity of the `return` keyword.
64	pub is_in_fn: bool,
65
66	comments: Vec<String>,
67	}
68
69	impl Parser {
70	pub fn new(tokens: Vec<Token>) -> Parser {
71	Parser {
72	tokens,
73	cursor: 0,
74	index: 0,
75	is_top_level: true,
76	is_in_fn: false,
77	comments: vec![],
78	}
79	}
80
81	pub fn empty(&self) -> bool {
82	self.cursor == self.tokens.len()
83	}
84
85	pub fn next(&mut self) -> Option<TokenType> {
86	if self.cursor == self.tokens.len() {
87	return None;
88	}
89
90	loop {
91	let token = self.tokens[self.cursor].clone();
92	self.index = token.range.start;
93	match token.variant {
94	TokenType::Comment(comment) => {
95	self.comments.push(comment);
96	self.cursor += 1;
97	continue;
98	}
99	token => {
100	self.cursor += 1;
101	return Some(token);
102	}
103	}
104	}
105	}
106
107	pub fn take_comments(&mut self) -> Option<Comment> {
108	if self.comments.len() == 0 {
109	return None;
110	}
111	let comments = self.comments.clone();
112	self.comments = vec![];
113	Some(Comment {
114	data: comments.join("\n"),
115	})
116	}
117
118	pub fn has_comments(&mut self) -> bool {
119	self.peek();
120	self.comments.len() > 0
121	}
122
123	pub fn peek(&mut self) -> Option<TokenType> {
124	loop {
125	if self.cursor == self.tokens.len() {
126	return None;
127	}
128
129	match self.tokens[self.cursor].clone().variant {
130	TokenType::Comment(comment) => {
131	self.comments.push(comment);
132	self.cursor += 1;
133	continue;
134	}
135	token => {
136	return Some(token);
137	}
138	}
139	}
140	}
141
142	pub fn error(&self, error: String) -> ParseError {
143	// TODO(@lino-levan): Add location information
144	ParseError::new(error, None)
145	}
146
147	fn error_at(&self, error: String, range: Range) -> ParseError {
148	ParseError::new(error, Some(range))
149	}
150	}

crates/sema/Cargo.toml deleted-14

...	@@ -1,14 +0,0 @@
1	[package]
2	name = "sema"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path = "./sema.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12	num.workspace = true
13
14	ast = { path = "../ast" }
	\ No newline at end of file

crates/sema/ast_type_coersion.rs deleted-54

...	@@ -1,54 +0,0 @@
1	// use ast::{BinaryOperator, ExprData, Type};
2
3	impl TypeCoerceable for ExprData {
4	fn to_type(&self) -> Option<Type> {
5	match self {
6	ExprData::Block(block) => {
7	if (block.has_implicit_return) {
8	block.statements.last().unwrap().to_type()
9	} else {
10	Some(Type::Void)
11	}
12	}
13	ExprData::Binary { op, left, right } => {
14	// TODO: this likely has to do further type checking
15	let left_type = left.to_type();
16	match op {
17	BinaryOperator::Add => left_type,
18	BinaryOperator::Sub => left_type,
19	BinaryOperator::Mul => left_type,
20	BinaryOperator::Div => left_type,
21	BinaryOperator::Pow => left_type,
22	BinaryOperator::Eq => Some(Type::Bool),
23	BinaryOperator::Gt => Some(Type::Bool),
24	BinaryOperator::Lt => Some(Type::Bool),
25	BinaryOperator::Gte => Some(Type::Bool),
26	BinaryOperator::Lte => Some(Type::Bool),
27	BinaryOperator::Lshift => Some(Type::Int),
28	BinaryOperator::Rshift => Some(Type::Int),
29	BinaryOperator::BitAnd => Some(Type::Int),
30	BinaryOperator::BitOr => Some(Type::Int),
31	BinaryOperator::BitXor => Some(Type::Int),
32	BinaryOperator::Mod => Some(Type::Int),
33	BinaryOperator::NotEq => Some(Type::Int),
34
35	BinaryOperator::And => Some(Type::Bool),
36	BinaryOperator::Or => Some(Type::Bool),
37	}
38	}
39	ExprData::Unary { op, value } => todo!(),
40	ExprData::Function {
41	name,
42	params,
43	return_type,
44	block,
45	} => Some(return_type),
46	ExprData::Call { base, arguments } => None, // oh shit. yeah this trait isn't implementable
47	ExprData::Builtin { id, arguments } => todo!(),
48	ExprData::If(_) => todo!(),
49	ExprData::Return(_) => todo!(),
50	ExprData::Literal(_) => todo!(),
51	ExprData::Identifier(_) => todo!(),
52	}
53	}
54	}

crates/sema/sema.rs deleted-98

...	@@ -1,98 +0,0 @@
1	use ast::Module;
2	use miette::Result;
3
4	pub fn analyze_module(_block: &mut Module) -> Result<()> {
5	//
6	Ok(())
7	}
8
9	// for statement in &block.statements {
10	// check_expression(statement)?;
11	// }
12
13	// Ok(())
14	// }
15
16	// pub fn resolve_type(expr: &Ast) -> Result<Type> {
17	// match expr {
18	// Ast::Number(value) => Ok(Type::NumberLiteral(value.clone())),
19	// Ast::Declaration {
20	// name,
21	// decl_type,
22	// value,
23	// is_constant,
24	// } => Ok(Type::NoReturn),
25	// Ast::Binary {
26	// operator,
27	// left,
28	// right,
29	// } => merge_types(resolve_type(left)?, resolve_type(right)?),
30	// Ast::Function {
31	// name,
32	// arguments,
33	// return_type,
34	// block,
35	// } => todo!(),
36	// Ast::FunctionCall { name, arguments } => todo!(),
37	// Ast::If {
38	// condition,
39	// value,
40	// else_value,
41	// } => todo!(),
42	// Ast::Return(_) => Ok(Type::NoReturn),
43	// Ast::Public(_) => todo!(), // TODO: remove Public
44	// Ast::Block(_) => todo!(),
45	// Ast::Bool(_) => Ok(Type::Bool),
46	// Ast::String(_) => Ok(Type::String),
47	// Ast::Identifier(_) => todo!(),
48	// Ast::BuiltIn(_) => todo!(),
49
50	// Ast::Comment(_) => unreachable!(),
51	// Ast::DocComment(_) => unreachable!(),
52	// }
53	// }
54
55	// pub fn check_expression(expr: &Ast) -> Result<()> {
56	// panic!()
57	// match expr {
58	// Ast::Declaration {
59	// name,
60	// decl_type: return_type,
61	// value,
62	// is_constant,
63	// } => {
64	// check_expression(value)?;
65	// // coerce_type(return_type.clone(), resolve_type(value)?)?;
66	// }
67	// _ => {}
68	// // Ast::Operation {
69	// // operator,
70	// // left,
71	// // right,
72	// // } => Ok(()),
73	// // Ast::Function {
74	// // name,
75	// // arguments,
76	// // return_type,
77	// // block,
78	// // } => Ok(()),
79	// // Ast::FunctionCall { name, arguments } => Ok(()),
80	// // Ast::If {
81	// // condition,
82	// // value,
83	// // else_value,
84	// // } => Ok(()),
85	// // Ast::Return(_) => Ok(()),
86	// // Ast::Public(_) => Ok(()),
87	// // Ast::Block(_) => Ok(()),
88	// // Ast::Number(_) => Ok(()),
89	// // Ast::Bool(_) => Ok(()),
90	// // Ast::String(_) => Ok(()),
91	// // Ast::Identifier(_) => Ok(()),
92	// // Ast::BuiltIn(_) => Ok(()),
93	// // Ast::Comment(_) => Ok(()),
94	// // Ast::DocComment(_) => Ok(()),
95	// }
96
97	// Ok(())
98	// }

crates/sema/sema_scope.rs deleted

crates/tokenizer/Cargo.toml deleted-13

...	@@ -1,13 +0,0 @@
1	[package]
2	name = "tokenizer"
3	version = "0.1.0"
4	edition = "2021"
5
6	[lib]
7	path="./tokenizer.rs"
8
9	[dependencies]
10	miette.workspace = true
11	thiserror.workspace = true
12
13	log = { path = "../log" }

crates/tokenizer/tokenizer.rs deleted-443

...	@@ -1,443 +0,0 @@
1	use log::Range;
2	use miette::{Diagnostic, Result};
3	use thiserror::Error;
4
5	#[derive(Error, Diagnostic, Debug)]
6	#[error("UnknownCharacter")]
7	struct UnknownCharacter {
8	#[help]
9	error: String,
10	}
11
12	#[derive(Debug, Clone, PartialEq)]
13	pub enum TokenType {
14	// Literals
15	Identifier(String),
16	Number(f64),
17	String(String),
18	// Comments
19	Comment(String),
20	DocComment(String),
21	// Keywords
22	Defer,
23	Else,
24	Enum,
25	False,
26	Fn,
27	If,
28	Impl,
29	Let,
30	Pub,
31	Return,
32	Struct,
33	Trait,
34	True,
35	Var,
36	Void,
37	// Symbols
38	And, // &
39	AndAnd, // &&
40	Bang, // !
41	BangEqual, // !=
42	EqualLt, // =>
43	Caret, // ^
44	ClosedBrace, // }
45	ClosedBracket, // ]
46	ClosedParen, // )
47	Colon, // :
48	Comma, // ,
49	Dot, // .
50	Equal, // =
51	EqualEqual, // ==
52	Gt, // >
53	GtEqual, // >=
54	GtGt, // >>
55	GtGtGt, // >>>
56	Lt, // <
57	LtEqual, // <=
58	LtLt, // <<
59	Minus, // -
60	OpenBrace, // {
61	OpenBracket, // [
62	OpenParen, // (
63	Or, // \|
64	OrOr, // \|\|
65	Percent, // %
66	Plus, // +
67	MinusLt, // ->
68	Semicolon, // ;
69	Slash, // /
70	Star, // *
71	StarStar, // **
72	Tilde, // ~
73	}
74
75	impl TokenType {
76	pub fn is_start_of_statement_like_expression(&self) -> bool {
77	match &self {
78	TokenType::If \| TokenType::OpenBrace => true,
79	_ => false,
80	}
81	}
82	}
83
84	impl std::fmt::Display for TokenType {
85	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86	let token = match &self {
87	TokenType::Identifier(_) => "identifier".to_string(),
88
89	TokenType::Number(n) => n.to_string(),
90	TokenType::String(s) => format!("\"{}\"", s),
91	TokenType::Comment(_) => "comment".to_string(),
92	TokenType::DocComment(_) => "documentation comment".to_string(),
93
94	TokenType::Defer => "defer".to_string(),
95	TokenType::Else => "else".to_string(),
96	TokenType::Enum => "enum".to_string(),
97	TokenType::False => "false".to_string(),
98	TokenType::Fn => "fn".to_string(),
99	TokenType::If => "if".to_string(),
100	TokenType::Impl => "impl".to_string(),
101	TokenType::Let => "let".to_string(),
102	TokenType::Pub => "pub".to_string(),
103	TokenType::Return => "return".to_string(),
104	TokenType::Struct => "struct".to_string(),
105	TokenType::Trait => "trait".to_string(),
106	TokenType::True => "true".to_string(),
107	TokenType::Var => "var".to_string(),
108	TokenType::Void => "void".to_string(),
109
110	TokenType::And => "&".to_string(),
111	TokenType::AndAnd => "&&".to_string(),
112	TokenType::Bang => "!".to_string(),
113	TokenType::BangEqual => "!=".to_string(),
114	TokenType::EqualLt => "=>".to_string(),
115	TokenType::Caret => "^".to_string(),
116	TokenType::ClosedBrace => "}".to_string(),
117	TokenType::ClosedBracket => "]".to_string(),
118	TokenType::ClosedParen => ")".to_string(),
119	TokenType::Colon => ":".to_string(),
120	TokenType::Comma => ",".to_string(),
121	TokenType::Dot => ".".to_string(),
122	TokenType::Equal => "=".to_string(),
123	TokenType::EqualEqual => "==".to_string(),
124	TokenType::Gt => ">".to_string(),
125	TokenType::GtEqual => ">=".to_string(),
126	TokenType::GtGt => ">>".to_string(),
127	TokenType::GtGtGt => ">>>".to_string(),
128	TokenType::Lt => "<".to_string(),
129	TokenType::LtEqual => "<=".to_string(),
130	TokenType::LtLt => "<<".to_string(),
131	TokenType::Minus => "-".to_string(),
132	TokenType::OpenBrace => "{".to_string(),
133	TokenType::OpenBracket => "[".to_string(),
134	TokenType::OpenParen => "(".to_string(),
135	TokenType::Or => "\|".to_string(),
136	TokenType::OrOr => "\|\|".to_string(),
137	TokenType::Percent => "%".to_string(),
138	TokenType::Plus => "+".to_string(),
139	TokenType::MinusLt => "->".to_string(),
140	TokenType::Semicolon => ";".to_string(),
141	TokenType::Slash => "/".to_string(),
142	TokenType::Star => "*".to_string(),
143	TokenType::StarStar => "**".to_string(),
144	TokenType::Tilde => "~".to_string(),
145	};
146
147	write!(f, "{}", token)
148	}
149	}
150
151	#[derive(Debug, Clone)]
152	pub struct Token {
153	pub variant: TokenType,
154	pub range: Range,
155	}
156
157	impl Token {
158	pub fn new(variant: TokenType, start: usize, length: usize) -> Self {
159	Self {
160	variant,
161	range: Range::new(start, length),
162	}
163	}
164	}
165
166	// TODO(@paperdave): rewrite this to not use a Vec<Token>. we probably dont need it. also store Range
167	pub fn tokenize(content: String) -> Result<Vec<Token>> {
168	let mut tokens: Vec<Token> = Vec::new();
169	let mut chars = content.chars().enumerate().peekable();
170
171	loop {
172	match chars.next() {
173	Some((start, c)) => {
174	let token = match c {
175	'a'..='z' \| 'A'..='Z' \| '_' \| '@' => {
176	let mut current_token = String::new();
177	current_token.push(c);
178	loop {
179	let c = chars.peek();
180	if c.is_none() {
181	break;
182	}
183	let (_, c) = c.unwrap();
184	match c {
185	'a'..='z' \| 'A'..='Z' \| '_' \| '0'..='9' => {
186	current_token.push(*c);
187	chars.next();
188	}
189	_ => break,
190	}
191	}
192
193	let length = current_token.len();
194
195	let variant = match current_token.as_str() {
196	"defer" => TokenType::Defer,
197	"else" => TokenType::Else,
198	"enum" => TokenType::Enum,
199	"false" => TokenType::False,
200	"fn" => TokenType::Fn,
201	"if" => TokenType::If,
202	"impl" => TokenType::Impl,
203	"let" => TokenType::Let,
204	"pub" => TokenType::Pub,
205	"return" => TokenType::Return,
206	"struct" => TokenType::Struct,
207	"trait" => TokenType::Trait,
208	"true" => TokenType::True,
209	"var" => TokenType::Var,
210	"void" => TokenType::Void,
211	_ => TokenType::Identifier(current_token),
212	};
213
214	Token::new(variant, start, length)
215	}
216	'"' => {
217	let mut length = 1;
218	let mut current_token = String::new();
219	loop {
220	let c = chars.next();
221	if c.is_none() {
222	break;
223	}
224	let (_, c) = c.unwrap();
225	length += 1;
226	match c {
227	'"' => break,
228	_ => current_token.push(c),
229	}
230	}
231	Token::new(TokenType::String(current_token), start, length)
232	}
233	'0'..='9' => {
234	let mut current_token = String::new();
235	current_token.push(c);
236	loop {
237	let c = chars.peek();
238	if c.is_none() {
239	break;
240	}
241	let (_, c) = c.unwrap();
242	match c {
243	'0'..='9' => {
244	current_token.push(*c);
245	chars.next();
246	}
247	_ => break,
248	}
249	}
250	match current_token.parse::<f64>() {
251	Ok(v) => Token::new(TokenType::Number(v), start, current_token.len()),
252	Err(_) => {
253	return Err(UnknownCharacter {
254	error: format!("Unknown character: {}", current_token),
255	}
256	.into())
257	}
258	}
259	}
260	':' => Token::new(TokenType::Colon, start, 1),
261	'=' => match chars.peek() {
262	Some((_, '=')) => {
263	chars.next();
264	Token::new(TokenType::EqualEqual, start, 2)
265	}
266	Some((_, '>')) => {
267	chars.next();
268	Token::new(TokenType::EqualLt, start, 2)
269	}
270	_ => Token::new(TokenType::Equal, start, 1),
271	},
272	';' => Token::new(TokenType::Semicolon, start, 1),
273	'>' => match chars.peek() {
274	Some((_, '=')) => {
275	chars.next();
276	Token::new(TokenType::GtEqual, start, 2)
277	}
278	Some((_, '>')) => {
279	chars.next();
280	match chars.peek() {
281	Some((_, '>')) => {
282	chars.next();
283	Token::new(TokenType::GtGtGt, start, 3)
284	}
285	_ => Token::new(TokenType::GtGt, start, 2),
286	}
287	}
288	_ => Token::new(TokenType::Gt, start, 1),
289	},
290	'<' => match chars.peek() {
291	Some((_, '=')) => {
292	chars.next();
293	Token::new(TokenType::LtEqual, start, 2)
294	}
295	Some((_, '<')) => {
296	chars.next();
297	Token::new(TokenType::LtLt, start, 2)
298	}
299	_ => Token::new(TokenType::Lt, start, 1),
300	},
301	'+' => Token::new(TokenType::Plus, start, 1),
302	'-' => match chars.peek() {
303	Some((_, '>')) => {
304	chars.next();
305	Token::new(TokenType::MinusLt, start, 2)
306	}
307	_ => Token::new(TokenType::Minus, start, 1),
308	},
309	'*' => match chars.peek() {
310	Some((_, '*')) => {
311	chars.next();
312	Token::new(TokenType::StarStar, start, 2)
313	}
314	_ => Token::new(TokenType::Star, start, 1),
315	},
316	'/' => {
317	match chars.peek() {
318	Some((_, '/')) => {
319	chars.next(); // Consume the second slash
320	let mut length = 2;
321
322	let is_doc_comment = match chars.peek() {
323	Some((_, '/')) => {
324	chars.next(); // Consume the third slash
325	length += 1;
326	true
327	}
328	_ => false,
329	};
330
331	let mut comment = String::new();
332	loop {
333	// skip leading whitespace
334	while let Some((_, c)) = chars.peek() {
335	if !c.is_whitespace() {
336	break;
337	}
338	chars.next();
339	length += 1;
340	}
341	// read the comment
342	while let Some((_, c)) = chars.next() {
343	length += 1;
344	if c == '\n' \|\| c == '\r' {
345	break;
346	}
347	comment.push(c);
348	}
349	// trim trailing whitespace on line
350	comment = comment.trim_end().to_string();
351	// skip to next non-whitespace character
352	while let Some((_, c)) = chars.peek() {
353	if !c.is_whitespace() {
354	break;
355	}
356	chars.next();
357	length += 1;
358	}
359	// check if we've hit another comment
360	let mut tmp = chars.clone();
361
362	let is_another_comment = match tmp.peek() {
363	Some((_, '/')) => {
364	tmp.next(); // Consume the second slash
365	match tmp.peek() {
366	Some((_, '/')) => {
367	tmp.next(); // Consume the third slash
368	is_doc_comment
369	}
370	_ => !is_doc_comment,
371	}
372	}
373	_ => false,
374	};
375
376	if is_another_comment {
377	// apply our temporary iterator to the main iterator
378	comment.push('\n');
379	chars = tmp;
380	chars.next();
381	length += 1;
382	} else {
383	break;
384	}
385	}
386
387	if is_doc_comment {
388	Token::new(TokenType::DocComment(comment), start, length)
389	} else {
390	Token::new(TokenType::Comment(comment), start, length)
391	}
392	}
393	_ => Token::new(TokenType::Slash, start, 1),
394	}
395	}
396	',' => Token::new(TokenType::Comma, start, 1),
397	'(' => Token::new(TokenType::OpenParen, start, 1),
398	')' => Token::new(TokenType::ClosedParen, start, 1),
399	'{' => Token::new(TokenType::OpenBrace, start, 1),
400	'}' => Token::new(TokenType::ClosedBrace, start, 1),
401	'[' => Token::new(TokenType::OpenBracket, start, 1),
402	']' => Token::new(TokenType::ClosedBracket, start, 1),
403	'&' => match chars.peek() {
404	Some((_, '&')) => {
405	chars.next();
406	Token::new(TokenType::AndAnd, start, 2)
407	}
408	_ => Token::new(TokenType::And, start, 1),
409	},
410	'\|' => match chars.peek() {
411	Some((_, '\|')) => {
412	chars.next();
413	Token::new(TokenType::OrOr, start, 2)
414	}
415	_ => Token::new(TokenType::Or, start, 1),
416	},
417	'!' => match chars.peek() {
418	Some((_, '=')) => {
419	chars.next();
420	Token::new(TokenType::BangEqual, start, 2)
421	}
422	_ => Token::new(TokenType::Bang, start, 1),
423	},
424	'^' => Token::new(TokenType::Caret, start, 1),
425	'%' => Token::new(TokenType::Percent, start, 1),
426	'~' => Token::new(TokenType::Tilde, start, 1),
427	'.' => Token::new(TokenType::Dot, start, 1),
428	s if s.is_whitespace() => continue,
429	_ => {
430	return Err(UnknownCharacter {
431	error: format!("Unexpected '{}' while tokenizing", c),
432	}
433	.into())
434	}
435	};
436	tokens.push(token);
437	}
438	None => break,
439	}
440	}
441
442	Ok(tokens)
443	}

examples/basics/associativity.blight deleted-6

...	@@ -1,6 +0,0 @@
1	let print = @raw("console.log");
2	print(1 * 2 * 3);
3	print(1 * 2 / 3);
4	print(1 + 2 ** 3);
5	print(1 2 3);
6	print(1 * 2 + 3);

examples/basics/comments.blight deleted-9

...	@@ -1,9 +0,0 @@
1	// this is a regular comment
2	let x: int = 0;
3
4	/// this is a doc comment
5	let y: int = 0;
6
7	/// Doc comments can be multiple lines.
8	/// This can be done by simply adding another triple-slash to the next line.
9	let z: int = 0;

examples/basics/control.blight deleted-17

...	@@ -1,17 +0,0 @@
1	let print = @raw("console.log");
2
3	let x: bool = true;
4
5	if x {
6	print("x is true!");
7	}
8
9	let y: int = 3;
10
11	if y == 1 {
12	print("y is 1!");
13	} else if y == 2 {
14	print("y is 2!");
15	} else {
16	print("y is not 1 or 2!");
17	}

examples/basics/defer.blight deleted-12

...	@@ -1,12 +0,0 @@
1	let print = @raw("console.log");
2
3	{
4	print("1");
5	defer print("6");
6	{
7	print("2");
8	defer print("4");
9	print("3");
10	}
11	print("5");
12	}

examples/basics/expressions.blight deleted-11

...	@@ -1,11 +0,0 @@
1	// everything is an expression in blight
2	// everything.
3	// hehe
4
5	let x: int = 5 + {
6	let y: int = 5;
7	y
8	};
9
10	let print = @raw("console.log");
11	print(x); // 10

examples/basics/fib.blight deleted-12

...	@@ -1,12 +0,0 @@
1	pub fn fib(n: int) -> int {
2	if n == 0 {
3	0
4	} else if n == 1 {
5	1
6	} else {
7	fib(n - 1) + fib(n - 2)
8	}
9	}
10
11	let print = @raw("console.log");
12	print(fib(10));

examples/basics/functions.blight deleted-8

...	@@ -1,8 +0,0 @@
1	fn bingus(x: int, y: int) -> int {
2	x * y
3	}
4
5	let result: int = bingus(10, 25);
6
7	let print = @raw("console.log");
8	print(result);

examples/basics/operations.blight deleted-4

...	@@ -1,4 +0,0 @@
1	let a: int = 10 - 2 / 4 * 15;
2
3	let print = @raw("console.log");
4	print(a);

examples/basics/shadowing.blight deleted-14

...	@@ -1,14 +0,0 @@
1	let print = @raw("console.log");
2
3	let x: int = 0;
4	let x: int = 1;
5	print(x); // 1
6
7	let y: int = 0;
8	{
9	let y: int = 1;
10	{
11	let y: int = 2;
12	}
13	}
14	print(y); // 0

examples/basics/stupid.blight deleted-2

...	@@ -1,2 +0,0 @@
1
2	fn add(x: int, y: int) => x + y

examples/basics/variables.blight deleted-10

...	@@ -1,10 +0,0 @@
1	var x: int = 0;
2	var y: float = 0;
3	var z: bigint = 0;
4
5	let a: int = 0;
6	let b: float = 0;
7	let c: bigint = 0;
8
9	let print = @raw("console.log");
10	print(x);

examples/basics/weird.blight deleted-14

...	@@ -1,14 +0,0 @@
1	// Just trying to break the blight compiler
2	fn bingus(y: bool) -> int {
3	let x: int = 5 + {
4	if y {
5	return 100000;
6	}
7	6
8	};
9	x
10	}
11
12	let print = @raw("console.log");
13	print(bingus(true));
14	print(bingus(false));

examples/interop/blight_in_js/add.blight deleted-5

...	@@ -1,5 +0,0 @@
1	pub fn add(x: int, y: int) -> int {
2	x + y
3	}
4
5	pub let TOP_LEVEL: int = 100;

examples/interop/blight_in_js/main.js deleted-4

...	@@ -1,4 +0,0 @@
1	import { add, TOP_LEVEL } from "./add.js";
2
3	console.log(add(1, 2));
4	console.log(TOP_LEVEL);

examples/interop/deno.blight deleted-7

...	@@ -1,7 +0,0 @@
1	let writeTextFile = @raw("Deno.writeTextFile");
2
3	fn x() -> string {
4	"hello"
5	}
6
7	writeTextFile("hello.txt", x());

src/compiler/Air.Dump.zig created+299

...	@@ -0,0 +1,299 @@
1	//! This implements a debug formatter for Teg
2	air: Air,
3	ip: *const InternPool,
4
5	const State = struct {
6	arena: std.mem.Allocator,
7	air: *const Air,
8	ip: *const InternPool,
9	indent: usize = 0,
10	tty: std.io.tty.Config,
11	visited: std.AutoHashMapUnmanaged(Inst.Index, void) = .{},
12
13	fn newline(state: State, writer: anytype) !void {
14	try writer.writeByte('\n');
15	if (state.indent > 0)
16	try writer.writeByteNTimes(' ', state.indent * 2);
17	}
18
19	fn dumpInst(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
20	const air = s.air;
21	const tty = s.tty;
22	const tag = s.air.instTag(inst);
23
24	try tty.setColor(w, switch (s.air.instFlags(inst).uses) {
25	.none => .bright_black,
26	.multi => .red,
27	.single => .magenta,
28	});
29	try w.writeByte(switch (s.air.instFlags(inst).uses) {
30	.none => 'X',
31	.multi => 'M',
32	.single => 'S',
33	});
34	try tty.setColor(w, .cyan);
35
36	try w.print(" #{} = ", .{
37	@intFromEnum(inst),
38	});
39
40	try tty.setColor(w, .cyan);
41	try w.print("{s}", .{@tagName(tag)});
42	try tty.setColor(w, .reset);
43
44	if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
45	try tty.setColor(w, .red);
46	try w.writeAll(" [error: recursive tree]");
47	try tty.setColor(w, .reset);
48	return;
49	}
50
51	switch (tag) {
52	.block => {
53	const index_list: Air.IndexList = air.instData(.block, inst);
54	const insts = index_list.slice(air);
55	try s.dumpInstList(w, insts, '{', '}');
56	},
57	.js_global_ref => {
58	const name: InternPool.String.WithLen = air.instData(.js_global_ref, inst);
59	try tty.setColor(w, .magenta);
60	try w.print(" {s}", .{name.toSlice(s.ip)});
61	try tty.setColor(w, .reset);
62	},
63	.arg => {
64	const arg: Air.Arg = air.instData(.arg, inst);
65	try w.writeAll(" ");
66	try tty.setColor(w, .yellow);
67	try w.print("{d}", .{arg.arg_number});
68	try tty.setColor(w, .reset);
69	try w.writeAll(": ");
70	try s.dumpValue(w, arg.type);
71	},
72	.@"break" => {
73	const brk: Air.Break = air.instData(.@"break", inst);
74	try w.writeAll("(");
75	try s.dumpInstRef(w, brk.block_ref);
76	try w.writeAll(", ");
77	try s.dumpRef(w, brk.value);
78	try w.writeAll(")");
79	},
80	inline .add,
81	.mul,
82	.sub,
83	.div,
84	.lt,
85	.gt,
86	.eq,
87	=> \|comptime_tag\| {
88	const bin: Air.Bin = air.instData(comptime_tag, inst);
89	try w.writeAll("(");
90	try s.dumpRef(w, bin.lhs);
91	try w.writeAll(", ");
92	try s.dumpRef(w, bin.rhs);
93	try w.writeAll(")");
94	},
95	inline .negate => \|comptime_tag\| {
96	const bin: Air.Un = air.instData(comptime_tag, inst);
97	try w.writeAll("(");
98	try s.dumpRef(w, bin.operand);
99	try w.writeAll(")");
100	},
101	.field => {
102	const field: Air.Field = air.instData(.field, inst);
103	try w.writeAll("(");
104	try s.dumpRef(w, field.base);
105	try w.writeAll(", ");
106	try tty.setColor(w, .magenta);
107	try w.print("{s}", .{field.field.toSlice(s.ip)});
108	try tty.setColor(w, .reset);
109	try w.writeAll(")");
110	},
111	.call => {
112	const call: Air.Call = air.instData(.call, inst);
113	const call_data = call.call_data.get(air);
114	try w.writeAll("(");
115	try s.dumpRef(w, call.base);
116	try w.writeAll(", ");
117	try s.dumpRefList(w, call_data.args.slice(air), '[', ']');
118	try w.writeAll(")");
119	},
120	.@"if" => {
121	const if_inst: Air.If = air.instData(.@"if", inst);
122	const if_data: Air.If.Data = if_inst.data.get(air);
123	try w.writeAll("(");
124	try s.dumpRef(w, if_inst.cond);
125	try w.writeAll(", {");
126	s.indent += 1;
127	try s.newline(w);
128	try w.writeAll("yes: ");
129	try s.dumpRefRecurse(w, if_data.yes);
130	try s.newline(w);
131	try w.writeAll("no: ");
132	try s.dumpRefRecurse(w, if_data.no);
133	s.indent -= 1;
134	try s.newline(w);
135	try w.writeAll("}");
136	},
137	}
138	}
139
140	fn dumpRef(s: *State, w: anytype, ref: Inst.Ref) !void {
141	switch (ref.unwrap()) {
142	.inst => \|i\| try s.dumpInstRef(w, i),
143	.value => \|v\| try s.dumpValue(w, v),
144	}
145	}
146
147	fn dumpRefRecurse(s: *State, w: anytype, ref: Inst.Ref) !void {
148	switch (ref.unwrap()) {
149	.inst => \|i\| try s.dumpInst(w, i),
150	.value => \|v\| try s.dumpValue(w, v),
151	}
152	}
153
154	fn dumpInstRef(s: *State, w: anytype, inst: Inst.Index) @TypeOf(w).Error!void {
155	try s.tty.setColor(w, .red);
156	try w.print("#{d}", .{@intFromEnum(inst)});
157	try s.tty.setColor(w, .reset);
158	}
159
160	fn dumpValue(s: *State, w: anytype, val: Value) !void {
161	const tty = s.tty;
162	switch (val.toKey(s.ip)) {
163	.simple_type => \|t\| {
164	try tty.setColor(w, .blue);
165	try w.print("{s}", .{@tagName(t)});
166	try tty.setColor(w, .reset);
167	},
168	.simple_value => \|t\| {
169	try tty.setColor(w, .yellow);
170	try w.print("{s}", .{@tagName(t)});
171	try tty.setColor(w, .reset);
172	},
173	.type_function => \|ty\| {
174	try tty.setColor(w, .blue);
175	try w.writeAll("fn(");
176	try tty.setColor(w, .reset);
177	for (ty.param_types, 0..) \|param, i\| {
178	if (i != 0) {
179	try tty.setColor(w, .blue);
180	try w.writeAll(",");
181	try tty.setColor(w, .reset);
182	}
183	try s.dumpValue(w, Value.fromInterned(param));
184	}
185	try tty.setColor(w, .blue);
186	try w.writeAll(") -> ");
187	try tty.setColor(w, .reset);
188	try s.dumpValue(w, Value.fromInterned(ty.return_type));
189	},
190	inline .float, .int32 => \|data, tag\| {
191	try tty.setColor(w, .bright_black);
192	try w.writeAll("<");
193	try tty.setColor(w, .blue);
194	try w.writeAll(@tagName(tag));
195	try tty.setColor(w, .reset);
196	try tty.setColor(w, .bright_black);
197	try w.writeAll(", ");
198	try tty.setColor(w, .yellow);
199	try w.print("{d}", .{data});
200	try tty.setColor(w, .bright_black);
201	try w.writeAll(">");
202	try tty.setColor(w, .reset);
203	},
204	.string => \|data\| {
205	try w.writeAll("<");
206	try tty.setColor(w, .blue);
207	try w.writeAll("string");
208	try tty.setColor(w, .reset);
209	try w.writeAll(", ");
210	try tty.setColor(w, .green);
211	try w.writeAll("\"");
212	try w.writeAll(data);
213	try w.writeAll("\"");
214	try tty.setColor(w, .reset);
215	try w.writeAll(">");
216	},
217	.func => \|func\| {
218	try tty.setColor(w, .bright_black);
219	try w.writeAll("<");
220	try s.dumpValue(w, Value.fromInterned(func.ty));
221	try tty.setColor(w, .bright_black);
222	try w.writeAll(", ");
223	const decl_name = s.ip.declPtrConst(func.owner_decl).name.toSlice(s.ip);
224	try tty.setColor(w, .bright_green);
225	try w.print("decl '{s}'", .{decl_name});
226	try tty.setColor(w, .bright_black);
227	try w.writeAll(">");
228	try tty.setColor(w, .reset);
229	},
230	}
231	}
232
233	fn dumpInstList(state: *State, writer: anytype, list: []const Inst.Index, l: u8, r: u8) !void {
234	try writer.print(" {c}", .{l});
235	if (list.len > 0) {
236	state.indent += 1;
237
238	for (list) \|inst\| {
239	try state.newline(writer);
240	try state.dumpInst(writer, inst);
241	}
242
243	state.indent -= 1;
244	try state.newline(writer);
245	}
246	try writer.print("{c}", .{r});
247	}
248
249	fn dumpRefList(state: *State, writer: anytype, list: []const Inst.Ref, l: u8, r: u8) !void {
250	try writer.print(" {c}", .{l});
251	if (list.len > 0) {
252	state.indent += 1;
253
254	for (list) \|inst\| {
255	try state.newline(writer);
256	try state.dumpRef(writer, inst);
257	}
258
259	state.indent -= 1;
260	try state.newline(writer);
261	}
262	try writer.print("{c}", .{r});
263	}
264	};
265
266	pub fn format(
267	dump: @This(),
268	comptime unused_fmt: []const u8,
269	_: std.fmt.FormatOptions,
270	writer: anytype,
271	) !void {
272	comptime assert(unused_fmt.len == 0);
273
274	var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
275	defer arena.deinit();
276
277	const air = dump.air;
278	var state = State{
279	.air = &air,
280	.ip = dump.ip,
281	.tty = detectTTYFromWriter(writer),
282	.arena = arena.allocator(),
283	};
284	try state.dumpInst(writer, .main_block);
285	try writer.writeAll("\n");
286	assert(state.indent == 0);
287	}
288
289	const std = @import("std");
290	const assert = std.debug.assert;
291
292	const Air = @import("Air.zig");
293	const Inst = Air.Inst;
294
295	const Value = @import("Value.zig");
296
297	const InternPool = @import("InternPool.zig");
298
299	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Air.zig created+292

...	@@ -0,0 +1,292 @@
1	insts: Inst.List.Slice,
2	extra: ExtraStore = .{},
3
4	pub const ExtraStore = dod.ExtraStore(.air, .@"const");
5	pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
6	pub const ExtraIndex = ExtraStore.Index;
7
8	pub const Inst = struct {
9	tag: Tag,
10	data: EncodedData,
11	flags: Flags,
12
13	pub const List = std.MultiArrayList(Inst);
14
15	pub const Tag = enum {
16	block,
17	js_global_ref,
18	call,
19	@"break",
20	@"if",
21	arg,
22	field,
23
24	// bin
25	add,
26	sub,
27	mul,
28	div,
29
30	eq,
31
32	lt,
33	gt,
34
35	// un
36	negate,
37
38	pub fn Data(comptime tag: Tag) type {
39	return switch (tag) {
40	.block => IndexList,
41	.js_global_ref => InternPool.String.WithLen,
42	.@"break" => Break,
43	.@"if" => If,
44	.call => Call,
45	.arg => Arg,
46	.add,
47	.sub,
48	.mul,
49	.div,
50	.lt,
51	.gt,
52	.eq,
53	=> Bin,
54	.negate,
55	=> Un,
56	.field,
57	=> Field,
58	};
59	}
60	};
61
62	/// reference to either a value or another instruction in this air
63	pub const Ref = packed struct(u32) {
64	kind: Kind,
65	data: u31,
66
67	pub const Kind = enum(u1) { value, inst };
68
69	pub const Decoded = union(enum) {
70	value: Value,
71	inst: Index,
72	};
73
74	pub fn unwrap(ref: Ref) Decoded {
75	return switch (ref.kind) {
76	.value => .{ .value = Value.fromInterned(@enumFromInt(ref.data)) },
77	.inst => .{ .inst = @enumFromInt(ref.data) },
78	};
79	}
80
81	pub fn toOptional(ref: ?Ref) Optional {
82	return if (ref) \|r\| @enumFromInt(@as(u32, @bitCast(r))) else .none;
83	}
84
85	pub const Optional = enum(u32) {
86	none = std.math.maxInt(u32),
87	_,
88
89	pub fn unwrap(opt: Optional) ?Ref {
90	return if (opt != .none) @bitCast(@intFromEnum(opt)) else null;
91	}
92	};
93	};
94
95	/// reference to another instruction in this air
96	/// note that these cannot point to code in other functions, such makes no sense
97	pub const Index = enum(u32) {
98	main_block,
99	_,
100
101	pub fn toRef(index: Index) Ref {
102	return .{
103	.kind = .inst,
104	.data = @intCast(@intFromEnum(index)),
105	};
106	}
107
108	pub fn toOptional(index: Index) Optional {
109	return @enumFromInt(@intFromEnum(index));
110	}
111
112	pub const Optional = enum(u32) {
113	none = std.math.maxInt(u32),
114	_,
115
116	pub fn unwrap(oi: Optional) ?Index {
117	return if (oi != .none) @enumFromInt(@intFromEnum(oi)) else null;
118	}
119	};
120	};
121	};
122
123	pub const Flags = packed struct {
124	uses: Uses = .none,
125
126	pub const Uses = enum(u2) {
127	none,
128	single,
129	multi,
130
131	pub fn increment(uses: Uses) Uses {
132	return switch (uses) {
133	.none => .single,
134	.single, .multi => .multi,
135	};
136	}
137	};
138	};
139
140	pub const IndexList = struct {
141	start: u32,
142	len: u32,
143
144	pub fn slice(self: IndexList, air: *const Air) []const Inst.Index {
145	return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
146	}
147	};
148
149	pub const RefList = struct {
150	start: u32,
151	len: u32,
152
153	pub fn slice(self: RefList, air: *const Air) []const Inst.Ref {
154	return @ptrCast(air.extra.list.items[self.start..][0..self.len]);
155	}
156	};
157
158	pub const Break = struct {
159	block_ref: Inst.Index,
160	value: Inst.Ref,
161	};
162
163	pub const Bin = struct {
164	lhs: Inst.Ref,
165	rhs: Inst.Ref,
166	};
167
168	pub const Un = struct {
169	operand: Inst.Ref,
170	unused: Unused = .zero,
171	};
172
173	pub const Field = struct {
174	base: Inst.Ref,
175	field: InternPool.NullTerminatedString,
176	};
177
178	pub const Call = struct {
179	base: Inst.Ref,
180	call_data: ExtraIndex(Data),
181
182	pub const Data = struct {
183	args: RefList,
184	};
185	};
186
187	pub const Arg = struct {
188	type: Value,
189	arg_number: u32,
190	};
191
192	pub const If = struct {
193	cond: Inst.Ref,
194	data: ExtraIndex(Data),
195
196	pub const Data = struct {
197	yes: Inst.Ref,
198	no: Inst.Ref,
199	union_ty: Value,
200	};
201	};
202
203	pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
204	defer air.* = undefined;
205	air.insts.deinit(gpa);
206	air.extra.deinit(gpa);
207	}
208
209	pub fn instTag(air: *const Air, index: Inst.Index) Inst.Tag {
210	return air.insts.items(.tag)[@intFromEnum(index)];
211	}
212
213	pub fn instFlags(air: *const Air, index: Inst.Index) Flags {
214	return air.insts.items(.flags)[@intFromEnum(index)];
215	}
216
217	pub fn instEncodedData(air: *const Air, index: Inst.Index) EncodedData {
218	return air.insts.items(.data)[@intFromEnum(index)];
219	}
220
221	pub fn instData(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
222	return air.instEncodedData(index).decode(tag, air.instTag(index));
223	}
224
225	pub fn instPayload(air: *const Air, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
226	return air.instData(tag, index).extra_index.get(air);
227	}
228
229	pub fn typeOf(air: const Air, ref: Inst.Ref, ip: InternPool) InternPool.Index {
230	return switch (ref.unwrap()) {
231	.value => \|v\| v.type(ip).toIntern(),
232	.inst => \|i\| air.typeOfIndex(i, ip),
233	};
234	}
235
236	pub fn typeOfIndex(air: const Air, index: Inst.Index, ip: InternPool) InternPool.Index {
237	return switch (air.instTag(index)) {
238	.js_global_ref => .opaque_type,
239
240	.block => {
241	const items = air.instData(.block, index).slice(air);
242	return air.typeOfIndex(items[items.len - 1], ip);
243	},
244
245	inline .add, .sub, .mul, .div => \|t\| air.typeOf(air.instData(t, index).lhs, ip),
246
247	.lt, .gt, .eq => .bool_type,
248
249	.arg => air.instData(.arg, index).type.toIntern(),
250
251	.negate => air.typeOf(air.instData(.negate, index).operand, ip),
252
253	.@"break" => .noreturn_type,
254
255	// todo:
256	.field => .opaque_type,
257
258	.call => {
259	const base_type = air.typeOf(air.instData(.call, index).base, ip);
260	if (base_type == .opaque_type) return .opaque_type;
261	return base_type.toKey(ip).type_function.return_type;
262	},
263
264	.@"if" => air.instData(.@"if", index).data.get(air).union_ty.toIntern(),
265	};
266	}
267
268	pub fn paramCount(air: Air) u32 {
269	for (air.insts.items(.tag)[1..], 0..) \|tag, i\| {
270	if (tag != .arg) return @intCast(i);
271	}
272	return @intCast(air.insts.len);
273	}
274
275	pub fn dumpToWriter(air: Air, ip: *InternPool, writer: anytype) !void {
276	try writer.print("{}", .{Dump{ .air = air, .ip = ip }});
277	}
278
279	pub const Dump = @import("Air.Dump.zig");
280
281	const Air = @This();
282	const std = @import("std");
283	const mem = std.mem;
284	const assert = std.debug.assert;
285
286	const Teg = @import("Teg.zig");
287	const InternPool = @import("InternPool.zig");
288	const dod = @import("dod.zig");
289
290	const Unused = enum(u32) { zero = 0 };
291
292	const Value = @import("Value.zig");

src/compiler/Ast.Dump.zig created+300

...	@@ -0,0 +1,300 @@
1	//! A flat array is not very useful for debugging, so this structure implements
2	//! a debug formatter for Ast.
3	ast: Ast,
4
5	const State = struct {
6	ast: *const Ast,
7	indent: usize = 0,
8	node_tags: []const Node.Tag,
9	node_data: []const Node.Data,
10	node_tokens: []const TokenIndex,
11	tty: std.io.tty.Config,
12
13	fn newline(state: State, writer: anytype) !void {
14	try writer.writeByte('\n');
15	if (state.indent > 0)
16	try writer.writeByteNTimes(' ', state.indent * 2);
17	}
18	};
19
20	pub fn format(
21	dump: Dump,
22	comptime unused_fmt: []const u8,
23	_: std.fmt.FormatOptions,
24	writer: anytype,
25	) !void {
26	comptime assert(unused_fmt.len == 0);
27	const ast = dump.ast;
28	var state = State{
29	.ast = &ast,
30	.node_tags = ast.nodes.items(.tag),
31	.node_data = ast.nodes.items(.data),
32	.node_tokens = ast.nodes.items(.main_token),
33	.tty = detectTTYFromWriter(writer),
34	};
35	try dumpNode(writer, &state, 0);
36	try writer.writeAll("\n");
37	assert(state.indent == 0);
38	}
39
40	fn dumpNode(writer: anytype, state: *State, node: NodeIndex) @TypeOf(writer).Error!void {
41	const tag = state.node_tags[node];
42	const data = state.node_data[node];
43	const data_tag = Node.data_tags[@intFromEnum(tag)];
44	const main_token = state.node_tokens[node];
45	const tty = state.tty;
46
47	try state.tty.setColor(writer, .bright_black);
48	try writer.print("{d}", .{node});
49	try state.tty.setColor(writer, .reset);
50	try writer.writeAll(" ");
51
52	try tty.setColor(writer, .cyan);
53	try writer.print("{s}", .{@tagName(tag)});
54	try tty.setColor(writer, .reset);
55
56	switch (tag) {
57	.builtin_call => {
58	try tty.setColor(writer, .red);
59	try writer.print(" {s}", .{state.ast.tokenSlice(main_token)});
60	try tty.setColor(writer, .reset);
61	},
62	else => {},
63	}
64
65	switch (data_tag) {
66	.none => {
67	switch (tag) {
68	.id => try tty.setColor(writer, .red),
69	.string => try tty.setColor(writer, .green),
70	.number_int, .number_float => try tty.setColor(writer, .yellow),
71	else => {},
72	}
73	switch (tag) {
74	.id,
75	.string,
76	.number_int,
77	.number_float,
78	=> try writer.print(" {s}", .{state.ast.tokenSlice(main_token)}),
79
80	else => unreachable,
81	}
82	try tty.setColor(writer, .reset);
83	},
84	.fn_decl => {
85	try writer.writeAll(" ");
86	const fn_decl = state.ast.fullFnDecl(node);
87	if (fn_decl.public) {
88	try writer.writeAll("[pub] ");
89	}
90	try tty.setColor(writer, .magenta);
91	try writer.writeAll(fn_decl.id);
92	try tty.setColor(writer, .reset);
93	try writer.writeAll(" {");
94	{
95	state.indent += 1;
96	defer state.indent -= 1;
97	try state.newline(writer);
98	try writer.writeAll("params:");
99	try dumpNodeList(writer, state, fn_decl.params, '[', ']');
100	if (fn_decl.return_type) \|return_type\| {
101	try state.newline(writer);
102	try writer.writeAll("return_type: ");
103	try dumpNode(writer, state, return_type);
104	}
105	try state.newline(writer);
106	try writer.writeAll("body: ");
107	try dumpNode(writer, state, fn_decl.body);
108	}
109	try state.newline(writer);
110	try writer.writeAll("}");
111	},
112	.list => {
113	const list = data.list.slice(state.ast);
114	switch (tag) {
115	.root,
116	.block_void,
117	.block_implicit_return,
118	=> try dumpNodeList(writer, state, list, '{', '}'),
119	.builtin_call,
120	=> try dumpNodeList(writer, state, list, '(', ')'),
121	else => unreachable,
122	}
123	},
124	.var_decl => {
125	try writer.writeAll(" ");
126	const var_decl = state.ast.fullVarDecl(node);
127	if (var_decl.public) {
128	try writer.writeAll("[pub] ");
129	}
130	try writer.writeAll(switch (var_decl.is_const) {
131	true => "constant",
132	false => "mutable",
133	});
134	try tty.setColor(writer, .magenta);
135	try writer.print(" {s}", .{var_decl.id});
136	try tty.setColor(writer, .reset);
137	var indent = false;
138	if (var_decl.type) \|type_node\| {
139	state.indent += 1;
140	indent = true;
141	try state.newline(writer);
142	try writer.writeAll(": ");
143	try dumpNode(writer, state, type_node);
144	try writer.writeAll(", ");
145	}
146	try writer.writeAll(" = ");
147	try dumpNode(writer, state, var_decl.init);
148	if (indent) {
149	state.indent -= 1;
150	}
151	},
152	.param_decl => {
153	try writer.writeAll(" ");
154	const param_decl = state.ast.fullParamDecl(node);
155	try tty.setColor(writer, .magenta);
156	try writer.writeAll(param_decl.id);
157	try tty.setColor(writer, .reset);
158	try writer.writeAll(": ");
159	try dumpNode(writer, state, param_decl.type);
160	},
161	.bin => {
162	try writer.writeAll(" (");
163	{
164	state.indent += 1;
165	defer state.indent -= 1;
166	try state.newline(writer);
167	try dumpNode(writer, state, data.bin.lhs);
168	try state.newline(writer);
169	try dumpNode(writer, state, data.bin.rhs);
170	}
171	try state.newline(writer);
172	try writer.writeAll(")");
173	},
174	.un => {
175	try writer.writeAll(" (");
176	{
177	state.indent += 1;
178	defer state.indent -= 1;
179	try state.newline(writer);
180	try dumpNode(writer, state, data.un.expr);
181	}
182	try state.newline(writer);
183	try writer.writeAll(")");
184	},
185	.label => {
186	const label_text = state.ast.tokenSlice(main_token);
187	try writer.print(" {s}:", .{label_text});
188
189	try writer.writeAll(" (");
190	{
191	state.indent += 1;
192	defer state.indent -= 1;
193	try state.newline(writer);
194	try dumpNode(writer, state, data.label.expr);
195	}
196	try state.newline(writer);
197	try writer.writeAll(")");
198	},
199	.field => {
200	try writer.writeAll(" (");
201	{
202	state.indent += 1;
203	defer state.indent -= 1;
204	try state.newline(writer);
205	try dumpNode(writer, state, data.field.base);
206	}
207	try state.newline(writer);
208	try writer.writeAll(")");
209	},
210	.@"return" => {
211	if (data.@"return".expr.unwrap()) \|expr\| {
212	try writer.writeAll(" (");
213	try dumpNode(writer, state, expr);
214	try writer.writeAll(")");
215	}
216	},
217	.@"break" => {
218	const label = data.@"break".label;
219	const value = data.@"break".value;
220	if (label != .none or value != .none) {
221	// try writer.writeAll("(");
222	// try writer.writeAll(")");
223	if (label.unwrap()) \|l\| {
224	const label_text = state.ast.tokenSlice(l);
225	try writer.print(" :{s}", .{label_text});
226	}
227	if (value.unwrap()) \|expr\| {
228	try writer.writeAll(" (");
229	try dumpNode(writer, state, expr);
230	try writer.writeAll(")");
231	}
232	}
233	},
234	.call => {
235	const base = data.call.base;
236	const extra = state.ast.extraData(data.call.extra, Call);
237	const arg_list = extra.args.slice(state.ast);
238
239	try writer.writeAll("(");
240	try writer.writeAll("base: ");
241	try dumpNode(writer, state, base);
242	try writer.writeAll(", ");
243	try writer.writeAll("arg:");
244	try dumpNodeList(writer, state, arg_list, '[', ']');
245	try writer.writeAll(")");
246	},
247	.@"if" => {
248	const branch = data.@"if";
249	const if_data = state.ast.extraData(branch.extra, Ast.If);
250
251	try writer.writeAll(" (");
252	try dumpNode(writer, state, branch.cond);
253	try writer.writeAll(") {");
254	{
255	state.indent += 1;
256	defer state.indent -= 1;
257	try state.newline(writer);
258	try writer.writeAll("yes: ");
259	try dumpNode(writer, state, if_data.yes);
260	try state.newline(writer);
261	if (if_data.no.unwrap()) \|no\| {
262	try writer.writeAll("no: ");
263	try dumpNode(writer, state, no);
264	} else {
265	try writer.writeAll("no: [none]");
266	}
267	}
268	try state.newline(writer);
269	try writer.writeAll("}");
270	},
271	}
272	}
273
274	fn dumpNodeList(writer: anytype, state: *State, list: []const NodeIndex, l: u8, r: u8) !void {
275	try writer.print(" {c}", .{l});
276	{
277	state.indent += 1;
278	defer state.indent -= 1;
279
280	for (list) \|node\| {
281	try state.newline(writer);
282	try dumpNode(writer, state, node);
283	}
284	}
285	try state.newline(writer);
286	try writer.print("{c}", .{r});
287	}
288
289	const Dump = @This();
290
291	const std = @import("std");
292	const assert = std.debug.assert;
293
294	const Ast = @import("Ast.zig");
295	const NodeIndex = Ast.NodeIndex;
296	const Node = Ast.Node;
297	const TokenIndex = Ast.TokenIndex;
298	const Call = Ast.Call;
299
300	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Ast.zig created+701

...	@@ -0,0 +1,701 @@
1	//! The Abstract Syntax Tree preserves most details of the source code,
2	//! as it is used for formatting. Before semantic analysis, this
3	//! structure is normalized into `Teg`, which removes constructs like
4	//! grouping, unneded let bindings, and other transforms.
5	//!
6	//! `ast.nodes` contains nodes stored as three parts:
7	//! - tag: the type of the node
8	//! - main_token: index to it's primary token, depends on the tag
9	//! - data: two u32 values, depends on the tag
10	//!
11	//! Many nodes encode all their metadata with just `node.data`, using
12	//! facts about the language grammar to be able to infer data without
13	//! storing it.
14	//!
15	//! For example, `var_decl` has the following encoding:
16	//! - `main_token` is either a `kw_var` or `kw_let`
17	//! - the token after `main_token` is always the identifier
18	//! - `data.var_decl` contains `NodeIndex`es pointing at the type
19	//! and the initializer, which are both expressions.
20	//! - If the keyword before `main_token` is `kw_pub` it is a public variable.
21	//!
22	//! To make decoding these easier, many nodes contain a "full" variant,
23	//! such as `full.VarDecl`, which can be aquired with `ast.fullVarDecl(id)`.
24	//!
25	//! For nodes that must store more than 64 bits of payload data, an "extra"
26	//! struct is stored out-of-line in the `extra` field. These can be read
27	//! using `ast.extraData(extra_index, T)`. Again, it is recommended to use
28	//! the `full` structures with their convenience functions.
29	//!
30	//! Documentation comments are not stored anywhere, but you can find them
31	//! by iterating backwards from a declaration and seeing if a `.doc_comment`
32	//! is present. This is provided by `ast.declDocComment(id)`.
33	//!
34	//! The `Ast` owns token and node data, but not the source code.
35
36	/// Externally owned source code.
37	utf8_source: []const u8,
38
39	tokens: Token.List.Slice,
40	nodes: Node.List.Slice,
41	extra: []const u32,
42
43	pub fn parse(gpa: std.mem.Allocator, source: []const u8) !Ast {
44	const Parse = @import("Parse.zig");
45
46	var todo: ByteOffset = 0;
47	var tokens = try Tokenizer.tokenize(source, gpa, &todo);
48	errdefer tokens.deinit(gpa);
49
50	var parser = Parse.init(gpa, source, tokens);
51	errdefer parser.deinit();
52
53	try parser.nodes.ensureUnusedCapacity(gpa, @divFloor(tokens.len, 4));
54
55	try parser.parseRoot();
56
57	return try parser.finalize(tokens);
58	}
59
60	pub const Location = struct {
61	line: usize,
62	column: usize,
63	line_start: usize,
64	line_end: usize,
65	};
66
67	pub const Span = struct {
68	start: u32,
69	end: u32,
70	main: u32,
71	};
72
73	// This can be removed when the following issue is implemented:
74	// See: https://github.com/ziglang/zig/issues/104
75	// See: https://github.com/ziglang/zig/issues/3806
76	pub const OptionalNodeIndex = enum(u32) {
77	none = 0,
78	_,
79
80	pub fn init(idx: ?NodeIndex) OptionalNodeIndex {
81	assert(idx != 0);
82	return @enumFromInt(idx orelse return .none);
83	}
84
85	pub fn unwrap(idx: OptionalNodeIndex) ?NodeIndex {
86	return switch (idx) {
87	.none => null,
88	_ => @intFromEnum(idx),
89	};
90	}
91	};
92
93	pub const OptionalTokenIndex = enum(u32) {
94	none = 0,
95	_,
96
97	pub fn init(idx: ?TokenIndex) OptionalTokenIndex {
98	assert(idx != 0);
99	return @enumFromInt(idx orelse return .none);
100	}
101
102	pub fn unwrap(idx: OptionalTokenIndex) ?TokenIndex {
103	return switch (idx) {
104	.none => null,
105	_ => @intFromEnum(idx),
106	};
107	}
108	};
109
110	pub const OptionalExtraIndex = enum(u32) {
111	none = std.math.maxInt(u32),
112	_,
113
114	pub fn init(idx: ?ExtraIndex) OptionalExtraIndex {
115	assert(idx != std.math.maxInt(u32));
116	return @enumFromInt(idx orelse return .none);
117	}
118
119	pub fn unwrap(idx: OptionalExtraIndex) ?ExtraIndex {
120	return switch (idx) {
121	.none => null,
122	_ => @intFromEnum(idx),
123	};
124	}
125	};
126
127	pub const Node = struct {
128	tag: Tag,
129	main_token: TokenIndex,
130	data: Data,
131
132	pub const List = std.MultiArrayList(Node);
133
134	pub const Tag = enum {
135	/// Read with `ast.rootNodes()`
136	/// data is .list (containing stmt slice)
137	/// TODO: rename to `module`
138	root,
139	/// main_token is the {
140	/// data is .list (containing stmt slice)
141	///
142	/// This guarantees that the block either empty, or the last statement
143	/// ends with a semicolon.
144	block_void,
145	/// main_token is the {
146	/// data is .list (containing stmt slice)
147	///
148	/// This guarantees that the block is not empty and that the last
149	/// statement is an expression statement without a trailing semicolon.
150	block_implicit_return,
151	/// Variable or constant declaration
152	/// `var x: type = init;`
153	/// `let y = x;`
154	/// main_token is the keyword
155	/// data is .var_decl
156	var_decl,
157	/// Function declaration
158	/// `fn foo(x: i32) -> i32 { rhs }`
159	/// Params and return type are stored in extra data
160	fn_decl,
161	/// 'name: type' declaration
162	/// Main token is the identifier
163	param_decl,
164	/// An identifier. main_token is the identifier.
165	id,
166	/// Main token is the data. May not fit in i32 range,
167	/// but will contain no dots or e's.
168	number_int,
169	/// Main token is data. Will contain '.' and/or 'e'.
170	number_float,
171	/// A string literal
172	string,
173	/// A multi-line string literal
174	/// data is .token_list of the tokens, which is guaranteed to be contiguous
175	multiline_string,
176	/// Call to a function
177	/// `foo(x, y);`
178	/// main_token is the `(`
179	/// data is .call
180	call,
181	/// Call to a builtin function
182	/// `@raw("hello");`
183	/// data is .list (containing arg slice)
184	builtin_call,
185	/// `if cond { ... } else ...`
186	/// main_token is the `if`
187	/// data is .@"if"
188	@"if",
189	/// `if let id = ... { ... } else ...`
190	/// main_token is the `if`
191	/// data is .@"if"
192	// if_let_nonvoid,
193	/// `id: ...`
194	/// data is un
195	label,
196	/// `expr.field`
197	/// main_token is the `.`
198	/// data is .field
199	field,
200	/// `lhs[rhs]`
201	/// main_token is the `[`
202	/// data is .bin
203	index,
204	/// `return expr;`
205	/// main_token is the `return`
206	/// data is .@"return"
207	@"return",
208	/// `break :brk expr;`
209	/// main_token is the `break`
210	/// data is .@"break"
211	@"break",
212	/// `(expr)`
213	/// main_token is the `(`
214	/// data is .un
215	group,
216	/// `lhs + rhs`. main_token is the operator.
217	add,
218	/// `lhs - rhs`. main_token is the operator.
219	sub,
220	/// `lhs * rhs`. main_token is the operator.
221	mul,
222	/// `lhs / rhs`. main_token is the operator.
223	div,
224	/// `lhs % rhs`. main_token is the operator.
225	rem,
226	/// `lhs ** rhs`. main_token is the operator.
227	pow,
228	/// `lhs && rhs`. main_token is the operator.
229	@"and",
230	/// `lhs \|\| rhs`. main_token is the operator.
231	@"or",
232	/// `lhs & rhs`. main_token is the operator.
233	bit_and,
234	/// `lhs \| rhs`. main_token is the operator.
235	bit_or,
236	/// `lhs ^ rhs`. main_token is the operator.
237	bit_xor,
238	/// `lhs << rhs`. main_token is the operator.
239	shl,
240	/// `lhs >> rhs`. main_token is the operator.
241	shr,
242	/// `lhs == rhs`. main_token is the operator.
243	eq,
244	/// `lhs != rhs`. main_token is the operator.
245	ne,
246	/// `lhs < rhs`. main_token is the operator.
247	lt,
248	/// `lhs <= rhs`. main_token is the operator.
249	le,
250	/// `lhs > rhs`. main_token is the operator.
251	gt,
252	/// `lhs >= rhs`. main_token is the operator.
253	ge,
254	/// `lhs = rhs`. main_token is the operator.
255	assign,
256	/// `lhs += rhs`. main_token is the operator.
257	add_assign,
258	/// `lhs -= rhs`. main_token is the operator.
259	sub_assign,
260	/// `lhs *= rhs`. main_token is the operator.
261	mul_assign,
262	/// `lhs /= rhs`. main_token is the operator.
263	div_assign,
264	/// `lhs %= rhs`. main_token is the operator.
265	rem_assign,
266	/// `lhs **= rhs`. main_token is the operator.
267	pow_assign,
268	/// `lhs &&= rhs`. main_token is the operator.
269	and_assign,
270	/// `lhs \|\|= rhs`. main_token is the operator.
271	or_assign,
272	/// `lhs &= rhs`. main_token is the operator.
273	bit_and_assign,
274	/// `lhs \|= rhs`. main_token is the operator.
275	bit_or_assign,
276	/// `lhs ^= rhs`. main_token is the operator.
277	bit_xor_assign,
278	/// `lhs <<= rhs`. main_token is the operator.
279	shl_assign,
280	/// `lhs >>= rhs`. main_token is the operator.
281	shr_assign,
282	/// `!expr`. main_token is the operator.
283	bool_not,
284	/// `-expr`. main_token is the operator.
285	negate,
286	/// `~expr`. main_token is the operator.
287	bit_not,
288	/// `comptime expr`. main_token is the operator.
289	@"comptime",
290	/// `pure expr`. main_token is the operator.
291	// pure,
292	/// `assert expr`. main_token is the operator.
293	// assert,
294
295	pub fn isAssignBin(op: Tag) bool {
296	return switch (op) {
297	.assign,
298	.add_assign,
299	.sub_assign,
300	.mul_assign,
301	.div_assign,
302	.rem_assign,
303	.pow_assign,
304	.and_assign,
305	.or_assign,
306	.bit_and_assign,
307	.bit_or_assign,
308	.bit_xor_assign,
309	.shl_assign,
310	.shr_assign,
311	=> true,
312	.add,
313	.sub,
314	.mul,
315	.div,
316	.rem,
317	.pow,
318	.@"and",
319	.@"or",
320	.bit_and,
321	.bit_or,
322	.bit_xor,
323	.shl,
324	.shr,
325	.eq,
326	.ne,
327	.lt,
328	.le,
329	.gt,
330	.ge,
331	=> false,
332	else => unreachable, // not a binary operator
333	};
334	}
335
336	pub fn isBasic(op: Tag) bool {
337	return switch (op) {
338	.id,
339	.string,
340	.number_int,
341	.number_float,
342	.builtin_call,
343	=> true,
344	else => false,
345	};
346	}
347	};
348
349	pub const data_tags = std.enums.directEnumArray(Tag, Data.FieldEnum, 0, .{
350	.root = .list,
351	.block_void = .list,
352	.block_implicit_return = .list,
353	.var_decl = .var_decl,
354	.fn_decl = .fn_decl,
355	.param_decl = .param_decl,
356	.id = .none,
357	.number_int = .none,
358	.number_float = .none,
359	.string = .none,
360	.multiline_string = .list,
361	.call = .call,
362	.builtin_call = .list,
363	.@"if" = .@"if",
364	.field = .field,
365	.index = .bin,
366	.@"return" = .@"return",
367	.@"break" = .@"break",
368	.group = .un,
369	.add = .bin,
370	.sub = .bin,
371	.mul = .bin,
372	.div = .bin,
373	.rem = .bin,
374	.pow = .bin,
375	.@"and" = .bin,
376	.@"or" = .bin,
377	.bit_and = .bin,
378	.bit_or = .bin,
379	.bit_xor = .bin,
380	.shl = .bin,
381	.shr = .bin,
382	.eq = .bin,
383	.ne = .bin,
384	.lt = .bin,
385	.le = .bin,
386	.gt = .bin,
387	.ge = .bin,
388	.assign = .bin,
389	.add_assign = .bin,
390	.sub_assign = .bin,
391	.mul_assign = .bin,
392	.div_assign = .bin,
393	.rem_assign = .bin,
394	.pow_assign = .bin,
395	.and_assign = .bin,
396	.or_assign = .bin,
397	.bit_and_assign = .bin,
398	.bit_or_assign = .bin,
399	.bit_xor_assign = .bin,
400	.shl_assign = .bin,
401	.shr_assign = .bin,
402	.bool_not = .un,
403	.negate = .un,
404	.bit_not = .un,
405	.@"comptime" = .un,
406	.label = .label,
407	});
408
409	pub const Data = union {
410	none: struct { Unused = .zero, Unused = .zero },
411	fn_decl: struct {
412	// A `FnDecl`
413	extra: ExtraIndex,
414	body: NodeIndex,
415	},
416	/// either is a node list or a token list, depending on the node's tag
417	list: SubList,
418	var_decl: struct {
419	type: OptionalNodeIndex,
420	init: NodeIndex,
421	},
422	param_decl: struct {
423	type: NodeIndex,
424	unused: Unused = .zero,
425	},
426	bin: struct {
427	lhs: NodeIndex,
428	rhs: NodeIndex,
429	},
430	field: struct {
431	base: NodeIndex,
432	field_token: TokenIndex,
433	},
434	un: struct {
435	expr: NodeIndex,
436	unused: Unused = .zero,
437	},
438	label: struct {
439	expr: NodeIndex,
440	unused: Unused = .zero,
441	},
442	@"return": struct {
443	expr: OptionalNodeIndex,
444	unused: Unused = .zero,
445	},
446	@"break": struct {
447	label: OptionalTokenIndex,
448	value: OptionalNodeIndex,
449	},
450	call: struct {
451	base: NodeIndex,
452	/// A `Call`
453	extra: ExtraIndex,
454	},
455	@"if": struct {
456	cond: NodeIndex,
457	/// A `If`
458	extra: ExtraIndex,
459	},
460
461	const FieldEnum = std.meta.FieldEnum(Data);
462	};
463	};
464
465	pub const SubList = struct {
466	/// Either NodeIndex or TokenIndex, depending on where this came from.
467	start: u32,
468	len: u32,
469
470	pub fn slice(self: SubList, ast: *const Ast) []const u32 {
471	return ast.extra[self.start..][0..self.len];
472	}
473	};
474
475	pub const FnDecl = struct {
476	params: SubList,
477	return_type: OptionalNodeIndex,
478	};
479
480	pub const Call = struct {
481	args: SubList,
482	};
483
484	pub const If = struct {
485	/// either .block_void or .block_implicit_return
486	yes: NodeIndex,
487	/// if it exists, it is either
488	/// .block_void, .block_implicit_return, or .@"if"
489	no: OptionalNodeIndex,
490	};
491
492	pub const full = struct {
493	pub const FnDecl = struct {
494	start_token: TokenIndex,
495	public: bool,
496	id: []const u8,
497	params: []const NodeIndex,
498	return_type: ?NodeIndex,
499	body: NodeIndex,
500	};
501
502	pub const VarDecl = struct {
503	start_token: TokenIndex,
504	public: bool,
505	is_const: bool,
506	id: []const u8,
507	type: ?NodeIndex,
508	init: NodeIndex,
509	};
510
511	pub const ParamDecl = struct {
512	start_token: TokenIndex,
513	id: []const u8,
514	type: NodeIndex,
515	};
516	};
517
518	pub fn deinit(ast: *Ast, gpa: std.mem.Allocator) void {
519	ast.tokens.deinit(gpa);
520	ast.nodes.deinit(gpa);
521	gpa.free(ast.extra);
522	ast.* = undefined;
523	}
524
525	pub fn nodeTag(ast: Ast, node: NodeIndex) Node.Tag {
526	return ast.nodes.items(.tag)[node];
527	}
528
529	pub fn nodeData(ast: Ast, node: NodeIndex) Node.Data {
530	return ast.nodes.items(.data)[node];
531	}
532
533	pub fn nodeToken(ast: Ast, node: NodeIndex) TokenIndex {
534	return ast.nodes.items(.main_token)[node];
535	}
536
537	pub fn fullFnDecl(ast: Ast, node: NodeIndex) full.FnDecl {
538	const tok_tags = ast.tokens.items(.tag);
539
540	const main_token = ast.nodes.items(.main_token)[node];
541	const public = tok_tags[main_token -\| 1] == .kw_pub;
542	assert(tok_tags[main_token] == .kw_fn);
543	assert(tok_tags[main_token + 1] == .id);
544	const id = ast.tokenSlice(main_token + 1);
545
546	const data = ast.nodes.items(.data)[node].fn_decl;
547	const extra = ast.extraData(data.extra, FnDecl);
548
549	return .{
550	.start_token = if (public) main_token - 1 else main_token,
551	.public = public,
552	.id = id,
553	.body = data.body,
554	.params = extra.params.slice(&ast),
555	.return_type = extra.return_type.unwrap(),
556	};
557	}
558
559	pub fn fullVarDecl(ast: Ast, node: NodeIndex) full.VarDecl {
560	const tok_tags = ast.tokens.items(.tag);
561
562	const main_token = ast.nodes.items(.main_token)[node];
563	const public = tok_tags[main_token -\| 1] == .kw_pub;
564	const is_const = switch (tok_tags[main_token]) {
565	.kw_let => true,
566	.kw_var => false,
567	else => \|tag\| {
568	if (std.debug.runtime_safety) {
569	std.debug.panic("Expected var decl, got {s}", .{@tagName(tag)});
570	}
571	unreachable;
572	},
573	};
574	assert(tok_tags[main_token + 1] == .id);
575	const id = ast.tokenSlice(main_token + 1);
576	const data = ast.nodes.items(.data)[node].var_decl;
577	return .{
578	.start_token = if (public) main_token - 1 else main_token,
579	.public = public,
580	.is_const = is_const,
581	.id = id,
582	.type = data.type.unwrap(),
583	.init = data.init,
584	};
585	}
586
587	pub fn fullParamDecl(ast: Ast, node: NodeIndex) full.ParamDecl {
588	const tok_tags = ast.tokens.items(.tag);
589
590	const main_token = ast.nodes.items(.main_token)[node];
591	assert(tok_tags[main_token] == .id);
592	const id = ast.tokenSlice(main_token);
593	const data = ast.nodes.items(.data)[node].param_decl;
594	return .{
595	.start_token = main_token,
596	.id = id,
597	.type = data.type,
598	};
599	}
600
601	pub fn nodeStatements(ast: *const Ast, node: NodeIndex) []const NodeIndex {
602	const data = ast.nodes.items(.data)[node].list;
603	return data.slice(ast);
604	}
605
606	pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location {
607	var loc = Location{
608	.line = 0,
609	.column = 0,
610	.line_start = start_offset,
611	.line_end = self.source.len,
612	};
613	const token_start = self.tokens.items(.start)[token_index];
614
615	// Scan to by line until we go past the token start
616	while (std.mem.indexOfScalarPos(u8, self.source, loc.line_start, '\n')) \|i\| {
617	if (i >= token_start) {
618	break; // Went past
619	}
620	loc.line += 1;
621	loc.line_start = i + 1;
622	}
623
624	const offset = loc.line_start;
625	for (self.source[offset..], 0..) \|c, i\| {
626	if (i + offset == token_start) {
627	loc.line_end = i + offset;
628	while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') {
629	loc.line_end += 1;
630	}
631	return loc;
632	}
633	if (c == '\n') {
634	loc.line += 1;
635	loc.column = 0;
636	loc.line_start = i + 1;
637	} else {
638	loc.column += 1;
639	}
640	}
641	return loc;
642	}
643
644	pub fn tokenSlice(ast: Ast, token_index: TokenIndex) []const u8 {
645	return ast.tokens.get(token_index).slice(ast.utf8_source);
646	}
647
648	pub fn basicNodeSlice(ast: Ast, node: NodeIndex) []const u8 {
649	assert(ast.nodes.items(.tag)[node].isBasic());
650	const token = ast.nodes.items(.main_token)[node];
651	return ast.tokenSlice(token);
652	}
653
654	pub fn extraData(tree: Ast, index: usize, comptime T: type) T {
655	const fields = std.meta.fields(T);
656	var result: T = undefined;
657	var i: usize = 0;
658	inline for (fields) \|field\| {
659	@field(result, field.name) = switch (field.type) {
660	NodeIndex => tree.extra[index + i],
661	SubList => SubList{
662	.start = tree.extra[index + i],
663	.len = tree.extra[index + i + 1],
664	},
665	OptionalNodeIndex => @enumFromInt(tree.extra[index + i]),
666	OptionalTokenIndex => @enumFromInt(tree.extra[index + i]),
667	OptionalExtraIndex => @enumFromInt(tree.extra[index + i]),
668	else => @compileError("ExtraData type not supported: " ++ @typeName(field.type)),
669	};
670	i += switch (field.type) {
671	NodeIndex,
672	OptionalNodeIndex,
673	OptionalTokenIndex,
674	OptionalExtraIndex,
675	=> 1,
676	SubList => 2,
677	else => comptime unreachable,
678	};
679	}
680	return result;
681	}
682
683	pub fn dumpToWriter(ast: Ast, writer: anytype) !void {
684	try writer.print("{}", .{Dump{ .ast = ast }});
685	}
686
687	pub const Dump = @import("Ast.Dump.zig");
688
689	pub const Unused = enum(u32) { zero = 0 };
690	pub const TokenIndex = u32;
691	pub const ExtraIndex = u32;
692	pub const NodeIndex = u32;
693
694	const Ast = @This();
695	const std = @import("std");
696	const assert = std.debug.assert;
697
698	const Tokenizer = @import("Tokenizer.zig");
699	const Token = Tokenizer.Token;
700	const TokenList = Tokenizer.Token.List;
701	const ByteOffset = Tokenizer.ByteOffset;

src/compiler/Cache.zig created+9

...	@@ -0,0 +1,9 @@
1	const std = @import("std");
2
3	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
4	pub const bin_digest_len = 16;
5	pub const hex_digest_len = bin_digest_len * 2;
6	pub const BinDigest = [bin_digest_len]u8;
7	pub const HexDigest = [hex_digest_len]u8;
8
9	pub const HashHelper = std.Build.Cache.HashHelper;

src/compiler/Comp.zig created+395

...	@@ -0,0 +1,395 @@
1	//! Compilation
2
3	/// General purpose allocator
4	gpa: Allocator,
5	/// Stores all Type and Value objects.
6	intern_pool: InternPool = .{},
7	/// All files in the compilation
8	files: std.StringArrayHashMapUnmanaged(*File) = .{},
9
10	/// # of sema instructions in this compilation cycle
11	analyzed_instructions: usize = 0,
12
13	// TODO: consider computing this dynamicly per-link
14	current_compile_decls: std.ArrayListUnmanaged(Decl.Index) = .{},
15
16	pub fn init(comp: *Comp) !void {
17	try comp.intern_pool.init(comp.gpa);
18	}
19
20	pub fn deinit(comp: *Comp) void {
21	comp.intern_pool.deinit(comp.gpa);
22	for (comp.files.values()) \|file\| {
23	file.deinit(comp.gpa);
24	}
25	comp.files.deinit(comp.gpa);
26	comp.current_compile_decls.deinit(comp.gpa);
27	}
28
29	pub fn getFile(comp: Comp, absolute_file_path: []const u8) !File {
30	assert(std.fs.path.isAbsolute(absolute_file_path));
31	const gop = try comp.files.getOrPut(comp.gpa, absolute_file_path);
32	if (gop.found_existing) {
33	return gop.value_ptr.*;
34	}
35	const file = try comp.gpa.create(File);
36	file.* = .{
37	.file_path = absolute_file_path,
38	.path_digest = null,
39	.root_module = .none,
40	.status = .never_loader,
41	.source = null,
42	.ast = null,
43	.teg = null,
44	};
45	gop.value_ptr.* = file;
46	return file;
47	}
48
49	pub const Source = struct {
50	code: [:0]const u8,
51	stat: Stat,
52	};
53
54	pub const File = struct {
55	/// TODO: change this to a URL
56	file_path: []const u8,
57
58	root_module: Decl.Index.Optional,
59	status: enum {
60	never_loader,
61	/// A failure occurred during loading the file. Retrying might succeed.
62	retryable_failure,
63	/// Errors during parse
64	parse_failure,
65	/// Errors during transform
66	transform_failure,
67	/// Teg is valid
68	success,
69	},
70
71	source: ?Source,
72	ast: ?Ast,
73	teg: ?Teg,
74
75	path_digest: ?Cache.BinDigest,
76
77	pub fn getSource(file: *File, gpa: Allocator) !Source {
78	if (file.source) \|source\| {
79	return source;
80	}
81
82	// Keep track of inode, file size, mtime, hash so we can detect which files
83	// have been modified when an incremental update is requested.
84	var f = try std.fs.openFileAbsolute(file.file_path, .{});
85	defer f.close();
86
87	const stat = try f.stat();
88
89	if (stat.size > std.math.maxInt(u32))
90	return error.FileTooBig;
91
92	const source = try gpa.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
93	errdefer gpa.free(source);
94	const amt = try f.readAll(source);
95	if (amt != stat.size)
96	return error.UnexpectedEndOfFile;
97
98	// Here we do not modify stat fields because this function is the one
99	// used for error reporting. We need to keep the stat fields stale so that
100	// transformFile can know to regenerate Teg.
101
102	const loaded_source = Source{
103	.code = source,
104	.stat = .{
105	.size = stat.size,
106	.inode = stat.inode,
107	.mtime = stat.mtime,
108	},
109	};
110	file.source = loaded_source;
111	return loaded_source;
112	}
113
114	pub fn getAst(file: *File, gpa: Allocator) !Ast {
115	if (file.ast) \|ast\| {
116	return ast;
117	}
118
119	const ast = try Ast.parse(gpa, (try file.getSource(gpa)).code);
120	file.ast = ast;
121	return ast;
122	}
123
124	pub fn getTeg(file: *File, gpa: Allocator) !Teg {
125	if (file.teg) \|teg\| {
126	return teg;
127	}
128
129	const bin_digest = hash: {
130	var path_hash: Cache.HashHelper = .{};
131	path_hash.addBytes("0.1.0");
132	path_hash.addBytes(file.file_path);
133	var bin: Cache.BinDigest = undefined;
134	path_hash.hasher.final(&bin);
135	break :hash bin;
136	};
137	file.path_digest = bin_digest;
138
139	const teg = try Transform.transform(try file.getAst(gpa), gpa);
140	file.teg = teg;
141	return teg;
142	}
143
144	pub fn unloadSource(file: *File, gpa: Allocator) void {
145	if (file.source) \|source\| {
146	gpa.free(source.code);
147	file.source = null;
148	}
149	}
150
151	pub fn unloadAst(file: *File, gpa: Allocator) void {
152	if (file.ast) \|*ast\| {
153	ast.deinit(gpa);
154	file.ast = null;
155	}
156	}
157
158	pub fn unloadTeg(file: *File, gpa: Allocator) void {
159	if (file.teg) \|*teg\| {
160	teg.deinit(gpa);
161	file.teg = null;
162	}
163	}
164
165	pub fn deinit(file: *File, gpa: Allocator) void {
166	file.unloadTeg(gpa);
167	file.unloadAst(gpa);
168	file.unloadSource(gpa);
169	gpa.destroy(file);
170	}
171	};
172
173	pub const Decl = struct {
174	name: InternPool.NullTerminatedString,
175	val: Value,
176	/// The direct parent namespace of the Decl. In the case of the Decl
177	/// corresponding to a file, this is the namespace of the struct, since
178	/// there is no parent.
179	src_namespace: Namespace.Index,
180
181	/// The AST node index of this declaration.
182	/// Must be recomputed when the corresponding source file is modified.
183	// src_node: Ast.NodeIndex,
184	/// Line number corresponding to `src_node`. Stored separately so that source files
185	/// do not need to be loaded into memory in order to compute debug line numbers.
186	/// This value is absolute.
187	// src_line: u32,
188	/// Index of the ZIR `declaration` instruction from which this `Decl` was created.
189	/// For the root `Decl` of a `File` and legacy anonymous decls, this is `.none`.
190	teg_decl_index: InternPool.TrackedInst.Index.Optional,
191
192	/// Represents the "shallow" analysis status. For example, for decls that are functions,
193	/// the function type is analyzed with this set to `in_progress`, however, the semantic
194	/// analysis of the function body is performed with this value set to `success`. Functions
195	/// have their own analysis status field.
196	analysis: enum {
197	/// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
198	/// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
199	unreferenced,
200	/// Semantic analysis for this Decl is running right now.
201	/// This state detects dependency loops.
202	in_progress,
203	/// The file corresponding to this Decl had a parse error or ZIR error.
204	/// There will be a corresponding ErrorMsg in Zcu.failed_files.
205	file_failure,
206	/// This Decl might be OK but it depends on another one which did not
207	/// successfully complete semantic analysis.
208	dependency_failure,
209	/// Semantic analysis failure.
210	/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
211	sema_failure,
212	/// There will be a corresponding ErrorMsg in Zcu.failed_decls.
213	codegen_failure,
214	/// Sematic analysis and constant value codegen of this Decl has
215	/// succeeded. However, the Decl may be outdated due to an in-progress
216	/// update. Note that for a function, this does not mean codegen of the
217	/// function body succeded: that state is indicated by the function's
218	/// `analysis` field.
219	complete,
220	},
221
222	codegen: ?JsTree = null,
223
224	is_pub: bool,
225
226	/// What kind of a declaration is this.
227	kind: Kind,
228
229	pub fn deinit(decl: *Decl, gpa: Allocator) void {
230	defer decl.* = undefined;
231	if (decl.codegen) \|*cg\| cg.deinit(gpa);
232	}
233
234	pub const Kind = enum {
235	named,
236	};
237
238	pub const Index = InternPool.DeclIndex;
239	};
240
241	pub const Namespace = struct {
242	parent: Index.Optional,
243	file_scope: *File,
244	/// Direct children of the namespace.
245	/// Declaration order is preserved via entry order.
246	/// These are only declarations named directly by the AST; anonymous
247	/// declarations are not stored here.
248	decls: std.ArrayHashMapUnmanaged(Decl.Index, void, DeclContext, true) = .{},
249
250	pub const DeclContext = struct {
251	comp: *Comp,
252
253	pub fn hash(ctx: @This(), decl_index: Decl.Index) u32 {
254	const decl = ctx.comp.intern_pool.declPtr(decl_index);
255	return std.hash.uint32(@intFromEnum(decl.name));
256	}
257
258	pub fn eql(ctx: @This(), a_decl_index: Decl.Index, b_decl_index: Decl.Index, b_index: usize) bool {
259	_ = b_index;
260	const a_decl = ctx.comp.intern_pool.declPtr(a_decl_index);
261	const b_decl = ctx.comp.intern_pool.declPtr(b_decl_index);
262	return a_decl.name == b_decl.name;
263	}
264	};
265
266	pub const DeclNameAdapter = struct {
267	comp: *Comp,
268
269	pub fn hash(self: @This(), s: InternPool.NullTerminatedString) u32 {
270	_ = self;
271	return std.hash.uint32(@intFromEnum(s));
272	}
273
274	pub fn eql(self: @This(), a: InternPool.NullTerminatedString, b_decl_index: Decl.Index, b_index: usize) bool {
275	_ = b_index;
276	return a == self.comp.intern_pool.declPtr(b_decl_index).name;
277	}
278	};
279
280	pub fn lookupDirect(ns: Namespace, comp: Comp, name: InternPool.NullTerminatedString) ?Decl.Index {
281	return ns.decls.getKeyAdapted(name, Namespace.DeclNameAdapter{ .comp = comp });
282	}
283
284	pub const Index = InternPool.NamespaceIndex;
285	};
286
287	/// Index a module in a Teg.
288	/// When a File's Teg is loaded, we call this on the root module.
289	/// submodules are analyzed lazily.
290	pub fn indexTegModule(comp: Comp, file: File, module_index: Teg.Inst.Index) !Namespace.Index {
291	const teg = file.teg.?;
292
293	log.info("Index {} @ {d}", .{ std.fmt.fmtSliceHexLower(file.path_digest.?[0..8]), @intFromEnum(module_index) });
294
295	const ns = try comp.intern_pool.createNamespace(comp.gpa, .{
296	.parent = .none,
297	.file_scope = file,
298	});
299	const ns_ptr = comp.intern_pool.namespacePtr(ns);
300	errdefer comp.intern_pool.destroyNamespace(comp.gpa, ns);
301
302	for (teg.moduleDeclarations(module_index)) \|decl_index\| {
303	const name = try comp.intern_pool.getOrPutString(
304	comp.gpa,
305	teg.declName(decl_index),
306	.no_embedded_nulls,
307	);
308	const tracked_inst = try comp.intern_pool.trackInst(comp.gpa, file, decl_index);
309	const decl = try comp.intern_pool.createDecl(comp.gpa, .{
310	.name = name,
311	.val = undefined,
312	.src_namespace = ns,
313	.teg_decl_index = tracked_inst.toOptional(),
314	.analysis = .unreferenced,
315	.is_pub = false, // todo:
316	.kind = .named,
317	});
318	try ns_ptr.decls.putNoClobberContext(comp.gpa, decl, void{}, .{ .comp = comp });
319	}
320
321	return ns;
322	}
323
324	pub fn semaEntryPoint(comp: Comp, file: File) !void {
325	const teg = try file.getTeg(comp.gpa);
326
327	const ns = try comp.indexTegModule(file, .main_module_inst);
328	const decl_index = comp.intern_pool.namespacePtr(ns).lookupDirect(comp, .empty) orelse
329	@panic("no top level code?");
330	const decl = comp.intern_pool.declPtr(decl_index);
331
332	assert(comp.current_compile_decls.items.len == 0);
333
334	var arena = std.heap.ArenaAllocator.init(comp.gpa);
335	defer arena.deinit();
336
337	var sema: Sema = .{
338	.comp = comp,
339	.gpa = comp.gpa,
340	.code = teg,
341	.arena = arena.allocator(),
342	.ip = &comp.intern_pool,
343	.decl = decl,
344	.decl_index = decl_index,
345	.namespace_index = ns,
346	};
347	try sema.init();
348	defer sema.deinit();
349	_ = try sema.analyzeDeclaration();
350	try sema.finalize();
351	}
352
353	pub fn codeGenDecl(c: Comp, decl: Decl, air: *const Air) !void {
354	const t = tracer.trace(@src());
355	defer t.end();
356
357	std.debug.print("Start Codegen\n", .{});
358	defer std.debug.print("End Codegen\n", .{});
359
360	var arena_state = std.heap.ArenaAllocator.init(c.gpa);
361	defer arena_state.deinit();
362	const arena = arena_state.allocator();
363
364	var cg = try JsNodeGen.init(c.gpa, arena, &c.intern_pool, air);
365	errdefer cg.deinit();
366	try cg.build();
367	decl.codegen = try cg.finalize();
368	}
369
370	pub fn link(c: *Comp, writer: anytype) !void {
371	try JsLink.linkJsCode(c, c.current_compile_decls.items, writer);
372	}
373
374	const Comp = @This();
375
376	const std = @import("std");
377	const Allocator = std.mem.Allocator;
378	const assert = std.debug.assert;
379	const Stat = std.Build.Cache.File.Stat;
380	const log = std.log.scoped(.comp);
381
382	const InternPool = @import("InternPool.zig");
383
384	const Ast = @import("Ast.zig");
385	const Air = @import("Air.zig");
386	const Teg = @import("Teg.zig");
387	const Cache = @import("Cache.zig");
388	const Value = @import("Value.zig");
389	const Sema = @import("Sema.zig");
390	const Transform = @import("Transform.zig");
391	const JsNodeGen = @import("JsNodeGen.zig");
392	const JsTree = @import("JsTree.zig");
393	const JsLink = @import("JsLink.zig");
394
395	const tracer = @import("tracer");

src/compiler/InternPool.zig created+1018

...	@@ -0,0 +1,1018 @@
1	//! Large parts of this file are based on the Zig compiler's InternPool.
2	//! https://github.com/ziglang/zig/blob/0.12.0/src/InternPool.zig
3
4	/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
5	/// constructed lazily.
6	map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
7	items: std.MultiArrayList(Item) = .{},
8	extra: ExtraStore = .{},
9
10	/// In order to store references to strings in fewer bytes, we copy all
11	/// string bytes into here. String bytes can be null. It is up to whomever
12	/// is referencing the data here whether they want to store both index and length,
13	/// thus allowing null bytes, or store only index, and use null-termination. The
14	/// `string_bytes` array is agnostic to either usage.
15	string_bytes: std.ArrayListUnmanaged(u8) = .{},
16
17	/// Used for finding the index inside `string_bytes`.
18	string_table: std.HashMapUnmanaged(
19	u32,
20	void,
21	std.hash_map.StringIndexContext,
22	std.hash_map.default_max_load_percentage,
23	) = .{},
24
25	/// An index into `tracked_insts` gives a reference to a single ZIR instruction which
26	/// persists across incremental updates.
27	tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{},
28
29	/// Rather than allocating Decl objects with an Allocator, we instead allocate
30	/// them with this SegmentedList. This provides four advantages:
31	/// * Stable memory so that one thread can access a Decl object while another
32	/// thread allocates additional Decl objects from this list.
33	/// * It allows us to use u32 indexes to reference Decl objects rather than
34	/// pointers, saving memory in Type, Value, and dependency sets.
35	/// * Using integers to reference Decl objects rather than pointers makes
36	/// serialization trivial.
37	/// * It provides a unique integer to be used for anonymous symbol names, avoiding
38	/// multi-threaded contention on an atomic counter.
39	allocated_decls: std.SegmentedList(Comp.Decl, 0) = .{},
40	/// When a Decl object is freed from `allocated_decls`, it is pushed into this stack.
41	decls_free_list: std.ArrayListUnmanaged(DeclIndex) = .{},
42
43	/// Same pattern as with `allocated_decls`.
44	allocated_namespaces: std.SegmentedList(Comp.Namespace, 0) = .{},
45	/// Same pattern as with `decls_free_list`.
46	namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{},
47
48	/// How many items in the InternPool are statically known.
49	/// This is specified with an integer literal and a corresponding comptime
50	/// assert below to break an unfortunate and arguably incorrect dependency loop
51	/// when compiling.
52	pub const static_len = Teg.Inst.Index.static_len;
53	comptime {
54	// @compileLog(static_len, static_keys.len);
55	assert(static_len == static_keys.len);
56	}
57
58	pub const Item = struct {
59	tag: Tag,
60	data: EncodedData,
61
62	pub fn encode(comptime tag: Tag, data: tag.Data()) Item {
63	return .{ .tag = tag, .data = EncodedData.encode(tag, data) };
64	}
65	};
66
67	pub const ExtraStore = dod.ExtraStore(.intern_pool, .@"var");
68	pub const EncodedData = dod.EncodedData(Tag, u32);
69	pub const ExtraIndex = ExtraStore.Index;
70
71	pub const Tag = enum(u8) {
72	/// This special tag represents a value which was removed from this pool via
73	/// `InternPool.remove`. The item remains allocated to preserve indices, but
74	/// lookups will consider it not equal to any other item, and all queries
75	/// assert not this tag. `data` is unused.
76	removed,
77
78	/// A function type
79	/// data is an extra index into a `FuncType`
80	type_function,
81
82	// --- before this are types ---
83	// --- past this are values ---
84
85	/// A type that can be represented with only an enum tag.
86	/// data is SimpleType enum value.
87	simple_type,
88	/// A value that can be represented with only an enum tag.
89	/// data is SimpleValue enum value.
90	simple_value,
91	/// An int32 value
92	/// data is i32
93	int32,
94	/// A float value
95	/// data is an extra index into an `Float`
96	float,
97	/// A function
98	/// data is an extra index into a `FuncDecl`
99	func,
100	/// A string
101	/// data is an extra index into a `String`
102	string,
103
104	pub fn Data(comptime tag: Tag) type {
105	return switch (tag) {
106	.removed,
107	=> unreachable, // does not have a data
108
109	.simple_type => SimpleType,
110	.simple_value => SimpleValue,
111	.int32 => i32,
112
113	else => ExtraIndex(tag.Payload()),
114	};
115	}
116
117	/// When `data` is an extra index, this is the type of the payload.
118	pub fn Payload(comptime tag: Tag) type {
119	return switch (tag) {
120	.removed,
121	.simple_type,
122	.simple_value,
123	.int32,
124	=> unreachable, // does not have a payload
125	.float => Float,
126	.func => FuncDecl,
127	.type_function => TypeFunction,
128	.string => String.WithLen,
129	};
130	}
131
132	/// Trailing: N/A
133	pub const FuncDecl = struct {
134	analysis: FuncAnalysis,
135	owner_decl: DeclIndex,
136	ty: Index,
137	teg_body_inst: TrackedInst.Index,
138	};
139
140	/// Trailing:
141	/// 0. param_type: Index for each params_len
142	pub const TypeFunction = struct {
143	params_len: u32,
144	return_type: Index,
145	};
146	};
147
148	pub const Index = enum(u32) {
149	pub const first_type: Index = .opaque_type;
150	pub const last_type: Index = .noreturn_type;
151	pub const first_value: Index = .void_value;
152	pub const last_value: Index = .empty_string;
153
154	opaque_type,
155	bool_type,
156	int32_type,
157	float_type,
158	string_type,
159	void_type,
160	noreturn_type,
161	type_type,
162
163	void_value,
164	bool_true,
165	bool_false,
166	int32_zero,
167	float_zero,
168	nan,
169	empty_string,
170
171	none = std.math.maxInt(u32),
172	_,
173
174	/// An array of `Index` existing within the `extra` array.
175	/// This type exists to provide a struct with lifetime that is
176	/// not invalidated when items are added to the `InternPool`.
177	pub const Slice = struct {
178	start: u32,
179	len: u32,
180
181	pub fn get(slice: Slice, ip: *const InternPool) []Index {
182	return @ptrCast(ip.extra.list.items[slice.start..][0..slice.len]);
183	}
184	};
185
186	/// Used for a map of `Index` values to the index within a list of `Index` values.
187	const Adapter = struct {
188	indexes: []const Index,
189
190	pub fn eql(ctx: @This(), a: Index, b_void: void, b_map_index: usize) bool {
191	_ = b_void;
192	return a == ctx.indexes[b_map_index];
193	}
194
195	pub fn hash(ctx: @This(), a: Index) u32 {
196	_ = ctx;
197	return std.hash.uint32(@intFromEnum(a));
198	}
199	};
200
201	pub fn toAir(index: Index) Air.Inst.Ref {
202	return .{ .kind = .value, .data = @intCast(@intFromEnum(index)) };
203	}
204
205	pub fn toKey(index: Index, ip: *const InternPool) Key {
206	return ip.indexToKey(index);
207	}
208	};
209
210	pub const static_keys = [_]Key{
211	SimpleType.@"opaque".key(),
212	SimpleType.bool.key(),
213	SimpleType.int32.key(),
214	SimpleType.float.key(),
215	SimpleType.string.key(),
216	SimpleType.void.key(),
217	SimpleType.noreturn.key(),
218	SimpleType.type.key(),
219
220	SimpleValue.void.key(),
221	SimpleValue.true.key(),
222	SimpleValue.false.key(),
223	.{ .int32 = 0 },
224	.{ .float = 0.0 },
225	.{ .float = Float.pure_nan.decode() },
226	.{ .string = "" },
227	};
228
229	/// State that is mutable during semantic analysis. This data is not used for
230	/// equality or hashing.
231	pub const FuncAnalysis = packed struct(u32) {
232	state: State,
233
234	_: u24 = 0,
235
236	pub const State = enum(u8) {
237	/// This function has not yet undergone analysis, because we have not
238	/// seen a potential runtime call. It may be analyzed in future.
239	none,
240	/// Analysis for this function has been queued, but not yet completed.
241	queued,
242	/// This function intentionally only has ZIR generated because it is marked
243	/// inline, which means no runtime version of the function will be generated.
244	inline_only,
245	in_progress,
246	/// There will be a corresponding ErrorMsg in Module.failed_decls
247	sema_failure,
248	/// This function might be OK but it depends on another Decl which did not
249	/// successfully complete semantic analysis.
250	dependency_failure,
251	/// There will be a corresponding ErrorMsg in Module.failed_decls.
252	/// Indicates that semantic analysis succeeded, but code generation for
253	/// this function failed.
254	codegen_failure,
255	/// Semantic analysis and code generation of this function succeeded.
256	success,
257	};
258	};
259
260	/// An index into `string_bytes`.
261	pub const String = enum(u32) {
262	/// An empty string.
263	empty = 0,
264	_,
265
266	pub const WithLen = struct {
267	string: String,
268	len: u32,
269
270	pub fn toSlice(self: WithLen, ip: *const InternPool) []const u8 {
271	return ip.string_bytes.items[@intFromEnum(self.string)..][0..@intCast(self.len)];
272	}
273
274	pub fn tryToNullTerminatedString(string: WithLen, ip: *InternPool, gpa: Allocator) !NullTerminatedString {
275	return string.string.tryToNullTerminatedString(string.len, ip, gpa);
276	}
277
278	pub fn toNullTerminatedString(string: WithLen, ip: *const InternPool) NullTerminatedString {
279	return string.string.tryToNullTerminatedString(string.len, ip);
280	}
281	};
282
283	pub fn toSlice(string: String, len: u64, ip: *const InternPool) []const u8 {
284	return ip.string_bytes.items[@intFromEnum(string)..][0..@intCast(len)];
285	}
286
287	pub fn withLen(string: String, len: u64) WithLen {
288	return .{ .string = string, .len = @intCast(len) };
289	}
290
291	pub fn at(string: String, index: u64, ip: *const InternPool) u8 {
292	return ip.string_bytes.items[@intCast(@intFromEnum(string) + index)];
293	}
294
295	pub fn toNullTerminatedString(string: String, len: u64, ip: *const InternPool) NullTerminatedString {
296	assert(std.mem.indexOfScalar(u8, string.toSlice(len, ip), 0) == null);
297	assert(string.at(len, ip) == 0);
298	return @enumFromInt(@intFromEnum(string));
299	}
300
301	pub fn tryToNullTerminatedString(
302	string: String,
303	len: u64,
304	ip: *InternPool,
305	gpa: Allocator,
306	) !NullTerminatedString {
307	const slice = string.toSlice(len, ip);
308	if (std.mem.indexOfScalar(u8, slice, 0)) \|_\| {
309	return error.HasNulls;
310	}
311	if (string.at(len, ip) == 0) {
312	return @enumFromInt(@intFromEnum(string));
313	}
314	return ip.getOrPutString(gpa, slice, .no_embedded_nulls);
315	}
316	};
317
318	pub const NullTerminatedString = enum(u32) {
319	empty = 0,
320	_,
321
322	pub fn toString(self: NullTerminatedString) String {
323	return @enumFromInt(@intFromEnum(self));
324	}
325
326	// pub fn toOptional(self: NullTerminatedString) Optional{
327	// return @enumFromInt(@intFromEnum(self));
328	// }
329
330	pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
331	const slice = ip.string_bytes.items[@intFromEnum(string)..];
332	return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
333	}
334
335	pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
336	return @intCast(string.toSlice(ip).len);
337	}
338
339	pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
340	return std.mem.eql(u8, string.toSlice(ip), slice);
341	}
342
343	const Adapter = struct {
344	strings: []const NullTerminatedString,
345
346	pub fn eql(ctx: @This(), a: NullTerminatedString, b_void: void, b_map_index: usize) bool {
347	_ = b_void;
348	return a == ctx.strings[b_map_index];
349	}
350
351	pub fn hash(ctx: @This(), a: NullTerminatedString) u32 {
352	_ = ctx;
353	return std.hash.uint32(@intFromEnum(a));
354	}
355	};
356
357	const FormatData = struct {
358	string: NullTerminatedString,
359	ip: *const InternPool,
360	};
361	fn format(
362	data: FormatData,
363	comptime specifier: []const u8,
364	_: std.fmt.FormatOptions,
365	writer: anytype,
366	) @TypeOf(writer).Error!void {
367	const slice = data.string.toSlice(data.ip);
368	if (comptime std.mem.eql(u8, specifier, "")) {
369	try writer.writeAll(slice);
370	} else if (comptime std.mem.eql(u8, specifier, "i")) {
371	try writer.print("{p}", .{std.zig.fmtId(slice)});
372	} else @compileError("invalid format string '" ++ specifier ++ "' for '" ++ @typeName(NullTerminatedString) ++ "'");
373	}
374
375	pub fn fmt(self: NullTerminatedString, ip: *const InternPool) std.fmt.Formatter(format) {
376	return .{ .data = .{ .string = self, .ip = ip } };
377	}
378	};
379
380	pub const TrackedInst = extern struct {
381	path_digest: Cache.BinDigest,
382	inst: Teg.Inst.Index,
383
384	comptime {
385	// The fields should be tightly packed. See also serialiation logic in `Compilation.saveState`.
386	assert(@sizeOf(@This()) == Cache.bin_digest_len + @sizeOf(Teg.Inst.Index));
387	}
388
389	pub const Index = enum(u32) {
390	_,
391	pub fn resolve(i: TrackedInst.Index, ip: *const InternPool) Teg.Inst.Index {
392	return ip.tracked_insts.keys()[@intFromEnum(i)].inst;
393	}
394	pub fn toOptional(i: TrackedInst.Index) Optional {
395	return @enumFromInt(@intFromEnum(i));
396	}
397	pub const Optional = enum(u32) {
398	none = std.math.maxInt(u32),
399	_,
400	pub fn unwrap(opt: Optional) ?TrackedInst.Index {
401	return switch (opt) {
402	.none => null,
403	_ => @enumFromInt(@intFromEnum(opt)),
404	};
405	}
406	};
407	};
408	};
409
410	pub const DeclIndex = enum(u32) {
411	_,
412
413	pub fn toOptional(i: DeclIndex) Optional {
414	return @enumFromInt(@intFromEnum(i));
415	}
416
417	pub const Optional = enum(u32) {
418	none = std.math.maxInt(u32),
419	_,
420
421	pub fn init(oi: ?DeclIndex) Optional {
422	return @enumFromInt(@intFromEnum(oi orelse return .none));
423	}
424
425	pub fn unwrap(oi: Optional) ?DeclIndex {
426	if (oi == .none) return null;
427	return @enumFromInt(@intFromEnum(oi));
428	}
429	};
430	};
431
432	pub const NamespaceIndex = enum(u32) {
433	_,
434
435	pub fn toOptional(i: DeclIndex) Optional {
436	return @enumFromInt(@intFromEnum(i));
437	}
438
439	pub const Optional = enum(u32) {
440	none = std.math.maxInt(u32),
441	_,
442
443	pub fn init(oi: ?NamespaceIndex) Optional {
444	return @enumFromInt(@intFromEnum(oi orelse return .none));
445	}
446
447	pub fn unwrap(oi: Optional) ?NamespaceIndex {
448	if (oi == .none) return null;
449	return @enumFromInt(@intFromEnum(oi));
450	}
451	};
452	};
453
454	/// A single value captured in the closure of a namespace type. This is not a plain
455	/// `Index` because we must differentiate between the following cases:
456	/// * runtime-known value (where we store the type)
457	/// * comptime-known value (where we store the value)
458	/// * decl val (so that we can analyze the value lazily)
459	/// * decl ref (so that we can analyze the reference lazily)
460	pub const CaptureValue = packed struct(u32) {
461	tag: enum(u2) { @"comptime", runtime, decl_val, decl_ref },
462	idx: u30,
463
464	pub fn wrap(val: Unwrapped) CaptureValue {
465	return switch (val) {
466	.@"comptime" => \|i\| .{ .tag = .@"comptime", .idx = @intCast(@intFromEnum(i)) },
467	.runtime => \|i\| .{ .tag = .runtime, .idx = @intCast(@intFromEnum(i)) },
468	.decl_val => \|i\| .{ .tag = .decl_val, .idx = @intCast(@intFromEnum(i)) },
469	.decl_ref => \|i\| .{ .tag = .decl_ref, .idx = @intCast(@intFromEnum(i)) },
470	};
471	}
472	pub fn unwrap(val: CaptureValue) Unwrapped {
473	return switch (val.tag) {
474	.@"comptime" => .{ .@"comptime" = @enumFromInt(val.idx) },
475	.runtime => .{ .runtime = @enumFromInt(val.idx) },
476	.decl_val => .{ .decl_val = @enumFromInt(val.idx) },
477	.decl_ref => .{ .decl_ref = @enumFromInt(val.idx) },
478	};
479	}
480
481	pub const Unwrapped = union(enum) {
482	/// Index refers to the value.
483	@"comptime": Index,
484	/// Index refers to the type.
485	runtime: Index,
486	decl_val: DeclIndex,
487	decl_ref: DeclIndex,
488	};
489
490	pub const Slice = struct {
491	start: u32,
492	len: u32,
493	pub fn get(slice: Slice, ip: *const InternPool) []CaptureValue {
494	return @ptrCast(ip.extra.items[slice.start..][0..slice.len]);
495	}
496	};
497	};
498
499	const SimpleValue = enum {
500	true,
501	false,
502	void,
503
504	pub fn key(self: SimpleValue) Key {
505	return .{ .simple_value = self };
506	}
507	};
508
509	const SimpleType = enum {
510	@"opaque",
511	int32,
512	float,
513	string,
514	bool,
515	void,
516	noreturn,
517	type,
518
519	pub fn key(self: SimpleType) Key {
520	return .{ .simple_type = self };
521	}
522	};
523
524	pub const Key = union(enum) {
525	int32: i32,
526	float: f64,
527	string: []const u8,
528	simple_value: SimpleValue,
529	simple_type: SimpleType,
530	type_function: TypeFunction,
531	func: Func,
532
533	const Adapter = struct {
534	intern_pool: *const InternPool,
535
536	pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
537	_ = b_void;
538	if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
539	return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
540	}
541
542	pub fn hash(ctx: @This(), a: Key) u32 {
543	return a.hash32(ctx.intern_pool);
544	}
545	};
546
547	pub const TypeFunction = struct {
548	param_types: []Index,
549	return_type: Index,
550
551	pub fn hash(self: TypeFunction, hasher: *Hash) void {
552	const t = tracer.trace(@src());
553	defer t.end();
554
555	for (self.param_types) \|param_type\| {
556	std.hash.autoHash(hasher, param_type);
557	}
558	std.hash.autoHash(hasher, self.return_type);
559	}
560
561	pub fn eql(a: TypeFunction, b: TypeFunction) bool {
562	return std.mem.eql(Index, a.param_types, b.param_types) and
563	a.return_type == b.return_type;
564	}
565	};
566
567	pub const Func = struct {
568	ty: Index,
569	owner_decl: DeclIndex,
570	teg_body_inst: TrackedInst.Index,
571
572	pub fn hash(self: Func, hasher: *Hash) void {
573	std.hash.autoHash(hasher, self.ty);
574	std.hash.autoHash(hasher, self.owner_decl);
575	std.hash.autoHash(hasher, self.teg_body_inst);
576	}
577	};
578
579	pub fn hash32(key: Key, ip: *const InternPool) u32 {
580	return @truncate(key.hash64(ip));
581	}
582
583	pub fn hash64(key: Key, ip: *const InternPool) u64 {
584	const t = tracer.trace(@src());
585	defer t.end();
586
587	_ = ip; // autofix
588	const asBytes = std.mem.asBytes;
589	const KeyTag = @typeInfo(Key).Union.tag_type.?;
590	const seed = @intFromEnum(@as(KeyTag, key));
591
592	return switch (key) {
593	inline .int32,
594	.simple_value,
595	.simple_type,
596	=> \|x\| Hash.hash(seed, asBytes(&x)),
597	.string,
598	=> \|x\| Hash.hash(seed, x),
599	.float => \|x\| Hash.hash(seed, asBytes(&Float.assertPure(x))),
600	.type_function => \|x\| {
601	var hasher = Hash.init(seed);
602	x.hash(&hasher);
603	return hasher.final();
604	},
605	.func => \|x\| {
606	var hasher = Hash.init(seed);
607	x.hash(&hasher);
608	return hasher.final();
609	},
610	};
611	}
612
613	pub fn eql(a: Key, b: Key, ip: *const InternPool) bool {
614	const t = tracer.trace(@src());
615	defer t.end();
616
617	_ = ip; // autofix
618	const KeyTag = @typeInfo(Key).Union.tag_type.?;
619	const a_tag: KeyTag = a;
620	const b_tag: KeyTag = b;
621	if (a_tag != b_tag) return false;
622	return switch (a) {
623	inline .int32,
624	.simple_value,
625	.simple_type,
626	=> \|x, tag\| x == @field(b, @tagName(tag)),
627	.string,
628	=> \|x\| std.mem.eql(u8, x, b.string),
629	.float => \|x\| Float.assertPure(x) == Float.assertPure(b.float),
630	.type_function => \|x\| x.eql(b.type_function),
631	.func => \|a_info\| {
632	const b_info = b.func;
633	if (a_info.owner_decl != b_info.owner_decl)
634	return false;
635	if (a_info.ty == b_info.ty)
636	return true;
637
638	// TODO: zig has something extra here for inferred error sets
639	return false;
640	},
641	};
642	}
643	};
644
645	const EmbeddedNulls = enum {
646	no_embedded_nulls,
647	maybe_embedded_nulls,
648
649	const OptionalNullTerminatedString = @compileError("TODO");
650	const OptionalString = @compileError("TODO");
651
652	fn StringType(comptime embedded_nulls: EmbeddedNulls) type {
653	return switch (embedded_nulls) {
654	.no_embedded_nulls => NullTerminatedString,
655	.maybe_embedded_nulls => String,
656	};
657	}
658
659	fn OptionalStringType(comptime embedded_nulls: EmbeddedNulls) type {
660	return switch (embedded_nulls) {
661	.no_embedded_nulls => OptionalNullTerminatedString,
662	.maybe_embedded_nulls => OptionalString,
663	};
664	}
665	};
666
667	// implementations:
668
669	/// To construct, instantiate `.{ .gpa = gpa }` then call this.
670	pub fn init(ip: *InternPool, gpa: Allocator) !void {
671	const t = tracer.trace(@src());
672	defer t.end();
673
674	assert(ip.items.len == 0);
675
676	// Reserve string index 0 for an empty string.
677	assert((try ip.getOrPutString(gpa, "", .no_embedded_nulls)) == .empty);
678
679	// So that we can use `catch unreachable` below.
680	try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
681	try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
682	try ip.extra.list.ensureUnusedCapacity(gpa, static_keys.len);
683
684	// This inserts all the statically-known values into the intern pool in the
685	// order expected.
686	for (static_keys) \|key\| {
687	_ = ip.get(gpa, key) catch unreachable;
688	}
689
690	if (std.debug.runtime_safety) {
691	// Sanity check.
692	assert(ip.indexToKey(.bool_true).simple_value == .true);
693	assert(ip.indexToKey(.bool_false).simple_value == .false);
694	}
695
696	assert(ip.items.len == static_keys.len);
697	}
698
699	pub fn deinit(ip: *InternPool, gpa: Allocator) void {
700	const t = tracer.trace(@src());
701	defer t.end();
702
703	ip.map.deinit(gpa);
704	ip.items.deinit(gpa);
705	ip.extra.deinit(gpa);
706	// ip.limbs.deinit(gpa);
707	ip.string_bytes.deinit(gpa);
708
709	var decls = ip.allocated_decls.iterator(0);
710	while (decls.next()) \|decl\| {
711	decl.deinit(gpa);
712	}
713
714	ip.allocated_decls.deinit(gpa);
715	ip.decls_free_list.deinit(gpa);
716
717	var namespaces = ip.allocated_namespaces.iterator(0);
718	while (namespaces.next()) \|ns\| {
719	ns.decls.deinit(gpa);
720	}
721
722	ip.namespaces_free_list.deinit(gpa);
723	ip.allocated_namespaces.deinit(gpa);
724
725	// for (ip.maps.items) \|*map\| map.deinit(gpa);
726	// ip.maps.deinit(gpa);
727
728	ip.string_table.deinit(gpa);
729
730	ip.tracked_insts.deinit(gpa);
731
732	// ip.src_hash_deps.deinit(gpa);
733	// ip.decl_val_deps.deinit(gpa);
734	// ip.func_ies_deps.deinit(gpa);
735	// ip.namespace_deps.deinit(gpa);
736	// ip.namespace_name_deps.deinit(gpa);
737
738	// ip.first_dependency.deinit(gpa);
739
740	// ip.dep_entries.deinit(gpa);
741	// ip.free_dep_entries.deinit(gpa);
742
743	ip.* = undefined;
744	}
745
746	pub fn indexToKey(ip: *const InternPool, index: Index) Key {
747	assert(index != .none);
748	const item: Item = ip.items.get(@intFromEnum(index));
749	const data = item.data;
750	return switch (item.tag) {
751	.removed => unreachable,
752
753	.type_function => .{
754	.type_function = ip.extraTypeFunction(data.decode(.type_function, item.tag)),
755	},
756
757	.simple_type => .{ .simple_type = data.decode(.simple_type, item.tag) },
758	.simple_value => .{ .simple_value = data.decode(.simple_value, item.tag) },
759	.int32 => .{ .int32 = data.decode(.int32, item.tag) },
760	.float => .{ .float = data.decode(.float, item.tag).get(ip).decode() },
761	.string => .{ .string = data.decode(.string, item.tag).get(ip).toSlice(ip) },
762	.func => {
763	const func = data.decode(.func, item.tag).get(ip);
764	return .{ .func = .{
765	.ty = func.ty,
766	.owner_decl = func.owner_decl,
767	.teg_body_inst = func.teg_body_inst,
768	} };
769	},
770	};
771	}
772
773	pub fn get(ip: *InternPool, gpa: Allocator, key: Key) !Index {
774	const t = tracer.traceFmt(@src(), "{s}", .{@tagName(key)});
775	defer t.end();
776
777	const adapter: Key.Adapter = .{ .intern_pool = ip };
778	const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
779	if (gop.found_existing) return @enumFromInt(gop.index);
780	if (ip.items.len >= std.math.maxInt(u31) - 1) {
781	return error.TooManyValues;
782	}
783
784	const t2 = tracer.traceFmt(@src(), "(insert)", .{});
785	defer t2.end();
786
787	try ip.items.ensureUnusedCapacity(gpa, 1);
788	const item: Item = switch (key) {
789	inline .int32,
790	.simple_type,
791	.simple_value,
792	=> \|s, tag\| Item.encode(@field(Tag, @tagName(tag)), s),
793	.string => \|s\| Item.encode(
794	.string,
795	try ip.extra.append(
796	gpa,
797	String.WithLen,
798	(try ip.getOrPutString(gpa, s, .maybe_embedded_nulls)).withLen(s.len),
799	),
800	),
801	.float => \|f\| Item.encode(
802	.float,
803	try ip.extra.append(gpa, Float, Float.encode(f)),
804	),
805	.func => \|f\| brk: {
806	const extra = try ip.extra.append(gpa, Tag.FuncDecl, .{
807	.analysis = .{
808	.state = .none,
809	},
810	.owner_decl = f.owner_decl,
811	.ty = f.ty,
812	.teg_body_inst = f.teg_body_inst,
813	});
814
815	break :brk Item.encode(.func, extra);
816	},
817	.type_function => \|f\| brk: {
818	const extra, var trail = try ip.extra.appendTrailing(gpa, Tag.TypeFunction, .{
819	.params_len = @intCast(f.param_types.len),
820	.return_type = f.return_type,
821	}, f.param_types.len);
822	defer trail.end();
823	trail.writeSlice(f.param_types);
824	break :brk Item.encode(.type_function, extra);
825	},
826	};
827	ip.items.appendAssumeCapacity(item);
828	const index: Index = @enumFromInt(ip.items.len - 1);
829	if (std.debug.runtime_safety) {
830	assert(ip.indexToKey(index).eql(key, ip));
831	}
832	return index;
833	}
834
835	fn extraTypeFunction(
836	ip: *const InternPool,
837	extra_index: ExtraIndex(Tag.TypeFunction),
838	) Key.TypeFunction {
839	const type_function, var trail = extra_index.getTrail(ip);
840	const param_types = trail.readInternPoolIndexSlice(type_function.params_len);
841	const result: Key.TypeFunction = .{
842	.return_type = type_function.return_type,
843	.param_types = param_types.get(ip),
844	};
845	return result;
846	}
847
848	pub fn getOrPutString(
849	ip: *InternPool,
850	gpa: Allocator,
851	slice: []const u8,
852	comptime embedded_nulls: EmbeddedNulls,
853	) Allocator.Error!embedded_nulls.StringType() {
854	const t = tracer.trace(@src());
855	defer t.end();
856
857	try ip.string_bytes.ensureUnusedCapacity(gpa, slice.len + 1);
858	ip.string_bytes.appendSliceAssumeCapacity(slice);
859	ip.string_bytes.appendAssumeCapacity(0);
860	return ip.getOrPutTrailingString(gpa, slice.len + 1, embedded_nulls);
861	}
862
863	fn getOrPutTrailingString(
864	ip: *InternPool,
865	gpa: Allocator,
866	len: usize,
867	comptime embedded_nulls: EmbeddedNulls,
868	) Allocator.Error!embedded_nulls.StringType() {
869	const t = tracer.trace(@src());
870	defer t.end();
871
872	const string_bytes = &ip.string_bytes;
873	const str_index: u32 = @intCast(string_bytes.items.len - len);
874	if (len > 0 and string_bytes.getLast() == 0) {
875	_ = string_bytes.pop();
876	} else {
877	try string_bytes.ensureUnusedCapacity(gpa, 1);
878	}
879	const key: []const u8 = string_bytes.items[str_index..];
880	const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
881	switch (embedded_nulls) {
882	.no_embedded_nulls => assert(!has_embedded_null),
883	.maybe_embedded_nulls => if (has_embedded_null) {
884	string_bytes.appendAssumeCapacity(0);
885	return @enumFromInt(str_index);
886	},
887	}
888	const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{
889	.bytes = string_bytes,
890	}, std.hash_map.StringIndexContext{
891	.bytes = string_bytes,
892	});
893	if (gop.found_existing) {
894	string_bytes.shrinkRetainingCapacity(str_index);
895	return @enumFromInt(gop.key_ptr.*);
896	} else {
897	gop.key_ptr.* = str_index;
898	string_bytes.appendAssumeCapacity(0);
899	return @enumFromInt(str_index);
900	}
901	}
902
903	pub fn trackInst(ip: InternPool, gpa: Allocator, file: Comp.File, inst: Teg.Inst.Index) Allocator.Error!TrackedInst.Index {
904	const t = tracer.trace(@src());
905	defer t.end();
906
907	const key: TrackedInst = .{
908	.path_digest = file.path_digest.?,
909	.inst = inst,
910	};
911	const gop = try ip.tracked_insts.getOrPut(gpa, key);
912	return @enumFromInt(gop.index);
913	}
914
915	pub fn declPtr(ip: InternPool, index: DeclIndex) Comp.Decl {
916	return ip.allocated_decls.at(@intFromEnum(index));
917	}
918
919	pub fn declPtrConst(ip: const InternPool, index: DeclIndex) const Comp.Decl {
920	return ip.allocated_decls.at(@intFromEnum(index));
921	}
922
923	pub fn namespacePtr(ip: InternPool, index: NamespaceIndex) Comp.Namespace {
924	return ip.allocated_namespaces.at(@intFromEnum(index));
925	}
926
927	pub fn createDecl(
928	ip: *InternPool,
929	gpa: Allocator,
930	initialization: Comp.Decl,
931	) Allocator.Error!DeclIndex {
932	const t = tracer.trace(@src());
933	defer t.end();
934
935	if (ip.decls_free_list.popOrNull()) \|index\| {
936	ip.allocated_decls.at(@intFromEnum(index)).* = initialization;
937	return index;
938	}
939	const ptr = try ip.allocated_decls.addOne(gpa);
940	ptr.* = initialization;
941	return @enumFromInt(ip.allocated_decls.len - 1);
942	}
943
944	pub fn destroyDecl(ip: *InternPool, gpa: Allocator, index: DeclIndex) void {
945	const t = tracer.trace(@src());
946	defer t.end();
947
948	ip.declPtr(index).* = undefined;
949	ip.decls_free_list.append(gpa, index) catch {
950	// In order to keep `destroyDecl` a non-fallible function, we ignore memory
951	// allocation failures here, instead leaking the Decl until garbage collection.
952	};
953	}
954
955	pub fn createNamespace(
956	ip: *InternPool,
957	gpa: Allocator,
958	initialization: Comp.Namespace,
959	) Allocator.Error!NamespaceIndex {
960	const t = tracer.trace(@src());
961	defer t.end();
962
963	if (ip.namespaces_free_list.popOrNull()) \|index\| {
964	ip.allocated_namespaces.at(@intFromEnum(index)).* = initialization;
965	return index;
966	}
967	const ptr = try ip.allocated_namespaces.addOne(gpa);
968	ptr.* = initialization;
969	return @enumFromInt(ip.allocated_namespaces.len - 1);
970	}
971
972	pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex) void {
973	const t = tracer.trace(@src());
974	defer t.end();
975
976	ip.namespacePtr(index).* = undefined;
977	ip.namespaces_free_list.append(gpa, index) catch {
978	// In order to keep `destroyNamespace` a non-fallible function, we ignore memory
979	// allocation failures here, instead leaking the Namespace until garbage collection.
980	};
981	}
982
983	const InternPool = @This();
984
985	const std = @import("std");
986	const Allocator = std.mem.Allocator;
987	const Hash = std.hash.Wyhash;
988	const assert = std.debug.assert;
989
990	const Comp = @import("Comp.zig");
991	const Cache = @import("Cache.zig");
992
993	const Teg = @import("Teg.zig");
994	const Air = @import("Air.zig");
995
996	const dod = @import("dod.zig");
997	const Float = dod.Float;
998
999	const tracer = @import("tracer");
1000
1001	test InternPool {
1002	const testing = std.testing;
1003	std.testing.refAllDeclsRecursive(@This());
1004
1005	var ip_struct: InternPool = .{ .gpa = std.testing.allocator };
1006	const ip = &ip_struct;
1007	defer ip.deinit();
1008
1009	const str = try ip.getOrPutString("hello", .no_embedded_nulls);
1010	try testing.expectEqualStrings("hello", str.toSlice(ip));
1011	const str2 = try ip.getOrPutString("hello", .no_embedded_nulls);
1012	try testing.expectEqual(str, str2);
1013
1014	const int1 = try ip.get(.{ .int32 = 21483 });
1015	try testing.expectEqual(@as(i32, 21483), ip.indexToKey(int1).int32);
1016	const int2 = try ip.get(.{ .int32 = 21483 });
1017	try testing.expectEqual(int1, int2);
1018	}

src/compiler/JsLink.zig created+457

...	@@ -0,0 +1,457 @@
1	/// This is the JavaScript backend's linker.
2	/// It takes in a compilation, a list of `JsTree`s and creates a JavaScript
3	/// bundle out of them.
4	comp: *Comp,
5
6	/// Value is an index into `link_strings`
7	decls: std.AutoHashMapUnmanaged(Decl.Index, LinkString.Index) = .{},
8	link_strings: std.ArrayHashMapUnmanaged(
9	LinkString,
10	GlobalOwner,
11	LinkString.Context,
12	true,
13	) = .{},
14	string_bytes: std.ArrayListUnmanaged(u8) = .{},
15
16	/// The first declaration is the function containing top-level code.
17	pub fn linkJsCode(
18	comp: *Comp,
19	decls: []const Decl.Index,
20	writer: anytype,
21	) !void {
22	log.info("Linking {d} decls", .{decls.len});
23
24	// Arena allocator is used for the main flow
25	var link: JsLink = .{ .comp = comp };
26	defer link.deinit();
27
28	// Temp allocator is used for extremely temporary string work.
29	// var temp_sfa = std.heap.stackFallback(4096, comp.gpa);
30	// const temp_alloc = temp_sfa.get();
31
32	// 1. Resolve all global identifiers. Function names must not shadow external
33	// names. We also need this list prepared so that local identifier names do not
34	// shadow functions or externals.
35	const root_decl = comp.intern_pool.declPtrConst(decls[0]);
36	assert(root_decl.kind == .named);
37	assert(root_decl.name == .empty);
38
39	for (root_decl.codegen.?.extern_symbols) \|extern_symbol\| {
40	try link.reserveExternSymbol(extern_symbol);
41	}
42
43	for (decls[1..]) \|decl_index\| {
44	assert(decl_index != decls[0]);
45
46	const decl = comp.intern_pool.declPtrConst(decl_index);
47
48	// todo: add all externs first
49	for (decl.codegen.?.extern_symbols) \|extern_symbol\| {
50	try link.reserveExternSymbol(extern_symbol);
51	}
52
53	const name = decl.name.toSlice(&comp.intern_pool);
54	assert(name.len > 0);
55
56	const gop = try link.link_strings.getOrPutContextAdapted(
57	comp.gpa,
58	name,
59	LinkString.SliceContext{ .link = &link },
60	.{ .link = &link },
61	);
62	if (gop.found_existing) {
63	@panic("TODO: handle this shadowing case.");
64	}
65	const name_str = try link.appendLinkString(name);
66	gop.key_ptr.* = name_str;
67	gop.value_ptr.* = .decl;
68	try link.decls.putNoClobber(comp.gpa, decl_index, @enumFromInt(gop.index));
69	}
70
71	try writer.print("// Compiler Demo. Do not use in production\n", .{});
72
73	// 2. Print every function decl
74	for (decls[1..]) \|decl_index\| {
75	try link.printDecl(decl_index, writer);
76	}
77
78	// 3. Print the top-level code
79	try link.printDeclContents(&root_decl.codegen.?, writer);
80	}
81
82	fn deinit(link: *JsLink) void {
83	defer link.* = undefined;
84	link.decls.deinit(link.comp.gpa);
85	link.link_strings.deinit(link.comp.gpa);
86	link.string_bytes.deinit(link.comp.gpa);
87	}
88
89	fn reserveExternSymbol(link: *JsLink, extern_symbol: String.WithLen) !void {
90	const name = extern_symbol.toSlice(&link.comp.intern_pool);
91	const gop = try link.link_strings.getOrPutContextAdapted(
92	link.comp.gpa,
93	name,
94	LinkString.SliceContext{ .link = link },
95	.{ .link = link },
96	);
97
98	if (gop.found_existing) {
99	if (gop.value_ptr.* != .external) {
100	@panic("TODO: Rename Decl");
101	}
102	}
103	gop.key_ptr.* = try link.appendLinkString(name);
104	gop.value_ptr.* = .external;
105	}
106
107	fn appendLinkString(link: *JsLink, string: []const u8) !LinkString {
108	const start = link.string_bytes.items.len;
109	log.info("Append '{s}' -> .{{ .start = {d}, .len = {d} }}", .{ string, start, string.len });
110	try link.string_bytes.appendSlice(link.comp.gpa, string);
111	return .{ .start = @intCast(start), .len = @intCast(string.len) };
112	}
113
114	fn declIdentifier(link: *const JsLink, decl: Decl.Index) []const u8 {
115	return (link.decls.get(decl) orelse if (std.debug.runtime_safety) {
116	std.debug.panic("Decl {d} ({}) is not present in link decl map", .{
117	@intFromEnum(decl),
118	link.comp.intern_pool.declPtrConst(decl).name.fmt(&link.comp.intern_pool),
119	});
120	} else unreachable).toString(link).slice(link);
121	}
122
123	fn printDecl(link: *JsLink, decl_index: Decl.Index, writer: anytype) !void {
124	try writer.writeAll("let ");
125	try writer.writeAll(link.declIdentifier(decl_index));
126	try writer.writeAll(" = ");
127
128	const decl = link.comp.intern_pool.declPtrConst(decl_index);
129	const tree = &decl.codegen.?;
130
131	try writer.writeAll("(");
132	for (0..tree.param_count) \|id\| {
133	try link.printLocal(@intCast(id), writer);
134	if (id != tree.param_count - 1) try writer.writeAll(", ");
135	}
136	try writer.writeAll(") => {\n");
137	try link.printDeclContents(tree, writer);
138	try writer.writeAll("};\n");
139	}
140
141	fn printDeclContents(link: JsLink, tree: const JsTree, writer: anytype) !void {
142	for (tree.mainStatements()) \|stmt\| {
143	try link.printExpr(tree, stmt, writer);
144	try writer.writeAll(";\n");
145	}
146	}
147
148	fn printBlockContents(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
149	assert(list.slice(tree).len > 0);
150	for (list.slice(tree)) \|stmt\| {
151	try link.printExpr(tree, stmt, writer);
152	switch (tree.nodeTag(stmt)) {
153	else => try writer.writeAll(";"),
154	.stmt_block,
155	.stmt_if,
156	=> {},
157	}
158	try writer.writeAll("\n");
159	}
160	}
161
162	fn printBlock(link: JsLink, tree: const JsTree, list: JsTree.IndexList, writer: anytype) !void {
163	if (list.len != 1) {
164	try writer.writeAll("{");
165	try link.printBlockContents(tree, list, writer);
166	try writer.writeAll("}");
167	} else {
168	try link.printBlockContents(tree, list, writer);
169	}
170	}
171
172	fn printExpr(link: JsLink, tree: const JsTree, node: Node.Index, writer: anytype) @TypeOf(writer).Error!void {
173	log.info("print node {d}", .{@intFromEnum(node)});
174	switch (tree.nodeTag(node)) {
175	else => \|tag\| std.debug.panic("TODO: printExpr .{s}", .{@tagName(tag)}),
176	.root => unreachable,
177
178	.plain_value => {
179	const plain_value: JsTree.PlainValue = tree.nodeData(.plain_value, node);
180	switch (plain_value.value.toKey(&link.comp.intern_pool)) {
181	// TODO: minification
182	.int32 => \|val\| try writer.print("{d}", .{val}),
183	// TODO: correct printing of many value types
184	.float => \|val\| try writer.print("{d}", .{val}),
185	.simple_value => \|val\| try writer.writeAll(switch (val) {
186	.true => "true",
187	.false => "false",
188	// TODO: make codegen decide from null and undefined
189	// instead of emitting void nodes.
190	.void => "undefined",
191	}),
192	// TODO: escapes
193	.string => \|val\| try writer.print("\"{s}\"", .{val}),
194
195	else => unreachable, // this value is not a plain_value
196	}
197	},
198
199	.local => {
200	const ref: JsTree.LocalRef = tree.nodeData(.local, node);
201	try link.printLocal(ref.id, writer);
202	},
203
204	.global => {
205	const ref: JsTree.GlobalRef = tree.nodeData(.global, node);
206	try writer.writeAll(ref.name.toSlice(&link.comp.intern_pool));
207	},
208
209	.decl => {
210	const ref: JsTree.DeclRef = tree.nodeData(.decl, node);
211	try writer.writeAll(link.declIdentifier(ref.index));
212	},
213
214	.stmt_break => {
215	const stmt: JsTree.StmtBreak = tree.nodeData(.stmt_break, node);
216	try writer.writeAll("break");
217	if (true) {
218	try writer.writeAll(" ");
219	try link.printLabelIdentifier(stmt.label, writer);
220	}
221	},
222
223	.stmt_return => {
224	const stmt: JsTree.StmtReturn = tree.nodeData(.stmt_return, node);
225	try writer.writeAll("return");
226	if (stmt.operand.unwrap()) \|operand\| {
227	try writer.writeAll(" ");
228	try link.printExpr(tree, operand, writer);
229	}
230	},
231
232	.stmt_declare_local => {
233	const stmt: JsTree.StmtDeclareLocal = tree.nodeData(.stmt_declare_local, node);
234	try writer.writeAll("let ");
235	try link.printLocal(stmt.identifier_index, writer);
236	if (stmt.init.unwrap()) \|init\| {
237	try writer.writeAll(" = ");
238	try link.printExpr(tree, init, writer);
239	}
240	},
241
242	.stmt_if => {
243	const stmt: JsTree.StmtIf = tree.nodeData(.stmt_if, node);
244	const data = stmt.data.get(tree);
245	try writer.writeAll("if (");
246	try link.printExpr(tree, stmt.cond, writer);
247	try writer.writeAll(") ");
248	if (data.flags.yes_label) {
249	try link.printLabelIdentifier(data.flags.block_label, writer);
250	try writer.writeAll(":");
251	}
252	try link.printBlock(tree, data.yes, writer);
253	if (data.no.len > 0) {
254	try writer.writeAll(" else ");
255	if (data.flags.no_label) {
256	try link.printLabelIdentifier(data.flags.block_label, writer);
257	try writer.writeAll(":");
258	}
259	try link.printBlock(tree, data.no, writer);
260	}
261	},
262
263	.stmt_block => {
264	const stmt: JsTree.StmtBlock = tree.nodeData(.stmt_block, node);
265	try link.printLabelIdentifier(stmt.label, writer);
266	try writer.writeAll(": {\n");
267	try link.printBlockContents(tree, stmt.content.get(tree), writer);
268	try writer.writeAll("}");
269	},
270
271	inline .assign,
272	.add,
273	.sub,
274	.mul,
275	.div,
276	.eq,
277	.lt,
278	.gt,
279	=> \|tag\| {
280	const bin: JsTree.Bin = tree.nodeData(tag, node);
281	const op: Operator = @field(Operator, @tagName(tag));
282	try link.printExpr(tree, bin.lhs, writer);
283	try writer.writeAll(" ");
284	try writer.writeAll(op.symbol());
285	try writer.writeAll(" ");
286	try link.printExpr(tree, bin.rhs, writer);
287	},
288
289	inline .negate => \|tag\| {
290	const un: JsTree.Un = tree.nodeData(tag, node);
291	const op: Operator = @field(Operator, @tagName(tag));
292	try writer.writeAll(op.symbol());
293	try link.printExpr(tree, un.operand, writer);
294	},
295
296	.ternary => {
297	const ternary: JsTree.Ternary = tree.nodeData(.ternary, node);
298	const data = ternary.data.get(tree);
299	try writer.writeAll("(");
300	try link.printExpr(tree, ternary.cond, writer);
301	try writer.writeAll(" ? ");
302	try link.printExpr(tree, data.yes, writer);
303	try writer.writeAll(" : ");
304	try link.printExpr(tree, data.no, writer);
305	try writer.writeAll(")");
306	},
307
308	.call => {
309	const call: JsTree.Call = tree.nodeData(.call, node);
310	const args = call.args.get(tree);
311
312	try link.printExpr(tree, call.base, writer);
313	try writer.writeAll("(");
314
315	for (args.slice(tree), 0..) \|arg, i\| {
316	if (i != 0) try writer.writeAll(", ");
317	try link.printExpr(tree, arg, writer);
318	}
319
320	try writer.writeAll(")");
321	},
322
323	.field => {
324	const field: JsTree.Field = tree.nodeData(.field, node);
325
326	try link.printExpr(tree, field.base, writer);
327	try writer.writeAll(".");
328	try writer.writeAll(field.field.toSlice(&link.comp.intern_pool));
329	},
330	}
331	}
332
333	fn printLocal(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
334	_ = link; // todo: be smarter about this
335	try writer.print("v{d}", .{local});
336	}
337
338	fn printLabelIdentifier(link: *JsLink, local: JsTree.LocalIndex, writer: anytype) !void {
339	_ = link; // todo: be smarter about this
340	try writer.print("label{d}", .{local});
341	}
342
343	const GlobalOwner = enum {
344	external,
345	decl,
346	};
347
348	// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_precedence
349	const Level = enum {
350	comma,
351	assignment,
352	conditional,
353	logical_or,
354	logical_and,
355	bitwise_or,
356	bitwise_xor,
357	bitwise_and,
358	equality,
359	relational,
360	bitshift,
361	additive,
362	multiplicative,
363	exponentiation,
364	prefix,
365	postfix,
366	new,
367	access_and_call,
368	member,
369	};
370
371	const Operator = enum {
372	assign,
373	add,
374	sub,
375	mul,
376	div,
377	eq,
378	lt,
379	gt,
380	negate,
381
382	pub fn symbol(op: Operator) []const u8 {
383	return switch (op) {
384	.assign => "=",
385	.add => "+",
386	.sub => "-",
387	.mul => "*",
388	.div => "/",
389	.eq => "===",
390	.lt => "<",
391	.gt => ">",
392
393	.negate => "-",
394	};
395	}
396	};
397
398	const LinkString = struct {
399	start: u32,
400	len: u32,
401
402	fn slice(str: LinkString, link: *const JsLink) []const u8 {
403	return link.string_bytes.items[str.start..][0..str.len];
404	}
405
406	const Index = enum(u32) {
407	_,
408
409	pub fn toString(index: Index, link: *const JsLink) LinkString {
410	return link.link_strings.keys()[@intFromEnum(index)];
411	}
412	};
413
414	const Context = struct {
415	link: *const JsLink,
416
417	pub fn hash(ctx: Context, str: LinkString) u32 {
418	// assert(str.start + str.len < ctx.link.string_bytes.items.len);
419
420	return @truncate(Wyhash.hash(0, str.slice(ctx.link)));
421	}
422
423	pub fn eql(ctx: Context, a: LinkString, b: LinkString, _: usize) bool {
424	assert(a.start + a.len < ctx.link.string_bytes.items.len);
425	assert(b.start + b.len < ctx.link.string_bytes.items.len);
426	return a.start == b.start;
427	}
428	};
429
430	const SliceContext = struct {
431	link: *const JsLink,
432
433	pub fn hash(_: SliceContext, str: []const u8) u32 {
434	return @truncate(Wyhash.hash(0, str));
435	}
436
437	pub fn eql(ctx: SliceContext, a: []const u8, b: LinkString, _: usize) bool {
438	// since the given string is not in string_bytes we do comparisons
439	return std.mem.eql(u8, a, b.slice(ctx.link));
440	}
441	};
442	};
443
444	const JsLink = @This();
445	const std = @import("std");
446	const assert = std.debug.assert;
447	const Wyhash = std.hash.Wyhash;
448	const log = std.log.scoped(.js_link);
449
450	const Comp = @import("Comp.zig");
451	const Decl = Comp.Decl;
452
453	const JsTree = @import("JsTree.zig");
454	const Node = JsTree.Node;
455
456	const InternPool = @import("InternPool.zig");
457	const String = InternPool.String;

src/compiler/JsNodeGen.zig created+541

...	@@ -0,0 +1,541 @@
1	//! Air -> JsTree
2	gpa: Allocator,
3	arena: Allocator,
4	air: *const Air,
5	ip: *const InternPool,
6
7	nodes: Node.List,
8	extra: dod.ExtraStore(.tfr, .@"var"),
9	extern_symbols: std.AutoArrayHashMapUnmanaged(InternPool.String.WithLen, void),
10	decl_deps: std.AutoArrayHashMapUnmanaged(Decl.Index, void),
11
12	param_count: u32,
13	air_node_map: []MapEntry,
14	scratch: std.ArrayListUnmanaged(u32),
15	next_id: u32,
16	next_label: u32,
17
18	const Error = error{
19	OutOfMemory,
20	};
21
22	const MapEntry = union {
23	none: void,
24	node: Node.Index,
25	block: packed struct {
26	used: bool = false,
27	index: u31,
28	},
29	};
30
31	pub fn init(gpa: Allocator, arena: Allocator, ip: const InternPool, air: const Air) !JsNodeGen {
32	const air_node_map = try arena.alloc(MapEntry, air.insts.len);
33	if (std.debug.runtime_safety) {
34	@memset(air_node_map, .{ .none = {} });
35	}
36	const param_count = air.paramCount();
37	return .{
38	.gpa = gpa,
39	.arena = arena,
40	.air = air,
41	.ip = ip,
42	.param_count = param_count,
43	.nodes = .{},
44	.extern_symbols = .{},
45	.decl_deps = .{},
46	.air_node_map = air_node_map,
47	.extra = .{},
48	.scratch = .{},
49	.next_id = param_count,
50	.next_label = 0,
51	};
52	}
53
54	pub fn deinit(c: *JsNodeGen) void {
55	defer c.* = undefined;
56	c.nodes.deinit(c.gpa);
57	c.extra.deinit(c.gpa);
58	c.extern_symbols.deinit(c.gpa);
59	c.decl_deps.deinit(c.gpa);
60
61	// arena can deinit the rest
62	}
63
64	pub fn finalize(c: *JsNodeGen) !JsTree {
65	// todo: errdefer will probably leave memory in a fully invalid state
66	try c.nodes.setCapacity(c.gpa, c.nodes.len);
67
68	var extra = try c.extra.toOwnedStore(c.gpa);
69	errdefer extra.deinit(c.gpa);
70
71	// todo: i dont like this dupe
72	const extern_symbols = try c.gpa.dupe(InternPool.String.WithLen, c.extern_symbols.keys());
73	errdefer c.gpa.free(extern_symbols);
74
75	const decl_deps = try c.gpa.dupe(Decl.Index, c.decl_deps.keys());
76	errdefer c.gpa.free(decl_deps);
77
78	defer c.deinit();
79
80	return .{
81	.param_count = c.param_count,
82	.nodes = c.nodes.toOwnedSlice(),
83	.extern_symbols = extern_symbols,
84	.decl_deps = decl_deps,
85	.extra = extra,
86	};
87	}
88
89	pub fn build(c: *JsNodeGen) !void {
90	assert(c.air.instTag(.main_block) == .block);
91	const statements = c.air.instData(.block, .main_block).slice(c.air);
92
93	const scratch_top = c.scratch.items.len;
94	defer c.scratch.shrinkRetainingCapacity(scratch_top);
95
96	_ = try c.add(.root, undefined);
97
98	for (statements[0..c.param_count], 1..) \|stmt, expected_id\| {
99	assert(@intFromEnum(stmt) == expected_id);
100	assert(c.air.instTag(stmt) == .arg);
101	c.air_node_map[@intFromEnum(stmt)] = .{
102	.node = try c.add(.local, .{ .id = @intFromEnum(stmt) - 1 }),
103	};
104	}
105
106	for (statements[c.param_count..]) \|stmt\| {
107	try c.codeGenInst(stmt);
108	}
109
110	const items = c.scratch.items[scratch_top..];
111	const top = c.extra.list.items.len;
112	try c.extra.list.appendSlice(c.gpa, items);
113	c.nodes.items(.data)[0] = EncodedData.encode(.root, .{
114	.start = @intCast(top),
115	.len = @intCast(items.len),
116	});
117	}
118
119	fn codeGenInst(c: *JsNodeGen, inst: Air.Inst.Index) Error!void {
120	std.debug.print("JsNodeGen: #{d}\n", .{@intFromEnum(inst)});
121	switch (c.air.instTag(inst)) {
122	.arg => unreachable, // codegen must skip over these instructions.
123
124	.block => try c.codeGenBlock(inst),
125
126	.js_global_ref => {
127	const data: InternPool.String.WithLen = c.air.instData(.js_global_ref, inst);
128	try c.extern_symbols.put(c.gpa, data, {});
129	c.air_node_map[@intFromEnum(inst)] = .{
130	.node = try c.add(.global, .{ .name = data }),
131	};
132	},
133
134	inline .gt,
135	.lt,
136	.eq,
137	.add,
138	.sub,
139	.mul,
140	.div,
141	=> \|tag\| {
142	const data: Air.Bin = c.air.instData(tag, inst);
143	const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
144	.lhs = try c.fetchExpression(data.lhs),
145	.rhs = try c.fetchExpression(data.rhs),
146	});
147	try c.markExpression(inst, node);
148	},
149
150	inline .negate,
151	=> \|tag\| {
152	const data: Air.Un = c.air.instData(tag, inst);
153	const node = try c.add(@field(JsTree.Tag, @tagName(tag)), .{
154	.operand = try c.fetchExpression(data.operand),
155	});
156	try c.markExpression(inst, node);
157	},
158
159	.field => {
160	const data: Air.Field = c.air.instData(.field, inst);
161	const node = try c.add(.field, .{
162	.base = try c.fetchExpression(data.base),
163	.field = data.field,
164	});
165	try c.markExpression(inst, node);
166	},
167
168	.call => {
169	const data: Air.Call = c.air.instData(.call, inst);
170	const call_data = data.call_data.get(c.air);
171
172	try c.extra.list.ensureUnusedCapacity(c.gpa, call_data.args.len);
173	const start = c.extra.list.items.len;
174	c.extra.list.items.len += call_data.args.len;
175
176	const items = c.extra.list.items[start..][0..call_data.args.len];
177
178	for (call_data.args.slice(c.air), items) \|arg, *dest\| {
179	dest.* = @intFromEnum(try c.fetchExpression(arg));
180	}
181
182	const node = try c.add(.call, .{
183	.base = try c.fetchExpression(data.base),
184	.args = try c.extra.append(c.gpa, JsTree.IndexList, .{
185	.start = @intCast(start),
186	.len = call_data.args.len,
187	}),
188	});
189	try c.markExpression(inst, node);
190	},
191
192	.@"if" => {
193	const data: Air.If = c.air.instData(.@"if", inst);
194	const extra = data.data.get(c.air);
195	var cond = try c.fetchExpression(data.cond);
196
197	if (c.canLowerToExpression(extra.yes) and
198	c.canLowerToExpression(extra.no))
199	{
200	const yes = try c.lowerToExpression(extra.yes);
201	const no = try c.lowerToExpression(extra.yes);
202
203	const ternary = try c.add(.ternary, .{
204	.cond = cond,
205	.data = try c.extra.append(c.gpa, JsTree.Ternary.Data, .{
206	.yes = yes,
207	.no = no,
208	}),
209	});
210
211	try c.markExpression(inst, ternary);
212	} else {
213	const flags = c.air.instFlags(inst);
214	const uses = flags.uses;
215
216	const target = if (uses != .none)
217	try c.addTemporary()
218	else
219	null;
220
221	var yes = try c.codeGenBranch(extra.yes, target);
222	var no = try c.codeGenBranch(extra.no, target);
223	assert(!((yes == null) and (no == null))); // no "void else void" branches
224
225	// when `yes` is void, swap the condition so that the `else` is omitted
226	if (yes == null) {
227	const temp_yes = yes;
228	yes = no;
229	no = temp_yes;
230	cond = try c.add(.bool_not, .{
231	.operand = cond,
232	});
233	}
234
235	const main_expr = try c.add(.stmt_if, .{
236	.cond = cond,
237	.data = try c.extra.append(c.gpa, JsTree.StmtIf.Data, .{
238	.yes = yes.?.list,
239	.no = if (no) \|branch\|
240	branch.list
241	else
242	.{ .start = 0, .len = 0 },
243	.flags = .{
244	.block_label = @intCast(c.next_label + 1),
245	.yes_label = yes.?.requires_label,
246	.no_label = if (no) \|branch\| branch.requires_label else false,
247	},
248	}),
249	});
250
251	try c.addStmt(main_expr);
252
253	if (target) \|node\| {
254	try c.markExpression(inst, node);
255	}
256	}
257	},
258
259	.@"break" => {
260	const brk: Air.Break = c.air.instData(.@"break", inst);
261
262	const operand = try c.fetchExpression(brk.value);
263
264	// break #0 -> return
265	if (brk.block_ref == .main_block) {
266	try c.addStmt(try c.add(.stmt_return, .{
267	.operand = operand.toOptional(),
268	}));
269	} else {
270	const block = &c.air_node_map[@intFromEnum(brk.block_ref)].block;
271
272	const block_insts = c.air.instData(.block, brk.block_ref).slice(c.air);
273	if (block_insts[block_insts.len - 1] != inst) {
274	block.used = true;
275	}
276
277	const assignment: Node.Index.Optional = @enumFromInt(c.scratch.items[block.index]);
278	if (assignment.unwrap()) \|target\| {
279	try c.addStmt(try c.add(.assign, .{
280	.lhs = target,
281	.rhs = operand,
282	}));
283	}
284
285	try c.addStmt(try c.add(.stmt_break, .{
286	.label = c.scratch.items[block.index + 1],
287	}));
288	}
289	},
290	}
291	}
292
293	fn markExpression(c: *JsNodeGen, inst: Air.Inst.Index, node: Node.Index) !void {
294	const flags: Air.Flags = c.air.instFlags(inst);
295	const i = @intFromEnum(inst);
296	switch (flags.uses) {
297	.none => {
298	try c.addStmt(node);
299	},
300	.single => {
301	// TODO: this is not aware of side effect fences
302	c.air_node_map[i] = .{ .node = node };
303	},
304	.multi => {
305	const decl = try c.add(.stmt_declare_local, .{
306	.identifier_index = c.next_id,
307	.init = node.toOptional(),
308	});
309	try c.addStmt(decl);
310	const id = try c.add(.local, .{ .id = c.next_id });
311	c.next_id += 1;
312	c.air_node_map[i] = .{ .node = id };
313	},
314	}
315	}
316
317	fn fetchExpression(c: *JsNodeGen, inst: Air.Inst.Ref) !Node.Index {
318	return switch (inst.unwrap()) {
319	.value => \|v\| c.valueToExpression(v),
320	.inst => \|index\| {
321	const data = c.air_node_map[@intFromEnum(index)];
322	if (std.debug.runtime_safety) {
323	const flags: Air.Flags = c.air.instFlags(index);
324	switch (flags.uses) {
325	.none => unreachable, // incorrectly marked instruction as unused,
326	.single => {},
327	.multi => assert(c.nodes.items(.tag)[@intFromEnum(data.node)] == .local),
328	}
329	}
330	return data.node;
331	},
332	};
333	}
334
335	fn valueToExpression(c: *JsNodeGen, v: Value) !Node.Index {
336	const intern_tag = c.ip.items.items(.tag)[@intFromEnum(v.toIntern())];
337	switch (intern_tag) {
338	.removed => unreachable, // UAF
339
340	.type_function, .simple_type => unreachable, // Type values are not allowed here
341
342	.int32, .float, .string, .simple_value => {
343	return try c.add(.plain_value, .{ .value = v });
344	},
345
346	// funcs always point to decls
347	.func => {
348	const func = v.toKey(c.ip).func;
349	try c.decl_deps.put(c.gpa, func.owner_decl, {});
350	return try c.add(.decl, .{ .index = func.owner_decl });
351	},
352	}
353	}
354
355	fn addTemporary(c: *JsNodeGen) !Node.Index {
356	try c.addStmt(try c.add(.stmt_declare_local, .{
357	.identifier_index = c.next_id,
358	.init = .none,
359	}));
360	const id = try c.add(.local, .{ .id = c.next_id });
361	c.next_id += 1;
362	return id;
363	}
364
365	const Branch = struct {
366	requires_label: bool,
367	list: JsTree.IndexList,
368	};
369
370	fn codeGenBranch(c: *JsNodeGen, inst: Air.Inst.Ref, temporary: ?Node.Index) !?Branch {
371	const scratch_top = c.scratch.items.len;
372	defer c.scratch.shrinkRetainingCapacity(scratch_top);
373
374	c.next_label += 1;
375	defer c.next_label -= 1;
376	const this_label = c.next_label;
377
378	var requires_label = false;
379	var items = switch (inst.unwrap()) {
380	.value => \|v\| items: {
381	if (v.ip_index == .void_value)
382	return null;
383	if (temporary) \|temp\| {
384	const node = try c.valueToExpression(v);
385	try c.addStmt(try c.add(.assign, .{
386	.lhs = temp,
387	.rhs = node,
388	}));
389	}
390	break :items c.scratch.items[scratch_top..];
391	},
392	.inst => \|i\| items: {
393	assert(c.air.instTag(i) == .block);
394	c.air_node_map[@intFromEnum(i)] = .{
395	.block = .{ .index = @intCast(scratch_top) },
396	};
397
398	try c.scratch.appendSlice(c.arena, &.{
399	@intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
400	this_label,
401	});
402
403	std.debug.print("requires_label {}\n", .{requires_label});
404	const block: Air.IndexList = c.air.instData(.block, i);
405	for (block.slice(c.air)) \|item\| {
406	try c.codeGenInst(item);
407	}
408
409	requires_label = c.air_node_map[@intFromEnum(i)].block.used;
410
411	break :items c.scratch.items[scratch_top + 2 ..];
412	},
413	};
414
415	const last_node = c.scratch.items[c.scratch.items.len - 1];
416	if (c.nodes.items(.tag)[last_node] == .stmt_break and
417	c.nodes.items(.data)[last_node].decode(.stmt_break, .stmt_break).label == this_label)
418	{
419	items.len -= 1;
420	c.nodes.len -= 1;
421	}
422
423	const top = c.extra.list.items.len;
424	try c.extra.list.appendSlice(c.gpa, items);
425	return .{
426	.requires_label = requires_label,
427	.list = .{
428	.start = @intCast(top),
429	.len = @intCast(items.len),
430	},
431	};
432	}
433
434	fn codeGenBlock(c: *JsNodeGen, inst: Air.Inst.Index) !void {
435	const scratch_top = c.scratch.items.len;
436	errdefer c.scratch.shrinkRetainingCapacity(scratch_top);
437
438	c.next_label += 1;
439	defer c.next_label -= 1;
440
441	const next_label = c.next_label;
442
443	try c.scratch.appendSlice(c.arena, &.{
444	// @intFromEnum(if (temporary) \|temp\| temp.toOptional() else .none),
445	@intFromEnum(Node.Index.Optional.none),
446	next_label,
447	});
448
449	c.air_node_map[@intFromEnum(inst)] = .{
450	.block = .{ .index = @intCast(scratch_top) },
451	};
452	const block: Air.IndexList = c.air.instData(.block, inst);
453
454	for (block.slice(c.air)) \|item\| {
455	try c.codeGenInst(item);
456	}
457
458	const items = c.scratch.items[scratch_top + 2 ..];
459	const top = c.extra.list.items.len;
460	try c.extra.list.appendSlice(c.gpa, items);
461	c.scratch.shrinkRetainingCapacity(scratch_top);
462	try c.addStmt(try c.add(.stmt_block, .{
463	.label = c.next_label,
464	.content = try c.extra.append(c.gpa, JsTree.IndexList, .{
465	.start = @intCast(top),
466	.len = @intCast(items.len),
467	}),
468	}));
469	}
470
471	fn canLowerToExpression(c: *const JsNodeGen, inst: Air.Inst.Ref) bool {
472	return switch (inst.unwrap()) {
473	.value => true,
474	.inst => \|index\| switch (c.air.instTag(index)) {
475	.js_global_ref,
476	.call,
477	.arg,
478	.field,
479	.add,
480	.sub,
481	.mul,
482	.div,
483	.eq,
484	.lt,
485	.gt,
486	.negate,
487	=> true,
488	.@"break",
489	=> false,
490	// TODO: these depend
491	.block => false,
492	.@"if" => false,
493	},
494	};
495	}
496
497	fn lowerToExpression(c: *JsNodeGen, ref: Air.Inst.Ref) !Node.Index {
498	assert(c.canLowerToExpression(ref));
499	return switch (ref.unwrap()) {
500	.value => \|v\| try c.valueToExpression(v),
501	.inst => \|index\| {
502	try c.codeGenInst(index);
503	return c.fetchExpression(ref);
504	},
505	};
506	}
507
508	fn add(c: *JsNodeGen, comptime tag: JsTree.Tag, data: tag.Data()) !Node.Index {
509	try c.nodes.append(c.gpa, .{
510	.tag = tag,
511	.data = EncodedData.encode(tag, data),
512	});
513	return @enumFromInt(c.nodes.len - 1);
514	}
515
516	fn addStmt(c: *JsNodeGen, stmt: Node.Index) !void {
517	try c.scratch.append(c.arena, @intFromEnum(stmt));
518	}
519
520	const JsNodeGen = @This();
521
522	const std = @import("std");
523	const assert = std.debug.assert;
524	const Allocator = std.mem.Allocator;
525
526	const InternPool = @import("InternPool.zig");
527	const NullTerminatedString = InternPool.NullTerminatedString;
528
529	const Comp = @import("Comp.zig");
530	const Decl = Comp.Decl;
531
532	const JsTree = @import("JsTree.zig");
533	const EncodedData = JsTree.EncodedData;
534	const Node = JsTree.Node;
535	const LocalIndex = JsTree.LocalIndex;
536
537	const Air = @import("Air.zig");
538
539	const Value = @import("Value.zig");
540
541	const dod = @import("dod.zig");

src/compiler/JsTree.zig created+245

...	@@ -0,0 +1,245 @@
1	//! JavaScript Function Representation is the last representation before javascript
2	//! source bytes are created. One notable thing here is that all local identifiers
3	//! are numbered, instead of being named. The first `param_count` local identifiers
4	//! refer to parameters, and the rest refer to the respective block that defined them.
5
6	param_count: u32,
7	nodes: Node.List.Slice,
8	extern_symbols: []String.WithLen,
9	decl_deps: []Decl.Index,
10	extra: ExtraStore,
11
12	pub const LocalIndex = u32;
13	pub const ExtraStore = dod.ExtraStore(.tfr, .@"const");
14	pub const EncodedData = dod.EncodedData(Tag, u64);
15	pub const ExtraIndex = ExtraStore.Index;
16
17	pub const Tag = enum {
18	root,
19	/// Only allow simple value types: int, float, string, bool
20	plain_value,
21	/// Reference to a local identifier (give an VarId)
22	local,
23	/// Reference to a global identifier (give a NullTerminatedString)
24	global,
25	/// Reference to a declaration (contains a Decl.Index)
26	decl,
27	// bin ops
28	assign,
29	add,
30	sub,
31	mul,
32	div,
33	eq,
34	lt,
35	gt,
36	// exprs
37	call,
38	field,
39	ternary,
40	// un ops
41	negate,
42	bool_not,
43	// stmts
44	/// `let ... = ...`
45	stmt_declare_local,
46	/// `if (...) ... else ...`
47	stmt_if,
48	/// `break ... ...`
49	stmt_break,
50	/// `return ...` (break from block #0)
51	stmt_return,
52	/// `...: { ... }`
53	stmt_block,
54
55	pub fn Data(comptime tag: Tag) type {
56	return switch (tag) {
57	.root => IndexList,
58	.plain_value => PlainValue,
59	.local => LocalRef,
60	.global => GlobalRef,
61	.decl => DeclRef,
62	.assign,
63	.add,
64	.sub,
65	.mul,
66	.div,
67	.eq,
68	.lt,
69	.gt,
70	=> Bin,
71	.ternary => Ternary,
72	.negate,
73	.bool_not,
74	=> Un,
75	.call => Call,
76	.field => Field,
77	.stmt_if => StmtIf,
78	.stmt_break => StmtBreak,
79	.stmt_return => StmtReturn,
80	.stmt_declare_local => StmtDeclareLocal,
81	.stmt_block => StmtBlock,
82	};
83	}
84	};
85
86	pub const Node = struct {
87	tag: Tag,
88	data: EncodedData,
89
90	pub const List = std.MultiArrayList(Node);
91
92	pub const Index = enum(u32) {
93	root = 0,
94	_,
95
96	pub fn toOptional(i: Index) Optional {
97	return @enumFromInt(@intFromEnum(i));
98	}
99
100	pub const Optional = enum(u32) {
101	none = std.math.maxInt(u32),
102	_,
103
104	pub fn unwrap(oi: Optional) ?Index {
105	return if (oi == .none)
106	null
107	else
108	@enumFromInt(@intFromEnum(oi));
109	}
110	};
111	};
112	};
113
114	pub const PlainValue = struct {
115	value: Value,
116	unused: Unused = .zero,
117	};
118
119	pub const LocalRef = struct {
120	id: LocalIndex,
121	unused: Unused = .zero,
122	};
123
124	pub const GlobalRef = struct {
125	name: InternPool.String.WithLen,
126	};
127
128	pub const DeclRef = struct {
129	index: Decl.Index,
130	unused: Unused = .zero,
131	};
132
133	pub const StmtDeclareLocal = struct {
134	identifier_index: u32,
135	init: Node.Index.Optional,
136	};
137
138	pub const Bin = struct {
139	lhs: Node.Index,
140	rhs: Node.Index,
141	};
142
143	pub const Un = struct {
144	operand: Node.Index,
145	unused: Unused = .zero,
146	};
147
148	pub const Call = struct {
149	base: Node.Index,
150	args: ExtraIndex(IndexList),
151	};
152
153	pub const Field = struct {
154	base: Node.Index,
155	field: NullTerminatedString,
156	};
157
158	pub const IndexList = struct {
159	start: u32,
160	len: u32,
161
162	pub fn slice(list: IndexList, tree: *const JsTree) []const Node.Index {
163	return @ptrCast(tree.extra.list.items[list.start..][0..list.len]);
164	}
165	};
166
167	pub const StmtIf = struct {
168	cond: Node.Index,
169	data: ExtraIndex(Data),
170
171	pub const Data = struct {
172	yes: IndexList,
173	no: IndexList,
174	flags: Flags,
175	};
176
177	pub const Flags = packed struct(u32) {
178	block_label: u16,
179	yes_label: bool,
180	no_label: bool,
181	_: u14 = 0,
182	};
183	};
184
185	pub const Ternary = struct {
186	cond: Node.Index,
187	data: ExtraIndex(Data),
188
189	pub const Data = struct {
190	yes: Node.Index,
191	no: Node.Index,
192	};
193	};
194
195	pub const StmtBreak = struct {
196	label: u32,
197	unused: Unused = .zero,
198	};
199
200	pub const StmtReturn = struct {
201	operand: Node.Index.Optional,
202	unused: Unused = .zero,
203	};
204
205	pub const StmtBlock = struct {
206	label: u32,
207	content: ExtraIndex(IndexList),
208	};
209
210	pub fn deinit(tree: *JsTree, gpa: std.mem.Allocator) void {
211	tree.nodes.deinit(gpa);
212	gpa.free(tree.decl_deps);
213	gpa.free(tree.extern_symbols);
214	tree.extra.deinit(gpa);
215	}
216
217	pub fn nodeTag(tree: JsTree, index: Node.Index) Tag {
218	return tree.nodes.items(.tag)[@intFromEnum(index)];
219	}
220
221	pub fn nodeData(tree: JsTree, comptime tag: Tag, index: Node.Index) tag.Data() {
222	return tree.nodes.items(.data)[@intFromEnum(index)].decode(tag, tree.nodeTag(index));
223	}
224
225	pub fn mainStatements(tree: *const JsTree) []const Node.Index {
226	return tree.nodeData(.root, .root).slice(tree);
227	}
228
229	const JsTree = @This();
230
231	const std = @import("std");
232	const assert = std.debug.assert;
233
234	const InternPool = @import("InternPool.zig");
235	const NullTerminatedString = InternPool.NullTerminatedString;
236	const String = InternPool.String;
237
238	const Comp = @import("Comp.zig");
239	const Decl = Comp.Decl;
240
241	const dod = @import("dod.zig");
242
243	const Value = @import("Value.zig");
244
245	const Unused = @import("Ast.zig").Unused;

src/compiler/Parse.zig created+961

...	@@ -0,0 +1,961 @@
1	//! Parser takes a Token list and creates a Node list. It does not own the source code or
2	//! token data, however transferring into an `Ast` does move token list ownership.
3	gpa: Allocator,
4	source: []const u8,
5	token_tags: []const Token.Tag,
6	token_starts: []const ByteOffset,
7
8	pos: TokenIndex = 0,
9	nodes: Node.List = .{},
10	extra: ArrayListUnmanaged(u32) = .{},
11	scratch: ArrayListUnmanaged(NodeIndex) = .{},
12
13	ast_error: ?AstError = null,
14
15	pub fn init(gpa: Allocator, source: []const u8, tokens: Token.List.Slice) Parse {
16	return Parse{
17	.gpa = gpa,
18	.source = source,
19	.token_tags = tokens.items(.tag),
20	.token_starts = tokens.items(.offset),
21	};
22	}
23
24	/// This transfers the parse result into an Ast
25	pub fn finalize(p: *Parse, token_slice: Token.List.Slice) !Ast {
26	const extra = try p.extra.toOwnedSlice(p.gpa);
27	defer p.* = undefined;
28	assert(token_slice.items(.tag).ptr == p.token_tags.ptr);
29	assert(token_slice.items(.offset).ptr == p.token_starts.ptr);
30	p.scratch.deinit(p.gpa);
31
32	const ast: Ast = .{
33	.utf8_source = p.source,
34	.tokens = token_slice,
35	.nodes = p.nodes.toOwnedSlice(),
36	.extra = extra,
37	};
38
39	ast.dumpToWriter(std.io.getStdErr().writer()) catch {};
40
41	return ast;
42	}
43
44	/// This discards all parser work
45	pub fn deinit(p: *Parse) void {
46	defer p.* = undefined;
47	p.nodes.deinit(p.gpa);
48	p.extra.deinit(p.gpa);
49	p.scratch.deinit(p.gpa);
50	}
51
52	pub const Error = error{
53	SyntaxError,
54	};
55
56	/// When error.SyntaxError is returned, `p.err` is set to this as well as `p.pos`
57	pub const AstError = union(enum) {
58	expected_token: Token.Tag,
59	expected_statement,
60	expected_expr,
61	expected_else_value,
62	};
63
64	pub fn parseRoot(p: *Parse) !void {
65	const trace = t.trace(@src());
66	defer trace.end();
67
68	assert(p.nodes.len == 0);
69	assert(p.extra.items.len == 0);
70	assert(p.pos == 0);
71	assert(p.nodes.capacity > 0);
72
73	p.nodes.appendAssumeCapacity(.{
74	.tag = .root,
75	.main_token = 0,
76	.data = undefined,
77	});
78
79	const list = try p.parseModuleMembers();
80	p.nodes.items(.data)[0] = .{
81	.list = list,
82	};
83	}
84
85	fn parseModuleMembers(p: *Parse) !SubList {
86	const trace = t.trace(@src());
87	defer trace.end();
88
89	const scratch_top = p.scratch.items.len;
90	defer p.scratch.shrinkRetainingCapacity(scratch_top);
91
92	// skip module doc comment
93	while (p.eatToken(.module_doc_comment)) \|_\| {}
94
95	while (true) switch (p.currentTag()) {
96	.kw_let,
97	.kw_var,
98	.kw_fn,
99	.kw_pub,
100	=> try p.scratch.append(p.gpa, try p.parseModuleDecl()),
101
102	.kw_struct,
103	.kw_enum,
104	.kw_trait,
105	.kw_impl,
106	=> return error.Todo,
107
108	.eof => break,
109
110	else => {
111	if (!p.isExpressionStart()) {
112	return p.fail(.expected_statement);
113	}
114
115	try p.scratch.append(p.gpa, try p.parseExpression());
116	_ = try p.expect(.semi);
117	},
118	};
119
120	return p.scratchToSubList(scratch_top);
121	}
122
123	fn parseModuleDecl(p: *Parse) !NodeIndex {
124	const trace = t.trace(@src());
125	defer trace.end();
126
127	_ = p.eatToken(.kw_pub);
128
129	return switch (p.currentTag()) {
130	.kw_var, .kw_let => try p.parseVarDecl(),
131
132	.kw_fn => try p.parseFnDecl(),
133
134	else => unreachable,
135	};
136	}
137
138	fn parseFnDecl(p: *Parse) !NodeIndex {
139	const trace = t.trace(@src());
140	defer trace.end();
141
142	const main_token = p.assertToken(.kw_fn);
143
144	_ = try p.expect(.id);
145
146	_ = try p.expect(.start_paren);
147
148	const params = try p.parseParamList();
149
150	const return_type = if (p.eatToken(.minus_lt)) \|_\|
151	try p.parseExpression()
152	else
153	null;
154
155	try p.expectWithoutAdvance(.start_brace);
156	const body = try p.parseBlock();
157
158	const extra = try p.addExtra(Ast.FnDecl, .{
159	.params = params,
160	.return_type = optionalNodeIndex(return_type),
161	});
162
163	return try p.addNode(.{
164	.tag = .fn_decl,
165	.main_token = main_token,
166	.data = .{ .fn_decl = .{
167	.body = body,
168	.extra = extra,
169	} },
170	});
171	}
172
173	fn parseParamList(p: *Parse) !SubList {
174	const trace = t.trace(@src());
175	defer trace.end();
176
177	const scratch_top = p.scratch.items.len;
178	defer p.scratch.shrinkRetainingCapacity(scratch_top);
179
180	while (true) {
181	const trace_param = t.traceFmt(@src(), "Param {d}", .{p.scratch.items.len - scratch_top});
182	defer trace_param.end();
183
184	if (p.currentTag() == .end_paren) break;
185
186	try p.scratch.append(p.gpa, try p.parseParamDecl());
187
188	switch (p.currentTag()) {
189	.comma => {
190	_ = p.nextToken();
191	},
192	.end_paren => break,
193	else => return p.fail(.{ .expected_token = .comma }),
194	}
195	}
196
197	_ = p.assertToken(.end_paren);
198
199	return p.scratchToSubList(scratch_top);
200	}
201
202	fn parseParamDecl(p: *Parse) !NodeIndex {
203	const trace = t.trace(@src());
204	defer trace.end();
205
206	const main_token = p.assertToken(.id);
207
208	_ = try p.expect(.colon);
209
210	const @"type" = try p.parseExpression();
211
212	return try p.addNode(.{
213	.tag = .param_decl,
214	.main_token = main_token,
215	.data = .{ .param_decl = .{
216	.type = @"type",
217	} },
218	});
219	}
220
221	fn parseBlock(p: *Parse) !NodeIndex {
222	const trace = t.trace(@src());
223	defer trace.end();
224
225	const main_token = p.assertToken(.start_brace);
226
227	const tag, const list = try p.parseBlockContents();
228	assert(p.lastTag() == .end_brace);
229
230	return try p.addNode(.{
231	.tag = tag,
232	.main_token = main_token,
233	.data = .{
234	.list = list,
235	},
236	});
237	}
238
239	fn parseLabelExpression(p: *Parse) !NodeIndex {
240	const trace = t.trace(@src());
241	defer trace.end();
242
243	const main_token = p.assertToken(.id);
244	_ = try p.expect(.colon);
245	const expr = try p.parseExpression();
246	return try p.addNode(.{
247	.tag = .label,
248	.main_token = main_token,
249	.data = .{ .label = .{
250	.expr = expr,
251	} },
252	});
253	}
254
255	fn parseBlockContents(p: *Parse) !struct { Node.Tag, SubList } {
256	const trace = t.trace(@src());
257	defer trace.end();
258
259	assert(p.lastTag() == .start_brace);
260	const scratch_top = p.scratch.items.len;
261	defer p.scratch.shrinkRetainingCapacity(scratch_top);
262
263	const tag: Node.Tag = while (true) switch (p.currentTag()) {
264	.kw_let,
265	.kw_var,
266	=> try p.scratch.append(p.gpa, try p.parseVarDecl()),
267
268	.end_brace => break .block_void,
269
270	else => {
271	if (!p.isExpressionStart()) {
272	return p.fail(.expected_statement);
273	}
274
275	const first_tag = p.currentTag();
276
277	try p.scratch.append(p.gpa, try p.parseExpression());
278
279	switch (first_tag) {
280	// some control flow and blocks are allowed to not have a
281	// trailing semicolons. this is a stylistic choice.
282	.kw_if,
283	.start_brace,
284	=> {
285	// ending a block here should also count as an implicit return,
286	// even though voiding it with a semicolon is not allowed.
287	if (p.currentTag() == .end_brace) break .block_implicit_return;
288	},
289
290	// TODO: make kw_break and kw_return REQUIRE a semicolon, as
291	// they are noreturn statements, doing implicit break on
292	// them makes no sense
293
294	else => {
295	if (p.eatToken(.semi) == null) {
296	// this must be last statement in block if it isnt,
297	// the p.expect(.end_brace) later will fail.
298	break .block_implicit_return;
299	}
300	},
301	}
302	},
303	};
304
305	_ = try p.expect(.end_brace);
306
307	return .{ tag, try p.scratchToSubList(scratch_top) };
308	}
309
310	/// VarDecl <- (kw_pub)? (kw_var \| kw_let) id (colon expr)? equal expr semi
311	fn parseVarDecl(p: *Parse) !NodeIndex {
312	const trace = t.trace(@src());
313	defer trace.end();
314
315	const kind = p.currentTag();
316	assert(kind == .kw_var or kind == .kw_let);
317
318	const main_token = p.nextToken();
319
320	_ = try p.expect(.id);
321
322	const data_type: OptionalNodeIndex = if (p.eatToken(.colon)) \|_\|
323	optionalNodeIndex(try p.parseExpression())
324	else
325	.none;
326
327	_ = try p.expect(.equal);
328
329	const data_init = try p.parseExpression();
330
331	_ = try p.expect(.semi);
332
333	return try p.addNode(.{
334	.tag = .var_decl,
335	.main_token = main_token,
336	.data = .{ .var_decl = .{
337	.type = data_type,
338	.init = data_init,
339	} },
340	});
341	}
342
343	fn isExpressionStart(p: *Parse) bool {
344	return switch (p.currentTag()) {
345	.id,
346	.number_int,
347	.number_float,
348	.string,
349	.builtin_id,
350	.kw_return,
351	.kw_break,
352	.kw_if,
353	.start_paren,
354	.start_brace,
355	.bang,
356	.tilde,
357	.minus,
358	=> true,
359	else => false,
360	};
361	}
362
363	const Operator = enum(u8) {
364	add = @intFromEnum(Token.Tag.plus),
365	sub = @intFromEnum(Token.Tag.minus),
366	mul = @intFromEnum(Token.Tag.star),
367	div = @intFromEnum(Token.Tag.slash),
368	rem = @intFromEnum(Token.Tag.percent),
369	pow = @intFromEnum(Token.Tag.star_star),
370	@"and" = @intFromEnum(Token.Tag.and_and),
371	@"or" = @intFromEnum(Token.Tag.pipe_pipe),
372	bit_and = @intFromEnum(Token.Tag.@"and"),
373	bit_or = @intFromEnum(Token.Tag.pipe),
374	bit_xor = @intFromEnum(Token.Tag.caret),
375	shl = @intFromEnum(Token.Tag.lt_lt),
376	shr = @intFromEnum(Token.Tag.gt_gt),
377	eq = @intFromEnum(Token.Tag.equal_equal),
378	ne = @intFromEnum(Token.Tag.bang_equal),
379	lt = @intFromEnum(Token.Tag.lt),
380	le = @intFromEnum(Token.Tag.lt_equal),
381	gt = @intFromEnum(Token.Tag.gt),
382	ge = @intFromEnum(Token.Tag.gt_equal),
383	assign = @intFromEnum(Token.Tag.equal),
384	add_assign = @intFromEnum(Token.Tag.plus_equal),
385	sub_assign = @intFromEnum(Token.Tag.minus_equal),
386	mul_assign = @intFromEnum(Token.Tag.star_equal),
387	div_assign = @intFromEnum(Token.Tag.slash_equal),
388	rem_assign = @intFromEnum(Token.Tag.percent_equal),
389	pow_assign = @intFromEnum(Token.Tag.star_star_equal),
390	and_assign = @intFromEnum(Token.Tag.and_and_equal),
391	or_assign = @intFromEnum(Token.Tag.pipe_pipe_equal),
392	bit_and_assign = @intFromEnum(Token.Tag.and_equal),
393	bit_or_assign = @intFromEnum(Token.Tag.pipe_equal),
394	bit_xor_assign = @intFromEnum(Token.Tag.caret_equal),
395	shl_assign = @intFromEnum(Token.Tag.lt_lt_equal),
396	shr_assign = @intFromEnum(Token.Tag.gt_gt_equal),
397
398	const Direction = enum { ltr, rtl };
399
400	/// Only the binary operator levels are actually used,
401	/// but all are listed for reference.
402	const Level = enum {
403	/// e. e[] e()
404	accessor,
405	/// new
406	new,
407	/// ++e, --e, !e, ~e, +e, -e, typeof, void, delete
408	prefix,
409	/// **
410	exponentiation,
411	/// *, /, %
412	multiplicative,
413	/// +, -
414	additive,
415	/// <<, >>, >>>
416	bitwise_shift,
417	/// <, <=, >, >=
418	relational,
419	/// ==, !=
420	equality,
421	/// &
422	bit_and,
423	/// ^
424	bit_xor,
425	/// \|
426	bit_or,
427	/// &&
428	logical_and,
429	/// \|\|, ??
430	logical_or,
431	/// =, +=, -=, *=, /=, %=, / <<=, >>=, >>>=, &=, ^=, \|=
432	assign,
433	};
434
435	pub fn direction(op: Operator) Direction {
436	return switch (op) {
437	.pow => .rtl,
438
439	.assign,
440	.add_assign,
441	.sub_assign,
442	.mul_assign,
443	.div_assign,
444	.rem_assign,
445	.pow_assign,
446	.and_assign,
447	.or_assign,
448	.bit_and_assign,
449	.bit_or_assign,
450	.bit_xor_assign,
451	.shl_assign,
452	.shr_assign,
453	=> .rtl,
454
455	else => .ltr,
456	};
457	}
458
459	pub fn level(op: Operator) Level {
460	return switch (op) {
461	.pow => .exponentiation,
462	.mul, .div, .rem => .multiplicative,
463	.add, .sub => .additive,
464	.shl, .shr => .bitwise_shift,
465	.lt, .le, .gt, .ge => .relational,
466	.eq, .ne => .equality,
467	.@"and" => .logical_and,
468	.@"or" => .logical_or,
469	.bit_and => .bit_and,
470	.bit_xor => .bit_xor,
471	.bit_or => .bit_or,
472	.assign,
473	.add_assign,
474	.sub_assign,
475	.mul_assign,
476	.div_assign,
477	.rem_assign,
478	.pow_assign,
479	.and_assign,
480	.or_assign,
481	.bit_and_assign,
482	.bit_or_assign,
483	.bit_xor_assign,
484	.shl_assign,
485	.shr_assign,
486	=> .assign,
487	};
488	}
489
490	const lut = brk: {
491	const token_tags = std.enums.values(Token.Tag);
492	const op_tags = std.enums.values(Operator);
493
494	var is_operator_lut: [token_tags.len]bool = .{false} ** token_tags.len;
495	for (op_tags) \|op\| {
496	is_operator_lut[@intFromEnum(op)] = true;
497	}
498
499	var ast_tag_lut: [token_tags.len]Node.Tag = .{.root} ** token_tags.len;
500	for (op_tags) \|op\| {
501	ast_tag_lut[@intFromEnum(op)] = @field(Node.Tag, @tagName(op));
502	}
503
504	break :brk struct {
505	const is_operator = is_operator_lut;
506	const ast_tag = ast_tag_lut;
507	};
508	};
509
510	pub fn fromToken(tag: Token.Tag) ?Operator {
511	return if (lut.is_operator[@intFromEnum(tag)])
512	@enumFromInt(@intFromEnum(tag))
513	else
514	null;
515	}
516
517	pub fn astTag(op: Operator) Node.Tag {
518	const tag = lut.ast_tag[@intFromEnum(op)];
519	assert(tag != .root); // invalid operator or lut construction
520	return tag;
521	}
522	};
523
524	fn parseExpression(p: *Parse) anyerror!NodeIndex {
525	const trace = t.trace(@src());
526	defer trace.end();
527
528	assert(p.isExpressionStart());
529
530	// These unary operators are greedy, so handle them first.
531	switch (p.currentTag()) {
532	.kw_return => return p.parseReturn(),
533	.kw_break => return p.parseBreak(),
534	else => {},
535	}
536
537	var expr = try p.parseExpressionPart();
538
539	while (Operator.fromToken(p.currentTag())) \|op\| {
540	expr = try p.parseBinarySuffix(expr, op);
541	}
542
543	return expr;
544	}
545
546	fn parseExpressionPart(p: *Parse) !NodeIndex {
547	const trace = t.trace(@src());
548	defer trace.end();
549
550	var expr = switch (p.currentTag()) {
551	// Literals
552	.id => if (p.nextTag() == .colon)
553	try p.parseLabelExpression()
554	else
555	try p.blankNode(.id),
556	.number_int => try p.blankNode(.number_int),
557	.number_float => try p.blankNode(.number_float),
558	.string => try p.blankNode(.string),
559
560	// Complex expressions
561	.builtin_id => try p.parseBuiltinCall(),
562	.kw_return => try p.parseReturn(),
563	.kw_break => try p.parseBreak(),
564	.kw_if => try p.parseIf(),
565	.start_paren => try p.parseGroup(),
566	.start_brace => try p.parseBlock(),
567
568	// Unary operators
569	inline .bang, .tilde, .minus => \|tag\| unary: {
570	const main_token = p.nextToken();
571	const expr = try p.parseExpressionPart();
572
573	break :unary try p.addNode(.{
574	.tag = switch (tag) {
575	.bang => .bool_not,
576	.tilde => .bit_not,
577	.minus => .negate,
578	else => comptime unreachable,
579	},
580	.main_token = main_token,
581	.data = .{ .un = .{
582	.expr = expr,
583	} },
584	});
585	},
586
587	else => return p.fail(.expected_expr),
588	};
589
590	// parse field, calls, and index expressions
591	while (true) {
592	const trace2 = t.traceFmt(@src(), "(field/call)", .{});
593	defer trace2.end();
594
595	switch (p.currentTag()) {
596	.dot => {
597	const main_token = p.nextToken();
598	const field = try p.expect(.id);
599
600	expr = try p.addNode(.{
601	.tag = .field,
602	.main_token = main_token,
603	.data = .{ .field = .{
604	.base = expr,
605	.field_token = field,
606	} },
607	});
608	},
609	.start_paren => {
610	const main_token = p.nextToken();
611	const args = try p.parseCallArgs();
612
613	const extra = try p.addExtra(Ast.Call, .{
614	.args = args,
615	});
616
617	expr = try p.addNode(.{
618	.tag = .call,
619	.main_token = main_token,
620	.data = .{ .call = .{
621	.base = expr,
622	.extra = extra,
623	} },
624	});
625	},
626	.start_bracket => {
627	const main_token = p.nextToken();
628	const index = try p.parseExpression();
629
630	expr = try p.addNode(.{
631	.tag = .index,
632	.main_token = main_token,
633	.data = .{ .bin = .{
634	.lhs = expr,
635	.rhs = index,
636	} },
637	});
638	},
639
640	else => break,
641	}
642	}
643
644	return expr;
645	}
646
647	fn parseBinarySuffix(p: *Parse, lhs: NodeIndex, op: Operator) !NodeIndex {
648	const trace = t.trace(@src());
649	defer trace.end();
650
651	const main_token = p.nextToken();
652	const level = op.level();
653	var rhs = try p.parseExpressionPart();
654
655	while (Operator.fromToken(p.currentTag())) \|next_op\| {
656	const new_level = next_op.level();
657	const use = switch (op.direction()) {
658	.ltr => @intFromEnum(new_level) < @intFromEnum(level),
659	.rtl => @intFromEnum(new_level) <= @intFromEnum(level),
660	};
661	if (!use) break;
662
663	rhs = try p.parseBinarySuffix(rhs, next_op);
664	}
665
666	return try p.addNode(.{
667	.tag = op.astTag(),
668	.main_token = main_token,
669	.data = .{ .bin = .{
670	.lhs = lhs,
671	.rhs = rhs,
672	} },
673	});
674	}
675
676	fn parseReturn(p: *Parse) !NodeIndex {
677	const trace = t.trace(@src());
678	defer trace.end();
679
680	const main_token = p.assertToken(.kw_return);
681
682	const expr = if (p.isExpressionStart())
683	try p.parseExpression()
684	else
685	null;
686
687	return try p.addNode(.{
688	.tag = .@"return",
689	.main_token = main_token,
690	.data = .{ .@"return" = .{
691	.expr = optionalNodeIndex(expr),
692	} },
693	});
694	}
695
696	fn parseBreak(p: *Parse) !NodeIndex {
697	const trace = t.trace(@src());
698	defer trace.end();
699
700	const main_token = p.assertToken(.kw_break);
701
702	const label = if (p.eatToken(.colon)) \|_\|
703	try p.expect(.id)
704	else
705	null;
706
707	const expr = if (p.isExpressionStart())
708	try p.parseExpression()
709	else
710	null;
711
712	return try p.addNode(.{
713	.tag = .@"break",
714	.main_token = main_token,
715	.data = .{ .@"break" = .{
716	.label = optionalTokenIndex(label),
717	.value = optionalNodeIndex(expr),
718	} },
719	});
720	}
721
722	fn parseBuiltinCall(p: *Parse) !NodeIndex {
723	const trace = t.trace(@src());
724	defer trace.end();
725
726	const main_token = p.assertToken(.builtin_id);
727
728	_ = try p.expect(.start_paren);
729
730	const args = try p.parseCallArgs();
731
732	return try p.addNode(.{
733	.tag = .builtin_call,
734	.main_token = main_token,
735	.data = .{ .list = args },
736	});
737	}
738
739	fn parseCallArgs(p: *Parse) anyerror!SubList {
740	const trace = t.trace(@src());
741	defer trace.end();
742
743	const scratch_top = p.scratch.items.len;
744	defer p.scratch.shrinkRetainingCapacity(scratch_top);
745
746	while (p.isExpressionStart()) {
747	try p.scratch.append(p.gpa, try p.parseExpression());
748
749	switch (p.currentTag()) {
750	.end_paren => {
751	break;
752	},
753	.comma => {
754	_ = p.nextToken();
755	if (p.currentTag() == .end_paren)
756	break;
757	},
758	else => return p.fail(.{ .expected_token = .end_paren }),
759	}
760	}
761
762	_ = try p.expect(.end_paren);
763
764	return p.scratchToSubList(scratch_top);
765	}
766
767	fn parseIf(p: *Parse) !NodeIndex {
768	const trace = t.trace(@src());
769	defer trace.end();
770
771	const main_token = p.assertToken(.kw_if);
772
773	const cond = try p.parseExpression();
774
775	_ = try p.expectWithoutAdvance(.start_brace);
776	const yes = try p.parseBlock();
777
778	const no = if (p.eatToken(.kw_else)) \|_\|
779	try p.parseElseValue()
780	else
781	null;
782
783	const extra = try p.addExtra(Ast.If, .{
784	.yes = yes,
785	.no = optionalNodeIndex(no),
786	});
787
788	return try p.addNode(.{
789	.tag = .@"if",
790	.main_token = main_token,
791	.data = .{ .@"if" = .{
792	.cond = cond,
793	.extra = extra,
794	} },
795	});
796	}
797
798	fn parseElseValue(p: *Parse) anyerror!NodeIndex {
799	const trace = t.trace(@src());
800	defer trace.end();
801
802	assert(p.lastTag() == .kw_else);
803
804	return switch (p.currentTag()) {
805	.start_brace => p.parseBlock(),
806	.kw_if => p.parseIf(),
807	.id => p.parseLabelExpression(),
808
809	else => return p.fail(.expected_else_value),
810	};
811	}
812
813	fn parseGroup(p: *Parse) !NodeIndex {
814	const trace = t.trace(@src());
815	defer trace.end();
816
817	const main_token = p.assertToken(.start_paren);
818
819	const expr = try p.parseExpression();
820
821	_ = try p.expect(.end_paren);
822
823	return try p.addNode(.{
824	.tag = .group,
825	.main_token = main_token,
826	.data = .{ .un = .{
827	.expr = expr,
828	} },
829	});
830	}
831
832	// helpers
833
834	/// Append a node and return it's index.
835	fn addNode(p: *Parse, node: Node) !NodeIndex {
836	const trace = t.trace(@src());
837	defer trace.end();
838
839	try p.nodes.append(p.gpa, node);
840	// @intCast is always safe because 2^32 tokens alwayss produce less than
841	// 2^32 nodes, and there is a 2^32 limit on source file bytes.
842	return @intCast(p.nodes.len - 1);
843	}
844
845	fn addExtra(p: *Parse, comptime T: type, extra: T) !ExtraIndex {
846	const trace = t.trace(@src());
847	defer trace.end();
848
849	const extra_top = p.extra.items.len;
850	const bytes = std.mem.asBytes(&extra);
851	try p.extra.appendSlice(p.gpa, std.mem.bytesAsSlice(u32, bytes));
852	return @intCast(extra_top);
853	}
854
855	fn fail(p: *Parse, ast_error: AstError) Error {
856	p.ast_error = ast_error;
857	std.debug.print("got {s}", .{@tagName(p.currentTag())});
858	return Error.SyntaxError;
859	}
860
861	fn blankNode(p: *Parse, comptime tag: Token.Tag) !NodeIndex {
862	const main_token = p.assertToken(tag);
863	return try p.addNode(.{
864	.tag = @field(Node.Tag, @tagName(tag)),
865	.main_token = main_token,
866	.data = .{ .none = .{} },
867	});
868	}
869
870	fn currentTag(p: *Parse) Token.Tag {
871	return p.token_tags[p.pos];
872	}
873
874	fn lastTag(p: *Parse) Token.Tag {
875	return p.token_tags[p.pos - 1];
876	}
877
878	fn nextTag(p: *Parse) Token.Tag {
879	return p.token_tags[p.pos + 1];
880	}
881
882	fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex {
883	return if (p.token_tags[p.pos] == tag) p.nextToken() else null;
884	}
885
886	fn expect(p: *Parse, tag: Token.Tag) !TokenIndex {
887	if (p.currentTag() != tag) {
888	return p.fail(.{ .expected_token = tag });
889	}
890	return p.nextToken();
891	}
892
893	fn expectWithoutAdvance(p: *Parse, tag: Token.Tag) !void {
894	if (p.currentTag() != tag) {
895	return p.fail(.{ .expected_token = tag });
896	}
897	}
898
899	fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex {
900	const token = p.nextToken();
901	assert(p.token_tags[token] == tag);
902	return token;
903	}
904
905	fn nextToken(p: *Parse) TokenIndex {
906	const result = p.pos;
907	p.pos += 1;
908	return result;
909	}
910
911	fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool {
912	return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
913	}
914
915	fn fullToken(p: *Parse, token: TokenIndex) Token {
916	return Token{
917	.tag = p.token_tags[token],
918	.offset = p.token_starts[token],
919	};
920	}
921
922	fn scratchToSubList(p: *Parse, scratch_top: usize) !SubList {
923	const trace = t.trace(@src());
924	defer trace.end();
925
926	const items = p.scratch.items[scratch_top..];
927	const extra_top = p.extra.items.len;
928	try p.extra.appendSlice(p.gpa, p.scratch.items[scratch_top..]);
929	return .{
930	.start = @intCast(extra_top),
931	.len = @intCast(items.len),
932	};
933	}
934
935	const Parse = @This();
936
937	const std = @import("std");
938	const assert = std.debug.assert;
939	const mem = std.mem;
940	const Allocator = mem.Allocator;
941	const ArrayListUnmanaged = std.ArrayListUnmanaged;
942
943	const Tokenizer = @import("Tokenizer.zig");
944	const Token = Tokenizer.Token;
945	const ByteOffset = Tokenizer.ByteOffset;
946
947	const Ast = @import("Ast.zig");
948	const Node = Ast.Node;
949	const Data = Node.Data;
950	const SubList = Ast.SubList;
951	const NodeIndex = Ast.NodeIndex;
952	const TokenIndex = Ast.TokenIndex;
953	const ExtraIndex = Ast.ExtraIndex;
954	const OptionalNodeIndex = Ast.OptionalNodeIndex;
955	const OptionalTokenIndex = Ast.OptionalTokenIndex;
956	const OptionalExtraIndex = Ast.OptionalExtraIndex;
957	const optionalTokenIndex = OptionalTokenIndex.init;
958	const optionalNodeIndex = OptionalNodeIndex.init;
959	const optionalExtraIndex = OptionalExtraIndex.init;
960
961	const t = @import("tracer");

src/compiler/Report.zig created+38

...	@@ -0,0 +1,38 @@
1	errors: []Item = &.{},
2	warns: []Item = &.{},
3	string_bytes: []u8 = &.{},
4
5	pub const Item = struct {
6	offset: ByteOffset,
7	message: String,
8	};
9
10	pub const String = struct {
11	start: u32,
12	len: u32,
13
14	pub fn message(item: Item, report: Report) []const u8 {
15	return report.string_bytes[item.message_start..][0..item.message_len];
16	}
17	};
18
19	pub fn hasErrors(report: Report) bool {
20	return report.errors.len > 0;
21	}
22
23	pub fn hasWarns(report: Report) bool {
24	return report.warns.len > 0;
25	}
26
27	pub const Wip = struct {
28	errors: ArrayListUnmanaged(Item),
29	warns: ArrayListUnmanaged(Item),
30	string_bytes: ArrayListUnmanaged(u8),
31	};
32
33	const Report = @This();
34	const std = @import("std");
35	const ArrayListUnmanaged = std.ArrayListUnmanaged;
36	const assert = std.debug.assert;
37	pub const Tokenizer = @import("Tokenizer.zig");
38	pub const ByteOffset = Tokenizer.ByteOffset;

src/compiler/Sema.zig created+693

...	@@ -0,0 +1,693 @@
1	/// absolutely stealing zig on this one haha
2	/// This struct is created per declaration and chills
3
4	// inputs:
5	comp: *Comp,
6	gpa: Allocator,
7	arena: Allocator,
8	code: Teg,
9	ip: *InternPool,
10	decl: *Decl,
11	decl_index: Decl.Index,
12	namespace_index: Namespace.Index,
13
14	// temp data:
15	inst_map: InstMap = undefined,
16	/// scratch memory. can be used in any place you might temporarily allocate an arraylist
17	/// this is most notably used to store block contents before the block is finalized
18	scratch: std.ArrayListUnmanaged(u32) = .{},
19	params: []InternPool.Index = undefined,
20
21	// outputs
22	air_instructions: std.MultiArrayList(Air.Inst) = .{},
23	air_extra: dod.ExtraStore(.air, .@"var") = .{},
24
25	pub const Error = error{
26	/// Analysis has failed, see error list
27	AnalysisFail,
28	/// This analysis is waiting on other dependencies to finish
29	Interrupt,
30	};
31
32	/// Instead of using a hash map for this, just pre-allocate a list long enough to hold everything.
33	/// This is fine because all instructions are going to be adacent in a single declaration.
34	const InstMap = struct {
35	items: []Air.Inst.Ref.Optional,
36	start: Teg.Inst.Index,
37
38	pub fn init(allocator: Allocator, code: *const Teg, inst_start: Teg.Inst.Index) !InstMap {
39	const decl = code.declData(inst_start);
40	log.info("pre-allocating {d} map-indicies for InstMap", .{decl.instruction_count});
41	const insts = try allocator.alloc(Air.Inst.Ref.Optional, decl.instruction_count);
42	@memset(insts, .none);
43	return .{
44	.items = insts,
45	.start = inst_start,
46	};
47	}
48
49	pub fn deinit(map: *InstMap, allocator: Allocator) void {
50	allocator.free(map.items);
51	map.* = undefined;
52	}
53
54	pub fn get(map: *InstMap, teg_code: Teg.Inst.Index) ?Air.Inst.Ref {
55	const source_i = @intFromEnum(teg_code);
56	const start_i = @intFromEnum(map.start);
57	assert(source_i >= start_i and source_i <= (start_i + map.items.len));
58	return map.items[source_i - start_i].unwrap();
59	}
60
61	pub fn add(map: *InstMap, from: Teg.Inst.Index, to: Air.Inst.Ref) !void {
62	assert(map.get(from) == null);
63	const source_i = @intFromEnum(from);
64	const start_i = @intFromEnum(map.start);
65	map.items[source_i - start_i] = to.toOptional();
66	}
67	};
68
69	pub fn init(s: *Sema) !void {
70	const t = tracer.trace(@src());
71	defer t.end();
72
73	try s.comp.current_compile_decls.append(s.gpa, s.decl_index);
74
75	log.info("init sema on index {}", .{s.decl_index});
76	s.inst_map = try InstMap.init(
77	s.arena,
78	&s.code,
79	s.ip.declPtr(s.decl_index).teg_decl_index.unwrap().?.resolve(s.ip),
80	);
81	}
82
83	pub fn deinit(s: *Sema) void {
84	const t = tracer.trace(@src());
85	defer t.end();
86
87	defer s.* = undefined;
88	s.inst_map.deinit(s.arena);
89	s.scratch.deinit(s.arena);
90	s.air_extra.deinit(s.gpa);
91	s.air_instructions.deinit(s.gpa);
92	}
93
94	pub fn finalize(s: *Sema) !void {
95	const air: Air = s.tempAir();
96
97	std.debug.print("=" ** 20, .{});
98	std.debug.print("\nAir for @ {d}\n", .{@intFromEnum(s.decl.teg_decl_index)});
99	air.dumpToWriter(s.ip, std.io.getStdErr().writer()) catch {};
100
101	try s.comp.codeGenDecl(s.decl, &air);
102	}
103
104	// --- analyze functions ---
105
106	/// This is the entrypoint to semantic analysis: It all starts at a declaration.
107	pub fn analyzeDeclaration(s: *Sema) anyerror!Value {
108	const t = tracer.trace(@src());
109	defer t.end();
110
111	// TODO: turn this entire function into a nonrecursive state machine, so
112	// that analysis can pause and continue at any point. for example, when a
113	// dependency on another declaration is required, but that declaration has
114	// not been resolved yet. or, we are waiting on a file read to happen
115
116	const decl, const fn_decl = s.code.fullFuncDecl(s.inst_map.start);
117	assert(decl.kind == .function);
118
119	s.params = try s.gpa.alloc(InternPool.Index, fn_decl.params.len);
120	defer s.gpa.free(s.params);
121
122	for (fn_decl.params.slice(&s.code), 0..) \|param, i\| {
123	s.params[i] = (try s.resolveRefComptime(param)).toIntern();
124	}
125
126	const return_type = try s.resolveRefComptime(fn_decl.return_type);
127
128	const root = (try s.analyzeBlock(
129	fn_decl.body.toIndex() orelse
130	@panic("TODO: please change this to Index. Ref is not allowed here anymore"),
131	)).unwrap().inst;
132	assert(@intFromEnum(root) == 0);
133
134	const func_prototype = try s.ip.get(s.gpa, .{
135	.type_function = .{
136	.param_types = s.params,
137	.return_type = return_type.toIntern(),
138	},
139	});
140	const func_value = try s.ip.get(s.gpa, .{
141	.func = .{
142	.ty = func_prototype,
143	.owner_decl = s.decl_index,
144	.teg_body_inst = s.decl.teg_decl_index.unwrap().?,
145	},
146	});
147	const value = Value.fromInterned(func_value);
148	s.decl.val = value;
149	return value;
150	}
151
152	pub fn analyzeBlock(s: *Sema, in: Teg.Inst.Index) anyerror!Air.Inst.Ref {
153	const t = tracer.trace(@src());
154	defer t.end();
155
156	const block = try s.addBlock();
157
158	try s.inst_map.add(in, block.block_index.toRef());
159
160	const scratch_top = s.scratch.items.len;
161	defer s.scratch.shrinkRetainingCapacity(scratch_top);
162
163	var last: Air.Inst.Ref = undefined;
164	const instructions = s.code.blockInstructions(in);
165	assert(instructions.len != 0);
166	for (instructions) \|inst\| {
167	last = try s.analyzeCode(inst);
168	}
169
170	if (scratch_top + 1 == s.scratch.items.len) {
171	const single_instruction = s.air_instructions.items(.tag)[@intFromEnum(block.block_index) + 1];
172	if (single_instruction == .@"break") {
173	const data: Air.Break = s.air_instructions.items(.data)[@intFromEnum(block.block_index) + 1]
174	.decode(.@"break", single_instruction);
175	if (data.block_ref == block.block_index) {
176	// This is a block that resolved to just a single value ref with
177	// no side effects. This means we can just lower this to the ref
178	// itself, and give codegen an easier time.
179
180	// Delete the block and the single instruction
181	// Break does not use any extra data, so this call removes both items
182	s.air_instructions.shrinkRetainingCapacity(@intFromEnum(block.block_index));
183
184	return data.value;
185	}
186	}
187	}
188
189	return (try block.finish(s, scratch_top)).toRef();
190	}
191
192	fn analyzeCode(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
193	const t = tracer.traceFmt(@src(), "%{d}, {s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
194	defer t.end();
195
196	log.info("runtime %{d} = .{s}", .{ @intFromEnum(in), @tagName(s.code.instTag(in)) });
197
198	const out = try s.analyzeCodeInner(in);
199	if (s.code.instTag(in) != .block)
200	try s.inst_map.add(in, out);
201
202	return out;
203	}
204
205	fn analyzeCodeInner(s: *Sema, in: Teg.Inst.Index) !Air.Inst.Ref {
206	switch (s.code.instTag(in)) {
207	.string => {
208	const string: Teg.SlicedString = s.code.instPayload(.string, in);
209	return s.makeValueAir(.{ .string = string.slice(s.code.string_bytes) });
210	},
211	.number => {
212	const number: dod.Float = s.code.instPayload(.number, in);
213	return s.makeValueAir(.{ .float = number.decode() });
214	},
215	.js => {
216	const operand = try s.resolveRefComptime(s.code.instData(.js, in).operand);
217	try s.validateType(operand.type(s.ip), .string_type);
218	return s.addInstRef(.js_global_ref, operand.asString(s.ip));
219	},
220	.field => {
221	const field: Teg.Inst.Bin = s.code.instPayload(.field, in);
222	const base = try s.resolveRefRuntime(field.lhs);
223	const rhs = try s.resolveRefComptime(field.rhs);
224
225	try s.validateType(rhs.type(s.ip), .string_type);
226	const string = rhs.asString(s.ip);
227
228	const base_type = s.typeOfRef(base);
229	log.info("process field with type {any} . {s}", .{ base_type, string.toSlice(s.ip) });
230
231	switch (base_type) {
232	.opaque_type => {
233	return s.addInstRef(.field, .{
234	.base = base,
235	.field = try string.tryToNullTerminatedString(s.ip, s.gpa),
236	});
237	},
238	else => \|tag\| std.debug.panic("TODO: field analyze with base .{s}", .{@tagName(tag)}),
239	}
240	},
241	.load_decl => {
242	const request: Teg.Inst.StrNode = s.code.instData(.load_decl, in);
243	const decl_name = try s.ip.getOrPutString(
244	s.gpa,
245	request.string.slice(s.code.string_bytes),
246	.no_embedded_nulls,
247	);
248	const decl_index = s.lookupDecl(decl_name) orelse {
249	std.debug.print("did not find {s}\n", .{decl_name.toSlice(s.ip)});
250	return error.AnalysisFail;
251	};
252	const resolved = try s.resolveDecl(decl_index);
253	return resolved.toIntern().toAir();
254	},
255	.param => {
256	const param: Teg.Inst.DataNode = s.code.instData(.param, in);
257	return s.addInstRef(.arg, .{
258	.type = Value.fromInterned(s.params[param.data]),
259	.arg_number = param.data,
260	});
261	},
262	inline .add,
263	.sub,
264	.mul,
265	.div,
266	.lt,
267	.gt,
268	.eq,
269	=> \|tag\| {
270	const bin: Teg.Inst.Bin = s.code.instPayload(tag, in);
271
272	const lhs = try s.resolveRefRuntime(bin.lhs);
273	const rhs = try s.resolveRefRuntime(bin.rhs);
274
275	if (lhs.kind == .value and rhs.kind == .value) {
276	@panic("TODO: comptime math operation");
277	}
278
279	const left_type = s.typeOfRef(lhs);
280	const right_type = s.typeOfRef(rhs);
281
282	// TODO: validate type for operation
283
284	log.info("runtime {s} operation on {} and {}", .{
285	@tagName(tag),
286	left_type.toKey(s.ip),
287	right_type.toKey(s.ip),
288	});
289
290	// todo: type check
291
292	return s.addInstRef(tag.airTag(), .{
293	.lhs = lhs,
294	.rhs = rhs,
295	});
296	},
297	inline .negate => \|tag\| {
298	const un: Teg.Inst.RefNode = s.code.instData(tag, in);
299
300	const operand = try s.resolveRefRuntime(un.operand);
301
302	const operand_type = s.typeOfRef(operand);
303
304	// TODO: validate type for operation
305
306	log.info("runtime {s} operation on {}", .{
307	@tagName(tag),
308	operand_type.toKey(s.ip),
309	});
310
311	return s.addInstRef(tag.airTag(), .{
312	.operand = operand,
313	});
314	},
315	.@"break" => {
316	const brk: Teg.Inst.Break = s.code.instPayload(.@"break", in);
317	const operand = try s.resolveRefRuntime(brk.operand);
318	const block_ref = s.inst_map.get(brk.block).?.unwrap().inst; // block ref is resolved by teg
319	return s.addInstRef(.@"break", .{
320	.block_ref = block_ref,
321	.value = operand,
322	});
323	},
324	.call => {
325	const call: Teg.Inst.Call = s.code.instPayload(.call, in);
326	const base = try s.resolveRefRuntime(call.base);
327	const base_type = s.typeOfRef(base).toKey(s.ip);
328
329	const is_opaque = base_type == .simple_type and base_type.simple_type == .@"opaque";
330
331	const scratch = try s.arena.alloc(Air.Inst.Ref, call.args.len);
332	defer s.arena.free(scratch);
333	if (!is_opaque) {
334	if (base_type != .type_function) {
335	std.debug.print("Attempt to call non-function {}", .{base_type});
336	return error.AnalysisFail;
337	}
338
339	const proto = base_type.type_function;
340	if (proto.param_types.len != call.args.len) {
341	std.debug.print("Incorrect args. Expect {d}, Got {d}", .{
342	proto.param_types.len,
343	call.args.len,
344	});
345	return error.AnalysisFail;
346	}
347
348	// validate function args
349	for (proto.param_types, call.args.slice(&s.code), 0..) \|param, arg, i\| {
350	const ref = try s.resolveRefRuntime(arg);
351	scratch[i] = ref;
352	try s.validateType(
353	Value.fromInterned(s.typeOfRef(ref)),
354	param,
355	);
356	}
357	} else {
358	// just copy the args in without checking
359	for (call.args.slice(&s.code), 0..) \|arg, i\| {
360	const ref = try s.resolveRefRuntime(arg);
361	scratch[i] = ref;
362	}
363	}
364
365	const start = s.air_extra.list.items.len;
366	try s.air_extra.list.appendSlice(s.gpa, @ptrCast(scratch));
367
368	return s.addInstRef(.call, .{
369	.base = base,
370	.call_data = try s.air_extra.append(s.gpa, Air.Call.Data, .{
371	.args = .{
372	.start = @intCast(start),
373	.len = @intCast(scratch.len),
374	},
375	}),
376	});
377	},
378	.ensure_void => {
379	const operand = try s.resolveRefRuntimeExtra(s.code.instData(.ensure_void, in).operand, false);
380	const t = Value.fromInterned(s.typeOfRef(operand));
381	if (!t.isEnsureVoid(s.ip)) {
382	std.debug.print("oh no, we are discarding {} on accident\n", .{
383	t.fmt(s.ip),
384	});
385	}
386	return InternPool.Index.void_value.toAir();
387	},
388	.@"if" => {
389	const data: Teg.Inst.If = s.code.instData(.@"if", in).extra_index.get(&s.code);
390
391	const cond = try s.resolveRefRuntime(data.cond);
392
393	try s.validateType(Value.fromInterned(s.typeOfRef(cond)), .bool_type);
394
395	// If known at comptime, only analyze the taken path
396	if (cond.kind == .value) {
397	@panic("TODO: comptime IF");
398	}
399
400	// Emit a runtime branch
401	const yes_block = try s.analyzeRefToBlock(data.yes);
402	const no_block = try s.analyzeRefToBlock(data.no);
403
404	return try s.addInstRef(.@"if", .{
405	.cond = cond,
406	.data = try s.air_extra.append(s.gpa, Air.If.Data, .{
407	.yes = yes_block,
408	.no = no_block,
409	.union_ty = s.unionType(Value.fromInterned(s.typeOfRef(yes_block)), Value.fromInterned(s.typeOfRef(no_block))),
410	}),
411	});
412	},
413	.block => {
414	const block = try s.analyzeBlock(in);
415	switch (block.unwrap()) {
416	.inst => \|t\| try s.scratch.append(s.gpa, @intFromEnum(t)),
417	else => {},
418	}
419	return block;
420	},
421	else => \|tag\| std.debug.panic("TODO: analyze of .{s}", .{@tagName(tag)}),
422	}
423	}
424
425	fn markInst(s: *Sema, inst: Air.Inst.Index) void {
426	const flags: *Air.Flags = &s.air_instructions.items(.flags)[@intFromEnum(inst)];
427	flags.uses = flags.uses.increment();
428	log.info("markInst {}", .{inst});
429	}
430
431	fn analyzeRefToBlock(s: *Sema, in: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
432	const t = tracer.trace(@src());
433	defer t.end();
434
435	const scratch_top = s.scratch.items.len;
436	defer s.scratch.shrinkRetainingCapacity(scratch_top);
437
438	const visited = try s.resolveRefRuntime(in);
439
440	if (scratch_top == s.scratch.items.len) {
441	return visited;
442	}
443
444	if (visited.kind == .inst and
445	s.air_instructions.items(.tag)[@intFromEnum(visited.unwrap().inst)] == .block)
446	{
447	return visited;
448	}
449
450	const block = try s.addBlock();
451	return (try block.finish(s, scratch_top)).toRef();
452	}
453
454	fn resolveRefRuntimeExtra(s: *Sema, ref: Teg.Inst.Ref, mark: bool) anyerror!Air.Inst.Ref {
455	const index = ref.toIndex() orelse {
456	return ref.toInterned().?.toAir();
457	};
458
459	if (s.inst_map.get(index)) \|air\| {
460	if (mark) {
461	switch (air.unwrap()) {
462	.inst => \|i\| s.markInst(i),
463	else => {},
464	}
465	}
466	return air;
467	}
468
469	return s.analyzeCode(index);
470	}
471
472	fn resolveRefRuntime(s: *Sema, ref: Teg.Inst.Ref) anyerror!Air.Inst.Ref {
473	return resolveRefRuntimeExtra(s, ref, true);
474	}
475
476	fn resolveRefComptime(s: *Sema, ref: Teg.Inst.Ref) !Value {
477	const in = ref.toIndex() orelse {
478	return Value{ .ip_index = ref.toInterned().? };
479	};
480
481	if (s.inst_map.get(in)) \|air\| {
482	switch (air.unwrap()) {
483	.value => \|value\| return value,
484	else => {},
485	}
486	}
487
488	const tag = s.code.instTag(in);
489	log.info("comptime %{d} = .{s}", .{ @intFromEnum(in), @tagName(tag) });
490	switch (tag) {
491	.string => {
492	const string: Teg.SlicedString = s.code.instPayload(.string, in);
493	return s.makeValue(.{ .string = string.slice(s.code.string_bytes) });
494	},
495
496	else => std.debug.panic("TODO: comptime analyze of .{s}", .{@tagName(tag)}),
497	}
498	}
499
500	fn resolveDecl(s: *Sema, decl_index: Decl.Index) !Value {
501	const decl = s.ip.declPtr(decl_index);
502	switch (decl.analysis) {
503	.unreferenced => {
504	decl.analysis = .in_progress;
505
506	// TODO: make this short-circuit the current analysis, move this analyze
507	// job into a "deferred" list, and then make this thread analyze something else
508
509	var arena = std.heap.ArenaAllocator.init(s.gpa);
510	defer arena.deinit();
511
512	var sema: Sema = .{
513	.comp = s.comp,
514	.gpa = s.gpa,
515	.ip = s.ip,
516	.arena = arena.allocator(),
517	.code = try s.ip.namespacePtr(decl.src_namespace).file_scope.getTeg(s.gpa),
518	.decl = decl,
519	.decl_index = decl_index,
520	.namespace_index = decl.src_namespace,
521	};
522
523	try sema.init();
524	defer sema.deinit();
525	const value = try sema.analyzeDeclaration();
526	try sema.finalize();
527
528	decl.analysis = .complete;
529
530	return value;
531	},
532	.complete => {
533	return decl.val;
534	},
535	else => \|tag\| std.debug.panic("TODO: load_decl when decl is in state .{s}", .{@tagName(tag)}),
536	}
537	}
538
539	fn unionType(s: *Sema, a: Value, b: Value) Value {
540	if (std.debug.runtime_safety) {
541	assert(a.type(s.ip).toIntern() == .type_type);
542	assert(b.type(s.ip).toIntern() == .type_type);
543	}
544
545	if (a.ip_index == b.ip_index) {
546	return a;
547	}
548
549	if (a.ip_index == .noreturn_type)
550	return b;
551
552	if (b.ip_index == .noreturn_type)
553	return a;
554
555	std.debug.panic(
556	"TODO: when two non-equal types implicitly merge (if/match), this should generate a union type.\na: {}\nb: {}",
557	.{
558	a.fmt(s.ip),
559	b.fmt(s.ip),
560	},
561	);
562	// return s.ip.get(s.gpa, .{ .@"union" = &.{ a, b } });
563	}
564
565	/// --- helper ---
566	const IncompleteBlock = struct {
567	block_index: Air.Inst.Index,
568
569	pub fn finish(block: IncompleteBlock, s: *Sema, scratch_top: usize) !Air.Inst.Index {
570	s.setInstData(.block, block.block_index, try s.scratchToIndexList(scratch_top));
571	return block.block_index;
572	}
573	};
574
575	fn addBlock(s: *Sema) !IncompleteBlock {
576	log.info("start block at @{d}", .{s.air_instructions.len});
577	try s.air_instructions.append(s.gpa, .{
578	.tag = .block,
579	.data = undefined,
580	.flags = .{},
581	});
582	return .{ .block_index = @enumFromInt(s.air_instructions.len - 1) };
583	}
584
585	fn addInstRef(s: *Sema, comptime tag: Air.Inst.Tag, data: tag.Data()) !Air.Inst.Ref {
586	log.info("add air .{s}, {}", .{ @tagName(tag), data });
587	try s.air_instructions.append(s.gpa, .{
588	.tag = tag,
589	.data = Air.EncodedData.encode(tag, data),
590	.flags = .{},
591	});
592	const index: Air.Inst.Index = @enumFromInt(s.air_instructions.len - 1);
593	try s.scratch.append(s.arena, @intFromEnum(index));
594	return index.toRef();
595	}
596
597	fn setInstData(
598	s: *Sema,
599	comptime tag: Air.Inst.Tag,
600	index: Air.Inst.Index,
601	data: tag.Data(),
602	) void {
603	const i = @intFromEnum(index);
604	assert(s.air_instructions.items(.tag)[i] == tag);
605	s.air_instructions.items(.data)[i] = Air.EncodedData.encode(tag, data);
606	}
607
608	fn makeValue(s: *Sema, key: InternPool.Key) !Value {
609	return .{ .ip_index = try s.ip.get(s.gpa, key) };
610	}
611
612	fn makeValueAir(s: *Sema, key: InternPool.Key) !Air.Inst.Ref {
613	return (try s.makeValue(key)).ip_index.toAir();
614	}
615
616	fn mapTeg(s: *Sema, inst: Teg.Inst.Ref) Air.Inst.Ref {
617	if (inst.toInterned()) \|interned\| {
618	return interned.toAir();
619	}
620	const index = inst.toIndex().?;
621	return s.inst_map.get(index) orelse {
622	if (std.debug.runtime_safety) {
623	std.debug.panic("Unmapped operand %{d}", .{@intFromEnum(index)});
624	}
625	unreachable;
626	};
627	}
628
629	fn validateType(s: *Sema, checked_type: Value, wanted_type: InternPool.Index) !void {
630	if (checked_type.toIntern() != wanted_type) {
631	log.info("failed type check {}, got {}", .{ Value.fromInterned(wanted_type).fmt(s.ip), checked_type.fmt(s.ip) });
632	return error.TypeCheckFailed;
633	} else {
634	log.info("passed type check {}", .{checked_type.fmt(s.ip)});
635	}
636	}
637
638	fn scratchToIndexList(s: *Sema, scratch_top: usize) !Air.IndexList {
639	const items = s.scratch.items[scratch_top..];
640	const extra_top = s.air_extra.list.items.len;
641	try s.air_extra.list.appendSlice(s.gpa, s.scratch.items[scratch_top..]);
642	return .{
643	.start = @intCast(extra_top),
644	.len = @intCast(items.len),
645	};
646	}
647
648	fn tempAir(s: *const Sema) Air {
649	return .{
650	.insts = s.air_instructions.slice(),
651	.extra = s.air_extra.constant(),
652	};
653	}
654
655	fn typeOfRef(s: *Sema, ref: Air.Inst.Ref) InternPool.Index {
656	return s.tempAir().typeOf(ref, s.ip);
657	}
658
659	fn lookupDecl(s: *Sema, name: NullTerminatedString) ?InternPool.DeclIndex {
660	var nsi = s.namespace_index;
661	while (true) {
662	const ns = s.ip.namespacePtr(nsi);
663	if (ns.lookupDirect(s.comp, name)) \|decl_id\| {
664	return decl_id;
665	}
666	nsi = ns.parent.unwrap() orelse
667	unreachable; // transform ensures all identifiers get resolved
668	}
669	}
670
671	const Sema = @This();
672
673	const std = @import("std");
674	const mem = std.mem;
675	const Allocator = std.mem.Allocator;
676	const assert = std.debug.assert;
677	const log = std.log.scoped(.sema);
678
679	const Comp = @import("Comp.zig");
680	const File = Comp.File;
681	const Decl = Comp.Decl;
682	const Namespace = Comp.Namespace;
683
684	const Air = @import("Air.zig");
685
686	const Teg = @import("Teg.zig");
687
688	const Value = @import("Value.zig");
689	const InternPool = @import("InternPool.zig");
690	const NullTerminatedString = InternPool.NullTerminatedString;
691	const dod = @import("dod.zig");
692
693	const tracer = @import("tracer");

src/compiler/Teg.Dump.zig created+292

...	@@ -0,0 +1,292 @@
1	//! This implements a debug formatter for Teg
2	teg: Teg,
3
4	const State = struct {
5	arena: std.mem.Allocator,
6	teg: *const Teg,
7	indent: usize = 0,
8	tty: std.io.tty.Config,
9	visited: std.AutoHashMapUnmanaged(Teg.Inst.Index, void) = .{},
10
11	fn newline(state: State, writer: anytype) !void {
12	try writer.writeByte('\n');
13	if (state.indent > 0)
14	try writer.writeByteNTimes(' ', state.indent * 2);
15	}
16
17	fn dumpInst(s: *State, writer: anytype, inst: Teg.Inst.Index) @TypeOf(writer).Error!void {
18	const teg = s.teg;
19	const tty = s.tty;
20
21	try writer.print("%{} = ", .{@intFromEnum(inst)});
22
23	const tag = s.teg.instTag(inst);
24
25	try tty.setColor(writer, .cyan);
26	try writer.print("{s}", .{@tagName(tag)});
27	try tty.setColor(writer, .reset);
28
29	if ((try s.visited.fetchPut(s.arena, inst, {})) != null) {
30	try tty.setColor(writer, .red);
31	try writer.writeAll(" [error: recursive tree]");
32	try tty.setColor(writer, .reset);
33	return;
34	}
35
36	switch (tag) {
37	.module => {
38	try s.dumpInstList(writer, teg.moduleDeclarations(inst), '{', '}');
39	},
40	.declaration => {
41	const decl_info = teg.declData(inst);
42	try writer.print(" ({s}) ", .{@tagName(decl_info.kind)});
43
44	const name = decl_info.name.slice(teg.string_bytes);
45	if (name.len == 0) {
46	try tty.setColor(writer, .bold);
47	try writer.writeAll("<top level code>");
48	} else {
49	try tty.setColor(writer, .magenta);
50	try writer.writeAll(name);
51	}
52	try tty.setColor(writer, .reset);
53
54	try writer.writeAll(" {");
55	s.indent += 1;
56	try s.newline(writer);
57
58	switch (decl_info.kind) {
59	.function => {
60	const fn_data: Teg.Inst.DeclFn = teg.fullFuncDecl(inst).@"1";
61	try writer.writeAll("params: ");
62	try s.dumpRefList(writer, fn_data.params.slice(teg), '[', ']');
63	try s.newline(writer);
64	try writer.writeAll("return_type: ");
65	try s.dumpRef(writer, fn_data.return_type);
66	try s.newline(writer);
67	try writer.writeAll("body: ");
68	try s.dumpRef(writer, fn_data.body);
69	},
70	}
71	s.indent -= 1;
72	try s.newline(writer);
73	try writer.writeAll("}");
74	},
75	.param => {
76	const data = teg.instData(.param, inst);
77	try writer.writeAll("(");
78	try tty.setColor(writer, .yellow);
79	try writer.print("{d}", .{data.data});
80	try tty.setColor(writer, .reset);
81	try writer.writeAll(")");
82	},
83	.block => {
84	const block = teg.instData(.block, inst);
85	const index_list = block.extra_index.get(teg);
86	try s.dumpInstList(writer, index_list.slice(teg), '[', ']');
87	},
88	.number => {
89	const number = teg.instData(.number, inst);
90	const float = number.extra_index.get(teg).decode();
91	try tty.setColor(writer, .yellow);
92	try writer.print(" {d}", .{float});
93	try tty.setColor(writer, .reset);
94	},
95	inline .field,
96	.store,
97	.add,
98	.sub,
99	.mul,
100	.div,
101	.rem,
102	.pow,
103	.@"and",
104	.@"or",
105	.bit_and,
106	.bit_or,
107	.bit_xor,
108	.shl,
109	.shr,
110	.eq,
111	.ne,
112	.lt,
113	.le,
114	.gt,
115	.ge,
116	=> \|comptime_tag\| {
117	const data = teg.instData(comptime_tag, inst).extra_index.get(teg);
118	try writer.writeAll("(");
119	try s.dumpRefMarker(writer, data.lhs);
120	try writer.writeAll(", ");
121	try s.dumpRefMarker(writer, data.rhs);
122	try writer.writeAll(")");
123	},
124	.@"break" => {
125	const data: Teg.Inst.Break = teg.instData(.@"break", inst).extra_index.get(teg);
126	try writer.writeAll("(");
127	try s.dumpRefMarker(writer, data.block.toRef());
128	try writer.writeAll(", ");
129	try s.dumpRefMarker(writer, data.operand);
130	try writer.writeAll(")");
131	},
132	.string => {
133	const data: Teg.SlicedString = teg.instData(.string, inst).extra_index.get(teg);
134	try tty.setColor(writer, .green);
135	try writer.writeAll(" \"");
136	try writer.writeAll(data.slice(teg.string_bytes));
137	try writer.writeAll("\"");
138	try tty.setColor(writer, .reset);
139	},
140	inline .memory,
141	.discard,
142	.ensure_void,
143	.ensure_bool,
144	.negate,
145	.bool_not,
146	.bit_not,
147	.js,
148	.import,
149	.fetch,
150	.compile_error,
151	=> \|comptime_tag\| {
152	const data = teg.instData(comptime_tag, inst);
153	try writer.writeAll("(");
154	try s.dumpRefMarker(writer, data.operand);
155	try writer.writeAll(")");
156	},
157	.call => {
158	const call: Teg.Inst.Call = teg.instData(.call, inst).extra_index.get(teg);
159	try writer.writeAll("(");
160	try s.dumpRefMarker(writer, call.base);
161	try writer.writeAll(", ");
162	try s.dumpRefMarkerList(writer, call.args.slice(teg), '[', ']');
163	try writer.writeAll(")");
164	},
165	.load_decl => {
166	const call: Teg.Inst.StrNode = teg.instData(.load_decl, inst);
167
168	try tty.setColor(writer, .red);
169	try writer.print(" {s}", .{call.string.slice(teg.string_bytes)});
170	try tty.setColor(writer, .reset);
171	},
172	.@"if" => {
173	const data: Teg.Inst.If = teg.instPayload(.@"if", inst);
174	try writer.writeAll("(");
175	try s.dumpRefMarker(writer, data.cond);
176	try writer.writeAll(") {");
177	s.indent += 1;
178	try s.newline(writer);
179	try writer.writeAll("yes: ");
180	try s.dumpRef(writer, data.yes);
181	try s.newline(writer);
182	try writer.writeAll("no: ");
183	try s.dumpRef(writer, data.no);
184	s.indent -= 1;
185	try s.newline(writer);
186	try writer.writeAll("}");
187	},
188	else => std.debug.panic("TODO format {s}", .{@tagName(tag)}),
189	}
190	}
191
192	fn dumpRef(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
193	if (ref.toIndex()) \|index\| {
194	try s.dumpInst(writer, index);
195	} else {
196	try s.tty.setColor(writer, .blue);
197	try writer.print("%{s}", .{@tagName(ref)});
198	try s.tty.setColor(writer, .reset);
199	}
200	}
201
202	fn dumpRefMarker(s: *State, writer: anytype, ref: Teg.Inst.Ref) !void {
203	if (ref.toIndex()) \|index\| {
204	try s.tty.setColor(writer, .blue);
205	try writer.print("%{d}", .{@intFromEnum(index)});
206	try s.tty.setColor(writer, .reset);
207	} else {
208	try s.tty.setColor(writer, .blue);
209	try writer.print("%{s}", .{@tagName(ref)});
210	try s.tty.setColor(writer, .reset);
211	}
212	}
213
214	fn dumpInstList(state: *State, writer: anytype, list: []const Teg.Inst.Index, l: u8, r: u8) !void {
215	try writer.print(" {c}", .{l});
216	if (list.len > 0) {
217	state.indent += 1;
218
219	for (list) \|inst\| {
220	try state.newline(writer);
221	try state.dumpInst(writer, inst);
222	}
223
224	state.indent -= 1;
225	try state.newline(writer);
226	}
227	try writer.print("{c}", .{r});
228	}
229
230	fn dumpRefList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
231	try writer.print(" {c}", .{l});
232	if (list.len > 0) {
233	state.indent += 1;
234
235	for (list) \|ref\| {
236	try state.newline(writer);
237	try state.dumpRef(writer, ref);
238	}
239
240	state.indent -= 1;
241
242	try state.newline(writer);
243	}
244	try writer.print("{c}", .{r});
245	}
246
247	fn dumpRefMarkerList(state: *State, writer: anytype, list: []const Teg.Inst.Ref, l: u8, r: u8) !void {
248	try writer.print(" {c}", .{l});
249	if (list.len > 0) {
250	state.indent += 1;
251
252	for (list) \|ref\| {
253	try state.newline(writer);
254	try state.dumpRefMarker(writer, ref);
255	}
256
257	state.indent -= 1;
258
259	try state.newline(writer);
260	}
261	try writer.print("{c}", .{r});
262	}
263	};
264
265	pub fn format(
266	dump: @This(),
267	comptime unused_fmt: []const u8,
268	_: std.fmt.FormatOptions,
269	writer: anytype,
270	) !void {
271	comptime assert(unused_fmt.len == 0);
272
273	var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
274	defer arena.deinit();
275
276	const teg = dump.teg;
277	var state = State{
278	.teg = &teg,
279	.tty = detectTTYFromWriter(writer),
280	.arena = arena.allocator(),
281	};
282	try state.dumpInst(writer, .main_module_inst);
283	try writer.writeAll("\n");
284	assert(state.indent == 0);
285	}
286
287	const std = @import("std");
288	const assert = std.debug.assert;
289
290	const Teg = @import("Teg.zig");
291
292	const detectTTYFromWriter = @import("util.zig").detectTTYFromWriter;

src/compiler/Teg.zig created+435

...	@@ -0,0 +1,435 @@
1	//! TODO: rename this to something else. Teg is not a graph
2
3	insts: Inst.List.Slice,
4	string_bytes: []const u8,
5	extra: ExtraStore = .{},
6
7	pub const ExtraStore = dod.ExtraStore(.teg, .@"const");
8	pub const EncodedData = dod.EncodedData(Inst.Tag, u64);
9	pub const ExtraIndex = ExtraStore.Index;
10	pub const Dump = @import("Teg.Dump.zig");
11
12	pub const Inst = struct {
13	pub const List = std.MultiArrayList(Inst);
14
15	tag: Tag,
16	data: EncodedData,
17
18	// todo: fully document every thing
19	pub const Tag = enum {
20	module,
21	declaration,
22
23	block,
24	/// Creates a memory cell in the current scope.
25	memory,
26	/// Performs a type coersion, such as in variable declarations.
27	/// Emits a sema error if the type conversion fails.
28	/// `let x: i32 = 1`
29	as,
30	load_decl,
31	/// Load a parameter
32	param,
33	store,
34	discard,
35	ensure_void,
36	ensure_bool,
37	/// lowered from `break` or `return`
38	/// ends a block with a resulting value
39	@"break",
40	number,
41	string,
42
43	@"if",
44
45	add,
46	sub,
47	mul,
48	div,
49	rem,
50	pow,
51	@"and",
52	@"or",
53	bit_and,
54	bit_or,
55	bit_xor,
56	shl,
57	shr,
58	eq,
59	ne,
60	lt,
61	le,
62	gt,
63	ge,
64
65	negate,
66	bool_not,
67	bit_not,
68	field,
69	call,
70
71	/// Implements the `@js` builtin
72	js,
73	/// Implements the `@import` builtin
74	import,
75	/// Implements the `@fetch` builtin
76	fetch,
77	/// Implements the `@compileError` builtin
78	compile_error,
79
80	pub fn Data(comptime tag: Tag) type {
81	return switch (tag) {
82	.module => PlNode(IndexList),
83	.declaration => PlNode(Decl),
84	.block => PlNode(IndexList),
85	.as => PlNode(As),
86	.@"break" => PlNode(Break),
87	.number => PlNode(Float),
88	.string => PlNode(SlicedString),
89	.call => PlNode(Call),
90	.@"if" => PlNode(If),
91	.load_decl => StrNode,
92	.param => DataNode,
93
94	// Single instruction operand
95	.memory,
96	.discard,
97	.ensure_void,
98	.ensure_bool,
99	.negate,
100	.bool_not,
101	.bit_not,
102	.js,
103	.import,
104	.fetch,
105	.compile_error,
106	=> RefNode,
107
108	// Two instruction operands
109	.field,
110	.store,
111	.add,
112	.sub,
113	.mul,
114	.div,
115	.rem,
116	.pow,
117	.@"and",
118	.@"or",
119	.bit_and,
120	.bit_or,
121	.bit_xor,
122	.shl,
123	.shr,
124	.eq,
125	.ne,
126	.lt,
127	.le,
128	.gt,
129	.ge,
130	=> PlNode(Bin),
131	};
132	}
133
134	pub fn airTag(comptime tag: Tag) Air.Inst.Tag {
135	return @field(Air.Inst.Tag, @tagName(tag));
136	}
137	};
138
139	/// TODO: make enum? idk
140	const NodeOffset = i32;
141
142	pub fn PlNode(comptime Payload: type) type {
143	return struct {
144	pub const T = Payload;
145
146	src_node: NodeOffset,
147	extra_index: ExtraIndex(Payload),
148	};
149	}
150
151	pub const RefNode = struct {
152	src_node: NodeOffset,
153	operand: Inst.Ref,
154	};
155
156	pub const StrNode = struct {
157	src_node: NodeOffset,
158	string: NullTerminatedString,
159	};
160
161	pub const DataNode = struct {
162	src_node: NodeOffset,
163	data: u32,
164	};
165
166	/// The position of a Teg instruction within the `Teg` instructions array.
167	pub const Index = enum(u32) {
168	/// Teg is structured so that the outermost "main" struct of any file
169	/// is always at index 0.
170	main_module_inst = 0,
171	_,
172
173	pub const static_len = @intFromEnum(Ref.empty_string) + 1;
174
175	pub fn toRef(i: Index) Inst.Ref {
176	return @enumFromInt(static_len + @intFromEnum(i));
177	}
178
179	pub fn toOptional(i: Index) OptionalIndex {
180	return @enumFromInt(@intFromEnum(i));
181	}
182	};
183
184	pub const OptionalIndex = enum(u32) {
185	/// Teg is structured so that the outermost "main" module of any file
186	/// is always at index 0.
187	main_module_inst = 0,
188	ref_start_index = Index.static_len,
189	none = std.math.maxInt(u32),
190	_,
191
192	pub fn unwrap(oi: OptionalIndex) ?Index {
193	return if (oi == .none) null else @enumFromInt(@intFromEnum(oi));
194	}
195	};
196
197	/// A reference to instruction, or to an InternPool index, or neither.
198	///
199	/// If the integer tag value is < InternPool.static_len, then it
200	/// corresponds to an InternPool index. Otherwise, this refers to a ZIR
201	/// instruction.
202	///
203	/// The tag type is specified so that it is safe to bitcast between `[]u32`
204	/// and `[]Ref`.
205	pub const Ref = enum(u32) {
206	opaque_type,
207	bool_type,
208	int32_type,
209	float_type,
210	string_type,
211	void_type,
212	noreturn_type,
213	type_type,
214
215	void_value,
216	bool_true,
217	bool_false,
218	int32_zero,
219	float_zero,
220	nan,
221	empty_string,
222
223	/// This Ref does not correspond to any instruction or constant
224	/// value and may instead be used as a sentinel to indicate null.
225	none = std.math.maxInt(u32),
226
227	_,
228
229	pub fn toIndex(inst: Ref) ?Index {
230	assert(inst != .none);
231	const ref_int = @intFromEnum(inst);
232	if (ref_int >= Index.static_len) {
233	return @enumFromInt(ref_int - Index.static_len);
234	} else {
235	return null;
236	}
237	}
238
239	pub fn toIndexAllowNone(inst: Ref) ?Index {
240	if (inst == .none) return null;
241	return toIndex(inst);
242	}
243
244	pub fn toInterned(inst: Ref) ?InternPool.Index {
245	const ref_int = @intFromEnum(inst);
246
247	// todo: is this > or >=
248	if (ref_int > Index.static_len)
249	return null;
250
251	return @enumFromInt(ref_int);
252	}
253	};
254
255	pub const Bin = struct {
256	lhs: Inst.Ref,
257	rhs: Inst.Ref,
258	};
259
260	pub const As = struct {
261	type: Inst.Ref,
262	operand: Inst.Ref,
263	};
264
265	pub const Break = struct {
266	block: Inst.Index,
267	operand: Inst.Ref,
268	};
269
270	pub const Call = struct {
271	base: Inst.Ref,
272	args: RefList,
273	};
274
275	pub const If = struct {
276	cond: Inst.Ref,
277	yes: Inst.Ref,
278	no: Inst.Ref,
279	};
280
281	/// This will contain trailing data depending on the .kind
282	pub const Decl = struct {
283	name: NullTerminatedString,
284	hash: Cache.BinDigest,
285	kind: Kind,
286	instruction_count: u32,
287
288	pub const Kind = enum {
289	function,
290	// future: struct, let, var, etc
291	};
292	};
293
294	/// Trailng data for function declarations
295	pub const DeclFn = struct {
296	params: RefList,
297	return_type: Ref,
298	body: Ref,
299	};
300	};
301
302	pub const NullTerminatedString = enum(u32) {
303	empty = 0,
304	_,
305
306	pub fn slice(str: NullTerminatedString, string_bytes: []const u8) [:0]const u8 {
307	if (str == .empty) return "";
308	const sub = string_bytes[@intFromEnum(str)..];
309	const i = mem.indexOfScalar(u8, sub, 0) orelse unreachable;
310
311	return sub[0..i :0];
312	}
313	};
314
315	pub const SlicedString = struct {
316	start: u32,
317	len: u32,
318
319	pub fn slice(str: SlicedString, string_bytes: []const u8) []const u8 {
320	return string_bytes[str.start..][0..str.len];
321	}
322	};
323
324	pub const RefList = struct {
325	start: u32,
326	len: u32,
327
328	pub fn slice(self: RefList, teg: *const Teg) []const Inst.Ref {
329	return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
330	}
331	};
332
333	pub const IndexList = struct {
334	start: u32,
335	len: u32,
336
337	pub fn slice(self: IndexList, teg: *const Teg) []const Inst.Index {
338	return @ptrCast(teg.extra.list.items[self.start..][0..self.len]);
339	}
340	};
341
342	pub fn deinit(teg: *Teg, gpa: std.mem.Allocator) void {
343	gpa.free(teg.string_bytes);
344	teg.extra.deinit(gpa);
345	teg.insts.deinit(gpa);
346	teg.* = undefined;
347	}
348
349	pub fn instTag(teg: *const Teg, index: Inst.Index) Inst.Tag {
350	return teg.insts.items(.tag)[@intFromEnum(index)];
351	}
352
353	pub fn instEncodedData(teg: *const Teg, index: Inst.Index) EncodedData {
354	return teg.insts.items(.data)[@intFromEnum(index)];
355	}
356
357	pub fn instData(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data() {
358	return teg.instEncodedData(index).decode(tag, teg.instTag(index));
359	}
360
361	pub fn instPayload(teg: *const Teg, comptime tag: Inst.Tag, index: Inst.Index) tag.Data().T {
362	return teg.instData(tag, index).extra_index.get(teg);
363	}
364
365	pub fn blockInstructions(teg: *const Teg, index: Inst.Index) []const Inst.Index {
366	return teg.instData(.block, index).extra_index.get(teg).slice(teg);
367	}
368
369	pub fn fullFuncDecl(teg: *const Teg, index: Inst.Index) struct { Inst.Decl, Inst.DeclFn } {
370	const decl, var trail = teg.instData(.declaration, index).extra_index.getTrail(teg);
371	const decl_fn = trail.read(&teg.extra, Inst.DeclFn);
372	return .{ decl, decl_fn };
373	}
374
375	pub fn declData(teg: *const Teg, index: Inst.Index) Inst.Decl {
376	return teg.instData(.declaration, index).extra_index.get(teg);
377	}
378
379	pub fn moduleDeclarations(teg: *const Teg, module_index: Inst.Index) []const Inst.Index {
380	const module = teg.instData(.module, module_index);
381	const module_data: IndexList = module.extra_index.get(teg);
382	const slice = module_data.slice(teg);
383	// if (std.debug.runtime_safety) {
384	// for (slice) \|item\| {
385	// if (teg.instTag(item) != .declaration) {
386	// std.debug.print("\ngot unknown tag {s}\n", .{@tagName(teg.instTag(item))});
387	// unreachable;
388	// }
389	// }
390	// }
391	return slice;
392	}
393
394	pub fn rootFunction(teg: *const Teg) Inst.Index {
395	const decls = teg.moduleDeclarations(.main_module_inst);
396	const root_fn = decls[decls.len - 1];
397	assert(teg.declName(root_fn).len == 0);
398	return root_fn;
399	}
400
401	pub fn declName(teg: *const Teg, decl_index: Inst.Index) []const u8 {
402	const decl = teg.instData(.declaration, decl_index);
403	const decl_data: Inst.Decl = decl.extra_index.get(teg);
404	return decl_data.name.slice(teg.string_bytes);
405	}
406
407	pub fn dumpToWriter(teg: Teg, writer: anytype) !void {
408	try writer.print("{}", .{Dump{ .teg = teg }});
409	}
410
411	const Teg = @This();
412
413	const std = @import("std");
414	const mem = std.mem;
415	const assert = std.debug.assert;
416
417	const Cache = @import("Cache.zig");
418
419	const Ast = @import("Ast.zig");
420	const Unused = Ast.Unused;
421	const NodeIndex = Ast.NodeIndex;
422	const TokenIndex = Ast.TokenIndex;
423	const OptionalNodeIndex = Ast.OptionalNodeIndex;
424	const OptionalTokenIndex = Ast.OptionalTokenIndex;
425	const OptionalExtraIndex = Ast.OptionalExtraIndex;
426	const optionalTokenIndex = OptionalTokenIndex.init;
427	const optionalNodeIndex = OptionalNodeIndex.init;
428	const optionalExtraIndex = OptionalExtraIndex.init;
429
430	const dod = @import("dod.zig");
431	const Float = dod.Float;
432
433	const InternPool = @import("InternPool.zig");
434
435	const Air = @import("Air.zig");

src/compiler/Tokenizer.zig created+913

...	@@ -0,0 +1,913 @@
1	//! The tokenizer is extremely simple structure that, given a view of source
2	//! code, can stream tokens one at a time. A token is an 8-bit tag and a 32-bit
3	//! offset to where that token is located.
4	//!
5	//! Retriving payloads like string/numeric values, identifier names, etc is
6	//! done separately, either in Parse or Transform, when the respective data
7	//! is actually needed. Tokenization does guarante that such data is valid.
8
9	// TODO: make `code` a [:0]const u8. this doesnt disallow null bytes, but
10	// it requires the source to end with one. This can make byteAt(1) return u8
11	// instead of an optional. I am told this improves performance.
12	code: []const u8,
13	pos: ByteOffset = 0,
14
15	/// Tokenize source code into a list of tokens.
16	/// When an error occurs, the position of the error is returned in `error_pos`.
17	/// Caller owns returned slice.
18	pub fn tokenize(
19	source_code: []const u8,
20	gpa: mem.Allocator,
21	error_pos: *ByteOffset,
22	) TokenizeError!Token.List.Slice {
23	const trace = T.trace(@src());
24	defer trace.end();
25
26	var tokens = Token.List{};
27	errdefer tokens.deinit(gpa);
28
29	// pre-allocate some space based on a very rough guess
30	try tokens.ensureTotalCapacity(gpa, @divFloor(source_code.len, 8));
31
32	var tokenizer: Tokenizer = .{ .code = source_code, .pos = 0 };
33	errdefer error_pos.* = tokenizer.pos;
34
35	tokenizer.skipWhitespace();
36	while (try tokenizer.next()) \|token\| {
37	try tokens.append(gpa, token);
38	tokenizer.skipWhitespace();
39	}
40
41	try tokens.append(gpa, .{
42	.tag = .eof,
43	.offset = @intCast(source_code.len),
44	});
45
46	return tokens.toOwnedSlice();
47	}
48
49	pub const ByteOffset = u32;
50
51	pub const Token = struct {
52	tag: Tag,
53	offset: ByteOffset,
54
55	pub const List = std.MultiArrayList(Token);
56
57	pub const Tag = enum {
58	id,
59	builtin_id,
60	number_int,
61	number_float,
62	string,
63	multiline_string,
64	doc_comment,
65	module_doc_comment,
66
67	// sort-lines: start
68	kw_assert,
69	kw_break,
70	kw_comptime,
71	kw_defer,
72	kw_else,
73	kw_enum,
74	kw_fn,
75	kw_infer,
76	kw_for,
77	kw_while,
78	kw_if,
79	kw_impl,
80	kw_json,
81	kw_let,
82	kw_match,
83	kw_module,
84	kw_pub,
85	kw_pure,
86	kw_return,
87	kw_struct,
88	kw_trait,
89	kw_var,
90	// sort-lines: end
91
92	// sort-lines: start
93	@"and",
94	and_and,
95	and_and_equal,
96	and_equal,
97	bang,
98	bang_equal,
99	caret,
100	caret_equal,
101	colon,
102	comma,
103	dot,
104	end_brace,
105	end_bracket,
106	end_paren,
107	equal,
108	equal_equal,
109	equal_lt,
110	gt,
111	gt_equal,
112	gt_gt,
113	gt_gt_equal,
114	gt_gt_gt,
115	gt_gt_gt_equal,
116	lt,
117	lt_equal,
118	lt_lt,
119	lt_lt_equal,
120	minus,
121	minus_equal,
122	minus_lt,
123	percent,
124	percent_equal,
125	pipe,
126	pipe_equal,
127	pipe_pipe,
128	pipe_pipe_equal,
129	plus,
130	plus_equal,
131	semi,
132	slash,
133	slash_equal,
134	star,
135	star_equal,
136	star_star,
137	star_star_equal,
138	start_brace,
139	start_bracket,
140	start_paren,
141	tilde,
142	tilde_equal,
143	// sort-lines: end
144
145	/// Represents a tokenization error. Instead of reporting an error during
146	/// tokenization, parsing will begin, and an error will be reported when
147	/// this token is hit. That way, the error can tell you what it expected.
148	invalid,
149	/// End of file
150	/// Used so that parser does not need to do out of bounds checks
151	eof,
152
153	/// Most tokens have a static length, such as "struct" or "+="
154	pub fn staticLen(tag: Tag) ?u32 {
155	return switch (tag) {
156	// do not have a static length
157	.id,
158	.builtin_id,
159	.number_int,
160	.number_float,
161	.string,
162	.multiline_string,
163	.doc_comment,
164	.module_doc_comment,
165	=> null,
166
167	.@"and" => 1,
168	.and_and => 2,
169	.bang => 1,
170	.bang_equal => 2,
171	.equal_lt => 2,
172	.caret => 1,
173	.end_brace => 1,
174	.end_bracket => 1,
175	.end_paren => 1,
176	.colon => 1,
177	.comma => 1,
178	.dot => 1,
179	.equal => 1,
180	.equal_equal => 2,
181	.gt => 1,
182	.gt_equal => 2,
183	.gt_gt => 2,
184	.gt_gt_gt => 3,
185	.lt => 1,
186	.lt_equal => 2,
187	.lt_lt => 2,
188	.minus => 1,
189	.start_brace => 1,
190	.start_bracket => 1,
191	.start_paren => 1,
192	.pipe => 1,
193	.pipe_pipe => 2,
194	.percent => 1,
195	.plus => 1,
196	.minus_lt => 2,
197	.semi => 1,
198	.slash => 1,
199	.star => 1,
200	.star_star => 2,
201	.tilde => 1,
202	.slash_equal => 2,
203	.percent_equal => 2,
204	.tilde_equal => 2,
205	.caret_equal => 2,
206	.and_equal => 2,
207	.pipe_equal => 2,
208	.lt_lt_equal => 3,
209	.gt_gt_equal => 3,
210	.gt_gt_gt_equal => 4,
211	.plus_equal => 2,
212	.minus_equal => 2,
213	.star_equal => 2,
214	.star_star_equal => 3,
215	.pipe_pipe_equal => 3,
216	.and_and_equal => 3,
217
218	.invalid => 0,
219	.eof => 0,
220
221	inline else => \|keyword\| comptime len: {
222	if (!mem.startsWith(u8, @tagName(keyword), "kw_")) {
223	@compileError("Token tag is not a keyword: " ++ @tagName(keyword));
224	}
225	break :len @tagName(keyword).len - 3;
226	},
227	};
228	}
229	};
230
231	/// Instead of storing the length of a token, we can recompute this
232	/// information on demand, as it is not often needed.
233	///
234	/// If you are computing the length of something that is knowable
235	/// without the source code, use token.tag.staticLen() instead.
236	pub fn len(token: Token, utf8_source: []const u8) u32 {
237	return token.tag.staticLen() orelse {
238	var tokenizer = Tokenizer{ .code = utf8_source, .pos = token.offset };
239	assert(
240	(tokenizer.next() catch
241	unreachable // data corruption
242	orelse unreachable // data corruption
243	).tag == token.tag,
244	);
245	return tokenizer.pos - token.offset;
246	};
247	}
248
249	pub fn slice(token: Token, utf8_source: []const u8) []const u8 {
250	return utf8_source[token.offset..][0..token.len(utf8_source)];
251	}
252
253	const keywords = std.ComptimeStringMap(Tag, keywords: {
254	const fields = std.enums.values(Tag);
255	const KV = struct { []const u8, Tag };
256	var kws: [fields.len]KV = undefined;
257	var i = 0;
258	for (fields) \|field\| {
259	if (mem.startsWith(u8, @tagName(field), "kw_")) {
260	kws[i] = .{ @tagName(field)[3..], field };
261	i += 1;
262	}
263	}
264	break :keywords kws[0..i];
265	});
266	};
267
268	/// When any of these error occurs, the tokenizer's position is set to
269	/// the start of the offending token.
270	pub const NextTokenError = error{
271	/// Generic token syntax error
272	InvalidSyntax,
273	/// A trailing '_' was seen in a numeric literal.
274	TrailingNumericSeparator,
275	/// Newline or EOF encountered while parsing a string literal.
276	UnterminatedString,
277	/// A string escape sequence was invalid.
278	InvalidEscapeSequence,
279	/// The codepoint represented by a unicode escape sequence
280	/// is not a valid unicode codepoint.
281	InvalidEscapeCodepoint,
282	/// There are unresolved git conflicts in the source code.
283	GitConflict,
284	/// The source code is not valid UTF-8.
285	InvalidUtf8,
286	};
287
288	pub const TokenizeError = mem.Allocator.Error \|\| NextTokenError;
289
290	/// Next assumes the position is already at the start of a token.
291	/// You must call `skipWhitespace` before calling this.
292	pub fn next(t: *Tokenizer) !?Token {
293	while (true) {
294	const trace = T.trace(@src());
295	defer trace.end();
296
297	switch (t.byte() orelse return null) {
298	' ', '\t', '\n', '\r' => unreachable, // call skipWhitespace
299
300	'a'...'z', 'A'...'Z', '_', '$' => {
301	const start = t.pos;
302	t.pos += 1;
303	return try t.tokenizeIdentifier(start);
304	},
305
306	'@' => switch (t.byteAt(1) orelse 0) {
307	'"' => {
308	const start = t.pos;
309	t.pos += 1;
310	try t.consumeString();
311	return .{ .tag = .id, .offset = start };
312	},
313	else => {
314	const start = t.pos;
315	t.pos += 1;
316	return try t.tokenizeIdentifier(start);
317	},
318	},
319
320	'0'...'9' => return try t.tokenizeNumber(),
321
322	'.' => switch (t.byteAt(1) orelse 0) {
323	'0'...'9' => return try t.tokenizeNumber(),
324	else => return t.simpleToken(.dot),
325	},
326
327	'/' => switch (t.byteAt(1) orelse 0) {
328	'=' => return t.simpleToken(.slash_equal),
329	'/' => {
330	if ((t.byteAt(2) orelse 0 == '/') and
331	(t.byteAt(3) orelse 0 != '/'))
332	{
333	const start = t.pos;
334	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 3, '\n') orelse t.code.len);
335	return .{ .tag = .doc_comment, .offset = start };
336	}
337
338	if (t.byteAt(2) orelse 0 == '!') {
339	const start = t.pos;
340	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
341	return .{ .tag = .module_doc_comment, .offset = start };
342	}
343
344	// skip the comment
345	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
346	t.skipWhitespace();
347	continue;
348	},
349	else => return t.simpleToken(.slash),
350	},
351
352	'\\' => switch (t.byteAt(1) orelse 0) {
353	'\\' => {
354	const start = t.pos;
355	t.pos = @intCast(mem.indexOfScalarPos(u8, t.code, t.pos + 2, '\n') orelse t.code.len);
356	return .{ .tag = .multiline_string, .offset = start };
357	},
358	else => return t.maybeUTF8Error(error.InvalidSyntax),
359	},
360
361	'"' => return try t.tokenizeString(),
362
363	'&' => switch (t.byteAt(1) orelse 0) {
364	'&' => return t.simpleToken(.and_and),
365	'=' => return t.simpleToken(.and_equal),
366	else => return t.simpleToken(.@"and"),
367	},
368	'\|' => switch (t.byteAt(1) orelse 0) {
369	'\|' => return t.simpleToken(.pipe_pipe),
370	'=' => return t.simpleToken(.pipe_equal),
371	else => return t.simpleToken(.pipe),
372	},
373	'^' => switch (t.byteAt(1) orelse 0) {
374	'^' => return t.simpleToken(.caret),
375	'=' => return t.simpleToken(.caret_equal),
376	else => return t.simpleToken(.caret),
377	},
378	'!' => switch (t.byteAt(1) orelse 0) {
379	'=' => return t.simpleToken(.equal_equal),
380	else => return t.simpleToken(.bang_equal),
381	},
382	'=' => switch (t.byteAt(1) orelse 0) {
383	'=' => return t.simpleToken(.equal_equal),
384	else => return t.simpleToken(.equal),
385	},
386	'>' => switch (t.byteAt(1) orelse 0) {
387	'=' => return t.simpleToken(.gt_equal),
388	'>' => switch (t.byteAt(2) orelse 0) {
389	'>' => return t.simpleToken(.gt_gt_gt),
390	else => return t.simpleToken(.gt_gt),
391	},
392	else => return t.simpleToken(.gt),
393	},
394	'<' => switch (t.byteAt(1) orelse 0) {
395	'=' => return t.simpleToken(.lt_equal),
396	'<' => switch (t.byteAt(2) orelse 0) {
397	'=' => return t.simpleToken(.lt_lt_equal),
398	else => {
399	// Git merge conflicts are <<<<<<<
400	const start = t.pos;
401	t.pos += 2;
402	brk: {
403	var remain: u3 = 4;
404	while (t.byte() orelse break :brk == '<') {
405	t.pos += 1;
406	remain -= 1;
407	if (remain == 0) {
408	return error.GitConflict;
409	}
410	}
411	}
412	t.pos = start;
413	return t.simpleToken(.lt_lt);
414	},
415	},
416	else => return t.simpleToken(.lt),
417	},
418	'+' => switch (t.byteAt(1) orelse 0) {
419	'=' => return t.simpleToken(.plus_equal),
420	else => return t.simpleToken(.plus),
421	},
422	'-' => switch (t.byteAt(1) orelse 0) {
423	'=' => return t.simpleToken(.minus_equal),
424	'>' => return t.simpleToken(.minus_lt),
425	else => return t.simpleToken(.minus),
426	},
427	'*' => switch (t.byteAt(1) orelse 0) {
428	'*' => return t.simpleToken(.star_star),
429	'=' => return t.simpleToken(.star_equal),
430	else => return t.simpleToken(.star),
431	},
432	'%' => switch (t.byteAt(1) orelse 0) {
433	'=' => return t.simpleToken(.percent_equal),
434	else => return t.simpleToken(.percent),
435	},
436	'~' => switch (t.byteAt(1) orelse 0) {
437	'=' => return t.simpleToken(.tilde_equal),
438	else => return t.simpleToken(.tilde),
439	},
440
441	';' => return t.simpleToken(.semi),
442	':' => return t.simpleToken(.colon),
443	'(' => return t.simpleToken(.start_paren),
444	')' => return t.simpleToken(.end_paren),
445	'[' => return t.simpleToken(.start_bracket),
446	']' => return t.simpleToken(.end_bracket),
447	'{' => return t.simpleToken(.start_brace),
448	'}' => return t.simpleToken(.end_brace),
449	',' => return t.simpleToken(.comma),
450
451	0x80...0xFF => {
452	// need to decode utf-8
453	const byte_len, const code_point = try t.codePoint();
454
455	if (unicode_props.isXidStart(code_point)) {
456	const start = t.pos;
457	t.pos += byte_len;
458	return try t.tokenizeIdentifier(start);
459	}
460
461	return error.InvalidSyntax;
462	},
463
464	else => return error.InvalidSyntax,
465	}
466	}
467	}
468
469	fn byte(t: Tokenizer) ?u8 {
470	if (t.pos >= t.code.len) return null;
471	return t.code[t.pos];
472	}
473
474	/// Remember that using this only makes sense if you know for certain that all
475	/// previous bytes are ASCII, otherwise you may hit UB
476	fn byteAt(t: Tokenizer, offset: u32) ?u8 {
477	if (t.pos + offset >= t.code.len) return null;
478	// assert that the previous bytes are ASCII
479	// current byte is allowed to be a continuation byte
480	for (t.code[t.pos .. t.pos + offset]) \|b\| {
481	assert(b < 0x80);
482	}
483	return t.code[t.pos + offset];
484	}
485
486	fn codePoint(t: Tokenizer) !struct { u3, u21 } {
487	const b = t.byte() orelse unreachable; // must check the first byte manually
488	assert(b >= 0x80); // must be a continuation byte
489
490	const byte_len = unicode.utf8ByteSequenceLength(b) catch
491	return error.InvalidUtf8;
492	assert(byte_len > 1);
493
494	const code_point = unicode.utf8Decode(t.code[t.pos..][0..byte_len]) catch
495	return error.InvalidUtf8;
496
497	return .{ byte_len, code_point };
498	}
499
500	fn isAtIdentifierStart(t: *Tokenizer) !bool {
501	switch (t.byte() orelse return false) {
502	'a'...'z', 'A'...'Z', '_', '$' => return true,
503	else => return false,
504	0x80...0xFF => {
505	_, const code_point = try t.codePoint();
506	return unicode_props.isXidStart(code_point);
507	},
508	}
509	}
510
511	fn tokenizeIdentifier(t: *Tokenizer, start_byte: u32) !Token {
512	const trace = T.trace(@src());
513	defer trace.end();
514
515	while (true) {
516	switch (t.byte() orelse break) {
517	'a'...'z', 'A'...'Z', '0'...'9', '_', '$' => {
518	t.pos += 1;
519	},
520	0x80...0xFF => {
521	const byte_len, const code_point = try t.codePoint();
522
523	t.pos += byte_len;
524
525	if (!unicode_props.isXidContinue(code_point)) {
526	break;
527	}
528	},
529	else => break,
530	}
531	}
532
533	return .{
534	.tag = Token.keywords.get(t.code[start_byte..t.pos]) orelse
535	if (t.code[start_byte] == '@')
536	.builtin_id
537	else
538	.id,
539	.offset = start_byte,
540	};
541	}
542
543	/// Numbers have the following rules:
544	/// - May start with a `.`, but cannot end with `.`
545	/// - May contain leading zeros, but transform pass will warn.
546	/// - May contain a `_` separator, cannot have two in a row.
547	/// - May contain `e` or `E` for exponents.
548	/// - May use 0x, 0b, 0o prefixes for hex, binary, octal numbers.
549	/// The token is tagged as `number_int` or `number_float` depending
550	/// on if it contained a `.` This is to help later re-parses.
551	//
552	// TODO: is it a zig compiler bug that `?` is required here?
553	fn tokenizeNumber(t: *Tokenizer) !?Token {
554	const trace = T.trace(@src());
555	defer trace.end();
556
557	// Caller ensures the following invariant:
558	switch (t.byte() orelse unreachable) {
559	'.' => switch (t.byteAt(1) orelse unreachable) {
560	'0'...'9' => {},
561	else => unreachable,
562	},
563	'0'...'9' => {},
564	else => unreachable,
565	}
566
567	// Parse Hex, Binary, Octal numbers
568	if (t.byte() == '0') {
569	switch (t.byteAt(1) orelse 0) {
570	'x', 'X' => {
571	t.pos += 2;
572	try t.consumeIntegerRadix(.hex);
573	},
574	'b', 'B' => {
575	t.pos += 2;
576	try t.consumeIntegerRadix(.binary);
577	},
578	'o', 'O' => {
579	t.pos += 2;
580	try t.consumeIntegerRadix(.octal);
581	},
582	else => {},
583	}
584	}
585
586	const start = t.pos;
587	var tag: Token.Tag = .number_int;
588
589	if (t.byte() == '.') {
590	t.pos += 1;
591	tag = .number_float;
592	try t.consumeInteger();
593	} else {
594	try t.consumeInteger();
595
596	// decimal point notation
597	switch (t.byte() orelse 0) {
598	'.' => switch (t.byteAt(1) orelse 0) {
599	'0'...'9' => {
600	t.pos += 1;
601	tag = .number_float;
602	try t.consumeInteger();
603	},
604	else => {
605	// we have not advanced the tokenizer past the `.`
606	assert(tag == .number_int);
607	return .{ .tag = tag, .offset = start };
608	},
609	},
610	else => {},
611	}
612	}
613
614	switch (t.byte() orelse 0) {
615	'e', 'E' => {
616	tag = .number_float;
617	t.pos += 1;
618
619	switch (t.byte() orelse 0) {
620	'+', '-' => {
621	t.pos += 1;
622	switch (t.byte() orelse 0) {
623	'0'...'9' => {},
624	else => return t.maybeUTF8Error(error.InvalidSyntax),
625	}
626	},
627	'0'...'9' => {},
628	else => return t.maybeUTF8Error(error.InvalidSyntax),
629	}
630
631	try t.consumeInteger();
632
633	// decimal point notation
634	switch (t.byte() orelse 0) {
635	'.' => switch (t.byteAt(1) orelse 0) {
636	'0'...'9' => {
637	t.pos += 1;
638	tag = .number_float;
639	try t.consumeInteger();
640	},
641	else => {},
642	},
643	else => {},
644	}
645	},
646	else => {},
647	}
648
649	// after numbers must be a valid word break the above returns are right
650	// before parsing `.` so those can be unchecked.
651	if (try t.isAtIdentifierStart()) {
652	return error.InvalidSyntax;
653	}
654
655	return .{
656	.tag = tag,
657	.offset = start,
658	};
659	}
660
661	fn maybeConsumeInteger(t: *Tokenizer) !?void {
662	const trace = T.trace(@src());
663	defer trace.end();
664
665	switch (t.byte() orelse return null) {
666	'0'...'9' => {
667	t.pos += 1;
668	},
669	else => return null,
670	}
671	t.consumeInteger();
672	}
673
674	/// Requires the first byte to be a digit
675	fn consumeInteger(t: *Tokenizer) !void {
676	const trace = T.trace(@src());
677	defer trace.end();
678
679	switch (t.byte() orelse unreachable) {
680	'0'...'9' => {
681	t.pos += 1;
682	},
683	else => unreachable,
684	}
685
686	while (true) {
687	switch (t.byte() orelse return) {
688	'0'...'9' => {
689	t.pos += 1;
690	},
691	'_' => {
692	t.pos += 1;
693	switch (t.byte() orelse 0) {
694	'0'...'9' => {
695	t.pos += 1;
696	},
697	else => return t.maybeUTF8Error(error.TrailingNumericSeparator),
698	}
699	},
700	else => break,
701	}
702	}
703	}
704
705	const Radix = enum { hex, binary, octal };
706
707	fn consumeIntegerRadix(t: *Tokenizer, comptime radix: Radix) !void {
708	const trace = T.trace(@src());
709	defer trace.end();
710
711	const digits = switch (radix) {
712	.hex => "0123456789abcdefABCDEF",
713	.binary => "01",
714	.octal => "01234567",
715	};
716
717	first: {
718	const b = t.byte() orelse return;
719	inline for (digits) \|digit\| {
720	if (b == digit) {
721	break :first;
722	}
723	}
724	return t.maybeUTF8Error(error.InvalidSyntax);
725	}
726
727	t.pos += 1;
728
729	while (true) {
730	brk: {
731	const b = t.byte() orelse return;
732	inline for (digits) \|digit\| {
733	if (b == digit) {
734	break :brk;
735	}
736	}
737	return;
738	}
739	t.pos += 1;
740	}
741	}
742
743	fn tokenizeString(t: *Tokenizer) !Token {
744	assert(t.byte().? == '"');
745	const start = t.pos;
746	try t.consumeString();
747	return .{ .tag = .string, .offset = start };
748	}
749
750	fn consumeString(t: *Tokenizer) !void {
751	const trace = T.trace(@src());
752	defer trace.end();
753
754	assert(t.byte().? == '"');
755
756	t.pos += 1;
757
758	while (true) {
759	const b = t.byte() orelse return error.UnterminatedString;
760	switch (b) {
761	'"' => {
762	t.pos += 1;
763	break;
764	},
765	'\\' => {
766	switch (t.byteAt(1) orelse {
767	// we want error.InvalidEscapeSequence to be at the '\' char,
768	// but we want error.UnterminatedString to be at the end of the source code
769	t.pos += 1;
770	return error.UnterminatedString;
771	}) {
772	'\n' => return error.UnterminatedString,
773	'\\' => t.pos += 2,
774	'"' => t.pos += 2,
775	'n' => t.pos += 2,
776	'r' => t.pos += 2,
777	't' => t.pos += 2,
778	'0' => switch (t.byteAt(2) orelse {
779	t.pos += 2;
780	return error.UnterminatedString;
781	}) {
782	// '\0xFF' hex escape
783	'x' => {},
784	// '\0' is special cased
785	else => t.pos += 2,
786	},
787	'u' => {
788	const esc_start = t.pos;
789	t.consumeStringUnicodeEscape() catch \|err\| switch (err) {
790	error.InvalidEscapeSequence => {
791	if (t.byte().? == '\n')
792	return error.UnterminatedString;
793	t.pos = esc_start;
794	return err;
795	},
796	error.InvalidEscapeCodepoint => {
797	t.pos = esc_start;
798	return err;
799	},
800	error.UnterminatedString => return err,
801	};
802	},
803	else => return t.maybeUTF8ErrorAt(error.InvalidEscapeSequence, 1),
804	}
805	},
806	else => {
807	t.pos += 1;
808	},
809	0x80...0xFF => {
810	const byte_len, _ = try t.codePoint();
811	t.pos += byte_len;
812	},
813	}
814	}
815	}
816
817	fn consumeStringUnicodeEscape(t: *Tokenizer) !void {
818	const trace = T.trace(@src());
819	defer trace.end();
820
821	assert(t.byte().? == '\\');
822	assert(t.byteAt(1).? == 'u');
823	t.pos += 2;
824
825	if ((t.byte() orelse return error.UnterminatedString) != '{')
826	return error.InvalidEscapeSequence;
827	t.pos += 1;
828
829	if ((t.byte() orelse return error.UnterminatedString) == '0') {
830	if ((t.byteAt(1) orelse return error.UnterminatedString) == 'x') {
831	t.pos += 2;
832	}
833	}
834
835	const start = t.pos;
836	t.consumeIntegerRadix(.hex) catch return error.InvalidEscapeSequence;
837
838	const slice = t.code[start..t.pos];
839
840	if ((t.byte() orelse return error.UnterminatedString) != '}')
841	return error.InvalidEscapeSequence;
842
843	const code_point = std.fmt.parseInt(u21, slice, 16) catch \|err\| switch (err) {
844	error.Overflow => return error.InvalidEscapeCodepoint,
845	error.InvalidCharacter => unreachable,
846	};
847	_ = std.unicode.utf8CodepointSequenceLength(code_point) catch \|err\| switch (err) {
848	error.CodepointTooLarge => return error.InvalidEscapeCodepoint,
849	};
850
851	t.pos += 1;
852	}
853
854	fn simpleToken(t: *Tokenizer, comptime tag: Token.Tag) Token {
855	const start = t.pos;
856	t.pos += comptime (tag.staticLen() orelse unreachable);
857	return .{ .tag = tag, .offset = start };
858	}
859
860	/// We are about to return an error, such as error.TrailingNumericSeparator.
861	/// However, the character we ran into might actually be invalid UTF-8.
862	/// This utility just checks for that to make sure we dont mask error.InvalidUtf8
863	fn maybeUTF8Error(t: *Tokenizer, comptime real_err: anytype) @Type(.{
864	.ErrorSet = &[_]std.builtin.Type.Error{
865	.{ .name = "InvalidUtf8" },
866	.{ .name = @errorName(real_err) },
867	},
868	}) {
869	switch (t.byte() orelse return real_err) {
870	0x80...0xFF => {
871	_ = try t.codePoint();
872	return real_err;
873	},
874	else => return real_err,
875	}
876	}
877
878	fn maybeUTF8ErrorAt(t: *Tokenizer, comptime real_err: anytype, at: u32) @Type(.{
879	.ErrorSet = &[_]std.builtin.Type.Error{
880	.{ .name = "InvalidUtf8" },
881	.{ .name = @errorName(real_err) },
882	},
883	}) {
884	switch (t.byteAt(at) orelse return real_err) {
885	0x80...0xFF => {
886	_ = try t.codePoint();
887	return real_err;
888	},
889	else => return real_err,
890	}
891	}
892
893	pub fn skipWhitespace(t: *Tokenizer) void {
894	const trace = T.trace(@src());
895	defer trace.end();
896
897	while (true) {
898	switch (t.byte() orelse break) {
899	' ', '\t', '\n', '\r' => {
900	t.pos += 1;
901	},
902	else => break,
903	}
904	}
905	}
906
907	const Tokenizer = @This();
908	const std = @import("std");
909	const unicode = std.unicode;
910	const assert = std.debug.assert;
911	const mem = std.mem;
912	const unicode_props = @import("unicode_props.zig");
913	const T = @import("tracer");

src/compiler/Transform.zig created+1276

...	@@ -0,0 +1,1276 @@
1	//! Transform pass
2	gpa: Allocator,
3	ast: Ast,
4	insts: Inst.List = .{},
5	string_bytes: std.ArrayListUnmanaged(u8) = .{},
6	extra: ExtraStore = .{},
7
8	scratch: std.ArrayListUnmanaged(u32) = .{},
9
10	// TODO: refactor this away
11	// The new home for this memory should be pre-allocated on scratch,
12	// but then remove the elements, moving scratch items on top of this
13	// backwards a few.
14	call_args: std.ArrayListUnmanaged(Inst.Ref) = .{},
15	scopes: std.ArrayListUnmanaged(Scope) = .{},
16
17	pub fn transform(ast: Ast, gpa: Allocator) !Teg {
18	const trace = tracer.trace(@src());
19	defer trace.end();
20
21	var t: Transform = .{
22	.ast = ast,
23	.gpa = gpa,
24	};
25	errdefer t.deinit();
26
27	try t.string_bytes.append(t.gpa, 0);
28
29	const module = try t.transformModule(0);
30	assert(@intFromEnum(module) == 0);
31
32	try t.insts.setCapacity(t.gpa, t.insts.len);
33
34	defer t.deinit();
35
36	return Teg{
37	.insts = t.insts.toOwnedSlice(),
38	.string_bytes = try t.string_bytes.toOwnedSlice(t.gpa),
39	.extra = try t.extra.toOwnedStore(t.gpa),
40	};
41	}
42
43	pub fn deinit(t: *Transform) void {
44	t.scratch.deinit(t.gpa);
45	t.call_args.deinit(t.gpa);
46	assert(t.scopes.items.len == 0);
47	t.scopes.deinit(t.gpa);
48	t.* = undefined;
49	}
50
51	const Scope = struct {
52	symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
53
54	const Symbol = struct {
55	node: NodeIndex,
56	data: Symbol.Data,
57	used: bool = false,
58	mutated: bool = false,
59
60	const Data = union(enum) {
61	declaration,
62	param,
63	mutable: Inst.Ref,
64	immutable: Inst.Ref,
65	};
66	};
67
68	fn deinit(s: *Scope, gpa: Allocator) void {
69	s.symbols.deinit(gpa);
70	s.* = undefined;
71	}
72	};
73
74	const ScratchType = enum {
75	ref,
76	index,
77	};
78
79	const BlockContext = struct {
80	node_base: i32,
81	return_target: ?*?Inst.Index = null,
82	break_target: ?*BreakTarget = null,
83
84	/// This struct is always stored on the stack; the idea is to pass
85	/// mutable pointers to it. When `break :brk` is encountered
86	const BreakTarget = struct {
87	label: ?[]const u8,
88	/// This allows `continue` as well as implicit break targetting.
89	is_loop: bool,
90	/// If `null`, this label was unused.
91	block_index: ?Inst.Index = null,
92	parent: ?*BreakTarget = null,
93
94	fn addChild(parent: *BreakTarget, label: ?[]const u8, is_loop: bool) BreakTarget {
95	return .{
96	.parent = parent,
97	.label = label,
98	.is_loop = is_loop,
99	};
100	}
101
102	fn getInst(target: BreakTarget, t: Transform) !Inst.Index {
103	return target.block_index orelse {
104	const inst = try t.addRootInst(.{
105	.tag = .block,
106	.data = undefined,
107	});
108	target.block_index = inst;
109	return inst;
110	};
111	}
112	};
113
114	fn init(root_node: NodeIndex) BlockContext {
115	return .{
116	.node_base = @intCast(root_node),
117	};
118	}
119
120	fn nodeOffset(self: BlockContext, node: NodeIndex) i32 {
121	return @as(i32, @intCast(node)) - self.node_base;
122	}
123
124	fn declOffset(self: BlockContext, node: NodeIndex) BlockContext {
125	return .{
126	.node_base = self.nodeOffset(node),
127	};
128	}
129
130	fn withReturn(self: BlockContext, return_target: ?*?Inst.Index) BlockContext {
131	return .{
132	.node_base = self.node_base,
133	.return_target = return_target,
134	};
135	}
136
137	fn withBreak(self: BlockContext, break_target: *BreakTarget) BlockContext {
138	return .{
139	.node_base = self.node_base,
140	.return_target = self.return_target,
141	.break_target = break_target,
142	};
143	}
144
145	fn findAndMarkBreak(self: BlockContext, t: *Transform, label: []const u8) !?Inst.Index {
146	var break_target = self.break_target;
147	while (break_target) \|target\| : (break_target = target.parent) {
148	if (target.label) \|target_label\| {
149	if (mem.eql(u8, label, target_label)) {
150	return try target.getInst(t);
151	}
152	}
153	}
154	return null;
155	}
156	};
157
158	const Liveness = enum { alive, dead };
159
160	const BuiltInIds = std.ComptimeStringMap(Inst.Ref, .{
161	.{ "void", .void_value },
162	.{ "true", .bool_true },
163	.{ "false", .bool_false },
164	.{ "int", .int32_type },
165	.{ "float", .float_type },
166	.{ "string", .string_type },
167	// .{ "bigint", .bigint_type },
168	.{ "bool", .bool_type },
169	.{ "noreturn", .noreturn_type },
170	.{ "type", .type_type },
171	// todo: nan/infinity?
172	});
173
174	fn transformModule(t: *Transform, module_node: NodeIndex) !Inst.Index {
175	const trace = tracer.trace(@src());
176	defer trace.end();
177
178	const module = try t.addRootInst(.{
179	.tag = .module,
180	.data = undefined,
181	});
182
183	const decl = BlockContext.init(module_node);
184
185	try t.pushScope();
186	defer t.popScope();
187
188	const statements = t.ast.nodeStatements(module_node);
189	for (statements) \|statement\| {
190	try t.previsitModuleStatement(statement);
191	}
192
193	const s = t.scope();
194
195	const temp = try t.gpa.alloc(Inst.Index, s.symbols.count() + 1);
196	defer t.gpa.free(temp);
197
198	{
199	// prevent concurrent access
200	s.symbols.lockPointers();
201	defer s.symbols.unlockPointers();
202	for (s.symbols.values(), 0..) \|value, i\| {
203	temp[i] = try t.transformFunction(decl, value.node);
204	}
205	}
206
207	const top_level_fn = try t.addRootInst(.{
208	.tag = .declaration,
209	.data = undefined,
210	});
211
212	temp[temp.len - 1] = top_level_fn;
213
214	const declarations: IndexList = .{
215	.start = @intCast(t.extra.list.items.len),
216	.len = @intCast(temp.len),
217	};
218	try t.extra.list.appendSlice(t.gpa, @ptrCast(temp));
219
220	const block = try t.addRootInst(.{
221	.tag = .block,
222	.data = undefined,
223	});
224
225	{
226	const scratch_top = t.scratch.items.len;
227	defer t.scratch.shrinkRetainingCapacity(scratch_top);
228
229	for (statements) \|statement\| {
230	try t.transformModuleStatement(decl, statement);
231	}
232
233	t.getInstData(block).* = EncodedData.encode(.block, .{
234	.src_node = decl.nodeOffset(module_node),
235	.extra_index = try t.extra.append(t.gpa, IndexList, try t.scratchToIndexList(scratch_top)),
236	});
237	}
238
239	const decl_extra, var trail = try t.extra.appendTrailing(t.gpa, Inst.Decl, .{
240	.kind = .function,
241	.name = NullTerminatedString.empty,
242	.hash = .{0} ** 16,
243	.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(top_level_fn),
244	}, dod.extraSizeOf(Inst.DeclFn));
245
246	trail.writeStruct(Inst.DeclFn, .{
247	.params = .{ .start = 0, .len = 0 },
248	.return_type = Inst.Ref.void_type,
249	.body = block.toRef(),
250	});
251
252	t.getInstData(top_level_fn).* = EncodedData.encode(.declaration, .{
253	.src_node = decl.nodeOffset(module_node),
254	.extra_index = decl_extra,
255	});
256
257	t.getInstData(module).* = EncodedData.encode(.module, .{
258	.src_node = 0,
259	.extra_index = try t.extra.append(t.gpa, IndexList, declarations),
260	});
261
262	return module;
263	}
264
265	fn previsitModuleStatement(t: *Transform, node: NodeIndex) !void {
266	const trace = tracer.trace(@src());
267	defer trace.end();
268
269	const tag = t.ast.nodes.items(.tag)[node];
270	switch (tag) {
271	.fn_decl => {
272	const fn_decl = t.ast.fullFnDecl(node);
273	const gop = try t.scope().symbols.getOrPut(t.gpa, fn_decl.id);
274	if (gop.found_existing) {
275	return error.DuplicateFunction;
276	}
277	gop.value_ptr.* = .{
278	.node = node,
279	.data = .declaration,
280	};
281	},
282	else => {},
283	}
284	}
285
286	fn transformModuleStatement(t: *Transform, decl: BlockContext, node: NodeIndex) !void {
287	const trace = tracer.trace(@src());
288	defer trace.end();
289
290	const tag = t.ast.nodes.items(.tag)[node];
291	switch (tag) {
292	.root,
293	.param_decl,
294	=> unreachable,
295
296	.fn_decl => {}, // functions analyzed separately
297
298	else => _ = try t.transformBlockStatement(decl, node, false),
299	}
300	}
301
302	fn transformFunction(t: *Transform, outer_decl: BlockContext, node: NodeIndex) !Inst.Index {
303	const trace = tracer.trace(@src());
304	defer trace.end();
305
306	const decl = outer_decl.declOffset(node);
307	const fn_decl = t.ast.fullFnDecl(node);
308
309	const func = try t.addRootInst(.{
310	.tag = .declaration,
311	.data = undefined,
312	});
313
314	try t.pushScope();
315	defer t.popScope();
316
317	const param_list = brk: {
318	const scratch_top = t.scratch.items.len;
319	defer t.scratch.shrinkRetainingCapacity(scratch_top);
320
321	for (fn_decl.params) \|param\| {
322	try t.addScratch(Inst.Ref, try t.transformParamDecl(decl, param));
323	}
324
325	break :brk try t.scratchToRefList(scratch_top);
326	};
327
328	const return_type = if (fn_decl.return_type) \|return_type_node\|
329	try t.transformExpressionToBlock(decl, return_type_node, false)
330	else
331	Inst.Ref.void_type;
332
333	const body = try t.transformExpressionToBlock(decl, fn_decl.body, true);
334
335	const decl_extra = try t.extra.append(t.gpa, Inst.Decl, .{
336	.kind = .function,
337	.name = try t.addNullTerminatedString(fn_decl.id),
338	.hash = .{0} ** 16,
339	.instruction_count = @as(u32, @intCast(t.insts.len)) - @intFromEnum(func),
340	});
341
342	_ = try t.extra.append(t.gpa, Inst.DeclFn, .{
343	.params = param_list,
344	.return_type = return_type,
345	.body = body,
346	});
347
348	t.getInstData(func).* = EncodedData.encode(.declaration, .{
349	.src_node = 0,
350	.extra_index = decl_extra,
351	});
352
353	return func;
354	}
355
356	const TransformFunctionCtx = struct {
357	param_list: RefList,
358	};
359
360	/// Converts expressions into blocks with a break. This is important
361	/// for places where a block is not allowed, such as a parameter list.
362	/// This is also used for the function body, even though it is a block.
363	/// Simple refs turn into just the ref
364	fn transformExpressionToBlock(
365	t: *Transform,
366	decl: BlockContext,
367	node: NodeIndex,
368	comptime is_function: bool,
369	) anyerror!Inst.Ref {
370	const trace = tracer.trace(@src());
371	defer trace.end();
372
373	const node_tag = t.ast.nodes.items(.tag)[node];
374	switch (node_tag) {
375	.id => {
376	// Lower builtin ids without using a block
377	const id = t.ast.basicNodeSlice(node);
378	if (BuiltInIds.get(id)) \|ref\| {
379	return ref;
380	}
381	},
382
383	else => {},
384	}
385
386	var block: ?Inst.Index = null;
387
388	const scratch_top = t.scratch.items.len;
389	defer t.scratch.shrinkRetainingCapacity(scratch_top);
390
391	if (is_function) {
392	const s = t.scopes.getLast();
393	for (s.symbols.values(), 0..) \|*param, i\| {
394	assert(param.data == .param);
395	const ref = try t.addInstRef(.{
396	.tag = .param,
397	.data = EncodedData.encode(.param, .{
398	.src_node = decl.nodeOffset(node),
399	.data = @intCast(i),
400	}),
401	});
402	param.data = .{ .immutable = ref };
403	}
404	}
405
406	const expr = try t.transformExpression(
407	if (is_function)
408	decl.withReturn(&block)
409	else
410	decl,
411	node,
412	);
413
414	// if there are no new instructions, we dont have to create a block context
415	if (scratch_top == t.scratch.items.len) {
416	const result = expr orelse unreachable; // null means we added break or return
417
418	// TODO: evaluate if `if` blocks would be fine in sema with instruction refs
419	// with this extra if check here, we filter out non-blocks.
420	if (result.toIndex() == null) {
421	return result;
422	}
423	}
424
425	if (expr) \|result\| {
426	if (result.toIndex()) \|index\| {
427	if (t.insts.items(.tag)[@intFromEnum(index)] == .block) {
428	return result;
429	}
430	}
431	}
432
433	const ret_block = if (block) \|b\|
434	b
435	else
436	try t.addRootInst(.{
437	.tag = .block,
438	.data = undefined,
439	});
440
441	if (expr) \|result\| {
442	_ = try t.addInstRef(.{
443	.tag = .@"break",
444	.data = EncodedData.encode(.@"break", .{
445	.src_node = decl.nodeOffset(node),
446	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
447	.block = ret_block,
448	.operand = result,
449	}),
450	}),
451	});
452	}
453
454	const content = try t.scratchToIndexList(scratch_top);
455
456	t.getInstData(ret_block).* = EncodedData.encode(.block, .{
457	.src_node = decl.nodeOffset(node),
458	.extra_index = try t.extra.append(t.gpa, IndexList, content),
459	});
460
461	return ret_block.toRef();
462	}
463
464	fn transformParamDecl(t: *Transform, decl: BlockContext, param: NodeIndex) !Inst.Ref {
465	const trace = tracer.trace(@src());
466	defer trace.end();
467
468	const param_decl = t.ast.fullParamDecl(param);
469	const value = try t.transformExpressionToBlock(decl, param_decl.type, false);
470
471	// add `sym` into the scope so that the next param's type
472	// can use this param's value, if known at compile time.
473	const sym = try t.scope().symbols.getOrPut(t.gpa, param_decl.id);
474	if (sym.found_existing) {
475	return error.DuplicateParam;
476	}
477
478	sym.value_ptr.* = .{
479	.node = param,
480	.data = .param,
481	};
482
483	return value;
484	}
485
486	const Builtin = enum {
487	js,
488	import,
489	fetch,
490	compile_error,
491
492	const map = std.ComptimeStringMap(Builtin, .{
493	.{ "js", .js },
494	.{ "import", .import },
495	.{ "fetch", .fetch },
496	.{ "compileError", .compile_error },
497	});
498
499	fn builtin_js(t: *Transform, decl: BlockContext, args: []const NodeIndex) !?Inst.Ref {
500	const trace = tracer.trace(@src());
501	defer trace.end();
502
503	if (args.len != 1) {
504	return error.InvalidBuiltin;
505	}
506
507	const source = try t.transformExpression(decl, args[0]) orelse return null;
508
509	return try t.addInstRef(.{
510	.tag = .js,
511	.data = EncodedData.encode(.js, .{
512	.src_node = decl.nodeOffset(args[0]),
513	.operand = source,
514	}),
515	});
516	}
517	};
518
519	/// This is the primary transformation function: converting an AST expression into instructions.
520	/// `null` represents a `return` or `break`, aka this branch of code is dead.
521	fn transformExpression(t: *Transform, decl: BlockContext, node: NodeIndex) anyerror!?Inst.Ref {
522	const node_tag = t.ast.nodes.items(.tag)[node];
523
524	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
525	defer trace.end();
526
527	switch (node_tag) {
528	.root,
529	.param_decl,
530	.var_decl,
531	.fn_decl,
532	=> unreachable, // not an expression
533
534	.id => {
535	const id = t.ast.basicNodeSlice(node);
536	if (BuiltInIds.get(id)) \|ref\| {
537	return ref;
538	}
539
540	if (mem.eql(u8, id, "_")) {
541	return error.InvalidDiscard;
542	}
543
544	if (mem.startsWith(u8, id, "@")) {
545	@panic("TODO: string identifiers");
546	}
547
548	const lookup = t.lookupId(id) orelse {
549	std.debug.print("lookupId failed for {s}\n", .{id});
550	return error.UnknownIdentifier;
551	};
552
553	return switch (lookup.data) {
554	.declaration => try t.addInstRef(.{
555	.tag = .load_decl,
556	.data = EncodedData.encode(.load_decl, .{
557	.src_node = decl.nodeOffset(node),
558	.string = try t.addNullTerminatedString(id),
559	}),
560	}),
561	.mutable,
562	.immutable,
563	=> \|ref\| ref,
564	.param => unreachable, //unresolved param
565	};
566	},
567
568	inline .number_int, .number_float => \|tag\| {
569	const text = t.ast.basicNodeSlice(node);
570	const value = switch (tag) {
571	.number_int => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
572	.number_float => std.fmt.parseFloat(f64, text) catch @panic("TODO: write my own float parser"),
573	else => comptime unreachable,
574	};
575	return try t.addInstRef(.{
576	.tag = .number,
577	.data = EncodedData.encode(.number, .{
578	.src_node = decl.nodeOffset(node),
579	.extra_index = try t.extra.append(t.gpa, Float, Float.encode(value)),
580	}),
581	});
582	},
583
584	.string => {
585	const str = t.ast.basicNodeSlice(node);
586	const sliced = try t.addSlicedString(str[1 .. str.len - 1]);
587	return try t.addInstRef(.{
588	.tag = .string,
589	.data = EncodedData.encode(.string, .{
590	.src_node = decl.nodeOffset(node),
591	.extra_index = try t.extra.append(t.gpa, SlicedString, sliced),
592	}),
593	});
594	},
595
596	.assign => {
597	const ast_bin = t.ast.nodeData(node).bin;
598	const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
599	const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
600	const assign_inst = try t.addInstRef(.{
601	.tag = .store,
602	.data = EncodedData.encode(.store, .{
603	.src_node = decl.nodeOffset(node),
604	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
605	.lhs = lhs,
606	.rhs = rhs,
607	}),
608	}),
609	});
610	return assign_inst;
611	},
612
613	inline .add,
614	.sub,
615	.mul,
616	.div,
617	.rem,
618	.pow,
619	.@"and",
620	.@"or",
621	.bit_and,
622	.bit_or,
623	.bit_xor,
624	.shl,
625	.shr,
626	.eq,
627	.ne,
628	.lt,
629	.le,
630	.gt,
631	.ge,
632	.add_assign,
633	.sub_assign,
634	.mul_assign,
635	.div_assign,
636	.rem_assign,
637	.pow_assign,
638	.and_assign,
639	.or_assign,
640	.bit_and_assign,
641	.bit_or_assign,
642	.bit_xor_assign,
643	.shl_assign,
644	.shr_assign,
645	=> \|tag\| {
646	// This code path implements binary operations, as well as the
647	// compound compute + assign operation.
648	const is_assign = comptime tag.isAssignBin();
649	const teg_tag = @field(
650	Teg.Inst.Tag,
651	if (is_assign)
652	@tagName(tag)[0 .. @tagName(tag).len - "_assign".len]
653	else
654	@tagName(tag),
655	);
656	const ast_bin = t.ast.nodeData(node).bin;
657	const lhs = try t.transformExpression(decl, ast_bin.lhs) orelse return null;
658	const rhs = try t.transformExpression(decl, ast_bin.rhs) orelse return null;
659
660	const compute_inst = try t.addInstRef(.{
661	.tag = teg_tag,
662	.data = EncodedData.encode(teg_tag, .{
663	.src_node = decl.nodeOffset(node),
664	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
665	.lhs = lhs,
666	.rhs = rhs,
667	}),
668	}),
669	});
670
671	if (is_assign) {
672	const assign_inst = try t.addInstRef(.{
673	.tag = .store,
674	.data = EncodedData.encode(.store, .{
675	.src_node = decl.nodeOffset(node),
676	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
677	.lhs = lhs,
678	.rhs = compute_inst,
679	}),
680	}),
681	});
682	return assign_inst;
683	}
684
685	return compute_inst;
686	},
687
688	inline .negate,
689	.bool_not,
690	.bit_not,
691	=> \|tag\| {
692	const ast_un = t.ast.nodeData(node).un;
693	const teg_tag = @field(Teg.Inst.Tag, @tagName(tag));
694
695	const operand = try t.transformExpression(decl, ast_un.expr) orelse
696	return null;
697
698	return try t.addInstRef(.{
699	.tag = teg_tag,
700	.data = EncodedData.encode(teg_tag, .{
701	.src_node = decl.nodeOffset(node),
702	.operand = operand,
703	}),
704	});
705	},
706
707	.label => {
708	const expr = t.ast.nodeData(node).label.expr;
709	const label = t.ast.tokenSlice(t.ast.nodes.items(.main_token)[node]);
710
711	var target: BlockContext.BreakTarget = .{
712	.parent = decl.break_target,
713	.label = label,
714	.is_loop = false,
715	};
716	const inner_ctx = decl.withBreak(&target);
717
718	const scratch_top = t.scratch.items.len;
719
720	const inner_expr = try t.transformExpression(inner_ctx, expr);
721
722	// if there are no new instructions, we dont have to create a block context
723	if (scratch_top == t.scratch.items.len)
724	return inner_expr orelse unreachable; // null means we added break or return
725
726	if (target.block_index) \|block\| {
727	if (inner_expr) \|result\| {
728	_ = try t.addInstRef(.{
729	.tag = .@"break",
730	.data = EncodedData.encode(.@"break", .{
731	.src_node = decl.nodeOffset(node),
732	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
733	.block = block,
734	.operand = result,
735	}),
736	}),
737	});
738	}
739
740	const content = try t.scratchToIndexList(scratch_top);
741	t.scratch.shrinkRetainingCapacity(scratch_top);
742
743	t.getInstData(block).* = EncodedData.encode(.block, .{
744	.src_node = decl.nodeOffset(node),
745	.extra_index = try t.extra.append(t.gpa, IndexList, content),
746	});
747
748	try t.addScratch(Inst.Index, block);
749
750	return block.toRef();
751	} else {
752	std.debug.print("Warn, unused label {s}", .{label});
753	}
754
755	return inner_expr;
756	},
757
758	inline .block_void,
759	.block_implicit_return,
760	=> \|tag\| {
761	const statements = t.ast.nodeStatements(node);
762
763	var last: Inst.Ref = .void_value;
764	for (statements, 1..) \|statement, i\| {
765	const is_last = i == statements.len;
766	last = try t.transformBlockStatement(
767	decl,
768	statement,
769	tag == .block_implicit_return and is_last,
770	) orelse {
771	return null;
772	};
773	}
774
775	return if (tag == .block_implicit_return) last else .void_value;
776	},
777
778	.builtin_call,
779	=> {
780	const builtin = Builtin.map.get(t.ast.basicNodeSlice(node)[1..]) orelse {
781	return error.InvalidBuiltin;
782	};
783
784	const base = t.ast.nodeData(node).list;
785	const args = base.slice(&t.ast);
786
787	return switch (builtin) {
788	.js => Builtin.builtin_js(t, decl, args),
789	// .import => Builtin.import(t, decl, args),
790	// .fetch => Builtin.fetch(t, decl, args),
791	// .compile_error => Builtin.compile_error(t, decl, args),
792	else => @panic("TODO"),
793	};
794	},
795
796	.call => {
797	const call = t.ast.nodeData(node).call;
798	const args = t.ast.extraData(call.extra, Ast.Call).args.slice(&t.ast);
799
800	const base = try t.transformExpression(decl, call.base) orelse return null;
801
802	const args_list: RefList = brk: {
803	const args_top = t.call_args.items.len;
804	defer t.call_args.shrinkRetainingCapacity(args_top);
805
806	for (args) \|arg\| {
807	try t.call_args.append(
808	t.gpa,
809	try t.transformExpression(decl, arg) orelse
810	return null,
811	);
812	}
813
814	const items = t.call_args.items[args_top..];
815	const extra_top = t.extra.list.items.len;
816	try t.extra.list.appendSlice(t.gpa, @ptrCast(items));
817
818	break :brk .{
819	.start = @intCast(extra_top),
820	.len = @intCast(items.len),
821	};
822	};
823
824	const call_inst = try t.addInstRef(.{
825	.tag = .call,
826	.data = EncodedData.encode(.call, .{
827	.src_node = decl.nodeOffset(node),
828	.extra_index = try t.extra.append(t.gpa, Inst.Call, .{
829	.base = base,
830	.args = args_list,
831	}),
832	}),
833	});
834
835	return call_inst;
836	},
837
838	.field => {
839	const field = t.ast.nodeData(node).field;
840
841	const base = try t.transformExpression(decl, field.base) orelse return null;
842
843	// TODO: handle identifier escapes
844	const id = t.ast.tokenSlice(field.field_token);
845	const prop_name_str = try t.addSlicedString(id);
846
847	const string_ref = try t.addInstRef(.{
848	.tag = .string,
849	.data = EncodedData.encode(.string, .{
850	.src_node = decl.nodeOffset(node), // TODO: this is incorrect, there is no node
851	.extra_index = try t.extra.append(t.gpa, SlicedString, prop_name_str),
852	}),
853	});
854
855	return try t.addInstRef(.{
856	.tag = .field,
857	.data = EncodedData.encode(.field, .{
858	.src_node = decl.nodeOffset(node),
859	.extra_index = try t.extra.append(t.gpa, Inst.Bin, .{
860	.lhs = base,
861	.rhs = string_ref,
862	}),
863	}),
864	});
865	},
866
867	.@"if" => {
868	const if_node = t.ast.nodeData(node).@"if";
869	const if_data = t.ast.extraData(if_node.extra, Ast.If);
870
871	const cond = try t.transformExpression(decl, if_node.cond) orelse
872	return null;
873
874	const yes = try t.transformExpressionToBlock(
875	decl,
876	if_data.yes,
877	false,
878	);
879	const no = if (if_data.no.unwrap()) \|no\|
880	try t.transformExpressionToBlock(
881	decl,
882	no,
883	false,
884	)
885	else
886	Inst.Ref.void_value;
887
888	if (yes == .void_value and no == .void_value) {
889	// when both branches do nothing, we can omit the branch,
890	// however we still need to type-check the condition operand
891	_ = try t.addInstRef(.{
892	.tag = .ensure_bool,
893	.data = EncodedData.encode(.ensure_bool, .{
894	.src_node = decl.nodeOffset(node),
895	.operand = cond,
896	}),
897	});
898	return .void_value;
899	}
900
901	return try t.addInstRef(.{
902	.tag = .@"if",
903	.data = EncodedData.encode(.@"if", .{
904	.src_node = decl.nodeOffset(node),
905	.extra_index = try t.extra.append(t.gpa, Inst.If, .{
906	.cond = cond,
907	.yes = yes,
908	.no = no,
909	}),
910	}),
911	});
912	},
913
914	.@"return" => {
915	const ret_node = t.ast.nodeData(node).@"return";
916
917	const ret_block_ptr = decl.return_target orelse {
918	return error.InvalidReturn;
919	};
920
921	const ret_block = if (ret_block_ptr.*) \|block\|
922	block
923	else append_block: {
924	const block = try t.addRootInst(.{
925	.tag = .block,
926	.data = undefined,
927	});
928	ret_block_ptr.* = block;
929	break :append_block block;
930	};
931
932	const operand: Inst.Ref = if (ret_node.expr.unwrap()) \|expr\|
933	try t.transformExpression(decl.withReturn(null), expr) orelse
934	return null
935	else
936	.void_value;
937
938	_ = try t.addInstRef(.{
939	.tag = .@"break",
940	.data = EncodedData.encode(.@"break", .{
941	.src_node = decl.nodeOffset(node),
942	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
943	.block = ret_block,
944	.operand = operand,
945	}),
946	}),
947	});
948
949	return null; // control flow ends
950	},
951
952	.@"break" => {
953	const brk_node = t.ast.nodeData(node).@"break";
954	const label = t.ast.tokenSlice(
955	brk_node.label.unwrap() orelse
956	@panic("TODO: unlabelled breaks"),
957	);
958
959	const block = try decl.findAndMarkBreak(t, label) orelse {
960	return error.UnknownBreakTarget;
961	};
962
963	const operand: Inst.Ref = if (brk_node.value.unwrap()) \|expr\|
964	try t.transformExpression(decl.withReturn(null), expr) orelse
965	return null
966	else
967	.void_value;
968
969	_ = try t.addInstRef(.{
970	.tag = .@"break",
971	.data = EncodedData.encode(.@"break", .{
972	.src_node = decl.nodeOffset(node),
973	.extra_index = try t.extra.append(t.gpa, Inst.Break, .{
974	.block = block,
975	.operand = operand,
976	}),
977	}),
978	});
979
980	return null; // control flow ends
981	},
982
983	else => \|tag\| std.debug.panic("TODO: expr {s}", .{@tagName(tag)}),
984	}
985	}
986
987	fn transformBlockStatement(
988	t: *Transform,
989	decl: BlockContext,
990	node: NodeIndex,
991	is_implicit_return_expression: bool,
992	) !?Inst.Ref {
993	const node_tag = t.ast.nodes.items(.tag)[node];
994
995	const trace = tracer.traceFmt(@src(), "{s}", .{@tagName(node_tag)});
996	defer trace.end();
997
998	switch (node_tag) {
999	.root => unreachable, // cannot appear in a block
1000	.param_decl => unreachable, // cannot appear in a block
1001
1002	.var_decl => {
1003	const var_decl = t.ast.fullVarDecl(node);
1004	const type_ref = if (var_decl.type) \|data_type\|
1005	try t.transformExpression(decl, data_type)
1006	else
1007	null;
1008	const data_ref = try t.transformExpression(decl, var_decl.init) orelse
1009	return null;
1010	const data_coerced_ref = if (type_ref) \|ref\|
1011	try t.addInstRef(.{
1012	.tag = .as,
1013	.data = EncodedData.encode(.as, .{
1014	.src_node = decl.nodeOffset(node),
1015	.extra_index = try t.extra.append(t.gpa, Inst.As, .{
1016	.type = ref,
1017	.operand = data_ref,
1018	}),
1019	}),
1020	})
1021	else
1022	data_ref;
1023
1024	const mem_ref = if (var_decl.is_const)
1025	data_coerced_ref
1026	else
1027	try t.addInstRef(.{
1028	.tag = .memory,
1029	.data = EncodedData.encode(.memory, .{
1030	.src_node = decl.nodeOffset(node),
1031	.operand = data_coerced_ref,
1032	}),
1033	});
1034
1035	try t.scope().symbols.put(t.gpa, var_decl.id, .{
1036	.node = node,
1037	.data = switch (var_decl.is_const) {
1038	true => .{ .immutable = mem_ref },
1039	false => .{ .mutable = mem_ref },
1040	},
1041	});
1042
1043	return mem_ref;
1044	},
1045
1046	.fn_decl,
1047	=> \|tag\| {
1048	std.debug.panic("TODO: block statement {s}", .{@tagName(tag)});
1049	},
1050
1051	.assign,
1052	=> {
1053	const bin = t.ast.nodeData(node).bin;
1054	const lhs = bin.lhs;
1055
1056	if (t.ast.nodeTag(lhs) == .id and
1057	mem.eql(u8, t.ast.basicNodeSlice(lhs), "_"))
1058	{
1059	// discard, instead of appending an .ensure_void check,
1060	// we will add a .discard instruction instead.
1061	// this will emit a warning if given void.
1062	const result = try t.transformExpression(decl, bin.rhs) orelse
1063	return null;
1064	return try t.addInstRef(.{
1065	.tag = .discard,
1066	.data = EncodedData.encode(.discard, .{
1067	.src_node = decl.nodeOffset(node),
1068	.operand = result,
1069	}),
1070	});
1071	} else {
1072	// process this assignment as usual, do not append .ensure_void
1073	return try t.transformExpression(decl, node);
1074	}
1075	},
1076
1077	.add_assign,
1078	.sub_assign,
1079	.mul_assign,
1080	.div_assign,
1081	.rem_assign,
1082	.pow_assign,
1083	.and_assign,
1084	.or_assign,
1085	.bit_and_assign,
1086	.bit_or_assign,
1087	.bit_xor_assign,
1088	.shl_assign,
1089	.shr_assign,
1090	=> {
1091	// do not append a .ensure_void check
1092	return try t.transformExpression(decl, node);
1093	},
1094
1095	else => {
1096	const result = try t.transformExpression(decl, node) orelse
1097	return null;
1098
1099	if (!is_implicit_return_expression and result != .void_value) {
1100	return try t.addInstRef(.{
1101	.tag = .ensure_void,
1102	.data = EncodedData.encode(.ensure_void, .{
1103	.src_node = decl.nodeOffset(node),
1104	.operand = result,
1105	}),
1106	});
1107	}
1108
1109	return result;
1110	},
1111	}
1112	}
1113
1114	// ---
1115
1116	fn addRootInst(t: *Transform, inst: Inst) !Inst.Index {
1117	try t.insts.append(t.gpa, inst);
1118	return @enumFromInt(t.insts.len - 1);
1119	}
1120
1121	fn addInstRef(t: *Transform, inst: Inst) !Inst.Ref {
1122	try t.scratch.ensureUnusedCapacity(t.gpa, 1);
1123	try t.insts.append(t.gpa, inst);
1124	const index: Inst.Index = @enumFromInt(t.insts.len - 1);
1125	t.scratch.appendAssumeCapacity(@intFromEnum(index));
1126	return index.toRef();
1127	}
1128
1129	fn pushScope(t: *Transform) !void {
1130	try t.scopes.append(t.gpa, .{});
1131	}
1132
1133	fn popScope(t: *Transform) void {
1134	t.scope().deinit(t.gpa);
1135	t.scopes.items[t.scopes.items.len - 1] = undefined;
1136	t.scopes.items.len -= 1;
1137	}
1138
1139	fn scope(t: Transform) Scope {
1140	return &t.scopes.items[t.scopes.items.len - 1];
1141	}
1142
1143	fn getInstData(t: Transform, index: Inst.Index) EncodedData {
1144	return &t.insts.items(.data)[@intFromEnum(index)];
1145	}
1146
1147	fn addScratch(t: *Transform, comptime T: type, value: T) !void {
1148	try t.scratch.append(t.gpa, switch (T) {
1149	Inst.Ref, Inst.Index => @intFromEnum(value),
1150	else => @compileError("addScratch not implemented for " ++ @typeName(T)),
1151	});
1152	}
1153
1154	fn scratchToRefList(t: *Transform, scratch_top: usize) !RefList {
1155	const items = t.scratch.items[scratch_top..];
1156	const extra_top = t.extra.list.items.len;
1157	try t.extra.list.appendSlice(t.gpa, t.scratch.items[scratch_top..]);
1158	return .{
1159	.start = @intCast(extra_top),
1160	.len = @intCast(items.len),
1161	};
1162	}
1163
1164	fn scratchToIndexList(t: *Transform, scratch_top: usize) !IndexList {
1165	const items = t.scratch.items[scratch_top..];
1166	const extra_top = t.extra.list.items.len;
1167	try t.extra.list.appendSlice(t.gpa, items);
1168	return .{
1169	.start = @intCast(extra_top),
1170	.len = @intCast(items.len),
1171	};
1172	}
1173
1174	fn lookupId(t: *Transform, id: []const u8) ?Scope.Symbol {
1175	var i = t.scopes.items.len;
1176	while (i > 0) {
1177	i -= 1;
1178	if (t.scopes.items[i].symbols.get(id)) \|sym\| {
1179	return sym;
1180	}
1181	}
1182	return null;
1183	}
1184
1185	fn addNullTerminatedString(t: *Transform, str: []const u8) !NullTerminatedString {
1186	if (str.len == 0) {
1187	return .empty;
1188	}
1189	// loop through strings trying to find an existing match, and dedupe memory
1190	var pos: usize = 0;
1191	while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
1192	const start_index = i - str.len;
1193	if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1194	const string: NullTerminatedString = @enumFromInt(pos);
1195	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1196	return string;
1197	}
1198	}
1199
1200	const string: NullTerminatedString = @enumFromInt(t.string_bytes.items.len);
1201	try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len + 1);
1202	t.string_bytes.appendSliceAssumeCapacity(str);
1203	t.string_bytes.appendAssumeCapacity(0);
1204	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1205	return string;
1206	}
1207
1208	fn addSlicedString(t: *Transform, str: []const u8) !SlicedString {
1209	if (str.len == 0) {
1210	return .{ .start = 0, .len = 0 };
1211	}
1212
1213	// loop through strings trying to find an existing match, and dedupe memory
1214	var pos: usize = 0;
1215	while (mem.indexOfScalarPos(u8, str, pos, 0)) \|i\| : (pos = i + 1) {
1216	const start_index = i - str.len;
1217	if (mem.eql(u8, t.string_bytes.items[start_index..][0..str.len], str)) {
1218	const string: SlicedString = .{
1219	.start = @intCast(start_index),
1220	.len = @intCast(str.len),
1221	};
1222	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1223	return string;
1224	}
1225	}
1226
1227	const string: SlicedString = .{
1228	.start = @intCast(t.string_bytes.items.len),
1229	.len = @intCast(str.len),
1230	};
1231	try t.string_bytes.ensureUnusedCapacity(t.gpa, str.len);
1232	t.string_bytes.appendSliceAssumeCapacity(str);
1233	assert(mem.eql(u8, string.slice(t.string_bytes.items), str));
1234	return string;
1235	}
1236
1237	const Transform = @This();
1238	const std = @import("std");
1239	const mem = std.mem;
1240	const Allocator = mem.Allocator;
1241	const assert = std.debug.assert;
1242
1243	const Teg = @import("Teg.zig");
1244	const Inst = Teg.Inst;
1245	const NullTerminatedString = Teg.NullTerminatedString;
1246	const SlicedString = Teg.SlicedString;
1247	const RefList = Teg.RefList;
1248	const IndexList = Teg.IndexList;
1249	const EncodedData = Teg.EncodedData;
1250	const ExtraIndex = Teg.ExtraIndex;
1251
1252	const dod = @import("dod.zig");
1253	const ExtraStore = dod.ExtraStore(.teg, .@"var");
1254	const Float = dod.Float;
1255
1256	const Tokenizer = @import("Tokenizer.zig");
1257	const Token = Tokenizer.Token;
1258	const ByteOffset = Tokenizer.ByteOffset;
1259
1260	const Ast = @import("Ast.zig");
1261	const Unused = Ast.Unused;
1262	const Node = Ast.Node;
1263	const Data = Node.Data;
1264	const SubList = Ast.SubList;
1265	const NodeIndex = Ast.NodeIndex;
1266	const TokenIndex = Ast.TokenIndex;
1267	const OptionalNodeIndex = Ast.OptionalNodeIndex;
1268	const OptionalTokenIndex = Ast.OptionalTokenIndex;
1269	const OptionalExtraIndex = Ast.OptionalExtraIndex;
1270	const optionalTokenIndex = OptionalTokenIndex.init;
1271	const optionalNodeIndex = OptionalNodeIndex.init;
1272	const optionalExtraIndex = OptionalExtraIndex.init;
1273
1274	const InternPool = @import("InternPool.zig");
1275
1276	const tracer = @import("tracer");

src/compiler/Value.zig created+89

...	@@ -0,0 +1,89 @@
1	ip_index: InternPool.Index,
2
3	pub fn fromInterned(i: InternPool.Index) Value {
4	assert(i != .none);
5	return .{ .ip_index = i };
6	}
7
8	pub fn toIntern(val: Value) InternPool.Index {
9	assert(val.ip_index != .none);
10	return val.ip_index;
11	}
12
13	pub fn toKey(val: Value, ip: *const InternPool) InternPool.Key {
14	return val.toIntern().toKey(ip);
15	}
16
17	pub fn asBool(val: Value) i32 {
18	return switch (val.toIntern()) {
19	.bool_true => true,
20	.bool_false => false,
21	else => unreachable,
22	};
23	}
24
25	pub fn asString(val: Value, ip: *InternPool) InternPool.String.WithLen {
26	const index = val.toIntern();
27	return ip.items.items(.data)[@intFromEnum(index)].decode(
28	.string,
29	ip.items.items(.tag)[@intFromEnum(index)],
30	)
31	.get(ip);
32	}
33
34	pub fn @"type"(val: Value, ip: *InternPool) Value {
35	std.debug.print("typeOfValue {any}\n", .{val.toKey(ip)});
36	const v: InternPool.Index = switch (val.toKey(ip)) {
37	.simple_type, .type_function => .type_type,
38
39	.int32 => .int32_type,
40	.float => .float_type,
41	.string => .string_type,
42	.simple_value => \|v\| switch (v) {
43	.true, .false => .bool_type,
44	.void => .void_type,
45	},
46	.func => \|f\| f.ty,
47	};
48	return Value.fromInterned(v);
49	}
50
51	/// used for validating ensure_void instructions
52	pub fn isEnsureVoid(val: Value, ip: *InternPool) bool {
53	const intern = val.toIntern();
54	const i = @intFromEnum(intern);
55
56	return switch (ip.items.items(.tag)[i]) {
57	.simple_type => switch (ip.items.items(.data)[i].decode(.simple_type, .simple_type)) {
58	.@"opaque", .void => true,
59	else => true,
60	},
61	else => false,
62	};
63	}
64
65	const Value = @This();
66
67	const std = @import("std");
68	const assert = std.debug.assert;
69
70	pub const InternPool = @import("InternPool.zig");
71
72	pub fn fmt(v: Value, ip: *InternPool) Format {
73	return .{ .value = v, .ip = ip };
74	}
75
76	pub const Format = struct {
77	value: Value,
78	ip: *InternPool,
79
80	pub fn format(
81	self: Format,
82	comptime unused_fmt_str: []const u8,
83	_: std.fmt.FormatOptions,
84	writer: anytype,
85	) !void {
86	comptime assert(unused_fmt_str.len == 0);
87	try writer.print("{any}", .{self.value.toKey(self.ip)});
88	}
89	};

src/compiler/dod.zig created+382

...	@@ -0,0 +1,382 @@
1	const extra = @This();
2
3	pub const Mutability = enum { @"const", @"var" };
4
5	fn setAsU32(items: []u32, comptime T: type, index: usize, payload: T) void {
6	const fields = std.meta.fields(@TypeOf(payload));
7	var i = index;
8	inline for (fields) \|field\| {
9	const value = @field(payload, field.name);
10	switch (field.type) {
11	u32 => items[i] = value,
12	i32 => items[i] = @bitCast(value),
13
14	[16]u8 => {
15	const u32s: [4]u32 = @bitCast(value);
16	items[i] = u32s[0];
17	items[i + 1] = u32s[1];
18	items[i + 2] = u32s[2];
19	items[i + 3] = u32s[3];
20	},
21
22	else => brk: {
23	const info = @typeInfo(field.type);
24	if (info == .Enum) {
25	items[i] = @intFromEnum(value);
26	break :brk;
27	}
28
29	const struct_data = info.Struct;
30	if (struct_data.layout == .@"packed") {
31	items[i] = @bitCast(value);
32	break :brk;
33	}
34
35	@call(.always_inline, setAsU32, .{ items, field.type, i, value });
36	},
37	}
38	i += extraSizeOf(field.type);
39	}
40	}
41
42	fn getAsU32(items: []const u32, comptime T: type, index: usize) T {
43	const fields = std.meta.fields(T);
44	var i = index;
45	var result: T = undefined;
46	inline for (fields) \|field\| {
47	@field(result, field.name) = switch (field.type) {
48	u32 => items[i],
49	i32 => @bitCast(items[i]),
50
51	[16]u8 => @bitCast(items[i..][0..4].*),
52
53	else => brk: {
54	const info = @typeInfo(field.type);
55	if (info == .Enum) {
56	break :brk @enumFromInt(items[i]);
57	}
58
59	const struct_data = info.Struct;
60	if (struct_data.layout == .@"packed") {
61	break :brk @bitCast(items[i]);
62	}
63
64	break :brk @call(
65	.always_inline,
66	getAsU32,
67	.{ items, field.type, i },
68	);
69	},
70	};
71	i += extraSizeOf(field.type);
72	}
73	return result;
74	}
75
76	pub const StoreKind = enum {
77	intern_pool,
78	teg,
79	air,
80	tfr,
81
82	pub fn resolveStoreConst(
83	comptime kind: StoreKind,
84	owner: anytype,
85	) *const kind.ResolvedStore(@TypeOf(owner)) {
86	return &owner.extra;
87	}
88
89	pub fn resolveStoreMut(
90	comptime kind: StoreKind,
91	owner: anytype,
92	) *kind.ResolvedStore(@TypeOf(owner)) {
93	return &owner.extra;
94	}
95
96	pub fn ResolvedStore(
97	comptime kind: StoreKind,
98	comptime Owner: type,
99	) type {
100	const ptr = @typeInfo(Owner).Pointer;
101	assert(ptr.size == .One);
102	const mutability = brk: for (@typeInfo(ptr.child).Struct.fields) \|field\| {
103	if (std.mem.eql(u8, field.name, "extra")) {
104	break :brk field.type.mutability;
105	}
106	} else unreachable;
107	return ExtraStore(kind, mutability);
108	}
109	};
110
111	fn ExtraWrap(comptime kind: StoreKind) type {
112	return struct {
113	pub fn Index(comptime T: type) type {
114	return enum(u32) {
115	_,
116
117	// Reads the data from `store.extra` and returns it.
118	pub fn get(index: @This(), owner: anytype) T {
119	return kind.resolveStoreConst(owner).get(T, @intFromEnum(index));
120	}
121
122	/// Returns data with a trailing reader to read data stored immediately after
123	/// the main data structure. What is stored should be documented on `T`
124	pub fn getTrail(index: @This(), owner: anytype) struct {
125	T,
126	kind.ResolvedStore(@TypeOf(owner)).TrailReader,
127	} {
128	const advance = comptime extraSizeOf(T);
129	return .{
130	index.get(owner),
131	.{ .index = @intFromEnum(index) + advance },
132	};
133	}
134	};
135	}
136	};
137	}
138
139	pub fn EncodedData(comptime Tag: type, backing_type: type) type {
140	return enum(backing_type) {
141	_,
142
143	pub fn decode(data: @This(), comptime tag: Tag, proof: Tag) tag.Data() {
144	assert(proof == tag); // incorrect memory access
145	const Data = tag.Data();
146	const info = @typeInfo(Data);
147	if (info == .Enum) {
148	return @enumFromInt(@intFromEnum(data));
149	}
150	if (info == .Struct) {
151	assert(backing_type == u64);
152	var fields: [2]u32 = @bitCast(@intFromEnum(data));
153	return getAsU32(&fields, Data, 0);
154	}
155	return @bitCast(@intFromEnum(data));
156	}
157
158	pub fn encode(comptime tag: Tag, data: tag.Data()) @This() {
159	const Data = tag.Data();
160	const info = @typeInfo(Data);
161	if (info == .Enum) {
162	return @enumFromInt(@intFromEnum(data));
163	}
164	if (info == .Struct) {
165	assert(backing_type == u64);
166	var fields: [2]u32 = undefined;
167	setAsU32(&fields, Data, 0, data);
168	return @enumFromInt(@as(backing_type, @bitCast(fields)));
169	}
170	return @enumFromInt(@as(backing_type, @bitCast(data)));
171	}
172	};
173	}
174
175	/// The number of 'u32's this type consumes in `ip.extra`.
176	pub fn extraSizeOf(comptime T: type) usize {
177	return switch (T) {
178	u32,
179	i32,
180	=> return 1,
181
182	[16]u8 => return 4,
183
184	else => {
185	const info = @typeInfo(T);
186	if (info == .Enum) {
187	comptime assert(@sizeOf(info.Enum.tag_type) <= 32);
188	return 1;
189	}
190
191	const struct_data = info.Struct;
192	if (struct_data.layout == .@"packed") {
193	comptime assert(struct_data.backing_integer == u32);
194	return 1;
195	}
196
197	const fields = std.meta.fields(T);
198	var size: usize = 0;
199	inline for (fields) \|field\| {
200	size += extraSizeOf(field.type);
201	}
202	return size;
203	},
204	};
205	}
206
207	pub fn ExtraStore(comptime set_kind: StoreKind, comptime set_mutability: Mutability) type {
208	return struct {
209	list: List = .{},
210
211	pub const kind = set_kind;
212	pub const mutability = set_mutability;
213
214	const Store = @This();
215
216	const index_holder = ExtraWrap(kind);
217	pub const Index = index_holder.Index;
218
219	pub const List = switch (mutability) {
220	.@"const" => struct {
221	items: []const u32 = &.{},
222
223	pub fn deinit(slice: *@This(), gpa: Allocator) void {
224	gpa.free(slice.items);
225	}
226	},
227	.@"var" => std.ArrayListUnmanaged(u32),
228	};
229
230	pub fn deinit(store: *Store, gpa: Allocator) void {
231	store.list.deinit(gpa);
232	store.* = undefined;
233	}
234
235	pub const TrailReader = struct {
236	index: usize,
237
238	pub fn read(trail: TrailReader, ip: const Store, comptime T: type) T {
239	defer trail.index += comptime extraSizeOf(T);
240	return ip.get(T, trail.index);
241	}
242
243	pub fn readInternPoolIndexSlice(trail: *TrailReader, len: u32) @import("InternPool.zig").Index.Slice {
244	defer trail.index += len;
245	return .{
246	.start = @intCast(trail.index),
247	.len = len,
248	};
249	}
250	};
251
252	pub const TrailWriter = struct {
253	store: *Store,
254	leftover: usize,
255
256	pub fn init(store: *Store, len: usize) TrailWriter {
257	return .{ .store = store, .leftover = len };
258	}
259
260	pub fn end(w: *TrailWriter) void {
261	assert(w.leftover == 0);
262	}
263
264	pub fn writeU32(w: *TrailWriter, value: u32) void {
265	assert(w.leftover > 0);
266	w.ip.extra.appendAssumeCapacity(value);
267	w.leftover -= 1;
268	}
269
270	pub fn writeSlice(w: *TrailWriter, slice: anytype) void {
271	const Child = std.meta.Child(@TypeOf(slice));
272	const child = @typeInfo(Child);
273	comptime assert(Child == u32 or (child == .Enum and child.Enum.tag_type == u32));
274	assert(w.leftover >= slice.len);
275	w.store.list.appendSliceAssumeCapacity(@ptrCast(slice));
276	w.leftover -= slice.len;
277	}
278
279	pub fn writeStruct(w: *TrailWriter, comptime T: type, data: T) void {
280	_ = appendAssumeCapacity(w.store, T, data);
281	}
282	};
283
284	pub fn append(store: *Store, gpa: Allocator, comptime T: type, payload: T) Allocator.Error!Index(T) {
285	comptime assert(mutability == .@"var");
286	try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)));
287	return store.appendAssumeCapacity(T, payload);
288	}
289
290	pub fn appendAssumeCapacity(ip: *Store, comptime T: type, payload: T) Index(T) {
291	comptime assert(mutability == .@"var");
292	const extra_index: u32 = @intCast(ip.list.items.len);
293	ip.list.items.len += extraSizeOf(@TypeOf(payload));
294	ip.set(T, extra_index, payload);
295	return @enumFromInt(extra_index);
296	}
297
298	pub fn appendTrailing(
299	store: *Store,
300	gpa: Allocator,
301	comptime T: type,
302	payload: T,
303	trailing_len: usize,
304	) Allocator.Error!struct { Index(T), TrailWriter } {
305	comptime assert(mutability == .@"var");
306	try store.list.ensureUnusedCapacity(gpa, extraSizeOf(@TypeOf(payload)) + trailing_len);
307	const extra_index: u32 = @intCast(store.list.items.len);
308	store.list.items.len += extraSizeOf(@TypeOf(payload));
309	store.set(T, extra_index, payload);
310	return .{
311	@enumFromInt(extra_index),
312	TrailWriter.init(store, trailing_len),
313	};
314	}
315
316	fn set(store: *Store, comptime T: type, index: usize, payload: T) void {
317	comptime assert(mutability == .@"var");
318	setAsU32(store.list.items, T, index, payload);
319	}
320
321	fn get(store: *const Store, comptime T: type, index: usize) T {
322	return getAsU32(store.list.items, T, index);
323	}
324
325	pub fn toOwnedStore(store: *Store, gpa: Allocator) !ExtraStore(set_kind, .@"const") {
326	comptime assert(mutability == .@"var");
327	return .{ .list = .{ .items = try store.list.toOwnedSlice(gpa) } };
328	}
329
330	pub fn constant(store: *const Store) ExtraStore(set_kind, .@"const") {
331	return .{ .list = .{ .items = store.list.items } };
332	}
333	};
334	}
335
336	pub const Float = struct {
337	low: u32,
338	high: u32,
339
340	pub const pure_nan = Float{ .low = 0x7f800000, .high = 0 };
341
342	pub fn purifyNaN(f: f64) f64 {
343	if (std.math.isNan(f)) {
344	return comptime Float.pure_nan.decode();
345	}
346	return f;
347	}
348
349	pub fn assertPure(f: f64) f64 {
350	if (std.math.isNan(f)) {
351	assert(Float.pure_nan.eql(encode(f)));
352	}
353	return f;
354	}
355
356	pub fn encode(decoded: f64) Float {
357	if (std.math.isNan(decoded)) {
358	return pure_nan;
359	}
360	const as_bytes: [2]u32 = @bitCast(decoded);
361	return .{
362	.low = as_bytes[0],
363	.high = as_bytes[1],
364	};
365	}
366
367	pub fn decode(encoded: Float) f64 {
368	const as_bytes: [2]u32 = .{
369	encoded.low,
370	encoded.high,
371	};
372	return @bitCast(as_bytes);
373	}
374
375	pub fn eql(a: Float, b: Float) bool {
376	return a.low == b.low and a.high == b.high;
377	}
378	};
379
380	const std = @import("std");
381	const assert = std.debug.assert;
382	const Allocator = std.mem.Allocator;

src/compiler/main.zig created+180

...	@@ -0,0 +1,180 @@
1	const std = @import("std");
2	const Tokenizer = @import("Tokenizer.zig");
3	const Parse = @import("Parse.zig");
4	const Transform = @import("Transform.zig");
5	const Comp = @import("Comp.zig");
6
7	const T = @import("tracer");
8	pub const tracer_impl = @field(T, "none");
9
10	pub fn main() !void {
11	std.debug.print("\r\x1b[K", .{});
12
13	try T.init();
14	defer T.deinit();
15
16	try T.init_thread(null);
17	defer T.deinit_thread();
18
19	var gpa: std.heap.GeneralPurposeAllocator(.{
20	.stack_trace_frames = 100,
21	}) = .{};
22	defer _ = gpa.deinit();
23	const alloc = gpa.allocator();
24
25	var maybe_file_path: ?[]const u8 = null;
26	defer if (maybe_file_path) \|path\| alloc.free(path);
27
28	var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
29	const cwd = try std.posix.getcwd(&cwd_buf);
30
31	var exe_name: []const u8 = undefined;
32
33	{
34	var args = try std.process.argsWithAllocator(alloc);
35	defer args.deinit();
36
37	exe_name = try alloc.dupe(u8, std.fs.path.basename(args.next() orelse "cli"));
38	errdefer alloc.free(exe_name);
39
40	while (args.next()) \|arg\| {
41	maybe_file_path = try std.fs.path.resolve(alloc, &.{ cwd, arg });
42	break;
43	}
44	}
45	defer alloc.free(exe_name);
46
47	const file_path = maybe_file_path orelse {
48	std.debug.print("Usage: {s} <input file>\n", .{exe_name});
49	std.process.exit(1);
50	};
51
52	var comp: Comp = .{ .gpa = T.tracingAllocator(&alloc) };
53	try comp.init();
54	defer comp.deinit();
55	const main_file = try comp.getFile(file_path);
56
57	const teg = try main_file.getTeg(comp.gpa);
58	{
59	const stderr = std.io.getStdErr().writer();
60	try teg.dumpToWriter(stderr);
61	}
62
63	try comp.semaEntryPoint(main_file);
64
65	const output = std.io.getStdOut().writer();
66	var bufw = std.io.bufferedWriter(output);
67	const writer = bufw.writer();
68	try comp.link(writer);
69	try bufw.flush();
70
71	// var error_pos: u32 = 0;
72	// var tokens = Tokenizer.tokenize(source, alloc.allocator(), &error_pos) catch \|err\| {
73	// var line: usize = 1;
74	// var byte: usize = 0;
75	// var line_data: []const u8 = undefined;
76	// while (true) {
77	// const b = std.mem.indexOfScalarPos(u8, source, byte, '\n') orelse {
78	// line_data = source[byte..];
79	// break;
80	// };
81	// if (b > error_pos) {
82	// line_data = source[byte..b];
83	// break;
84	// }
85	// byte = b + 1;
86	// line += 1;
87	// }
88	// var col: usize = 1;
89	// var arrow_pos: usize = 0;
90	// while (byte < error_pos) {
91	// const b = source[byte];
92	// std.debug.assert(b != '\n');
93	// byte += std.unicode.utf8ByteSequenceLength(b) catch break;
94	// col += 1;
95
96	// arrow_pos += 1; // TODO: grapheme cluster
97	// }
98
99	// const stderr = std.io.getStdErr();
100	// const config = std.io.tty.detectConfig(stderr);
101	// var buf = std.io.bufferedWriter(stderr.writer());
102	// const writer = buf.writer();
103
104	// try config.setColor(writer, .red);
105	// try writer.writeAll("error");
106	// try config.setColor(writer, .bright_black);
107	// try writer.writeAll(": ");
108	// try config.setColor(writer, .reset);
109	// switch (err) {
110	// error.OutOfMemory => try writer.writeAll("Out of Memory"),
111	// error.InvalidUtf8 => try writer.writeAll("Invalid UTF-8"),
112	// error.InvalidSyntax => try writer.writeAll("Syntax Error"),
113	// error.TrailingNumericSeparator => try writer.writeAll("Trailing Numeric Separator"),
114	// error.UnterminatedString => try writer.writeAll("Unterminated String"),
115	// error.InvalidEscapeSequence => try writer.writeAll("Invalid Escape Sequence"),
116	// error.InvalidEscapeCodepoint => try writer.writeAll("Invalid Escape Codepoint"),
117	// error.GitConflict => try writer.writeAll("Found Git Conflict"),
118	// }
119	// try writer.print(" at {d}:{d}\n ", .{ line, col });
120	// try writer.writeAll(line_data);
121	// try writer.writeAll("\n ");
122	// try writer.writeByteNTimes(' ', col);
123
124	// try config.setColor(writer, .bright_green);
125	// try writer.writeAll("^");
126	// try config.setColor(writer, .reset);
127	// try writer.writeAll("\n");
128
129	// try buf.flush();
130	// return err;
131	// };
132	// defer tokens.deinit(alloc.allocator());
133
134	// // std.debug.print("tokens.len = {d}\n", .{tokens.len});
135	// // std.debug.print("token list bytes = {d}\n", .{tokens.toMultiArrayList().allocatedBytes().len});
136	// // std.debug.print("source bytes = {d}\n", .{source.len});
137	// // for (
138	// // tokens.items(.tag),
139	// // tokens.items(.offset),
140	// // ) \|tag, offset\| {
141	// // std.debug.print(".tag = {s} .offset = {d}\n", .{
142	// // @tagName(tag),
143	// // offset,
144	// // });
145	// // }
146
147	// const ast = ast: {
148	// var p = Parse.init(alloc.allocator(), source, tokens);
149	// errdefer p.deinit();
150	// try p.nodes.ensureUnusedCapacity(alloc.allocator(), 100);
151	// p.parseRoot() catch \|err\| {
152	// const tok = p.fullToken(p.pos);
153
154	// std.debug.print("ast_error = {?}\n", .{p.ast_error});
155
156	// std.debug.print("tok: {s} - '{s}' - {d}\n", .{
157	// @tagName(tok.tag),
158	// tok.slice(p.source),
159	// tok.offset,
160	// });
161
162	// return err;
163	// };
164	// break :ast try p.finalize(tokens);
165	// };
166
167	// try ast.dumpToWriter(std.io.getStdOut().writer());
168
169	// const teg = try Transform.transform(ast, alloc.allocator());
170	// std.debug.print("teg.insts.len = {d}\n", .{teg.insts.len});
171	// std.debug.print("teg.string_bytes.len = {d}\n", .{teg.string_bytes.len});
172	// std.debug.print("teg.extra.len = {d}\n", .{teg.extra.len});
173
174	// for (teg.insts.items(.tag), teg.insts.items(.data)) \|tag, data\| {
175	// std.debug.print("{s} - {}\n", .{
176	// @tagName(tag),
177	// data,
178	// });
179	// }
180	}

src/compiler/unicode_props.zig created+103

...	@@ -0,0 +1,103 @@
1	//! Wrapper around the 'zg' PropsData library.
2	const std = @import("std");
3	const PropsData = @import("zg:PropsData");
4	const T = @import("tracer");
5
6	var props: ?PropsData = undefined;
7	var props_memory: []u8 = undefined;
8
9	const raw_allocator = if (@import("builtin").target.isWasm())
10	std.heap.wasm_allocator
11	else
12	std.heap.page_allocator;
13
14	pub fn init() !void {
15	if (props != null) return;
16
17	const trace = T.trace(@src());
18	defer trace.end();
19
20	// We know exactly how much this uses, so let's preallocate these pages and use a FBA.
21	const allocation_size = 8704 + 34560 + 8704 + 17664 + 8704 + 25856;
22
23	if (allocation_size == 0) {
24	var logging_alloc = std.heap.loggingAllocator(std.heap.page_allocator);
25	props = try PropsData.init(logging_alloc.allocator());
26	std.log.info("please write the above numbers into unicode.zig to prealloc this", .{});
27	std.process.exit(1);
28	}
29
30	props_memory = try raw_allocator.alloc(u8, allocation_size);
31	errdefer std.heap.page_allocator.free(props_memory);
32
33	var fba = std.heap.FixedBufferAllocator.init(props_memory);
34	props = try PropsData.init(fba.allocator());
35	std.debug.assert(fba.end_index == allocation_size);
36	}
37
38	pub fn deinit() void {
39	std.debug.assert(props != null);
40	raw_allocator.free(props_memory);
41	props = null;
42	props_memory = undefined;
43	}
44
45	/// True if `cp` is a mathematical symbol.
46	pub inline fn isMath(cp: u21) bool {
47	return props.?.isMath(cp);
48	}
49
50	/// True if `cp` is an alphabetic character.
51	pub inline fn isAlphabetic(cp: u21) bool {
52	return props.?.isAlphabetic(cp);
53	}
54
55	/// True if `cp` is a valid identifier start character.
56	pub inline fn isIdStart(cp: u21) bool {
57	return props.?.isIdStart(cp);
58	}
59
60	/// True if `cp` is a valid identifier continuation character.
61	pub inline fn isIdContinue(cp: u21) bool {
62	return props.?.isIdContinue(cp);
63	}
64
65	/// True if `cp` is a valid extended identifier start character.
66	pub inline fn isXidStart(cp: u21) bool {
67	return props.?.isXidStart(cp);
68	}
69
70	/// True if `cp` is a valid extended identifier continuation character.
71	pub inline fn isXidContinue(cp: u21) bool {
72	return props.?.isXidContinue(cp);
73	}
74
75	/// True if `cp` is a whitespace character.
76	pub inline fn isWhitespace(cp: u21) bool {
77	return props.?.isWhitespace(cp);
78	}
79
80	/// True if `cp` is a hexadecimal digit.
81	pub inline fn isHexDigit(cp: u21) bool {
82	return props.?.isHexDigit(cp);
83	}
84
85	/// True if `cp` is a diacritic mark.
86	pub inline fn isDiacritic(cp: u21) bool {
87	return props.?.isDiacritic(cp);
88	}
89
90	/// True if `cp` is numeric.
91	pub inline fn isNumeric(cp: u21) bool {
92	return props.?.isNumeric(cp);
93	}
94
95	/// True if `cp` is a digit.
96	pub inline fn isDigit(cp: u21) bool {
97	return props.?.isDigit(cp);
98	}
99
100	/// True if `cp` is decimal.
101	pub inline fn isDecimal(cp: u21) bool {
102	return props.?.isDecimal(cp);
103	}

src/compiler/util.zig created+20

...	@@ -0,0 +1,20 @@
1	/// TODO: move to io utility file
2	pub fn detectTTYFromWriter(writer: anytype) std.io.tty.Config {
3	if (@hasField(@TypeOf(writer), "unbuffered_writer")) {
4	return detectTTYFromWriter(writer.unbuffered_writer);
5	}
6	if (@TypeOf(writer) == std.fs.File.Writer) {
7	return std.io.tty.detectConfig(writer.context);
8	}
9	if (@TypeOf(writer) == std.io.AnyWriter) {
10	const writeFn = comptime (std.fs.File{ .handle = undefined })
11	.writer().any().writeFn;
12	if (writer.writeFn == writeFn) {
13	const file: *const std.fs.File = @alignCast(@ptrCast(writer.context));
14	return std.io.tty.detectConfig(file.*);
15	}
16	}
17	return .no_color;
18	}
19
20	const std = @import("std");

src/wasm/enum_codegen.zig created+28

...	@@ -0,0 +1,28 @@
1	const std = @import("std");
2	const Tokenizer = @import("Tokenizer.zig");
3	const Ast = @import("Ast.zig");
4	const Teg = @import("Teg.zig");
5	// const InternPool = @import("InternPool.zig");
6
7	pub fn main() !void {
8	var buf = std.io.bufferedWriter(std.io.getStdOut().writer());
9	defer buf.flush() catch {};
10	const w = buf.writer();
11
12	inline for (.{
13	.{ "TokenTag", Tokenizer.Token.Tag },
14	.{ "NodeTag", Ast.Node.Tag },
15	.{ "TegInstTag", Teg.Inst.Tag },
16	.{ "TegInstRef", Teg.Inst.Ref },
17	// .{ "InternPoolIndex", InternPool.Index },
18	}) \|decl\| {
19	const name, const enum_type = decl;
20
21	try w.print("export const {s} = {{\n", .{name});
22	for (std.enums.values(enum_type)) \|tag\| {
23	try w.print(" {d}: \"{s}\",\n", .{ @intFromEnum(tag), @tagName(tag) });
24	}
25	try w.writeAll("};\n");
26	try w.writeAll("\n");
27	}
28	}

src/wasm/wasm_api.zig created+32

...	@@ -0,0 +1,32 @@
1	//! A subset of the compiler is exposed to wasm
2	//! This powers the playground
3
4	extern fn iterateTokenCallback(
5	tag: i32,
6	offset: ByteOffset,
7	) void;
8
9	export fn iterateTokens(
10	source_bytes: [*]const u8,
11	source_len: u32,
12	) callconv(.C) i32 {
13	unicode_props.init() catch @panic("unicode_props.init failed");
14
15	var tokenizer: Tokenizer = .{ .code = source_bytes[0..source_len], .pos = 0 };
16	tokenizer.skipWhitespace();
17	while (tokenizer.next() catch {
18	return @intCast(tokenizer.pos);
19	}) \|token\| {
20	iterateTokenCallback(@intFromEnum(token.tag), token.offset);
21	tokenizer.skipWhitespace();
22	}
23
24	return -1;
25	}
26
27	const std = @import("std");
28	const wasm_allocator = std.heap.wasm_allocator;
29	const Tokenizer = @import("Tokenizer.zig");
30	const Token = Tokenizer.Token;
31	const ByteOffset = Tokenizer.ByteOffset;
32	const unicode_props = @import("unicode_props.zig");

author	dave caruso <me@paperdave.net> 2024-05-16 17:46:34-07:00
committer	GitHub <noreply@github.com> 2024-05-16 17:46:34-07:00
log	634a70376c0023f46a6aae6611ed672397eae557
tree	da7f7daddfe6575ce2771b5e1253ffda833a0910
parent	5a950c1982ddcab8e8d84ec27310eb1e0afdeb1f
signature	Signed by PGP key B5690EEEBB952194

rewrite entire pipeline in zig, implement sema (#30)